From 94b49c89987c6fab2e4f0793b611bee2a90797fa Mon Sep 17 00:00:00 2001 From: Josh Date: Tue, 28 May 2024 14:55:47 -0700 Subject: [PATCH] add elasticsearch_index_name to config --- ci/ci.sh | 3 ++ .../main/java/jpl/cws/task/CwsInstaller.java | 26 ++++++++++++-- dev.sh | 29 ++++++++-------- install/cws-ui/cws-ui.properties | 1 + install/installerPresets.properties | 1 + install/logging/cws-logstash.conf | 10 +++--- utils.sh | 34 ++++++++++--------- 7 files changed, 67 insertions(+), 37 deletions(-) diff --git a/ci/ci.sh b/ci/ci.sh index 297156a1..083e11e2 100755 --- a/ci/ci.sh +++ b/ci/ci.sh @@ -105,6 +105,7 @@ project_webapp_root=proj cws_enable_cloud_autoscaling=n elasticsearch_protocol=${ES_PROTOCOL} elasticsearch_host=${ES_HOST} +elasticsearch_index_name=${ES_INDEX_NAME} elasticsearch_use_auth=${ES_USE_AUTH} elasticsearch_port=${ES_PORT} elasticsearch_username=${ES_USERNAME} @@ -115,6 +116,7 @@ default_cws_ldap_url=ldap://localhost:389 cws_ldap_url=ldap://localhost:389 default_elasticsearch_use_auth=n aws_cloudwatch_endpoint=monitoring.us-west-1.amazonaws.com +default_elasticsearch_index_name default_elasticsearch_port=9200 default_aws_cloudwatch_endpoint=monitoring.us-west-1.amazonaws.com aws_sqs_dispatcher_msgFetchLimit=1 @@ -167,6 +169,7 @@ email_subject=[CWS] You have been assigned a task (CWS_TASK_NAME) email_body=fn:CWS_USER_FIRSTNAME
ln:CWS_USER_LASTNAME,
tn:(CWS_TASK_NAME), em:CWS_USER_EMAIL elasticsearch_protocol=${ES_PROTOCOL} elasticsearch_host=${ES_HOST} +elasticsearch_index_name=${ES_INDEX_NAME} elasticsearch_use_auth=${ES_USE_AUTH} elasticsearch_port=${ES_PORT} elasticsearch_username=${ES_USERNAME} diff --git a/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java b/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java index 71da8ac9..5b4093fb 100755 --- a/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java +++ b/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java @@ -222,6 +222,7 @@ public class CwsInstaller { private static String elasticsearch_protocol_init; private static String elasticsearch_host; private static String elasticsearch_host_init; + private static String elasticsearch_index_name; private static String elasticsearch_port; private static String elasticsearch_use_auth; private static String elasticsearch_username; @@ -1368,6 +1369,21 @@ private static void setupElasticsearch() { log.debug("elasticsearch_port: " + elasticsearch_port); + // PROMPT USER FOR ELASTICSEARCH INDEX + elasticsearch_index_name = getPreset("elasticsearch_index_name"); + + if (elasticsearch_index_name == null) { + elasticsearch_index_name = getPreset("default_elasticsearch_index_name"); + } + + if (cws_installer_mode.equals("interactive")) { + elasticsearch_index_name = readLine("Enter the Elasticsearch Index. " + + "Default is " + elasticsearch_index_name + ": ", elasticsearch_index_name); + } + + log.debug("elasticsearch_index_name: " + elasticsearch_index_name); + + // PROMPT USER ELASTICSEARCH AUTH elasticsearch_use_auth = getPreset("elasticsearch_use_auth"); @@ -1783,7 +1799,8 @@ private static void showInstallationInfo() { print("...................................................................................."); print("Elasticsearch Protocol = " + elasticsearch_protocol); print("Elasticsearch Host = " + elasticsearch_host); - print("Elasticsearch Port = " + elasticsearch_port); + print("Elasticsearch Index Name = " + elasticsearch_index_name); + print("Elasticsearch Port = " + elasticsearch_port); if (elasticsearch_use_auth.equalsIgnoreCase("Y")) { print("Elasticsearch User = " + elasticsearch_username); print("Elasticsearch Password = ****** (hidden) "); @@ -2442,6 +2459,7 @@ private static int validateElasticsearch() { print(" [ELASTICSEARCH]: Configuration Details"); print(" elasticsearch_protocol=" + elasticsearch_protocol_init + " "); print(" elasticsearch_host=" + elasticsearch_host_init + " "); + print(" elasticsearch_index_name=" + elasticsearch_index_name + " "); print(" elasticsearch_port=" + elasticsearch_port + " "); print(" ........................................................................................."); print(""); @@ -2964,6 +2982,7 @@ private static void updateCwsUiProperties() throws IOException { content = content.replace("__CWS_CONSOLE_SSL_PORT__", cws_console_ssl_port); content = content.replace("__CWS_ES_PROTOCOL__", elasticsearch_protocol); content = content.replace("__CWS_ES_HOST__", elasticsearch_host); + content = content.replace("__CWS_ES_INDEX_NAME__", elasticsearch_index_name); content = content.replace("__CWS_ES_PORT__", elasticsearch_port); content = content.replace("__CWS_ES_USE_AUTH__", elasticsearch_use_auth); content = content.replace("__CWS_ENABLE_CLOUD_AUTOSCALING__", cws_enable_cloud_autoscaling); @@ -3086,8 +3105,9 @@ private static void updateCwsUiConfig() throws IOException { content = getFileContents(path); content = content.replace("__ES_PROTOCOL__", elasticsearch_protocol); content = content.replace("__ES_HOST__", elasticsearch_host); + content = content.replace("__ES_INDEX_NAME__", elasticsearch_index_name); content = content.replace("__ES_PORT__", elasticsearch_port); - content = content.replace("__ES_USE_AUTH__", elasticsearch_use_auth); + content = content.replace("__ES_USE_AUTH__", elasticsearch_use_auth); if (elasticsearch_use_auth.equalsIgnoreCase("Y")) { content = content.replace("__ES_USERNAME__", elasticsearch_username); content = content.replace("__ES_PASSWORD__", elasticsearch_password); @@ -3330,6 +3350,7 @@ private static void installLogstash() throws IOException { logstashContent = logstashContent.replace("__CWS_ES_PROTOCOL__", elasticsearch_protocol); logstashContent = logstashContent.replace("__CWS_ES_HOST__", elasticsearch_host); + logstashContent = logstashContent.replace("__CWS_ES_INDEX_NAME__", elasticsearch_index_name); logstashContent = logstashContent.replace("__CWS_ES_PORT__", elasticsearch_port); if (elasticsearch_use_auth.equalsIgnoreCase(("Y"))) { // Construct the auth config for logstash @@ -3415,6 +3436,7 @@ private static void writeOutConfigurationFile() throws IOException { setPreset("cws_token_expiration_hours", cws_token_expiration_hours); setPreset("elasticsearch_protocol", elasticsearch_protocol); setPreset("elasticsearch_host", elasticsearch_host); + setPreset("elasticsearch_index_name", elasticsearch_index_name); setPreset("elasticsearch_port", elasticsearch_port); setPreset("elasticsearch_use_auth", elasticsearch_use_auth); setPreset("elasticsearch_username", elasticsearch_username); diff --git a/dev.sh b/dev.sh index 4e0116b2..4329efcd 100755 --- a/dev.sh +++ b/dev.sh @@ -14,20 +14,21 @@ DB_USER=${7} DB_PASS=${8} ES_PROTOCOL=${9} ES_HOST=${10} -ES_PORT=${11} -ES_USE_AUTH=${12} -ES_USERNAME=${13} -ES_PASSWORD=${14} -ENABLE_CLOUD_AS=${15} -SECURITY_SCHEME=${16} -THIS_HOSTNAME=${17} -NOTIFICATION_EMAILS=${18} -ADMIN_FIRSTNAME=${19} -ADMIN_LASTNAME=${20} -ADMIN_EMAIL=${21} -NUM_WORKERS=${22} -WORKER_MAX_NUM_RUNNING_PROCS=${23} -WORKER_ABANDONED_DAYS=${24} +ES_INDEX_NAME=${11} +ES_PORT=${12} +ES_USE_AUTH=${13} +ES_USERNAME=${14} +ES_PASSWORD=${15} +ENABLE_CLOUD_AS=${16} +SECURITY_SCHEME=${17} +THIS_HOSTNAME=${18} +NOTIFICATION_EMAILS=${19} +ADMIN_FIRSTNAME=${20} +ADMIN_LASTNAME=${21} +ADMIN_EMAIL=${22} +NUM_WORKERS=${23} +WORKER_MAX_NUM_RUNNING_PROCS=${24} +WORKER_ABANDONED_DAYS=${25} source ${ROOT}/utils.sh diff --git a/install/cws-ui/cws-ui.properties b/install/cws-ui/cws-ui.properties index 57600b5d..d2f1c44c 100755 --- a/install/cws-ui/cws-ui.properties +++ b/install/cws-ui/cws-ui.properties @@ -18,6 +18,7 @@ cws.db.password=__CWS_DB_PASSWORD__ cws.elasticsearch.protocol=__CWS_ES_PROTOCOL__ cws.elasticsearch.hostname=__CWS_ES_HOST__ +cws.elasticsearch.index=__CWS_ES_INDEX_NAME__ cws.elasticsearch.port=__CWS_ES_PORT__ cws.elasticsearch.use.auth=__CWS_ES_USE_AUTH__ cws.elasticsearch.username=__CWS_ES_USERNAME__ diff --git a/install/installerPresets.properties b/install/installerPresets.properties index c3a12470..1c4fdf61 100644 --- a/install/installerPresets.properties +++ b/install/installerPresets.properties @@ -15,6 +15,7 @@ default_cws_token_expiration_hours=24 default_smtp_hostname=smtp.localhost default_smtp_port=25 default_elasticsearch_use_auth=n +default_elasticsearch_index_name=cws default_elasticsearch_port=9200 default_user_provided_logstash=n default_history_level=full diff --git a/install/logging/cws-logstash.conf b/install/logging/cws-logstash.conf index f2efd5fb..0a805a25 100644 --- a/install/logging/cws-logstash.conf +++ b/install/logging/cws-logstash.conf @@ -22,7 +22,7 @@ filter { if "_grokparsefailure" in [tags] { drop { } } - + grok { match => [ "message", '%{TIMESTAMP_ISO8601:logdate} %{DATA:logLevel} +\[%{DATA:threadName}\] .* - \[%{DATA:cwsHost}\]\[%{DATA:cwsWorkerId}\]\[%{DATA:procDefKey}\]\[%{DATA:procInstId}\]\[%{DATA:actInstId}\] %{GREEDYDATA:msgBody}', @@ -32,20 +32,20 @@ filter { "message", '%{GREEDYDATA:msgBody}' ] } - + date { match => [ "logdate", "yyyy-MM-dd HH:mm:ss,SSS" ] timezone => "UTC" locale => en remove_field => [ "logdate" ] } - + if [logLevel] == "WARNING" { mutate { update => { "logLevel" => "WARN" } } } - + if [logLevel] == "SEVERE" { mutate { update => { "logLevel" => "ERROR" } @@ -57,7 +57,7 @@ output { elasticsearch { hosts => ["__CWS_ES_PROTOCOL__://__CWS_ES_HOST__:__CWS_ES_PORT__"] ilm_enabled => false - index => "logstash-%{+YYYY.MM.dd}" + index => "__CWS_ES_INDEX_NAME__-logstash-%{+YYYY.MM.dd}" ssl => __LOGSTASH_ES_USE_SSL__ __LOGSTASH_ES_USERNAME__ __LOGSTASH_ES_PASSWORD__ diff --git a/utils.sh b/utils.sh index a1c4c54d..26cbe019 100755 --- a/utils.sh +++ b/utils.sh @@ -191,22 +191,23 @@ function auto_conf_data () { DB_PASS=${9} ES_PROTOCOL=${10} ES_HOST=${11} - ES_PORT=${12} - ES_USE_AUTH=${13} - ES_USERNAME=${14} - ES_PASSWORD=${15} - ENABLE_CLOUD_AS=${16} - SECURITY_SCHEME=${17} - THIS_HOSTNAME=${18} - NOTIFICATION_EMAILS=${19} - ADMIN_FIRSTNAME=${20} - ADMIN_LASTNAME=${21} - ADMIN_EMAIL=${22} - NUM_WORKERS=${23} - WORKER_MAX_NUM_RUNNING_PROCS=${24} - WORKER_ABANDONED_DAYS=${25} - - OUTPUT_FILE=${26} + ES_INDEX_NAME=${12} + ES_PORT=${13} + ES_USE_AUTH=${14} + ES_USERNAME=${15} + ES_PASSWORD=${16} + ENABLE_CLOUD_AS=${17} + SECURITY_SCHEME=${18} + THIS_HOSTNAME=${19} + NOTIFICATION_EMAILS=${20} + ADMIN_FIRSTNAME=${21} + ADMIN_LASTNAME=${22} + ADMIN_EMAIL=${23} + NUM_WORKERS=${24} + WORKER_MAX_NUM_RUNNING_PROCS=${25} + WORKER_ABANDONED_DAYS=${26} + + OUTPUT_FILE=${27} source ${ROOT}/utils.sh @@ -281,6 +282,7 @@ function auto_conf_data () { database_password=${DB_PASS} elasticsearch_protocol=${ES_PROTOCOL} elasticsearch_host=${ES_HOST} + elasticsearch_index_name=${ES_INDEX_NAME} elasticsearch_port=${ES_PORT} elasticsearch_use_auth=${ES_USE_AUTH} elasticsearch_username=${ES_USERNAME}