Add streamlit data visualisation

.gitignore

@@ -48,3 +48,10 @@ hs_err_pid*
 # IntelliJ files
 .idea/
 *.iml
+
+# python
+__pycache__
+
+# prevent notebooks from being checked in
+*.ipynb
+.ipynb_checkpoints

core/Dockerfile.emap-portal

@@ -0,0 +1,10 @@
+FROM nginx:otel
+RUN apt update && \
+    apt install -y apache2-utils && \
+    apt clean
+COPY core/emap-portal/nginx.conf /etc/nginx/
+COPY core/emap-portal/conf.d/ /etc/nginx/conf.d/
+COPY core/emap-portal/www/* /usr/share/nginx/html/
+RUN --mount=type=secret,id=portal-build-secrets \
+    . /run/secrets/portal-build-secrets && \
+    htpasswd -b -B -c /etc/nginx/conf.d/htpasswd "$PORTAL_USERNAME" "$PORTAL_PASSWORD"

core/docker-compose.yml

@@ -54,4 +54,25 @@ services:
     restart: on-failure
     depends_on:
        - cassandra
+  emap-portal:
+    build:
+      context: ..
+      dockerfile: core/Dockerfile.emap-portal
+      args:
+        HTTP_PROXY: ${HTTP_PROXY}
+        http_proxy: ${http_proxy}
+        HTTPS_PROXY: ${HTTPS_PROXY}
+        https_proxy: ${https_proxy}
+      secrets:
+        - portal-build-secrets
+    env_file:
+      - ../../config/portal-config-envs
+    ports:
+      - "${PORTAL_PORT}:80"
+
+
+
+secrets:
+  portal-build-secrets:
+    file: ../../config/portal-config-envs
 

core/emap-portal/conf.d/default.conf

@@ -0,0 +1,43 @@
+server {
+    listen       80;
+    listen  [::]:80;
+    server_name  localhost;
+    # nginx is behind docker, so the browser is using a different port number which nginx doesn't know about.
+    # Use relative redirects to avoid redirecting to port 80. (301s are used when trailing slashes are omitted)
+    absolute_redirect off;
+    auth_basic           "Administrator’s Area";
+    auth_basic_user_file conf.d/htpasswd;
+
+
+    access_log  /var/log/nginx/host.access.log  main;
+
+    #error_page  404              /404.html;
+
+    # redirect server error pages to the static page /50x.html
+    #
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+
+    location / {
+        root   /usr/share/nginx/html;
+        include conf.d/shared/shared_location_config.conf;
+    }
+
+    location /glowroot/ {
+        include conf.d/shared/shared_location_config.conf;
+        proxy_pass   http://glowroot-central:4000/;
+        sub_filter 'href="/' 'href="/glowroot/';
+        sub_filter 'src="/' 'src="/glowroot/';
+    }
+
+    location /streamlit/ {
+        include conf.d/shared/shared_location_config.conf;
+        proxy_pass   http://streamlit:8501/streamlit/;
+        proxy_buffering off;
+
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+    }
+}

core/emap-portal/conf.d/shared/shared_location_config.conf

@@ -0,0 +1,11 @@
+sub_filter_once off;
+proxy_redirect off;
+proxy_set_header Host $host;
+proxy_set_header X-Real-IP $remote_addr;
+proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+proxy_set_header X-Forwarded-Proto $scheme;
+proxy_set_header Accept-Encoding ""; # turn off gzip for upstream so rewriting can work
+# needed for websockets
+proxy_http_version 1.1;
+proxy_read_timeout 86400;
+proxy_send_timeout 3600;

core/emap-portal/nginx.conf

@@ -0,0 +1,50 @@
+
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log debug;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    include /etc/nginx/conf.d/*.conf;
+}
+
+# pure TCP proxy?
+#
+# stream {
+#     upstream backend {
+#         server backend-server:12345;
+#     }
+#
+#     server {
+#         listen 12345;
+#         proxy_pass backend;
+# Allow specific IP addresses
+#     allow 192.168.1.1;  # Replace with the allowed IP address
+#     allow 192.168.1.2;  # Add more allowed IP addresses as needed
+#     deny all;           # Deny all other IP addresses
+
+#     }
+# }

core/emap-portal/www/index.html

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Emap admin page</title>
+</head>
+<body>
+You can access various Emap admin/visualisation/monitoring services:
+
+<ul>
+    <li><a href="/streamlit">streamlit</a>
+    <!-- experimental - still accessible directly so don't forget to keep auth enabled in the apps themselves! -->
+    <li><a href="/glowroot">Glowroot</a> (experimental)
+</ul>
+
+</body>
+</html>

emap-setup/emap_runner/docker/docker_runner.py

@@ -108,6 +108,8 @@ def docker_compose_paths(self) -> List[Path]:
             paths.append(Path(self.emap_dir, "waveform-reader", "docker-compose.yml"))
             if self.use_fake_waveform:
                 paths.append(Path(self.emap_dir, "waveform-generator", "docker-compose.yml"))
+        if self.config.get("monitoring", "use_streamlit"):
+            paths.append(Path(self.emap_dir, "monitoring", "docker-compose.yml"))
 
         # allow for hoover and to be optional compose path
         if "hoover" in self.config["repositories"]:

emap-setup/emap_runner/global_config.py

@@ -22,7 +22,8 @@ class GlobalConfiguration(dict):
         "glowroot",
         "common",
         "fake_uds",
-        "waveform"
+        "waveform",
+        "monitoring",
     )
 
     def __init__(self, filepath: Path):

emap-setup/global-configuration-EXAMPLE.yaml

@@ -109,3 +109,11 @@ waveform:
   WAVEFORM_SYNTHETIC_WARP_FACTOR: 6
   WAVEFORM_SYNTHETIC_START_DATETIME: "2024-01-02T12:00:00Z"
   WAVEFORM_SYNTHETIC_END_DATETIME: "2024-01-03T12:00:00Z"
+
+# The nginx portal and other monitoring/validation/visualisation services
+monitoring:
+  SERVER_EXTERNAL_HOSTNAME: server.fqdn.example
+  PORTAL_PORT: 7100
+  PORTAL_USERNAME: emap
+  PORTAL_PASSWORD: portal_password
+  use_streamlit: false

emap-star/emap-star/src/main/java/uk/ac/ucl/rits/inform/informdb/visit_recordings/Waveform.java

@@ -34,6 +34,7 @@
         @Index(name = "waveform_datetime", columnList = "observationDatetime"),
         @Index(name = "waveform_location", columnList = "sourceLocation"),
         @Index(name = "waveform_location_visit", columnList = "locationVisitId"),
+        @Index(name = "waveform_observation_type", columnList = "visitObservationTypeId"),
 })
 @Data
 @EqualsAndHashCode(callSuper = true)

global-config-envs.EXAMPLE

@@ -4,3 +4,4 @@ RABBITMQ_ADMIN_PORT=5674
 GLOWROOT_ADMIN_PORT=4000
 FAKEUDS_PORT=5433
 HL7_READER_PORT=9999
+PORTAL_PORT=

monitoring/docker-compose.yml

@@ -0,0 +1,15 @@
+services:
+  streamlit:
+    build:
+      context: ..
+      dockerfile: monitoring/streamlit/Dockerfile
+      args:
+        HTTP_PROXY: ${HTTP_PROXY}
+        http_proxy: ${http_proxy}
+        HTTPS_PROXY: ${HTTPS_PROXY}
+        https_proxy: ${https_proxy}
+    env_file:
+      - ../../config/streamlit-config-envs
+    logging:
+      driver: "json-file"
+    restart: "no"

monitoring/requirements.txt

@@ -0,0 +1,11 @@
+jupyter
+jupyterlab
+jupytext
+matplotlib
+pandas
+psycopg2-binary
+pytest
+scipy
+soundfile
+sqlalchemy
+streamlit

monitoring/streamlit-config-envs.EXAMPLE

@@ -0,0 +1,6 @@
+UDS_JDBC_URL=
+UDS_SCHEMA=
+UDS_USERNAME=
+UDS_PASSWORD=
+SERVER_EXTERNAL_HOSTNAME=
+PORTAL_PORT=

monitoring/streamlit/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.12-slim-bullseye
+WORKDIR /app/streamlit
+COPY monitoring/requirements.txt /app/streamlit
+RUN pip install -r requirements.txt
+COPY monitoring/streamlit/ /app/streamlit
+CMD streamlit run \
+    --browser.gatherUsageStats=false \
+    --server.enableWebsocketCompression=false \
+    --server.enableXsrfProtection=false \
+    # base URL to match where the proxy expects it to be - simpler than URL rewriting in the proxy
+    --server.baseUrlPath "streamlit" \
+    # Without this, websocket calls don't work behind nginx
+    --browser.serverAddress ${SERVER_EXTERNAL_HOSTNAME} \
+    --browser.serverPort ${PORTAL_PORT} \
+    st_home.py