YDA-5993: implement syslog for iRODS 4.3

Set up rsyslog so that iRODS messages are logged in a readable format.

.github/workflows/build-push-image-provider.yml

@@ -11,6 +11,7 @@ on:
       - '.github/workflows/build-push-image-provider.yml'
       - 'docker/images/yoda_irods_icat/**'
       - 'roles/irods_completion/files/irods_completion.sh'
+      - 'roles/irods_log/files/log-transform.py'
       - 'roles/pam_python/files/pam_python.so'
 
 jobs:

docker/images/yoda_irods_icat/Dockerfile

@@ -40,9 +40,13 @@ RUN apt-get update
 # hadolint ignore=DL3033
 RUN apt-get install -y wget git sudo netcat-traditional gcc vim
 
-# Install rsyslog
+# Install rsyslog and configure it for logging iRODS messages
 # hadolint ignore=DL3033
 RUN apt-get install -y rsyslog
+COPY stage/log-transform.py /var/lib/irods/scripts/log-transform.py
+RUN chmod +x /var/lib/irods/scripts/log-transform.py
+COPY rsyslogd-irods.conf /etc/rsyslog.d/rsyslogd-irods.conf
+COPY rsyslog.conf /etc/rsyslog.conf
 
 # Install Python 3.12
 # hadolint ignore=DL3033
@@ -141,7 +145,9 @@ COPY host_access_control_config.json /etc/irods/host_access_control_config.json
 COPY rules_uu.cfg /etc/irods/yoda-ruleset
 
 # Fix up file ownership iRODS files
-RUN chown -R irods:irods /var/lib/irods /etc/irods
+RUN chown -R irods:irods /var/lib/irods /etc/irods && \
+    chmod 0775 /var/lib/irods/log && \
+    chown irods:syslog /var/lib/irods/log
 
 # Pre-build ruleset dependencies
 RUN test "$PRE_BUILD_RULESET_DEPENDENCIES" = "yes" && git clone --branch "$YODA_RULESET_BRANCH" "$YODA_RULESET_REPO" /var/lib/irods/yoda-ruleset

docker/images/yoda_irods_icat/irods-icat-init.sh

@@ -22,6 +22,10 @@ function progress_update {
 }
 
 function start_service {
+  before_update "Starting rsyslogd"
+  sudo /usr/sbin/rsyslogd
+  progress_update "Rsyslogd started"
+
   before_update "Starting iRODS"
   sudo -u irods /var/lib/irods/irodsctl start || true
   progress_update "iRODS started"

docker/images/yoda_irods_icat/rsyslog.conf

@@ -0,0 +1,54 @@
+# /etc/rsyslog.conf configuration file for rsyslog
+#
+# For more information install rsyslog-doc and see
+# /usr/share/doc/rsyslog-doc/html/configuration/index.html
+#
+# Default logging rules can be found in /etc/rsyslog.d/50-default.conf
+
+
+#################
+#### MODULES ####
+#################
+
+module(load="imuxsock") # provides support for local system logging
+#module(load="immark")  # provides --MARK-- message capability
+
+# provides UDP syslog reception
+#module(load="imudp")
+#input(type="imudp" port="514")
+
+# provides TCP syslog reception
+#module(load="imtcp")
+#input(type="imtcp" port="514")
+
+# provides kernel logging support and enable non-kernel klog messages
+## Disabled, because not available in container
+##module(load="imklog" permitnonkernelfacility="on")
+
+###########################
+#### GLOBAL DIRECTIVES ####
+###########################
+
+# Filter duplicated messages
+$RepeatedMsgReduction on
+
+#
+# Set the default permissions for all log files.
+#
+$FileOwner syslog
+$FileGroup adm
+$FileCreateMode 0640
+$DirCreateMode 0755
+$Umask 0022
+$PrivDropToUser syslog
+$PrivDropToGroup syslog
+
+#
+# Where to place spool and state files
+#
+$WorkDirectory /var/spool/rsyslog
+
+#
+# Include all config files in /etc/rsyslog.d/
+#
+$IncludeConfig /etc/rsyslog.d/*.conf

docker/images/yoda_irods_icat/rsyslogd-irods.conf

@@ -0,0 +1,12 @@
+$WorkDirectory /var/lib/irods/log
+$FileOwner irods
+$FileGroup irods
+$FileCreateMode 0644
+$DirCreateMode 0755
+
+module(load="omprog")
+
+if ($programname == 'irodsServer' or $programname == "irodsReServer") then {
+	action(type="omprog" name="transformer" binary="/var/lib/irods/scripts/log-transform.py")
+	stop
+}

docker/images/yoda_irods_icat/stage-uploads.sh

@@ -16,3 +16,4 @@ fi
 
 cp ../../../roles/pam_python/files/pam_python.so stage
 cp ../../../roles/irods_completion/files/irods_completion.sh stage
+cp ../../../roles/irods_log/files/log-transform.py stage

playbook.yml

@@ -117,6 +117,7 @@
     - pam_python
     - postgresql_odbc
     - irods_icat
+    - irods_log
     - irods_runtime
     - role: irods_resource_plugin_s3
       when: enable_s3_resource
@@ -162,6 +163,7 @@
   become: true
   roles:
     - irods_resource
+    - irods_log
     - role: irods_resource_plugin_s3
       when: enable_s3_resource
     - irods_runtime

roles/irods_log/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+# copyright Utrecht University
+
+irods_service_account: irods

roles/irods_log/files/log-transform.py

@@ -0,0 +1,56 @@
+#!/usr/bin/python3
+#
+# This script is used by rsyslogd to transform iRODS
+# JSON log messages into a more human-readable format.
+
+import datetime
+import json
+import os
+import sys
+
+LOG_DIR = "/var/lib/irods/log"
+
+
+def get_log_file() -> str:
+    filename = datetime.datetime.now().strftime("rodsLog-%Y-%m-%d")
+    return os.path.join(LOG_DIR, filename)
+
+
+def write_message(message: str) -> None:
+    with open(get_log_file(), "a") as logfile:
+        logfile.write(message)
+        logfile.flush()
+
+
+def process_log() -> None:
+    for line in sys.stdin:
+        output_message = get_output_message(line)
+        write_message(output_message)
+
+
+def get_output_message(line: str) -> str:
+    start_json_message = line.index("{") if "{" in line else 0
+    try:
+        json_data = json.loads(line[start_json_message:])
+    except json.decoder.JSONDecodeError:
+        # If it's not JSON, it's probably an error message such as a Python exception
+        # Just write it as-is, for readability.
+        return line
+
+    datestamp = json_data.get("server_timestamp", "n.d.")
+    pid = json_data.get("server_pid", "N/A")
+    category = json_data.get("log_category", "N/A")
+    client_user = json_data.get("request_client_user", "N/A")
+    proxy_user = json_data.get("request_proxy_user", "N/A")
+    zone = json_data.get("server_zone", "N/A")
+    if client_user == "N/A":
+        user_string = "no_user"
+    else:
+        user_string = f"{client_user}#{zone}" if proxy_user == client_user else f"{client_user}:{proxy_user}#{zone}"
+    level = json_data.get("log_level", "N/A")
+    message = json_data.get("log_message", "No message")
+    return f"{datestamp} pid:{pid} {category}:{level} {{{user_string}}} {message}\n"
+
+
+if __name__ == "__main__":
+    process_log()

roles/irods_log/handlers/main.yml

@@ -0,0 +1,13 @@
+---
+# copyright Utrecht University
+
+- name: Restart AppArmor
+  ansible.builtin.service:
+    name: apparmor
+    state: restarted
+
+
+- name: Restart rsyslogd
+  ansible.builtin.service:
+    name: rsyslog
+    state: restarted

roles/irods_log/meta/main.yml

@@ -0,0 +1,11 @@
+---
+# copyright Utrecht University
+
+galaxy_info:
+  author: Sietse Snel
+  description: Set up syslog processing for irods
+  license: GPLv3
+  min_ansible_version: '2.11'
+  platforms:
+    - name: Ubuntu
+      version: noble

roles/irods_log/tasks/main.yml

@@ -0,0 +1,39 @@
+---
+# copyright Utrecht University
+
+- name: Ensure iRODS log directory is writable for rsyslogd
+  ansible.builtin.file:
+    path: /var/lib/irods/log
+    state: directory
+    owner: "{{ irods_service_account }}"
+    group: syslog
+    mode: "0775"
+
+
+- name: Upload iRODS log transform script for rsyslogd
+  ansible.builtin.copy:
+    src: log-transform.py
+    dest: /var/lib/irods/scripts/log-transform.py
+    owner: "{{ irods_service_account }}"
+    group: "{{ irods_service_account }}"
+    mode: "0755"
+
+
+- name: Add iRODS-specific rules to AppArmor profile rsyslogd
+  ansible.builtin.template:
+    src: rsyslogd-irods.profile.j2
+    dest: /etc/apparmor.d/rsyslog.d/irods
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart AppArmor
+
+
+- name: Add rsyslog configuration for iRODS
+  ansible.builtin.template:
+    src: rsyslogd-irods.conf.j2
+    dest: /etc/rsyslog.d/irods.conf
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart rsyslogd

roles/irods_log/templates/rsyslogd-irods.conf.j2

@@ -0,0 +1,14 @@
+# {{ ansible_managed }}
+
+$WorkDirectory /var/lib/irods/log
+$FileOwner {{ irods_service_account }}
+$FileGroup {{ irods_service_account }}
+$FileCreateMode 0644
+$DirCreateMode 0755
+
+module(load="omprog")
+
+if ($programname == 'irodsServer' or $programname == "irodsReServer") then {
+        action(type="omprog" name="transformer" binary="/var/lib/irods/scripts/log-transform.py")
+        stop
+}

roles/irods_log/templates/rsyslogd-irods.profile.j2

@@ -0,0 +1,10 @@
+# {{ ansible_managed }}
+#
+# This allows Rsyslogd to write iRODS log files via the
+# log transform script. The transform script rewrites
+# JSON log messages in a form that is easier to read for humans.
+/var/lib/irods/log/** rw,
+/var/lib/irods/scripts/log-transform.py mrix,
+/usr/local/lib/python3.12/** r,
+/usr/bin/python3.12 ix,
+/usr/bin/python3 ix,