pipeline.json

{
  "description": "Pipeline for parsing Postfix mail service logs based on Elastic Common Schema (ECS)",
  "processors": [
    {
      "grok": {
        "pattern_definitions": {
          "STATUS_WORD": "[\\w-]*",
          "SYSLOG_PREFIX": "%{SYSLOGTIMESTAMP:@timestamp} %{SYSLOGHOST:host.name} %{PROG:process.name}(?:\\[%{POSINT:process.pid}\\])?:",
          "POSTFIX_QUEUEID": "([0-9A-F]{6,}|[0-9a-zA-Z]{12,})|NOQUEUE",
          "POSTFIX_RELAY_INFO": "%{HOSTNAME:postfix.relay_hostname}?\\[(%{IP:postfix.relay_ip}|%{DATA:postfix.relay_service})\\](:%{INT:postfix.relay_port})?|%{WORD:postfix.relay_service}",
          "POSTFIX_SMTP_STAGE": "(CONNECT|HELO|EHLO|STARTTLS|AUTH|MAIL( FROM)?|RCPT( TO)?|(end of )?DATA|RSET|UNKNOWN|END-OF-MESSAGE|VRFY|\\.)",
          "POSTFIX_ACTION": "(accept|defer|discard|filter|header-redirect|reject)",
          "POSTFIX_STATUS_CODE": "\\d{3}",
          "POSTFIX_STATUS_CODE_ENHANCED": "\\d\\.\\d\\.\\d",
          "POSTFIX_DNSBL_MESSAGE": "Service unavailable; .* \\[%{GREEDYDATA:postfix.status_data}\\] %{GREEDYDATA:postfix.status_message};",
          "POSTFIX_PS_ACCESS_ACTION": "(DISCONNECT|BLACKLISTED|WHITELISTED|WHITELIST VETO|PASS NEW|PASS OLD)",
          "POSTFIX_PS_VIOLATION": "(BARE NEWLINE|COMMAND (TIME|COUNT|LENGTH) LIMIT|COMMAND PIPELINING|DNSBL|HANGUP|NON-SMTP COMMAND|PREGREET)",
          "POSTFIX_TIME_UNIT": "%{NUMBER}[smhd]",
          "POSTFIX_KEYVALUE_DATA": "[\\w-]+=[^;]*",
          "POSTFIX_KEYVALUE": "%{POSTFIX_QUEUEID:postfix.queue_id}: %{POSTFIX_KEYVALUE_DATA:postfix.keyvalue_data}",
          "POSTFIX_WARNING_LEVEL": "(warning|fatal|info)",
          "POSTFIX_CLIENT_INFO": "%{HOSTNAME:client.address}?\\[%{IP:client.ip}\\](:%{INT:client.port})?",
          "POSTFIX_LOSTCONN": "(Connection timed out|No route to host|Connection refused|Network is unreachable|lost connection|timeout|SSL_accept error|-1)",
          "POSTFIX_SMTPD_CONNECT": "connect from %{POSTFIX_CLIENT_INFO}",
          "POSTFIX_SMTPD_DISCONNECT": "disconnect from %{POSTFIX_CLIENT_INFO}( %{GREEDYDATA:postfix.command_counter_data})?",
          "POSTFIX_SMTPD_LOSTCONN": "%{POSTFIX_LOSTCONN:postfix.smtpd_lostconn_data}( after %{POSTFIX_SMTP_STAGE:postfix.smtp_stage}( \\(%{INT} bytes\\))?)? from %{POSTFIX_CLIENT_INFO}(: %{GREEDYDATA:postfix.smtpd_lostconn_reason})?",
          "POSTFIX_SMTPD_NOQUEUE": "NOQUEUE: %{POSTFIX_ACTION:postfix.action}: %{POSTFIX_SMTP_STAGE:postfix.smtp_stage} from %{POSTFIX_CLIENT_INFO}:( %{POSTFIX_STATUS_CODE:postfix.status_code} %{POSTFIX_STATUS_CODE_ENHANCED:postfix.status_code_enhanced})?( <%{DATA:postfix.status_data}>:)? (%{POSTFIX_DNSBL_MESSAGE}|%{GREEDYDATA:postfix.status_message};) %{POSTFIX_KEYVALUE_DATA:postfix.keyvalue_data}",
          "POSTFIX_SMTPD": "%{POSTFIX_SMTPD_CONNECT}|%{POSTFIX_SMTPD_DISCONNECT}|%{POSTFIX_SMTPD_LOSTCONN}|%{POSTFIX_SMTPD_NOQUEUE}|%{POSTFIX_KEYVALUE}",
          "POSTFIX_QMGR_REMOVED": "%{POSTFIX_QUEUEID:postfix.queue_id}: removed",
          "POSTFIX_QMGR_ACTIVE": "%{POSTFIX_QUEUEID:postfix.queue_id}: %{POSTFIX_KEYVALUE_DATA:postfix.keyvalue_data} (queue active)",
          "POSTFIX_QMGR_EXPIRED": "%{POSTFIX_QUEUEID:postfix.queue_id}: from=<%{DATA:postfix.from}>, status=%{STATUS_WORD:postfix.status}, returned to sender",
          "POSTFIX_QMGR": "%{POSTFIX_QMGR_REMOVED}|%{POSTFIX_QMGR_ACTIVE}|%{POSTFIX_QMGR_EXPIRED}",
          "POSTFIX_EVENT": "%{SYSLOG_PREFIX} %{POSTFIX_SMTPD}|%{POSTFIX_QMGR}",
          "CATCH_ALL": "%{SYSLOGTIMESTAMP:@timestamp} %{SYSLOGHOST:host.name} %{PROG:process.name}: %{GREEDYDATA}"
        },
        "patterns": [
          "%{POSTFIX_EVENT}",
          "%{CATCH_ALL}"
        ],
        "field": "message",
        "ignore_missing": true
      }
    },
    {
      "geoip": {
        "field": "client.ip",
        "target_field": "client.geo.location",
        "ignore_missing": true
      }
    }
  ],
  "on_failure": [
    {
      "set": {
        "field": "error.message",
        "value": "{{ _ingest.on_failure_message }}"
      }
    },
    {
      "set": {
        "field": "error.tag",
        "value": "parse_failure"
      }
    }
  ],
  "version": 1
}