Enhance dovecot-logreport script: improve statistics output format and clarity

dovecot-logreport.sh

@@ -26,7 +26,7 @@ set -euo pipefail
 #       * --oneline: 1 Zeile TSV (kein Leerzeilentrenner)
 #       * --stats: Statistik statt Events (inkl. Matrix/Toplisten)
 #   - Live-Modus:
-#       * --follow: tail -F (gz-Dateien werden dafür übersprungen)
+#       * --follow: tail -F, sudo nur falls nötig (gz-Dateien werden dafür übersprungen)
 #
 # Anforderungen/Annahmen:
 #   - Timestamp steht als erstes Feld in ISO-Format (z.B. 2026-01-17T12:34:56+01:00)
@@ -53,6 +53,10 @@ FOLLOW=0
 ALL=0
 ONELINE=0
 STATS=0
+SHOW_PROGRESS=0
+if [[ -t 2 ]]; then
+  SHOW_PROGRESS=1
+fi
 
 # Service-/Event-Filter (OR-verknüpft, wenn mehrere gesetzt sind)
 ONLY_LOGIN=0
@@ -111,7 +115,7 @@ INPUT OPTIONS
         You can combine --all and --file to add extra files.
 
   --follow
-      Follow the active logfile(s) (tail -F).
+      Follow the active logfile(s) (tail -F; sudo only if needed).
       .gz files cannot be followed and will be skipped in follow mode.
 
 TIME FILTERS (ISO8601 recommended)
@@ -336,6 +340,24 @@ if [[ $FOLLOW -eq 1 ]]; then
   [[ ${#FOLLOW_FILES[@]} -gt 0 ]] || die "--follow requested, but only .gz files selected. Add an active log with --file."
 fi
 
+NEED_SUDO=0
+declare -a UNREADABLE_FILES=()
+for f in "${LOGFILES[@]}"; do
+  if ! { : < "$f"; } 2>/dev/null; then
+    NEED_SUDO=1
+    UNREADABLE_FILES+=("$f")
+  fi
+done
+
+if [[ $NEED_SUDO -eq 1 ]]; then
+  {
+    echo "Using sudo because these selected log file(s) cannot be opened for reading by $(id -un):"
+    echo "Current groups: $(id -nG)"
+    printf "  %s\n" "${UNREADABLE_FILES[@]}"
+  } >&2
+  sudo -v || die "sudo authentication failed; cannot read Dovecot log files."
+fi
+
 # -------------------------------------------------------------------
 # POSIX-AWK Parser
 #   - kompatibel zu mawk/nawk/busybox awk
@@ -388,7 +410,7 @@ function extract_type(line, arr, t) {
 #   Tries to extract a user/mailbox from common patterns.
 function extract_user(line, arr, u) {
   if (match(line, /user=<[^>]+>/, arr)) {
-    return substr(arr[0], 7, length(arr[0])-7-1)
+    return substr(arr[0], 7, length(arr[0])-7)
   }
 
   if (match(line, /: (imap|pop3|lmtp)\([^)]*\)/, arr)) {
@@ -468,42 +490,49 @@ function type_regex_match(type, typere) {
   return 0
 }
 
-# ---- STATS helper: sort and print top N ----
-function print_top_sorted(title, arr, n,   k,i,j,keys,tmp) {
-  print title
-  i=0
-  for (k in arr) { i++; keys[i]=k }
+# ---- progress spinner on stderr ----
+function progress_tick(   now, ch) {
+  if (show_progress != 1) return
 
-  for (i=1; i<=length(keys); i++) {
-    for (j=i+1; j<=length(keys); j++) {
-      if (arr[keys[j]] > arr[keys[i]]) {
-        tmp=keys[i]; keys[i]=keys[j]; keys[j]=tmp
-      }
-    }
-  }
+  now = systime()
+  if (PROGRESS_LAST != "" && now - PROGRESS_LAST < 2) return
 
-  for (i=1; i<=length(keys) && i<=n; i++) {
-    printf "  %7d  %s\n", arr[keys[i]], keys[i]
-  }
-  print ""
+  PROGRESS_LAST = now
+  PROGRESS_IDX = (PROGRESS_IDX % 4) + 1
+  ch = substr("|/-\\", PROGRESS_IDX, 1)
+  printf "\r%s", ch > "/dev/stderr"
+  fflush("/dev/stderr")
+  PROGRESS_DRAWN = 1
 }
 
-# ---- STATS helper: sort and print top N for matrix-like keys ----
-function print_matrix(title, arr, n,   k,i,j,keys,tmp) {
-  print title
-  i=0
-  for (k in arr) { i++; keys[i]=k }
+function progress_clear() {
+  if (show_progress != 1 || PROGRESS_DRAWN != 1) return
+  printf "\r \r" > "/dev/stderr"
+  fflush("/dev/stderr")
+  PROGRESS_DRAWN = 0
+}
 
-  for (i=1; i<=length(keys); i++) {
-    for (j=i+1; j<=length(keys); j++) {
-      if (arr[keys[j]] > arr[keys[i]]) {
-        tmp=keys[i]; keys[i]=keys[j]; keys[j]=tmp
-      }
+# ---- STATS helper: keep only Top N while scanning the array ----
+function print_top_sorted(title, arr, n,   k,i,pos,count,topk,topv) {
+  print title
+
+  count=0
+  for (k in arr) {
+    pos=1
+    while (pos<=count && arr[k] <= topv[pos]) pos++
+    if (pos > n) continue
+
+    if (count < n) count++
+    for (i=count; i>pos; i--) {
+      topk[i]=topk[i-1]
+      topv[i]=topv[i-1]
     }
+    topk[pos]=k
+    topv[pos]=arr[k]
   }
 
-  for (i=1; i<=length(keys) && i<=n; i++) {
-    printf "  %7d  %s\n", arr[keys[i]], keys[i]
+  for (i=1; i<=count; i++) {
+    printf "  %7d  %s\n", topv[i], topk[i]
   }
   print ""
 }
@@ -524,10 +553,14 @@ BEGIN {
   onlypop3  = ENVIRON["ONLY_POP3"] + 0
 
   typere = ENVIRON["TYPE_RE"]
+  show_progress = ENVIRON["SHOW_PROGRESS"] + 0
   total=0
+  progress_tick()
 }
 
 {
+  progress_tick()
+
   line=$0
   if (line !~ /dovecot:/) next
 
@@ -566,10 +599,10 @@ BEGIN {
   if (emailneedle != "" && index(tolower(user), emailneedle) == 0) next
 
   total++
+  c_res[res]++
 
   # Stats sammeln
   if (stats == 1) {
-    c_res[res]++
     c_type[type]++
     c_res_type[res "@" type]++
 
@@ -592,10 +625,12 @@ BEGIN {
 
   # Eventausgabe
   if (oneline == 1) {
+    progress_clear()
     printf "%s\t%s\t%s\t%s\t%s\t%s\n", ts, type, res, ip, user, msg
     next
   }
 
+  progress_clear()
   print "time:      " ts
   print "type:      " type
   print "client-ip: " ip
@@ -606,7 +641,22 @@ BEGIN {
 }
 
 END {
-  if (stats != 1) exit
+  progress_clear()
+  if (stats != 1) {
+    if (oneline == 1) {
+      printf "stats\ttotal=%d\tsuccess=%d\tfail=%d\terror=%d\tinfo=%d\n", total+0, c_res["success"]+0, c_res["fail"]+0, c_res["error"]+0, c_res["info"]+0
+      exit
+    }
+
+    print ""
+    print "Statistics:"
+    printf "  total:   %7d\n", total+0
+    printf "  success: %7d\n", c_res["success"]+0
+    printf "  fail:    %7d\n", c_res["fail"]+0
+    printf "  error:   %7d\n", c_res["error"]+0
+    printf "  info:    %7d\n", c_res["info"]+0
+    exit
+  }
 
   print "Events: " total
   print ""
@@ -626,7 +676,7 @@ END {
   print_top_sorted("Top users (overall, Top 10)", c_user, 10)
   print_top_sorted("Top users (fail only, Top 10)", c_user_fail, 10)
 
-  print_matrix("Matrix result@type (Top 30)", c_res_type, 30)
+  print_top_sorted("Matrix result@type (Top 30)", c_res_type, 30)
 
   print_top_sorted("Fail combos IP -> user (Top 20)", c_fail_ip_user, 20)
 }
@@ -635,16 +685,25 @@ END {
 # -------------------------------------------------------------------
 # run_pipeline():
 #   Liest die ausgewählten Files (inkl. .gz über zcat -f) und pipe't
-#   in awk. In --follow Mode: tail -F auf nicht-gz Dateien.
+#   in awk. sudo wird nur verwendet, wenn mindestens eine Logdatei nicht
+#   direkt lesbar ist. In --follow Mode: tail -F auf nicht-gz Dateien.
 # -------------------------------------------------------------------
+run_read() {
+  if [[ $NEED_SUDO -eq 1 ]]; then
+    sudo "$@"
+  else
+    "$@"
+  fi
+}
+
 run_pipeline() {
   local f
 
   if [[ $FOLLOW -eq 1 ]]; then
-    tail -F "${FOLLOW_FILES[@]}" | \
+    run_read tail -F "${FOLLOW_FILES[@]}" | \
       env SINCE="$SINCE" UNTIL="$UNTIL" ADDR_RE="$ADDR_RE" EMAIL_NEEDLE="$EMAIL_NEEDLE" TYPE_RE="$TYPE_RE" \
           ONLY_LOGIN="$ONLY_LOGIN" ONLY_LMTP="$ONLY_LMTP" ONLY_IMAP="$ONLY_IMAP" ONLY_POP3="$ONLY_POP3" \
-          STATUS_FILTER="$STATUS_FILTER" ONELINE="$ONELINE" STATS="$STATS" \
+          STATUS_FILTER="$STATUS_FILTER" ONELINE="$ONELINE" STATS="$STATS" SHOW_PROGRESS="$SHOW_PROGRESS" \
       awk "$AWK_SCRIPT"
     return
   fi
@@ -652,16 +711,15 @@ run_pipeline() {
   {
     for f in "${LOGFILES[@]}"; do
       if [[ "$f" =~ \.gz$ ]]; then
-        zcat -f -- "$f" 2>/dev/null || true
+        run_read zcat -f -- "$f" || true
       else
-        cat -- "$f" 2>/dev/null || true
+        run_read cat -- "$f" || true
       fi
     done
   } | env SINCE="$SINCE" UNTIL="$UNTIL" ADDR_RE="$ADDR_RE" EMAIL_NEEDLE="$EMAIL_NEEDLE" TYPE_RE="$TYPE_RE" \
           ONLY_LOGIN="$ONLY_LOGIN" ONLY_LMTP="$ONLY_LMTP" ONLY_IMAP="$ONLY_IMAP" ONLY_POP3="$ONLY_POP3" \
-          STATUS_FILTER="$STATUS_FILTER" ONELINE="$ONELINE" STATS="$STATS" \
+          STATUS_FILTER="$STATUS_FILTER" ONELINE="$ONELINE" STATS="$STATS" SHOW_PROGRESS="$SHOW_PROGRESS" \
       awk "$AWK_SCRIPT"
 }
 
 run_pipeline
-

mailtrace.sh

@@ -73,6 +73,7 @@ set -euo pipefail
 
 FOLLOW=0
 ALL_LOGS=0
+LOG_FILE=""
 
 FAIL_ONLY=0
 SUCCESS_ONLY=0
@@ -82,6 +83,8 @@ FROM_FILTER=""
 TO_FILTER=""
 QID_FILTER=""
 MSGID_FILTER=""
+SENDER_HOST_FILTER=""
+SENDER_IP_FILTER=""
 
 SINCE_PREFIX=""
 UNTIL_PREFIX=""
@@ -91,7 +94,7 @@ NO_NOTE=0          # suppress note lines in block output when TLS unknown
 
 DEDUP=0
 DEDUP_PREFER="final"   # final|amavis|first|last
-FINAL_ONLY=0
+FINAL_ONLY=1
 
 SASL_ONLY=0
 NO_SASL=0
@@ -99,6 +102,21 @@ INBOUND_ONLY=0
 OUTBOUND_ONLY=0
 
 DEBUG=0
+SHOW_PROGRESS=0
+if [[ -t 2 ]]; then
+  SHOW_PROGRESS=1
+fi
+
+AWK_BIN="${AWK_BIN:-}"
+if [[ -z "$AWK_BIN" ]]; then
+  if command -v mawk >/dev/null 2>&1; then
+    AWK_BIN="mawk"
+  elif command -v gawk >/dev/null 2>&1; then
+    AWK_BIN="gawk"
+  else
+    AWK_BIN="awk"
+  fi
+fi
 
 usage() {
   cat <<'USAGE'
@@ -115,6 +133,8 @@ Filter:
   --to-addr <substr>         match nur in to
   --qid <substr>             match Queue-ID (Postfix Queue-ID / "qid")
   --message-id <substr>      match Header Message-ID (cleanup: message-id=<...>)
+  --sender-host <substr>     match absendenden Client-Host (client/orig_client)
+  --sender-ip <substr>       match absendende Client-IP (clientip/orig_clientip; IPv4/IPv6)
 
 Status-Filter:
   --success                  nur erfolgreiche Endzustellungen (status=success)
@@ -129,7 +149,11 @@ AUTH / Richtung:
 Dedup (ein Eintrag pro Empfänger):
   --dedup                    dedupliziere pro (message-id,to). Fallback: (qid,to) falls msgid fehlt
   --dedup-prefer <mode>      final|amavis|first|last   (Default: final)
-  --final-only               nur endgültige Zustellung ausgeben (remote SMTP oder lokales LMTP), keine Amavis-Handoffs
+
+Umfang:
+  (default)                  nur endgültige Zustellungen ausgeben (remote SMTP oder lokales LMTP), keine Amavis-Handoffs
+  --final-only               wie Default; nur finale Zustellungen anzeigen
+  --all-outcomes             alle SMTP/LMTP-Outcomes anzeigen, inkl. Amavis-Handoffs
 
 TLS/Notizen:
   --no-note                  im Blockformat keine Hinweiszeile ausgeben, wenn TLS nicht korrelierbar ist
@@ -137,11 +161,38 @@ TLS/Notizen:
 Debug:
   --debug                    ergänzt Debug-Details zur Heuristik/Klassifikation
 
+Fortschritt:
+  Bei interaktivem stderr zeigt mailtrace.sh waehrend der Verarbeitung einen
+  kleinen Spinner an. Der Spinner wird vor jeder Ausgabezeile entfernt.
+
 Input / Zeitraum:
   --follow                   tail -F /var/log/mail.log (live)
   --all-logs                 /var/log/mail.log* inkl. .gz (historisch)
+  --log-file <pfad>          einzelne Logdatei (auch .gz); nicht kombinierbar mit --follow/--all-logs
   --since <prefix>           Timestamp prefix-match (z.B. 2026-01-13T14:47)
   --until <prefix>           stoppe sobald ts >= prefix (ISO8601 string compare)
+
+Beispiele:
+  # Finale Zustellungen eines absendenden Hosts (Default)
+  mailtrace.sh --sender-host web0.warenform.de
+
+  # Finale Zustellungen einer IPv4-Adresse
+  mailtrace.sh --sender-ip 83.223.85.164
+
+  # Finale Zustellungen einer IPv6-Adresse
+  mailtrace.sh --sender-ip 2a01:30:0:13:286:96ff:fe4a:6eee
+
+  # Nur erfolgreiche finale Zustellungen dieses Hosts
+  mailtrace.sh --sender-host web0.warenform.de --success
+
+  # Nur fehlgeschlagene Zustellungen einer IP in alten und rotierten Logs
+  mailtrace.sh --all-logs --sender-ip 83.223.85.164 --fail
+
+  # Voller Verlauf inkl. Amavis-Handoffs und Reinject-Zustellungen
+  mailtrace.sh --sender-host web0.warenform.de --all-outcomes
+
+  # Kompakte Ausgabe fuer einen Zeitraum
+  mailtrace.sh --sender-host web0.warenform.de --since 2026-01-13T14 --one-line
 USAGE
 }
 
@@ -150,6 +201,7 @@ while [[ $# -gt 0 ]]; do
   case "$1" in
     --follow) FOLLOW=1 ;;
     --all-logs) ALL_LOGS=1 ;;
+    --log-file) LOG_FILE="${2:-}"; shift ;;
     --success) SUCCESS_ONLY=1 ;;
     --fail) FAIL_ONLY=1 ;;
     --addr) ADDR_FILTER="${2:-}"; shift ;;
@@ -157,6 +209,8 @@ while [[ $# -gt 0 ]]; do
     --to-addr) TO_FILTER="${2:-}"; shift ;;
     --qid) QID_FILTER="${2:-}"; shift ;;
     --message-id) MSGID_FILTER="${2:-}"; shift ;;
+    --sender-host) SENDER_HOST_FILTER="${2:-}"; shift ;;
+    --sender-ip) SENDER_IP_FILTER="${2:-}"; shift ;;
     --since) SINCE_PREFIX="${2:-}"; shift ;;
     --until) UNTIL_PREFIX="${2:-}"; shift ;;
     --one-line) FORMAT="one-line" ;;
@@ -165,6 +219,7 @@ while [[ $# -gt 0 ]]; do
     --dedup) DEDUP=1 ;;
     --dedup-prefer) DEDUP_PREFER="${2:-}"; shift ;;
     --final-only) FINAL_ONLY=1 ;;
+    --all-outcomes) FINAL_ONLY=0 ;;
     --sasl-only) SASL_ONLY=1 ;;
     --no-sasl) NO_SASL=1 ;;
     --inbound-only) INBOUND_ONLY=1 ;;
@@ -185,6 +240,14 @@ if [[ "$DEDUP_PREFER" != "final" && "$DEDUP_PREFER" != "amavis" && "$DEDUP_PREFE
   echo "Ungültiges --dedup-prefer: $DEDUP_PREFER (final|amavis|first|last)" >&2
   exit 2
 fi
+if [[ -n "$LOG_FILE" && ($FOLLOW -eq 1 || $ALL_LOGS -eq 1) ]]; then
+  echo "--log-file kann nicht zusammen mit --follow oder --all-logs verwendet werden." >&2
+  exit 2
+fi
+if [[ -n "$LOG_FILE" && ! -e "$LOG_FILE" ]]; then
+  echo "Datei nicht gefunden: $LOG_FILE" >&2
+  exit 2
+fi
 if [[ $SUCCESS_ONLY -eq 1 && $FAIL_ONLY -eq 1 ]]; then
   echo "Bitte nur eines von --success oder --fail verwenden." >&2
   exit 2
@@ -199,44 +262,120 @@ if [[ $INBOUND_ONLY -eq 1 && $OUTBOUND_ONLY -eq 1 ]]; then
 fi
 
 # --- Input selection ---------------------------------------------------------
-if [[ $FOLLOW -eq 1 ]]; then
-  INPUT_CMD=(sudo tail -F /var/log/mail.log)
+declare -a LOGFILES=()
+
+if [[ -n "$LOG_FILE" ]]; then
+  LOGFILES=("$LOG_FILE")
+elif [[ $ALL_LOGS -eq 1 ]]; then
+  for f in /var/log/mail.log*; do
+    [[ -e "$f" ]] && LOGFILES+=("$f")
+  done
 else
-  if [[ $ALL_LOGS -eq 1 ]]; then
-    INPUT_CMD=(sudo zcat -f /var/log/mail.log*)
-  else
-    INPUT_CMD=(sudo cat /var/log/mail.log)
+  LOGFILES=(/var/log/mail.log)
+fi
+
+if [[ ${#LOGFILES[@]} -eq 0 ]]; then
+  echo "Keine Logdateien gefunden." >&2
+  exit 2
+fi
+
+NEED_SUDO=0
+declare -a UNREADABLE_FILES=()
+for f in "${LOGFILES[@]}"; do
+  if ! { : < "$f"; } 2>/dev/null; then
+    NEED_SUDO=1
+    UNREADABLE_FILES+=("$f")
   fi
+done
+
+if [[ $NEED_SUDO -eq 1 ]]; then
+  {
+    echo "Using sudo because these selected log file(s) cannot be opened for reading by $(id -un):"
+    echo "Current groups: $(id -nG)"
+    for f in "${UNREADABLE_FILES[@]}"; do
+      if [[ -e "$f" ]]; then
+        ls -ld -- "$f"
+      else
+        printf "  missing: %s\n" "$f"
+      fi
+    done
+  } >&2
+  sudo -v || { echo "sudo authentication failed; cannot read mail log files." >&2; exit 2; }
+fi
+
+run_read() {
+  if [[ $NEED_SUDO -eq 1 ]]; then
+    sudo "$@"
+  else
+    "$@"
+  fi
+}
+
+if [[ $FOLLOW -eq 1 ]]; then
+  INPUT_CMD=(run_read tail -F /var/log/mail.log)
+else
+  INPUT_CMD=(run_read zcat -f "${LOGFILES[@]}")
 fi
 
 ###############################################################################
 # gawk core
 ###############################################################################
-"${INPUT_CMD[@]}" | gawk \
-  -v success_only="$SUCCESS_ONLY" \
-  -v fail_only="$FAIL_ONLY" \
-  -v addr_filter="$ADDR_FILTER" \
-  -v from_filter="$FROM_FILTER" \
-  -v to_filter="$TO_FILTER" \
-  -v qid_filter="$QID_FILTER" \
-  -v msgid_filter="$MSGID_FILTER" \
-  -v since_prefix="$SINCE_PREFIX" \
-  -v until_prefix="$UNTIL_PREFIX" \
-  -v out_format="$FORMAT" \
-  -v no_note="$NO_NOTE" \
-  -v dedup="$DEDUP" \
-  -v dedup_prefer="$DEDUP_PREFER" \
-  -v final_only="$FINAL_ONLY" \
-  -v follow_mode="$FOLLOW" \
-  -v sasl_only="$SASL_ONLY" \
-  -v no_sasl="$NO_SASL" \
-  -v inbound_only="$INBOUND_ONLY" \
-  -v outbound_only="$OUTBOUND_ONLY" \
-  -v debug_mode="$DEBUG" '
+AWK_PROGRAM="$(mktemp)"
+AWK_FIFO="$(mktemp -u)"
+GAWK_ERR="$(mktemp)"
+mkfifo "$AWK_FIFO"
+trap 'rm -f "$AWK_PROGRAM" "$AWK_FIFO" "$GAWK_ERR"' EXIT
+cat > "$AWK_PROGRAM" <<'AWK'
 ###############################################################################
 # Helferfunktionen
 ###############################################################################
-function grab(re, s,   m) { return match(s, re, m) ? m[1] : "" }
+function token_after(s, prefix,   n, r) {
+  n = index(s, prefix)
+  if (n == 0) return ""
+  r = substr(s, n + length(prefix))
+  sub(/[ ,;].*/, "", r)
+  return r
+}
+function angle_after(s, prefix,   n, r, e) {
+  n = index(s, prefix)
+  if (n == 0) return ""
+  r = substr(s, n + length(prefix))
+  e = index(r, ">")
+  if (e == 0) return ""
+  return substr(r, 1, e - 1)
+}
+function bracket_after(s, prefix,   n, r, b, e) {
+  n = index(s, prefix)
+  if (n == 0) return ""
+  r = substr(s, n + length(prefix))
+  b = index(r, "[")
+  if (b == 0) return ""
+  r = substr(r, b + 1)
+  e = index(r, "]")
+  if (e == 0) return ""
+  return substr(r, 1, e - 1)
+}
+function host_before_bracket_after(s, prefix,   n, r, b) {
+  n = index(s, prefix)
+  if (n == 0) return ""
+  r = substr(s, n + length(prefix))
+  b = index(r, "[")
+  if (b > 0) return substr(r, 1, b - 1)
+  sub(/[ ,;].*/, "", r)
+  return r
+}
+function qid_from_line(s) {
+  if (!match(s, / [A-F0-9]{7,20}:/)) return ""
+  return substr(s, RSTART + 1, RLENGTH - 2)
+}
+function status_message(s,   n, r, p) {
+  n = index(s, " status=")
+  if (n == 0) return ""
+  r = substr(s, n + length(" status="))
+  p = index(r, " (")
+  if (p == 0 || substr(r, length(r), 1) != ")") return ""
+  return substr(r, p + 2, length(r) - p - 2)
+}
 function is_amavis_handoff(relay) {
   return (relay ~ /(127\.0\.0\.1|localhost).*:1002[456]$/ || relay ~ /\[127\.0\.0\.1\]:1002[456]$/)
 }
@@ -262,6 +401,28 @@ function annotate_sender(from, qid,   reason) {
   return from
 }
 
+function delete_qid_state(qid) {
+  delete FROM[qid]
+  delete TO[qid]
+  delete MID[qid]
+  delete CLIENT[qid]
+  delete CLIENTIP[qid]
+  delete ORIGCLIENT[qid]
+  delete ORIGCLIENTIP[qid]
+  delete HELO[qid]
+  delete SASL[qid]
+  delete UID[qid]
+  delete TLS[qid]
+  delete TLS_P[qid]
+  delete TLS_C[qid]
+  delete TLS_B[qid]
+  delete TMP_EXTCLIENT[qid]
+  delete TMP_EXTCLIENTIP[qid]
+  delete DSN_KIND[qid]
+  delete DSN_REASON[qid]
+  delete LAST_FAILURE[qid]
+}
+
 function jesc(s) {
   gsub(/\\/,"\\\\",s); gsub(/"/,"\\\"",s)
   gsub(/\t/,"\\t",s); gsub(/\r/,"\\r",s); gsub(/\n/,"\\n",s)
@@ -280,6 +441,7 @@ BEGIN {
   maxw = 0
   for (i=1; i in labels; i++) { w = length(labels[i] ":"); if (w > maxw) maxw = w }
   LABEL_W = maxw
+  progress_tick()
 }
 function pl(label, value) { printf "%-*s %s\n", LABEL_W, label ":", value }
 
@@ -298,12 +460,23 @@ function classify_source(qid, mid, relay,   extip) {
 ###############################################################################
 # TLS Korrelation
 ###############################################################################
-function remember_tls(line,   ip, proto, cipher, bits) {
-  ip = grab(" from [^\\[]+\\[([^\\]]+)\\]", line)
+function remember_tls(line,   ip, proto, cipher, bits, n, r, p) {
+  ip = bracket_after(line, " from ")
   if (ip == "") return
-  proto  = grab(": (TLS[^ ]+) with cipher", line)
-  cipher = grab(" with cipher ([^ ]+)", line)
-  bits   = grab(" \\(([0-9]+/[0-9]+) bits\\)", line)
+
+  n = index(line, ": TLS")
+  if (n > 0) {
+    r = substr(line, n + 2)
+    p = index(r, " with cipher")
+    if (p > 0) proto = substr(r, 1, p - 1)
+  }
+  cipher = token_after(line, " with cipher ")
+  n = index(line, " (")
+  if (n > 0) {
+    r = substr(line, n + 2)
+    p = index(r, " bits)")
+    if (p > 0) bits = substr(r, 1, p - 1)
+  }
 
   TLS_SEEN[ip]   = 1
   TLS_PROTO[ip]  = (proto  != "" ? proto  : "-")
@@ -357,10 +530,67 @@ function is_inbound(sasl, orig_ip, src) {
   return 0
 }
 
+###############################################################################
+# Fortschrittsanzeige auf stderr
+###############################################################################
+function progress_tick(   now, ch) {
+  if (show_progress != 1) return
+
+  now = systime()
+  if (PROGRESS_LAST != "" && now - PROGRESS_LAST < 2) return
+
+  PROGRESS_LAST = now
+  PROGRESS_IDX = (PROGRESS_IDX % 4) + 1
+  ch = substr("|/-\\", PROGRESS_IDX, 1)
+  printf "\r%s", ch > "/dev/stderr"
+  fflush("/dev/stderr")
+  PROGRESS_DRAWN = 1
+}
+
+function progress_clear() {
+  if (show_progress != 1 || PROGRESS_DRAWN != 1) return
+  printf "\r \r" > "/dev/stderr"
+  fflush("/dev/stderr")
+  PROGRESS_DRAWN = 0
+}
+
 ###############################################################################
 # Ausgabe
 ###############################################################################
+function count_record(k) {
+  STAT_TOTAL++
+  if (REC_status[k] == "success") STAT_SUCCESS++
+  if (REC_status[k] == "failed") STAT_FAILED++
+}
+
+function emit_stats(   width) {
+  progress_clear()
+
+  if (out_format == "json") {
+    printf "{\"type\":\"stats\",\"total\":%d,\"success\":%d,\"failed\":%d}\n", STAT_TOTAL+0, STAT_SUCCESS+0, STAT_FAILED+0
+    return
+  }
+
+  if (out_format == "one-line") {
+    printf "stats\ttotal=%d\tsuccess=%d\tfailed=%d\n", STAT_TOTAL+0, STAT_SUCCESS+0, STAT_FAILED+0
+    return
+  }
+
+  width = length(STAT_TOTAL+0)
+  if (length(STAT_SUCCESS+0) > width) width = length(STAT_SUCCESS+0)
+  if (length(STAT_FAILED+0) > width) width = length(STAT_FAILED+0)
+
+  print ""
+  print "Statistics:"
+  printf "  total:   %*d\n", width, STAT_TOTAL+0
+  printf "  success: %*d\n", width, STAT_SUCCESS+0
+  printf "  failed:  %*d\n", width, STAT_FAILED+0
+}
+
 function emit_record(k,   tls_part) {
+  progress_clear()
+  count_record(k)
+
   if (out_format == "one-line") {
     tls_part = ""
     if (REC_tls[k] != "" && REC_tls[k] != "-") {
@@ -487,12 +717,15 @@ END {
     }
     for (i=1; i<=n; i++) emit_record(KEYS[i])
   }
+  emit_stats()
 }
 
 ###############################################################################
 # Hauptparser: pro Logzeile State sammeln, bei Zustell-Outcome ausgeben
 ###############################################################################
 {
+  progress_tick()
+
   ts = $1
 
   # Zeitraumfilter (prefix match; ISO8601)
@@ -503,17 +736,22 @@ END {
   if ($0 ~ /TLS connection established from /) { remember_tls($0); next }
 
   # Postfix Queue-ID (qid) aus Zeile ziehen
-  qid = grab(" ([A-F0-9]{7,20}):", $0)
+  qid = qid_from_line($0)
   if (qid == "") next
   if (qid_filter != "" && index(qid, qid_filter) == 0) next
 
+  if ($0 ~ / postfix\/qmgr\[/ && $0 ~ / removed$/) {
+    delete_qid_state(qid)
+    next
+  }
+
   # Envelope from/to sammeln
-  if ($0 ~ / from=<[^>]*>/) FROM[qid] = grab(" from=<([^>]*)>", $0)
-  if ($0 ~ / to=<[^>]*>/)   TO[qid]   = grab(" to=<([^>]*)>", $0)
+  if ($0 ~ / from=<[^>]*>/) FROM[qid] = angle_after($0, " from=<")
+  if ($0 ~ / to=<[^>]*>/)   TO[qid]   = angle_after($0, " to=<")
 
   # cleanup: Header Message-ID sammeln + Ursprung/TLS an msgid binden
   if ($0 ~ / postfix\/cleanup\[/ && $0 ~ / message-id=<[^>]+>/) {
-    MID[qid] = grab(" message-id=<([^>]+)>", $0)
+    MID[qid] = angle_after($0, " message-id=<")
 
     # Externen Ursprung puffern und an msgid binden (für spätere qids)
     if (MID[qid] != "" && TMP_EXTCLIENT[qid] != "") {
@@ -532,11 +770,11 @@ END {
   }
 
   # pickup uid: lokal erzeugt
-  if ($0 ~ / postfix\/pickup\[/ && $0 ~ / uid=/) UID[qid] = grab(" uid=([0-9]+)", $0)
+  if ($0 ~ / postfix\/pickup\[/ && $0 ~ / uid=/) UID[qid] = token_after($0, " uid=")
 
   # postfix/bounce: neue DSN-Queue-ID einer fehlgeschlagenen Ursprungszustellung zuordnen
   if ($0 ~ / postfix\/bounce\[/ && $0 ~ / sender non-delivery notification: /) {
-    newqid = grab(" sender non-delivery notification: ([A-F0-9]{7,20})", $0)
+    newqid = token_after($0, " sender non-delivery notification: ")
     if (newqid != "") {
       DSN_KIND[newqid] = "non-delivery notification"
       if (LAST_FAILURE[qid] != "") DSN_REASON[newqid] = LAST_FAILURE[qid]
@@ -545,12 +783,12 @@ END {
 
   # smtpd: client/orig_client, HELO fallback, TLS map via clientip
   if ($0 ~ / postfix\/smtpd\[/ && $0 ~ / client=/) {
-    CLIENT[qid]   = grab(" client=([^\\[]+)", $0)
-    CLIENTIP[qid] = grab(" client=[^\\[]+\\[([^\\]]+)\\]", $0)
+    CLIENT[qid]   = host_before_bracket_after($0, " client=")
+    CLIENTIP[qid] = bracket_after($0, " client=")
 
     if ($0 ~ / orig_client=/) {
-      ORIGCLIENT[qid]   = grab(" orig_client=([^\\[]+)", $0)
-      ORIGCLIENTIP[qid] = grab(" orig_client=[^\\[]+\\[([^\\]]+)\\]", $0)
+      ORIGCLIENT[qid]   = host_before_bracket_after($0, " orig_client=")
+      ORIGCLIENTIP[qid] = bracket_after($0, " orig_client=")
     }
 
     # externen Ursprung puffern bis cleanup (msgid) kommt
@@ -561,8 +799,8 @@ END {
 
     # HELO auslesen (falls vorhanden), sonst fallback auf client host
     if ($0 ~ / helo=/) {
-      he = grab(" helo=<([^>]+)>", $0)
-      if (he == "") he = grab(" helo=([^ ,;]+)", $0)
+      he = angle_after($0, " helo=<")
+      if (he == "") he = token_after($0, " helo=")
       if (he != "") HELO[qid] = he
     }
     if (!HELO[qid] && CLIENT[qid] != "") HELO[qid] = CLIENT[qid]
@@ -577,11 +815,11 @@ END {
   }
 
   # SMTP AUTH User
-  if ($0 ~ / postfix\/smtpd\[/ && $0 ~ / sasl_username=/) SASL[qid] = grab(" sasl_username=([^, ]+)", $0)
+  if ($0 ~ / postfix\/smtpd\[/ && $0 ~ / sasl_username=/) SASL[qid] = token_after($0, " sasl_username=")
 
   # ===================== Delivery Outcome (Ausgabezeitpunkt) =====================
   if ($0 ~ / postfix\/(lmtp|smtp)\[/ && $0 ~ / status=/) {
-    raw_status = grab(" status=([^ ]+)",  $0)
+    raw_status = token_after($0, " status=")
     transport = ($0 ~ / postfix\/smtp\[/ ? "smtp" : "lmtp")
 
     from = (FROM[qid] ? FROM[qid] : "-")
@@ -594,11 +832,11 @@ END {
     if (from_filter != "" && index(from, from_filter)==0) next
     if (to_filter   != "" && index(to,   to_filter)==0) next
 
-    dsn   = grab(" dsn=([^, ]+)",    $0)
-    relay = grab(" relay=([^, ]+)",  $0)
+    dsn   = token_after($0, " dsn=")
+    relay = token_after($0, " relay=")
     if (final_only == 1 && !is_final_relay(relay, transport)) next
     status = outcome_status(raw_status, relay)
-    msg   = grab(" status=[^ ]+ \\((.*)\\)$", $0)
+    msg   = status_message($0)
     if (status == "failed" && msg != "") LAST_FAILURE[qid] = msg
 
     # Statusfilter
@@ -620,6 +858,14 @@ END {
     orig_client   = (mid != "" && EXTCLIENT_BY_MSGID[mid]   ? EXTCLIENT_BY_MSGID[mid]   : (ORIGCLIENT[qid]?ORIGCLIENT[qid]:"-"))
     orig_clientip = (mid != "" && EXTCLIENTIP_BY_MSGID[mid] ? EXTCLIENTIP_BY_MSGID[mid] : (ORIGCLIENTIP[qid]?ORIGCLIENTIP[qid]:"-"))
 
+    # Absender-Host/IP Filter: direkter client plus rekonstruierter Ursprung.
+    if (sender_host_filter != "" &&
+        index(client, sender_host_filter)==0 &&
+        index(orig_client, sender_host_filter)==0) next
+    if (sender_ip_filter != "" &&
+        index(clientip, sender_ip_filter)==0 &&
+        index(orig_clientip, sender_ip_filter)==0) next
+
     # TLS: qid oder msgid-Vererbung
     tls       = (TLS[qid]   ? TLS[qid]   : "")
     tls_proto = (TLS_P[qid] ? TLS_P[qid] : "")
@@ -669,4 +915,66 @@ END {
     emit_record(k)
   }
 }
-'
+AWK
+
+set +e
+"${INPUT_CMD[@]}" > "$AWK_FIFO" &
+input_pid=$!
+
+bash -c '
+  awk_bin=$1
+  shift
+  "$awk_bin" "$@"
+  st=$?
+  exit "$st"
+' _ "$AWK_BIN" \
+  -v success_only="$SUCCESS_ONLY" \
+  -v fail_only="$FAIL_ONLY" \
+  -v addr_filter="$ADDR_FILTER" \
+  -v from_filter="$FROM_FILTER" \
+  -v to_filter="$TO_FILTER" \
+  -v qid_filter="$QID_FILTER" \
+  -v msgid_filter="$MSGID_FILTER" \
+  -v sender_host_filter="$SENDER_HOST_FILTER" \
+  -v sender_ip_filter="$SENDER_IP_FILTER" \
+  -v since_prefix="$SINCE_PREFIX" \
+  -v until_prefix="$UNTIL_PREFIX" \
+  -v out_format="$FORMAT" \
+  -v no_note="$NO_NOTE" \
+  -v dedup="$DEDUP" \
+  -v dedup_prefer="$DEDUP_PREFER" \
+  -v final_only="$FINAL_ONLY" \
+  -v follow_mode="$FOLLOW" \
+  -v sasl_only="$SASL_ONLY" \
+  -v no_sasl="$NO_SASL" \
+  -v inbound_only="$INBOUND_ONLY" \
+  -v outbound_only="$OUTBOUND_ONLY" \
+  -v debug_mode="$DEBUG" \
+  -v show_progress="$SHOW_PROGRESS" \
+  -f "$AWK_PROGRAM" < "$AWK_FIFO" 2>"$GAWK_ERR"
+gawk_status=$?
+
+wait "$input_pid"
+input_status=$?
+
+set -e
+
+if [[ $gawk_status -eq 139 ]]; then
+  grep -v -E '(Segmentation fault|Speicherzugriffsfehler).*(gawk|awk)' "$GAWK_ERR" >&2 || true
+  echo "ERROR: $AWK_BIN crashed with a segmentation fault while parsing the mail log." >&2
+  echo "       Try narrowing the input with --since/--until or --log-file, or set AWK_BIN=gawk/mawk explicitly." >&2
+  exit 1
+fi
+
+if [[ $gawk_status -ne 0 ]]; then
+  cat "$GAWK_ERR" >&2
+  echo "ERROR: $AWK_BIN failed while parsing the mail log (exit status: $gawk_status)." >&2
+  exit "$gawk_status"
+fi
+
+# 141 = SIGPIPE. This can happen when awk exits intentionally, for example
+# because --until stopped parsing before zcat/cat consumed all input.
+if [[ $input_status -ne 0 && $input_status -ne 141 ]]; then
+  echo "ERROR: input command failed while reading the mail log (exit status: $input_status)." >&2
+  exit "$input_status"
+fi

postfix.code-workspace

@@ -0,0 +1,7 @@
+{
+	"folders": [
+		{
+			"path": "."
+		}
+	]
+}