From 3b60d1b83e2ff51e74dc44d6ac87923ad229dc6c Mon Sep 17 00:00:00 2001 From: Christoph Date: Wed, 12 Nov 2025 17:22:11 +0100 Subject: [PATCH] Add documentation and service scripts for DMARC reporting. --- DOC/DMARC-Report/daily-run.sh | 14 + DOC/DMARC-Report/dmarc-collect.sh | 47 ++ DOC/DMARC-Report/dmarc-scan-setup.md | 382 +++++++++++++++ DOC/DMARC-Report/dmarc-scan.sh | 301 ++++++++++++ DOC/DMARC-Report/dmarc-server-setup.md | 616 +++++++++++++++++++++++++ 5 files changed, 1360 insertions(+) create mode 100755 DOC/DMARC-Report/daily-run.sh create mode 100755 DOC/DMARC-Report/dmarc-collect.sh create mode 100644 DOC/DMARC-Report/dmarc-scan-setup.md create mode 100755 DOC/DMARC-Report/dmarc-scan.sh create mode 100644 DOC/DMARC-Report/dmarc-server-setup.md diff --git a/DOC/DMARC-Report/daily-run.sh b/DOC/DMARC-Report/daily-run.sh new file mode 100755 index 0000000..13ca119 --- /dev/null +++ b/DOC/DMARC-Report/daily-run.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE="/var/lib/dmarc" +TODAY_DIR="$BASE/reports/$(date -u +%Y/%m/%d)" +OUTDIR="$BASE/exports" +CSV="$OUTDIR/records.csv" +LOGF="$BASE/logs/scan-$(date -u +%F).log" + +mkdir -p "$OUTDIR" + +if [[ -d "$TODAY_DIR" ]]; then + /usr/local/bin/dmarc-scan.sh "$TODAY_DIR" --domain fluechtlingsrat-brandenburg.de --csv "$CSV" --append --top 25 --outdir "$OUTDIR" >> "$LOGF" 2>&1 +fi diff --git a/DOC/DMARC-Report/dmarc-collect.sh b/DOC/DMARC-Report/dmarc-collect.sh new file mode 100755 index 0000000..71b6221 --- /dev/null +++ b/DOC/DMARC-Report/dmarc-collect.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE="/var/lib/dmarc" +INBOX="$BASE/reports" +PROC="$BASE/processed" +LOGF="$BASE/logs/collector.log" + +umask 027 + +TMPDIR="$(mktemp -d)" +EML="$TMPDIR/mail.eml" +cat > "$EML" + +ripmime --no-nameless --name-by-type --overwrite -i "$EML" -d "$TMPDIR" >>"$LOGF" 2>&1 || true + +TODAY="$(date -u +%Y/%m/%d)" +OUTDIR="$INBOX/$TODAY" +mkdir -p "$OUTDIR" + +moved=0 +shopt -s nullglob +for f in "$TMPDIR"/*; do + case "$f" in + *.xml|*.XML|*.gz|*.zip) + sha="$(sha256sum "$f" | awk '{print $1}')" + base="$(basename "$f")" + dst="$OUTDIR/$(date -u +%Y%m%dT%H%M%SZ)_${sha:0:12}_$base" + mv "$f" "$dst" + echo "$(date -Is) stored $dst" >> "$LOGF" + moved=$((moved+1)) + ;; + *) : ;; + esac +done + +mkdir -p "$PROC" +mv "$EML" "$PROC/$(date -u +%Y%m%dT%H%M%SZ)_mail.eml" || true +rm -rf "$TMPDIR" + +if (( moved > 0 )); then + exit 0 +else + echo "$(date -Is) no usable attachment in message" >> "$LOGF" + exit 0 +fi + diff --git a/DOC/DMARC-Report/dmarc-scan-setup.md b/DOC/DMARC-Report/dmarc-scan-setup.md new file mode 100644 index 0000000..0c4310a --- /dev/null +++ b/DOC/DMARC-Report/dmarc-scan-setup.md @@ -0,0 +1,382 @@ + +# DMARC-Auswertungsskript `dmarc-scan.sh` – Installation, Beschreibung & Verwendung + +## 📖 Zweck + +`dmarc-scan.sh` analysiert DMARC-Aggregatberichte (XML, .zip, .gz) und erzeugt: +- gut lesbare Tabellen im Terminal, +- eine fortschreibbare Records-CSV (`--append`), +- Top-Listen nach IPs (gesamt und je Fail-Kategorie) als CSV. + +--- + +## ⚙️ Installation + +```bash +sudo apt install -y xmlstarlet unzip gzip +sudo tee /usr/local/bin/dmarc-scan.sh >/dev/null <<'EOF' +#!/usr/bin/env bash +# +# dmarc-scan.sh — DMARC-XML-Reports (auch .gz/.zip) einlesen, tabellarisch anzeigen, +# Records als CSV exportieren (append optional), Top-IPs ermitteln +# und Top-Listen als CSV schreiben. +# +# Nutzung: +# ./dmarc-scan.sh /pfad/zu/reports \ +# [--domain DOMAIN] \ +# [--csv pfad/zur/records.csv] \ +# [--append] \ +# [--top N] \ +# [--outdir pfad/zum/ordner] +# +# Beispiele: +# ./dmarc-scan.sh /var/mail/dmarc +# ./dmarc-scan.sh /var/mail/dmarc --domain fluechtlingsrat-brandenburg.de --csv dmarc.csv --append --top 15 --outdir ./export +# +# Voraussetzungen: xmlstarlet, unzip (für .zip), gzip (für .gz) +# Debian/Ubuntu: sudo apt-get install xmlstarlet unzip gzip +# +set -euo pipefail + +REPORT_DIR="${1:-}" +shift || true + +# Defaults +WANT_DOMAIN="" +CSV_PATH="./dmarc-summary.csv" +APPEND=0 +TOP_N=10 +OUTDIR="." + +# Arg-Parsing (einfach) +while [[ $# -gt 0 ]]; do + case "${1:-}" in + --domain) + WANT_DOMAIN="${2:-}"; shift 2 || true ;; + --csv) + CSV_PATH="${2:-}"; shift 2 || true ;; + --append) + APPEND=1; shift || true ;; + --top) + TOP_N="${2:-10}"; shift 2 || true ;; + --outdir) + OUTDIR="${2:-.}"; shift 2 || true ;; + *) + # Unbekannte Option ignorieren + shift || true ;; + esac +done + +if [[ -z "$REPORT_DIR" || ! -d "$REPORT_DIR" ]]; then + echo "Fehler: Bitte ein Verzeichnis mit DMARC-Reports angeben." + echo "Beispiel: $0 /var/mail/dmarc --domain fluechtlingsrat-brandenburg.de --csv dmarc.csv --append --top 15 --outdir ./export" + exit 1 +fi + +if ! command -v xmlstarlet >/dev/null 2>&1; then + echo "Fehler: xmlstarlet nicht gefunden. Bitte installieren (z.B. 'sudo apt-get install xmlstarlet')." + exit 1 +fi + +mkdir -p "$OUTDIR" + +# CSV-Header für Records; bei --append nur schreiben, wenn Datei noch nicht existiert +RECORDS_HEADER="report_org,policy_domain,begin_utc,end_utc,source_ip,count,disposition,spf,dkim,header_from" +ensure_records_header() { + if [[ "$APPEND" -eq 1 ]]; then + if [[ ! -f "$CSV_PATH" ]]; then + echo "$RECORDS_HEADER" > "$CSV_PATH" + fi + else + echo "$RECORDS_HEADER" > "$CSV_PATH" + fi +} +ensure_records_header + +# Zähler & Sets +declare -A SEEN_IPS=() +declare -A IP_COUNTS=() # Summe pro IP +declare -A SPF_ONLY_FAIL_IP=() # nur SPF fail pro IP +declare -A DKIM_ONLY_FAIL_IP=() # nur DKIM fail pro IP +declare -A BOTH_FAIL_IP=() # SPF+DKIM fail pro IP + +total_msgs=0 +pass_msgs=0 +fail_msgs=0 +spf_only_fail=0 +dkim_only_fail=0 +both_fail=0 + +# Hilfsfunktion: Epoch -> Datum (UTC) +epoch2date() { + local e="$1" + if [[ -z "$e" ]]; then printf "-"; return; fi + date -u -d @"$e" +"%Y-%m-%d %H:%M:%S UTC" 2>/dev/null || printf "%s" "$e" +} + +# CSV-escape (Felder in Anführungszeichen, doppelte Anführungszeichen verdoppeln) +csv_escape() { + local s="${1:-}" + s="${s//\"/\"\"}" + printf "\"%s\"" "$s" +} + +# Eine einzelne XML-Datei parsen +parse_xml() { + local xml_input="$1" + + # Domain aus policy_published, ggf. für Filter + local domain + domain=$(xmlstarlet sel -T -t -v "/feedback/policy_published/domain" "$xml_input" 2>/dev/null || true) + if [[ -n "$WANT_DOMAIN" && "$domain" != "$WANT_DOMAIN" ]]; then + return 0 + fi + + local org begin end pol sp aspf adkim + org=$(xmlstarlet sel -T -t -v "/feedback/report_metadata/org_name" "$xml_input" 2>/dev/null || printf "-") + begin=$(xmlstarlet sel -T -t -v "/feedback/report_metadata/date_range/begin" "$xml_input" 2>/dev/null || printf "") + end=$(xmlstarlet sel -T -t -v "/feedback/report_metadata/date_range/end" "$xml_input" 2>/dev/null || printf "") + pol=$(xmlstarlet sel -T -t -v "/feedback/policy_published/p" "$xml_input" 2>/dev/null || printf "-") + sp=$(xmlstarlet sel -T -t -v "/feedback/policy_published/sp" "$xml_input" 2>/dev/null || printf "-") + aspf=$(xmlstarlet sel -T -t -v "/feedback/policy_published/aspf" "$xml_input" 2>/dev/null || printf "-") + adkim=$(xmlstarlet sel -T -t -v "/feedback/policy_published/adkim" "$xml_input" 2>/dev/null || printf "-") + + # Report-Header + echo "==============================================================================" + echo "Report von: ${org}" + echo "Domain (Policy): ${domain} (p=${pol}, sp=${sp}, aspf=${aspf}, adkim=${adkim})" + echo "Zeitraum: $(epoch2date "$begin") – $(epoch2date "$end")" + echo "------------------------------------------------------------------------------" + printf "%-16s %7s %-10s %-6s %-6s %s\n" "Source IP" "Count" "Disposition" "SPF" "DKIM" "Header-From" + echo "------------------------------------------------------------------------------" + + # Alle -Einträge tabellarisch ausgeben + while IFS='|' read -r ip cnt dispo spfres dkimres hfrom; do + [[ -z "$ip$cnt$dispo$spfres$dkimres$hfrom" ]] && continue + + local n=0 + if [[ -n "${cnt:-}" && "$cnt" =~ ^[0-9]+$ ]]; then n="$cnt"; fi + + total_msgs=$(( total_msgs + n )) + [[ -n "$ip" ]] && SEEN_IPS["$ip"]=1 + [[ -n "$ip" ]] && IP_COUNTS["$ip"]=$(( ${IP_COUNTS["$ip"]:-0} + n )) + + if [[ "${spfres:-}" == "pass" && "${dkimres:-}" == "pass" ]]; then + pass_msgs=$(( pass_msgs + n )) + else + fail_msgs=$(( fail_msgs + n )) + if [[ "${spfres:-}" != "pass" && "${dkimres:-}" == "pass" ]]; then + spf_only_fail=$(( spf_only_fail + n )) + [[ -n "$ip" ]] && SPF_ONLY_FAIL_IP["$ip"]=$(( ${SPF_ONLY_FAIL_IP["$ip"]:-0} + n )) + elif [[ "${spfres:-}" == "pass" && "${dkimres:-}" != "pass" ]]; then + dkim_only_fail=$(( dkim_only_fail + n )) + [[ -n "$ip" ]] && DKIM_ONLY_FAIL_IP["$ip"]=$(( ${DKIM_ONLY_FAIL_IP["$ip"]:-0} + n )) + else + both_fail=$(( both_fail + n )) + [[ -n "$ip" ]] && BOTH_FAIL_IP["$ip"]=$(( ${BOTH_FAIL_IP["$ip"]:-0} + n )) + fi + fi + + printf "%-16s %7s %-10s %-6s %-6s %s\n" "${ip:--}" "${n}" "${dispo:--}" "${spfres:--}" "${dkimres:--}" "${hfrom:--}" + + local begin_human end_human + begin_human="$(epoch2date "$begin")" + end_human="$(epoch2date "$end")" + { + csv_escape "$org"; printf "," + csv_escape "$domain"; printf "," + csv_escape "$begin_human"; printf "," + csv_escape "$end_human"; printf "," + csv_escape "${ip:-}"; printf "," + printf "%s," "$n" + csv_escape "${dispo:-}"; printf "," + csv_escape "${spfres:-}"; printf "," + csv_escape "${dkimres:-}"; printf "," + csv_escape "${hfrom:-}"; printf "\n" + } >> "$CSV_PATH" + + done < <(xmlstarlet sel -T -t \ + -m "/feedback/record" \ + -v "row/source_ip" -o "|" \ + -v "row/count" -o "|" \ + -v "row/policy_evaluated/disposition" -o "|" \ + -v "row/policy_evaluated/spf" -o "|" \ + -v "row/policy_evaluated/dkim" -o "|" \ + -v "identifiers/header_from" -n \ + "$xml_input" 2>/dev/null) + + echo +} + +# Alle Dateien im Verzeichnis verarbeiten +shopt -s nullglob +for f in "$REPORT_DIR"/*; do + case "$f" in + *.xml) + parse_xml "$f" + ;; + *.gz) + if command -v gzip >/dev/null 2>&1; then + tmp="$(mktemp)" + if gzip -cd "$f" > "$tmp"; then + parse_xml "$tmp" + else + echo "Warnung: Konnte $f nicht entpacken." + fi + rm -f "$tmp" + else + echo "Warnung: gzip nicht verfügbar, überspringe $f" + fi + ;; + *.zip) + if command -v unzip >/dev/null 2>&1; then + tmpdir="$(mktemp -d)" + if unzip -qq -j "$f" '*.xml' -d "$tmpdir" >/dev/null 2>&1; then + for x in "$tmpdir"/*.xml; do + [[ -e "$x" ]] || continue + parse_xml "$x" + done + else + echo "Warnung: Konnte $f nicht entpacken (oder keine XML darin)." + fi + rm -rf "$tmpdir" + else + echo "Warnung: unzip nicht verfügbar, überspringe $f" + fi + ;; + *) : ;; + esac +done + +# Zusammenfassung +unique_ips=${#SEEN_IPS[@]} +echo "==============================================================================" +echo "GESAMT-ZUSAMMENFASSUNG" +echo "Nachrichten gesamt: $total_msgs" +echo "Eindeutige Source-IPs: $unique_ips" +echo "Alle Auth PASS: $pass_msgs" +echo "SPF/DKIM Fehler gesamt: $fail_msgs" +echo " ├─ nur SPF-Fail: $spf_only_fail" +echo " ├─ nur DKIM-Fail: $dkim_only_fail" +echo " └─ SPF+DKIM-Fail: $both_fail" +[[ -n "$WANT_DOMAIN" ]] && echo "Gefilterte Domain: $WANT_DOMAIN" +echo "Records-CSV: $CSV_PATH" +echo "Hinweis: 'Fail' umfasst Records, bei denen SPF oder DKIM nicht 'pass' war." + +# Top-Listen auf STDOUT +echo +echo "TOP $TOP_N IPs nach Anzahl (über alle Reports):" +{ + for ip in "${!IP_COUNTS[@]}"; do + printf "%10d %s\n" "${IP_COUNTS[$ip]}" "$ip" + done +} | sort -rn | head -n "$TOP_N" | awk '{printf " %2d) %-16s %7s\n", NR, $2, $1}' + +if (( fail_msgs > 0 )); then + echo + echo "Top-IPs nur SPF-Fail:" + { for ip in "${!SPF_ONLY_FAIL_IP[@]}"; do printf "%10d %s\n" "${SPF_ONLY_FAIL_IP[$ip]}" "$ip"; done; } \ + | sort -rn | head -n "$TOP_N" | awk '{printf " %2d) %-16s %7s\n", NR, $2, $1}' + echo + echo "Top-IPs nur DKIM-Fail:" + { for ip in "${!DKIM_ONLY_FAIL_IP[@]}"; do printf "%10d %s\n" "${DKIM_ONLY_FAIL_IP[$ip]}" "$ip"; done; } \ + | sort -rn | head -n "$TOP_N" | awk '{printf " %2d) %-16s %7s\n", NR, $2, $1}' + echo + echo "Top-IPs SPF+DKIM-Fail:" + { for ip in "${!BOTH_FAIL_IP[@]}"; do printf "%10d %s\n" "${BOTH_FAIL_IP[$ip]}" "$ip"; done; } \ + | sort -rn | head -n "$TOP_N" | awk '{printf " %2d) %-16s %7s\n", NR, $2, $1}' +fi + +# ---- CSV-Exporte der Top-Listen --------------------------------------------- + +write_top_csv () { + local outfile="$1"; shift + local -n assoc="$1" # name reference auf assoziatives Array + echo "ip,count" > "$outfile" + if [[ "${#assoc[@]}" -eq 0 ]]; then + : # leer + else + for ip in "${!assoc[@]}"; do + printf "%s,%s\n" "$ip" "${assoc[$ip]}" + done | sort -t, -k2,2nr > "$outfile" + fi +} + +# Gesamt-Top-IPs +write_top_csv "$OUTDIR/top_ips.csv" IP_COUNTS + +# Top-Listen nach Fail-Kategorien +write_top_csv "$OUTDIR/top_spf_fail_ips.csv" SPF_ONLY_FAIL_IP +write_top_csv "$OUTDIR/top_dkim_fail_ips.csv" DKIM_ONLY_FAIL_IP +write_top_csv "$OUTDIR/top_both_fail_ips.csv" BOTH_FAIL_IP + +echo +echo "Exportierte Top-CSV-Dateien:" +echo " $OUTDIR/top_ips.csv" +echo " $OUTDIR/top_spf_fail_ips.csv" +echo " $OUTDIR/top_dkim_fail_ips.csv" +echo " $OUTDIR/top_both_fail_ips.csv" + +EOF +sudo chmod 750 /usr/local/bin/dmarc-scan.sh +``` + +--- + +## 🧩 Verwendung + +Grundbeispiel: +```bash +dmarc-scan.sh /var/lib/dmarc/reports/2025/11/12 +``` + +Mit Optionen: +```bash +dmarc-scan.sh /var/lib/dmarc/reports/2025/11/12 --domain fluechtlingsrat-brandenburg.de --csv /var/lib/dmarc/exports/records.csv --append --top 25 --outdir /var/lib/dmarc/exports/ +``` + +### Parameterübersicht + +| Parameter | Bedeutung | +|------------|------------| +| `--domain ` | Filtert Berichte auf bestimmte Domain | +| `--csv ` | Pfad zur Ausgabedatei (Records-CSV) | +| `--append` | Bestehende CSV fortschreiben statt überschreiben | +| `--top ` | Anzahl der angezeigten Top-IPs | +| `--outdir ` | Zielverzeichnis für Top-Listen (CSV) | + +--- + +## 📊 Ausgabedateien + +| Datei | Inhalt | +|-------|---------| +| `records.csv` | Alle Einzel-Records | +| `top_ips.csv` | IPs mit den meisten Mails | +| `top_spf_fail_ips.csv` | IPs mit nur SPF-Fails | +| `top_dkim_fail_ips.csv` | IPs mit nur DKIM-Fails | +| `top_both_fail_ips.csv` | IPs mit SPF+DKIM-Fails | + +--- + +## 🔁 Integration in den Serverbetrieb + +Das Skript wird typischerweise durch den täglichen Job `/usr/local/lib/dmarc/daily-run.sh` aufgerufen. +Manuelle Nutzung ist jederzeit möglich. + +--- + +## 🧩 Kontext – Verzeichnisstruktur (empfohlen) + +``` +/var/lib/dmarc/ +├── reports/ # Eingehende Rohdaten (XML, ZIP, GZ) – nach Datum +├── exports/ # CSV- und Top-Dateien +└── logs/ # Logausgaben der täglichen Auswertung +``` + +--- + +**Autor:** oopen.de / Systemkonfiguration +**Stand:** November 2025 +**Version:** 1.2 diff --git a/DOC/DMARC-Report/dmarc-scan.sh b/DOC/DMARC-Report/dmarc-scan.sh new file mode 100755 index 0000000..0276b94 --- /dev/null +++ b/DOC/DMARC-Report/dmarc-scan.sh @@ -0,0 +1,301 @@ +#!/usr/bin/env bash +# +# dmarc-scan.sh — DMARC-XML-Reports (auch .gz/.zip) einlesen, tabellarisch anzeigen, +# Records als CSV exportieren (append optional), Top-IPs ermitteln +# und Top-Listen als CSV schreiben. +# +# Nutzung: +# ./dmarc-scan.sh /pfad/zu/reports \ +# [--domain DOMAIN] \ +# [--csv pfad/zur/records.csv] \ +# [--append] \ +# [--top N] \ +# [--outdir pfad/zum/ordner] +# +# Beispiele: +# ./dmarc-scan.sh /var/mail/dmarc +# ./dmarc-scan.sh /var/mail/dmarc --domain fluechtlingsrat-brandenburg.de --csv dmarc.csv --append --top 15 --outdir ./export +# +# Voraussetzungen: xmlstarlet, unzip (für .zip), gzip (für .gz) +# Debian/Ubuntu: sudo apt-get install xmlstarlet unzip gzip +# +set -euo pipefail + +REPORT_DIR="${1:-}" +shift || true + +# Defaults +WANT_DOMAIN="" +CSV_PATH="./dmarc-summary.csv" +APPEND=0 +TOP_N=10 +OUTDIR="." + +# Arg-Parsing (einfach) +while [[ $# -gt 0 ]]; do + case "${1:-}" in + --domain) + WANT_DOMAIN="${2:-}"; shift 2 || true ;; + --csv) + CSV_PATH="${2:-}"; shift 2 || true ;; + --append) + APPEND=1; shift || true ;; + --top) + TOP_N="${2:-10}"; shift 2 || true ;; + --outdir) + OUTDIR="${2:-.}"; shift 2 || true ;; + *) + # Unbekannte Option ignorieren + shift || true ;; + esac +done + +if [[ -z "$REPORT_DIR" || ! -d "$REPORT_DIR" ]]; then + echo "Fehler: Bitte ein Verzeichnis mit DMARC-Reports angeben." + echo "Beispiel: $0 /var/mail/dmarc --domain fluechtlingsrat-brandenburg.de --csv dmarc.csv --append --top 15 --outdir ./export" + exit 1 +fi + +if ! command -v xmlstarlet >/dev/null 2>&1; then + echo "Fehler: xmlstarlet nicht gefunden. Bitte installieren (z.B. 'sudo apt-get install xmlstarlet')." + exit 1 +fi + +mkdir -p "$OUTDIR" + +# CSV-Header für Records; bei --append nur schreiben, wenn Datei noch nicht existiert +RECORDS_HEADER="report_org,policy_domain,begin_utc,end_utc,source_ip,count,disposition,spf,dkim,header_from" +ensure_records_header() { + if [[ "$APPEND" -eq 1 ]]; then + if [[ ! -f "$CSV_PATH" ]]; then + echo "$RECORDS_HEADER" > "$CSV_PATH" + fi + else + echo "$RECORDS_HEADER" > "$CSV_PATH" + fi +} +ensure_records_header + +# Zähler & Sets +declare -A SEEN_IPS=() +declare -A IP_COUNTS=() # Summe pro IP +declare -A SPF_ONLY_FAIL_IP=() # nur SPF fail pro IP +declare -A DKIM_ONLY_FAIL_IP=() # nur DKIM fail pro IP +declare -A BOTH_FAIL_IP=() # SPF+DKIM fail pro IP + +total_msgs=0 +pass_msgs=0 +fail_msgs=0 +spf_only_fail=0 +dkim_only_fail=0 +both_fail=0 + +# Hilfsfunktion: Epoch -> Datum (UTC) +epoch2date() { + local e="$1" + if [[ -z "$e" ]]; then printf "-"; return; fi + date -u -d @"$e" +"%Y-%m-%d %H:%M:%S UTC" 2>/dev/null || printf "%s" "$e" +} + +# CSV-escape (Felder in Anführungszeichen, doppelte Anführungszeichen verdoppeln) +csv_escape() { + local s="${1:-}" + s="${s//\"/\"\"}" + printf "\"%s\"" "$s" +} + +# Eine einzelne XML-Datei parsen +parse_xml() { + local xml_input="$1" + + # Domain aus policy_published, ggf. für Filter + local domain + domain=$(xmlstarlet sel -T -t -v "/feedback/policy_published/domain" "$xml_input" 2>/dev/null || true) + if [[ -n "$WANT_DOMAIN" && "$domain" != "$WANT_DOMAIN" ]]; then + return 0 + fi + + local org begin end pol sp aspf adkim + org=$(xmlstarlet sel -T -t -v "/feedback/report_metadata/org_name" "$xml_input" 2>/dev/null || printf "-") + begin=$(xmlstarlet sel -T -t -v "/feedback/report_metadata/date_range/begin" "$xml_input" 2>/dev/null || printf "") + end=$(xmlstarlet sel -T -t -v "/feedback/report_metadata/date_range/end" "$xml_input" 2>/dev/null || printf "") + pol=$(xmlstarlet sel -T -t -v "/feedback/policy_published/p" "$xml_input" 2>/dev/null || printf "-") + sp=$(xmlstarlet sel -T -t -v "/feedback/policy_published/sp" "$xml_input" 2>/dev/null || printf "-") + aspf=$(xmlstarlet sel -T -t -v "/feedback/policy_published/aspf" "$xml_input" 2>/dev/null || printf "-") + adkim=$(xmlstarlet sel -T -t -v "/feedback/policy_published/adkim" "$xml_input" 2>/dev/null || printf "-") + + # Report-Header + echo "==============================================================================" + echo "Report von: ${org}" + echo "Domain (Policy): ${domain} (p=${pol}, sp=${sp}, aspf=${aspf}, adkim=${adkim})" + echo "Zeitraum: $(epoch2date "$begin") – $(epoch2date "$end")" + echo "------------------------------------------------------------------------------" + printf "%-16s %7s %-10s %-6s %-6s %s\n" "Source IP" "Count" "Disposition" "SPF" "DKIM" "Header-From" + echo "------------------------------------------------------------------------------" + + # Alle -Einträge tabellarisch ausgeben + while IFS='|' read -r ip cnt dispo spfres dkimres hfrom; do + [[ -z "$ip$cnt$dispo$spfres$dkimres$hfrom" ]] && continue + + local n=0 + if [[ -n "${cnt:-}" && "$cnt" =~ ^[0-9]+$ ]]; then n="$cnt"; fi + + total_msgs=$(( total_msgs + n )) + [[ -n "$ip" ]] && SEEN_IPS["$ip"]=1 + [[ -n "$ip" ]] && IP_COUNTS["$ip"]=$(( ${IP_COUNTS["$ip"]:-0} + n )) + + if [[ "${spfres:-}" == "pass" && "${dkimres:-}" == "pass" ]]; then + pass_msgs=$(( pass_msgs + n )) + else + fail_msgs=$(( fail_msgs + n )) + if [[ "${spfres:-}" != "pass" && "${dkimres:-}" == "pass" ]]; then + spf_only_fail=$(( spf_only_fail + n )) + [[ -n "$ip" ]] && SPF_ONLY_FAIL_IP["$ip"]=$(( ${SPF_ONLY_FAIL_IP["$ip"]:-0} + n )) + elif [[ "${spfres:-}" == "pass" && "${dkimres:-}" != "pass" ]]; then + dkim_only_fail=$(( dkim_only_fail + n )) + [[ -n "$ip" ]] && DKIM_ONLY_FAIL_IP["$ip"]=$(( ${DKIM_ONLY_FAIL_IP["$ip"]:-0} + n )) + else + both_fail=$(( both_fail + n )) + [[ -n "$ip" ]] && BOTH_FAIL_IP["$ip"]=$(( ${BOTH_FAIL_IP["$ip"]:-0} + n )) + fi + fi + + printf "%-16s %7s %-10s %-6s %-6s %s\n" "${ip:--}" "${n}" "${dispo:--}" "${spfres:--}" "${dkimres:--}" "${hfrom:--}" + + local begin_human end_human + begin_human="$(epoch2date "$begin")" + end_human="$(epoch2date "$end")" + { + csv_escape "$org"; printf "," + csv_escape "$domain"; printf "," + csv_escape "$begin_human"; printf "," + csv_escape "$end_human"; printf "," + csv_escape "${ip:-}"; printf "," + printf "%s," "$n" + csv_escape "${dispo:-}"; printf "," + csv_escape "${spfres:-}"; printf "," + csv_escape "${dkimres:-}"; printf "," + csv_escape "${hfrom:-}"; printf "\n" + } >> "$CSV_PATH" + + done < <(xmlstarlet sel -T -t \ + -m "/feedback/record" \ + -v "row/source_ip" -o "|" \ + -v "row/count" -o "|" \ + -v "row/policy_evaluated/disposition" -o "|" \ + -v "row/policy_evaluated/spf" -o "|" \ + -v "row/policy_evaluated/dkim" -o "|" \ + -v "identifiers/header_from" -n \ + "$xml_input" 2>/dev/null) + + echo +} + +# Alle Dateien im Verzeichnis verarbeiten +shopt -s nullglob +for f in "$REPORT_DIR"/*; do + case "$f" in + *.xml) + parse_xml "$f" + ;; + *.gz) + if command -v gzip >/dev/null 2>&1; then + tmp="$(mktemp)" + if gzip -cd "$f" > "$tmp"; then + parse_xml "$tmp" + else + echo "Warnung: Konnte $f nicht entpacken." + fi + rm -f "$tmp" + else + echo "Warnung: gzip nicht verfügbar, überspringe $f" + fi + ;; + *.zip) + if command -v unzip >/dev/null 2>&1; then + tmpdir="$(mktemp -d)" + if unzip -qq -j "$f" '*.xml' -d "$tmpdir" >/dev/null 2>&1; then + for x in "$tmpdir"/*.xml; do + [[ -e "$x" ]] || continue + parse_xml "$x" + done + else + echo "Warnung: Konnte $f nicht entpacken (oder keine XML darin)." + fi + rm -rf "$tmpdir" + else + echo "Warnung: unzip nicht verfügbar, überspringe $f" + fi + ;; + *) : ;; + esac +done + +# Zusammenfassung +unique_ips=${#SEEN_IPS[@]} +echo "==============================================================================" +echo "GESAMT-ZUSAMMENFASSUNG" +echo "Nachrichten gesamt: $total_msgs" +echo "Eindeutige Source-IPs: $unique_ips" +echo "Alle Auth PASS: $pass_msgs" +echo "SPF/DKIM Fehler gesamt: $fail_msgs" +echo " ├─ nur SPF-Fail: $spf_only_fail" +echo " ├─ nur DKIM-Fail: $dkim_only_fail" +echo " └─ SPF+DKIM-Fail: $both_fail" +[[ -n "$WANT_DOMAIN" ]] && echo "Gefilterte Domain: $WANT_DOMAIN" +echo "Records-CSV: $CSV_PATH" +echo "Hinweis: 'Fail' umfasst Records, bei denen SPF oder DKIM nicht 'pass' war." + +# Top-Listen auf STDOUT +echo +echo "TOP $TOP_N IPs nach Anzahl (über alle Reports):" +{ + for ip in "${!IP_COUNTS[@]}"; do + printf "%10d %s\n" "${IP_COUNTS[$ip]}" "$ip" + done +} | sort -rn | head -n "$TOP_N" | awk '{printf " %2d) %-16s %7s\n", NR, $2, $1}' + +if (( fail_msgs > 0 )); then + echo + echo "Top-IPs nur SPF-Fail:" + { for ip in "${!SPF_ONLY_FAIL_IP[@]}"; do printf "%10d %s\n" "${SPF_ONLY_FAIL_IP[$ip]}" "$ip"; done; } \ + | sort -rn | head -n "$TOP_N" | awk '{printf " %2d) %-16s %7s\n", NR, $2, $1}' + echo + echo "Top-IPs nur DKIM-Fail:" + { for ip in "${!DKIM_ONLY_FAIL_IP[@]}"; do printf "%10d %s\n" "${DKIM_ONLY_FAIL_IP[$ip]}" "$ip"; done; } \ + | sort -rn | head -n "$TOP_N" | awk '{printf " %2d) %-16s %7s\n", NR, $2, $1}' + echo + echo "Top-IPs SPF+DKIM-Fail:" + { for ip in "${!BOTH_FAIL_IP[@]}"; do printf "%10d %s\n" "${BOTH_FAIL_IP[$ip]}" "$ip"; done; } \ + | sort -rn | head -n "$TOP_N" | awk '{printf " %2d) %-16s %7s\n", NR, $2, $1}' +fi + +# ---- CSV-Exporte der Top-Listen --------------------------------------------- + +write_top_csv () { + local outfile="$1"; shift + local -n assoc="$1" # name reference auf assoziatives Array + echo "ip,count" > "$outfile" + if [[ "${#assoc[@]}" -eq 0 ]]; then + : # leer + else + for ip in "${!assoc[@]}"; do + printf "%s,%s\n" "$ip" "${assoc[$ip]}" + done | sort -t, -k2,2nr > "$outfile" + fi +} + +# Gesamt-Top-IPs +write_top_csv "$OUTDIR/top_ips.csv" IP_COUNTS + +# Top-Listen nach Fail-Kategorien +write_top_csv "$OUTDIR/top_spf_fail_ips.csv" SPF_ONLY_FAIL_IP +write_top_csv "$OUTDIR/top_dkim_fail_ips.csv" DKIM_ONLY_FAIL_IP +write_top_csv "$OUTDIR/top_both_fail_ips.csv" BOTH_FAIL_IP + +echo +echo "Exportierte Top-CSV-Dateien:" +echo " $OUTDIR/top_ips.csv" +echo " $OUTDIR/top_spf_fail_ips.csv" +echo " $OUTDIR/top_dkim_fail_ips.csv" +echo " $OUTDIR/top_both_fail_ips.csv" diff --git a/DOC/DMARC-Report/dmarc-server-setup.md b/DOC/DMARC-Report/dmarc-server-setup.md new file mode 100644 index 0000000..3950a8b --- /dev/null +++ b/DOC/DMARC-Report/dmarc-server-setup.md @@ -0,0 +1,616 @@ + +# DMARC-Server-Sammelsystem – Einrichtung und Betrieb + +## 📖 Präambel + +Dieses Dokument beschreibt die vollständige Einrichtung eines automatisierten DMARC-Report-Sammelsystems auf einem Linux-Mailserver mit **Postfix**, **Amavis** und **Dovecot (LMTP)**. +Ziel: DMARC-Aggregatberichte (XML, .gz, .zip) serverseitig empfangen, speichern und regelmäßig auswerten. + +--- + +## 🎯 Ziel + +- DMARC-Reports zentral über `dmarc-reports@oopen.de` empfangen. +- Reports automatisch extrahieren und datumsbasiert speichern. +- Regelmäßige automatische Auswertung mit `dmarc-scan.sh`. +- Speicherung der Ergebnisse in CSV-Dateien. +- Wartungsarm, sicher und robust. + +--- + +## 📁 Verzeichnisstruktur + +Alle Dateien werden unter `/var/lib/dmarc` abgelegt: + +``` +/var/lib/dmarc/ +├── reports/ # Eingegangene XML-, GZ-, ZIP-Dateien +│ └── YYYY/MM/DD/ # Datumsbasierte Ablage +├── processed/ # Originalmails (Archiv) +├── exports/ # CSV- und Top-Auswertungen +└── logs/ # Logdateien +``` + +Verzeichnisse anlegen: + +```bash +sudo install -d -o root -g root -m 750 /var/lib/dmarc/{reports,processed,exports,logs} +sudo install -d -o root -g root -m 750 /usr/local/lib/dmarc +``` + +--- + +## ⚙️ 1. Postfix-Integration + +### 1.1 Transport und Master-Konfiguration + +In `/etc/postfix/master.cf` **am Ende einfügen:** + +```bash +dmarc-pipe unix - n n - - pipe + flags=Rq user=vmail argv=/usr/local/bin/dmarc-collect.sh +``` + +> Passe `user=` an, falls dein Mailbenutzer anders heißt (z. B. `mail` oder `amavis`). + +### 1.2 Transportregel definieren + +In `/etc/postfix/transport`: + +```bash +dmarc-reports@oopen.de dmarc-pipe: +``` + +Aktivieren: + +```bash +sudo postmap /etc/postfix/transport +sudo systemctl reload postfix +``` + +Damit weiß Postfix, dass Mails an diese Adresse an das Skript weitergegeben werden. + +### 1.3 (Optional) Kopie der Mail behalten + +Wenn du zusätzlich eine Kopie im IMAP-Postfach haben willst: + +In `/etc/postfix/virtual`: + +```bash +dmarc-reports@oopen.de reports@oopen.de, dmarc-pipe: +``` + +Dann: + +```bash +sudo postmap /etc/postfix/virtual +sudo systemctl reload postfix +``` + +> **Merke:** `/etc/postfix/transport` = ZustellWEG, `/etc/postfix/virtual` = EmpfängerALIAS. +> Nur Skript? → Nur `transport`-Eintrag. +> Skript + Kopie? → `virtual`-Alias **zusätzlich**. + +--- + +## 📨 2. DNS-Einträge (DMARC + External Reporting) + +### Beispiel für eine Domain + +```dns +_dmarc.fluechtlingsrat-brandenburg.de. IN TXT "v=DMARC1; p=none; rua=mailto:dmarc-reports@oopen.de; ruf=mailto:dmarc-reports@oopen.de; fo=1; aspf=r; adkim=r" +``` + +### External Reporting (oopen.de) + +Wenn du Berichte für mehrere Domains auf `@oopen.de` empfängst, erlaube externes Reporting: + +```dns +oopen.de._report._dmarc.oopen.de. IN TXT "v=DMARC1" +*.oopen.de._report._dmarc.oopen.de. IN TXT "v=DMARC1" +``` + +--- + +## 🧰 3. Sammelskript `/usr/local/bin/dmarc-collect.sh` + +**Datei anlegen:** +```bash +sudo tee /usr/local/bin/dmarc-collect.sh >/dev/null <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +BASE="/var/lib/dmarc" +INBOX="$BASE/reports" +PROC="$BASE/processed" +LOGF="$BASE/logs/collector.log" + +umask 027 + +TMPDIR="$(mktemp -d)" +EML="$TMPDIR/mail.eml" +cat > "$EML" + +ripmime --no-nameless --name-by-type --overwrite -i "$EML" -d "$TMPDIR" >>"$LOGF" 2>&1 || true + +TODAY="$(date -u +%Y/%m/%d)" +OUTDIR="$INBOX/$TODAY" +mkdir -p "$OUTDIR" + +moved=0 +shopt -s nullglob +for f in "$TMPDIR"/*; do + case "$f" in + *.xml|*.XML|*.gz|*.zip) + sha="$(sha256sum "$f" | awk '{print $1}')" + base="$(basename "$f")" + dst="$OUTDIR/$(date -u +%Y%m%dT%H%M%SZ)_${sha:0:12}_$base" + mv "$f" "$dst" + echo "$(date -Is) stored $dst" >> "$LOGF" + moved=$((moved+1)) + ;; + *) : ;; + esac +done + +mkdir -p "$PROC" +mv "$EML" "$PROC/$(date -u +%Y%m%dT%H%M%SZ)_mail.eml" || true +rm -rf "$TMPDIR" + +if (( moved > 0 )); then + exit 0 +else + echo "$(date -Is) no usable attachment in message" >> "$LOGF" + exit 0 +fi + +EOF +sudo apt install -y ripmime +sudo install -m 750 -o vmail -g vmail /usr/local/bin/dmarc-collect.sh +``` + +Inhalt von `dmarc-collect.sh`: +```bash +#!/usr/bin/env bash +set -euo pipefail + +BASE="/var/lib/dmarc" +INBOX="$BASE/reports" +PROC="$BASE/processed" +LOGF="$BASE/logs/collector.log" + +umask 027 + +TMPDIR="$(mktemp -d)" +EML="$TMPDIR/mail.eml" +cat > "$EML" + +ripmime --no-nameless --name-by-type --overwrite -i "$EML" -d "$TMPDIR" >>"$LOGF" 2>&1 || true + +TODAY="$(date -u +%Y/%m/%d)" +OUTDIR="$INBOX/$TODAY" +mkdir -p "$OUTDIR" + +moved=0 +shopt -s nullglob +for f in "$TMPDIR"/*; do + case "$f" in + *.xml|*.XML|*.gz|*.zip) + sha="$(sha256sum "$f" | awk '{print $1}')" + base="$(basename "$f")" + dst="$OUTDIR/$(date -u +%Y%m%dT%H%M%SZ)_${sha:0:12}_$base" + mv "$f" "$dst" + echo "$(date -Is) stored $dst" >> "$LOGF" + moved=$((moved+1)) + ;; + *) : ;; + esac +done + +mkdir -p "$PROC" +mv "$EML" "$PROC/$(date -u +%Y%m%dT%H%M%SZ)_mail.eml" || true +rm -rf "$TMPDIR" + +if (( moved > 0 )); then + exit 0 +else + echo "$(date -Is) no usable attachment in message" >> "$LOGF" + exit 0 +fi + +``` + +--- + +## ⏱️ 4. Automatische tägliche Auswertung + +**Datei:** `/usr/local/lib/dmarc/daily-run.sh` + +```bash +sudo tee /usr/local/lib/dmarc/daily-run.sh >/dev/null <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +BASE="/var/lib/dmarc" +TODAY_DIR="$BASE/reports/$(date -u +%Y/%m/%d)" +OUTDIR="$BASE/exports" +CSV="$OUTDIR/records.csv" +LOGF="$BASE/logs/scan-$(date -u +%F).log" + +mkdir -p "$OUTDIR" + +if [[ -d "$TODAY_DIR" ]]; then + /usr/local/bin/dmarc-scan.sh "$TODAY_DIR" --domain fluechtlingsrat-brandenburg.de --csv "$CSV" --append --top 25 --outdir "$OUTDIR" >> "$LOGF" 2>&1 +fi +EOF +sudo chmod 750 /usr/local/lib/dmarc/daily-run.sh +``` + +Cronjob anlegen: +```bash +echo '17 3 * * * root /usr/local/lib/dmarc/daily-run.sh' | sudo tee /etc/cron.d/dmarc-daily >/dev/null +``` + +--- + +## 🧮 5. Auswertungsskript `/usr/local/bin/dmarc-scan.sh` + +Installation: +```bash +sudo apt install -y xmlstarlet unzip gzip +sudo tee /usr/local/bin/dmarc-scan.sh >/dev/null <<'EOF' +#!/usr/bin/env bash +# +# dmarc-scan.sh — DMARC-XML-Reports (auch .gz/.zip) einlesen, tabellarisch anzeigen, +# Records als CSV exportieren (append optional), Top-IPs ermitteln +# und Top-Listen als CSV schreiben. +# +# Nutzung: +# ./dmarc-scan.sh /pfad/zu/reports \ +# [--domain DOMAIN] \ +# [--csv pfad/zur/records.csv] \ +# [--append] \ +# [--top N] \ +# [--outdir pfad/zum/ordner] +# +# Beispiele: +# ./dmarc-scan.sh /var/mail/dmarc +# ./dmarc-scan.sh /var/mail/dmarc --domain fluechtlingsrat-brandenburg.de --csv dmarc.csv --append --top 15 --outdir ./export +# +# Voraussetzungen: xmlstarlet, unzip (für .zip), gzip (für .gz) +# Debian/Ubuntu: sudo apt-get install xmlstarlet unzip gzip +# +set -euo pipefail + +REPORT_DIR="${1:-}" +shift || true + +# Defaults +WANT_DOMAIN="" +CSV_PATH="./dmarc-summary.csv" +APPEND=0 +TOP_N=10 +OUTDIR="." + +# Arg-Parsing (einfach) +while [[ $# -gt 0 ]]; do + case "${1:-}" in + --domain) + WANT_DOMAIN="${2:-}"; shift 2 || true ;; + --csv) + CSV_PATH="${2:-}"; shift 2 || true ;; + --append) + APPEND=1; shift || true ;; + --top) + TOP_N="${2:-10}"; shift 2 || true ;; + --outdir) + OUTDIR="${2:-.}"; shift 2 || true ;; + *) + # Unbekannte Option ignorieren + shift || true ;; + esac +done + +if [[ -z "$REPORT_DIR" || ! -d "$REPORT_DIR" ]]; then + echo "Fehler: Bitte ein Verzeichnis mit DMARC-Reports angeben." + echo "Beispiel: $0 /var/mail/dmarc --domain fluechtlingsrat-brandenburg.de --csv dmarc.csv --append --top 15 --outdir ./export" + exit 1 +fi + +if ! command -v xmlstarlet >/dev/null 2>&1; then + echo "Fehler: xmlstarlet nicht gefunden. Bitte installieren (z.B. 'sudo apt-get install xmlstarlet')." + exit 1 +fi + +mkdir -p "$OUTDIR" + +# CSV-Header für Records; bei --append nur schreiben, wenn Datei noch nicht existiert +RECORDS_HEADER="report_org,policy_domain,begin_utc,end_utc,source_ip,count,disposition,spf,dkim,header_from" +ensure_records_header() { + if [[ "$APPEND" -eq 1 ]]; then + if [[ ! -f "$CSV_PATH" ]]; then + echo "$RECORDS_HEADER" > "$CSV_PATH" + fi + else + echo "$RECORDS_HEADER" > "$CSV_PATH" + fi +} +ensure_records_header + +# Zähler & Sets +declare -A SEEN_IPS=() +declare -A IP_COUNTS=() # Summe pro IP +declare -A SPF_ONLY_FAIL_IP=() # nur SPF fail pro IP +declare -A DKIM_ONLY_FAIL_IP=() # nur DKIM fail pro IP +declare -A BOTH_FAIL_IP=() # SPF+DKIM fail pro IP + +total_msgs=0 +pass_msgs=0 +fail_msgs=0 +spf_only_fail=0 +dkim_only_fail=0 +both_fail=0 + +# Hilfsfunktion: Epoch -> Datum (UTC) +epoch2date() { + local e="$1" + if [[ -z "$e" ]]; then printf "-"; return; fi + date -u -d @"$e" +"%Y-%m-%d %H:%M:%S UTC" 2>/dev/null || printf "%s" "$e" +} + +# CSV-escape (Felder in Anführungszeichen, doppelte Anführungszeichen verdoppeln) +csv_escape() { + local s="${1:-}" + s="${s//\"/\"\"}" + printf "\"%s\"" "$s" +} + +# Eine einzelne XML-Datei parsen +parse_xml() { + local xml_input="$1" + + # Domain aus policy_published, ggf. für Filter + local domain + domain=$(xmlstarlet sel -T -t -v "/feedback/policy_published/domain" "$xml_input" 2>/dev/null || true) + if [[ -n "$WANT_DOMAIN" && "$domain" != "$WANT_DOMAIN" ]]; then + return 0 + fi + + local org begin end pol sp aspf adkim + org=$(xmlstarlet sel -T -t -v "/feedback/report_metadata/org_name" "$xml_input" 2>/dev/null || printf "-") + begin=$(xmlstarlet sel -T -t -v "/feedback/report_metadata/date_range/begin" "$xml_input" 2>/dev/null || printf "") + end=$(xmlstarlet sel -T -t -v "/feedback/report_metadata/date_range/end" "$xml_input" 2>/dev/null || printf "") + pol=$(xmlstarlet sel -T -t -v "/feedback/policy_published/p" "$xml_input" 2>/dev/null || printf "-") + sp=$(xmlstarlet sel -T -t -v "/feedback/policy_published/sp" "$xml_input" 2>/dev/null || printf "-") + aspf=$(xmlstarlet sel -T -t -v "/feedback/policy_published/aspf" "$xml_input" 2>/dev/null || printf "-") + adkim=$(xmlstarlet sel -T -t -v "/feedback/policy_published/adkim" "$xml_input" 2>/dev/null || printf "-") + + # Report-Header + echo "==============================================================================" + echo "Report von: ${org}" + echo "Domain (Policy): ${domain} (p=${pol}, sp=${sp}, aspf=${aspf}, adkim=${adkim})" + echo "Zeitraum: $(epoch2date "$begin") – $(epoch2date "$end")" + echo "------------------------------------------------------------------------------" + printf "%-16s %7s %-10s %-6s %-6s %s\n" "Source IP" "Count" "Disposition" "SPF" "DKIM" "Header-From" + echo "------------------------------------------------------------------------------" + + # Alle -Einträge tabellarisch ausgeben + while IFS='|' read -r ip cnt dispo spfres dkimres hfrom; do + [[ -z "$ip$cnt$dispo$spfres$dkimres$hfrom" ]] && continue + + local n=0 + if [[ -n "${cnt:-}" && "$cnt" =~ ^[0-9]+$ ]]; then n="$cnt"; fi + + total_msgs=$(( total_msgs + n )) + [[ -n "$ip" ]] && SEEN_IPS["$ip"]=1 + [[ -n "$ip" ]] && IP_COUNTS["$ip"]=$(( ${IP_COUNTS["$ip"]:-0} + n )) + + if [[ "${spfres:-}" == "pass" && "${dkimres:-}" == "pass" ]]; then + pass_msgs=$(( pass_msgs + n )) + else + fail_msgs=$(( fail_msgs + n )) + if [[ "${spfres:-}" != "pass" && "${dkimres:-}" == "pass" ]]; then + spf_only_fail=$(( spf_only_fail + n )) + [[ -n "$ip" ]] && SPF_ONLY_FAIL_IP["$ip"]=$(( ${SPF_ONLY_FAIL_IP["$ip"]:-0} + n )) + elif [[ "${spfres:-}" == "pass" && "${dkimres:-}" != "pass" ]]; then + dkim_only_fail=$(( dkim_only_fail + n )) + [[ -n "$ip" ]] && DKIM_ONLY_FAIL_IP["$ip"]=$(( ${DKIM_ONLY_FAIL_IP["$ip"]:-0} + n )) + else + both_fail=$(( both_fail + n )) + [[ -n "$ip" ]] && BOTH_FAIL_IP["$ip"]=$(( ${BOTH_FAIL_IP["$ip"]:-0} + n )) + fi + fi + + printf "%-16s %7s %-10s %-6s %-6s %s\n" "${ip:--}" "${n}" "${dispo:--}" "${spfres:--}" "${dkimres:--}" "${hfrom:--}" + + local begin_human end_human + begin_human="$(epoch2date "$begin")" + end_human="$(epoch2date "$end")" + { + csv_escape "$org"; printf "," + csv_escape "$domain"; printf "," + csv_escape "$begin_human"; printf "," + csv_escape "$end_human"; printf "," + csv_escape "${ip:-}"; printf "," + printf "%s," "$n" + csv_escape "${dispo:-}"; printf "," + csv_escape "${spfres:-}"; printf "," + csv_escape "${dkimres:-}"; printf "," + csv_escape "${hfrom:-}"; printf "\n" + } >> "$CSV_PATH" + + done < <(xmlstarlet sel -T -t \ + -m "/feedback/record" \ + -v "row/source_ip" -o "|" \ + -v "row/count" -o "|" \ + -v "row/policy_evaluated/disposition" -o "|" \ + -v "row/policy_evaluated/spf" -o "|" \ + -v "row/policy_evaluated/dkim" -o "|" \ + -v "identifiers/header_from" -n \ + "$xml_input" 2>/dev/null) + + echo +} + +# Alle Dateien im Verzeichnis verarbeiten +shopt -s nullglob +for f in "$REPORT_DIR"/*; do + case "$f" in + *.xml) + parse_xml "$f" + ;; + *.gz) + if command -v gzip >/dev/null 2>&1; then + tmp="$(mktemp)" + if gzip -cd "$f" > "$tmp"; then + parse_xml "$tmp" + else + echo "Warnung: Konnte $f nicht entpacken." + fi + rm -f "$tmp" + else + echo "Warnung: gzip nicht verfügbar, überspringe $f" + fi + ;; + *.zip) + if command -v unzip >/dev/null 2>&1; then + tmpdir="$(mktemp -d)" + if unzip -qq -j "$f" '*.xml' -d "$tmpdir" >/dev/null 2>&1; then + for x in "$tmpdir"/*.xml; do + [[ -e "$x" ]] || continue + parse_xml "$x" + done + else + echo "Warnung: Konnte $f nicht entpacken (oder keine XML darin)." + fi + rm -rf "$tmpdir" + else + echo "Warnung: unzip nicht verfügbar, überspringe $f" + fi + ;; + *) : ;; + esac +done + +# Zusammenfassung +unique_ips=${#SEEN_IPS[@]} +echo "==============================================================================" +echo "GESAMT-ZUSAMMENFASSUNG" +echo "Nachrichten gesamt: $total_msgs" +echo "Eindeutige Source-IPs: $unique_ips" +echo "Alle Auth PASS: $pass_msgs" +echo "SPF/DKIM Fehler gesamt: $fail_msgs" +echo " ├─ nur SPF-Fail: $spf_only_fail" +echo " ├─ nur DKIM-Fail: $dkim_only_fail" +echo " └─ SPF+DKIM-Fail: $both_fail" +[[ -n "$WANT_DOMAIN" ]] && echo "Gefilterte Domain: $WANT_DOMAIN" +echo "Records-CSV: $CSV_PATH" +echo "Hinweis: 'Fail' umfasst Records, bei denen SPF oder DKIM nicht 'pass' war." + +# Top-Listen auf STDOUT +echo +echo "TOP $TOP_N IPs nach Anzahl (über alle Reports):" +{ + for ip in "${!IP_COUNTS[@]}"; do + printf "%10d %s\n" "${IP_COUNTS[$ip]}" "$ip" + done +} | sort -rn | head -n "$TOP_N" | awk '{printf " %2d) %-16s %7s\n", NR, $2, $1}' + +if (( fail_msgs > 0 )); then + echo + echo "Top-IPs nur SPF-Fail:" + { for ip in "${!SPF_ONLY_FAIL_IP[@]}"; do printf "%10d %s\n" "${SPF_ONLY_FAIL_IP[$ip]}" "$ip"; done; } \ + | sort -rn | head -n "$TOP_N" | awk '{printf " %2d) %-16s %7s\n", NR, $2, $1}' + echo + echo "Top-IPs nur DKIM-Fail:" + { for ip in "${!DKIM_ONLY_FAIL_IP[@]}"; do printf "%10d %s\n" "${DKIM_ONLY_FAIL_IP[$ip]}" "$ip"; done; } \ + | sort -rn | head -n "$TOP_N" | awk '{printf " %2d) %-16s %7s\n", NR, $2, $1}' + echo + echo "Top-IPs SPF+DKIM-Fail:" + { for ip in "${!BOTH_FAIL_IP[@]}"; do printf "%10d %s\n" "${BOTH_FAIL_IP[$ip]}" "$ip"; done; } \ + | sort -rn | head -n "$TOP_N" | awk '{printf " %2d) %-16s %7s\n", NR, $2, $1}' +fi + +# ---- CSV-Exporte der Top-Listen --------------------------------------------- + +write_top_csv () { + local outfile="$1"; shift + local -n assoc="$1" # name reference auf assoziatives Array + echo "ip,count" > "$outfile" + if [[ "${#assoc[@]}" -eq 0 ]]; then + : # leer + else + for ip in "${!assoc[@]}"; do + printf "%s,%s\n" "$ip" "${assoc[$ip]}" + done | sort -t, -k2,2nr > "$outfile" + fi +} + +# Gesamt-Top-IPs +write_top_csv "$OUTDIR/top_ips.csv" IP_COUNTS + +# Top-Listen nach Fail-Kategorien +write_top_csv "$OUTDIR/top_spf_fail_ips.csv" SPF_ONLY_FAIL_IP +write_top_csv "$OUTDIR/top_dkim_fail_ips.csv" DKIM_ONLY_FAIL_IP +write_top_csv "$OUTDIR/top_both_fail_ips.csv" BOTH_FAIL_IP + +echo +echo "Exportierte Top-CSV-Dateien:" +echo " $OUTDIR/top_ips.csv" +echo " $OUTDIR/top_spf_fail_ips.csv" +echo " $OUTDIR/top_dkim_fail_ips.csv" +echo " $OUTDIR/top_both_fail_ips.csv" + +EOF +sudo chmod 750 /usr/local/bin/dmarc-scan.sh +``` + +Beschreibung: Das Skript liest XML/ZIP/GZ-Reports, zeigt eine Tabelle pro Report, schreibt eine Records-CSV (mit `--append` fortsetzbar) und exportiert Top-Listen als CSV in `--outdir`. + +**Wichtige Parameter:** +- `--domain DOMAIN` (Filter) +- `--csv PFAD` (Records-CSV) +- `--append` (anhängen statt überschreiben) +- `--top N` (Top-Liste Größe) +- `--outdir PFAD` (Top-CSV Ziel) + +**Beispiel:** +```bash +dmarc-scan.sh /var/lib/dmarc/reports/2025/11/12 --domain fluechtlingsrat-brandenburg.de --csv /var/lib/dmarc/exports/records.csv --append --top 25 --outdir /var/lib/dmarc/exports/ +``` + +--- + +## 🔁 6. Logrotate + +**Datei:** `/etc/logrotate.d/dmarc` + +```bash +/var/lib/dmarc/logs/*.log { + weekly + rotate 12 + compress + delaycompress + missingok + notifempty + create 640 vmail vmail + sharedscripts + postrotate + systemctl reload postfix >/dev/null 2>&1 || true + endscript +} +``` + +--- + +## ✅ 7. Test + +```bash +sudo -u vmail /usr/local/bin/dmarc-collect.sh < testmail.eml +``` + +--- + +**Autor:** oopen.de / Systemkonfiguration +**Stand:** November 2025 +**Version:** 1.2