From a71f7a0dd99b0967fb1866fc4b128bb16ad8f96e Mon Sep 17 00:00:00 2001 From: Nuno Duque Nunes Date: Sun, 24 May 2026 02:13:06 +0000 Subject: [PATCH] fix fw logs not showing, add hourly structuring to logs --- commands/group.command.sh | 2 +- commands/inspect.command.sh | 19 --- commands/logs.command.sh | 46 ++++-- core/json_helper.py | 320 +++++++++++++++++++++++++----------- 4 files changed, 253 insertions(+), 134 deletions(-) diff --git a/commands/group.command.sh b/commands/group.command.sh index d9d76dc..51499ed 100644 --- a/commands/group.command.sh +++ b/commands/group.command.sh @@ -115,7 +115,7 @@ function cmd::group::list() { fi local data - data=$(json::group_list_data "$groups_dir" "$(ctx::blocks)") + data=$(json::group_list_data "$groups_dir" "$(ctx::blocks)" "$(ctx::clients)") [[ -z "$data" ]] && log::wg "No groups configured" && return 0 # Measure column widths diff --git a/commands/inspect.command.sh b/commands/inspect.command.sh index e148b01..1e0a9ec 100644 --- a/commands/inspect.command.sh +++ b/commands/inspect.command.sh @@ -127,25 +127,6 @@ function cmd::inspect::_peer_info() { return 0 } -# function cmd::inspect::_rule_info() { -# local name="${1:-}" -# local rule -# rule=$(peers::get_meta "$name" "rule") -# [[ -z "$rule" ]] && return 0 -# rule::exists "$rule" || return 0 - -# cmd::inspect::_section "Rule: ${rule}" - -# if ui::rule::tree "$rule"; then -# # printf "\n" -# : # no-op -# else -# # No inheritance — flat view -# rule::render_flat "$rule" -# fi -# return 0 -# } - function cmd::inspect::_rule_separator() { local line_width=20 local total=$INSPECT_WIDTH diff --git a/commands/logs.command.sh b/commands/logs.command.sh index 49289a4..5aa19d8 100644 --- a/commands/logs.command.sh +++ b/commands/logs.command.sh @@ -17,6 +17,7 @@ function cmd::logs::on_load() { flag::register --force flag::register --days flag::register --raw + flag::register --detailed } function cmd::logs::help() { @@ -86,7 +87,8 @@ function cmd::logs::run() { function cmd::logs::show() { local name="" type="" limit=50 - local fw_only=false wg_only=false follow=false merged=false raw=false + local fw_only=false wg_only=false follow=false merged=false raw=false detailed=false + while [[ $# -gt 0 ]]; do case "$1" in @@ -98,6 +100,7 @@ function cmd::logs::show() { --merged) merged=true; shift ;; --follow|-f) follow=true; shift ;; --raw) raw=true; shift ;; + --detailed) detailed=true shift ;; --help) cmd::logs::help; return ;; *) log::error "Unknown flag: $1" @@ -106,6 +109,10 @@ function cmd::logs::show() { esac done + local collapse=1 + $detailed && collapse=0 + + if [[ -n "$name" && -n "$type" ]]; then name=$(peers::resolve_and_require "$name" "$type") || return 1 fi @@ -132,19 +139,19 @@ function cmd::logs::show() { return fi - $wg_only || cmd::logs::show_fw_events "$filter_ip" "$name" "$type" "$limit" "$net_file" - $fw_only || cmd::logs::show_wg_events "$filter_ip" "$name" "$type" "$limit" + $wg_only || cmd::logs::show_fw_events "$filter_ip" "$name" "$type" "$limit" "$net_file" "$collapse" + $fw_only || cmd::logs::show_wg_events "$filter_ip" "$name" "$type" "$limit" "$collapse" } function cmd::logs::show_fw_events() { local filter_ip="${1:-}" filter_name="${2:-}" filter_type="${3:-}" \ - limit="${4:-50}" net_file="${5:-}" + limit="${4:-50}" net_file="${5:-}" collapse="${6:-1}" [[ ! -f "$FW_EVENTS_LOG" ]] && return 0 local data data=$(json::fw_events "$FW_EVENTS_LOG" "$filter_ip" "$filter_type" \ - "$(ctx::clients)" "${net_file:-}" "$(ctx::hosts)" "$limit" 2>/dev/null) + "$(ctx::clients)" "${net_file:-}" "$limit" "$collapse" 2>/dev/null) [[ -z "$data" ]] && return 0 @@ -178,33 +185,40 @@ function cmd::logs::show_fw_events() { } function cmd::logs::show_wg_events() { - local filter_ip="${1:-}" filter_name="${2:-}" filter_type="${3:-}" limit="${4:-50}" - + local filter_ip="${1:-}" filter_name="${2:-}" filter_type="${3:-}" \ + limit="${4:-50}" collapse="${5:-1}" + [[ ! -f "$WG_EVENTS_LOG" ]] && return 0 - + local data - data=$(json::wg_events "$WG_EVENTS_LOG" "$filter_name" "$filter_type" "$limit" 2>/dev/null) - + data=$(json::wg_events "$WG_EVENTS_LOG" "$filter_name" "$filter_type" "$limit" "$collapse" 2>/dev/null) + [[ -z "$data" ]] && return 0 - - # Measure column widths + + # Resolve endpoints and measure column widths local w_client=16 w_endpoint=16 + local resolved_data="" while IFS='|' read -r ts client endpoint event count; do [[ -z "$ts" ]] && continue - (( ${#client} > w_client )) && w_client=${#client} - (( ${#endpoint} > w_endpoint )) && w_endpoint=${#endpoint} + local endpoint_display + endpoint_display=$(resolve::ip "$endpoint") + [[ -z "$endpoint_display" ]] && endpoint_display="$endpoint" + resolved_data+="${ts}|${client}|${endpoint_display}|${event}|${count}"$'\n' + (( ${#client} > w_client )) && w_client=${#client} + (( ${#endpoint_display} > w_endpoint )) && w_endpoint=${#endpoint_display} done <<< "$data" (( w_client += 2 )) (( w_endpoint += 2 )) - + ui::logs::wg_section_header while IFS='|' read -r ts client endpoint event count; do [[ -z "$ts" ]] && continue ui::logs::wg_row "$ts" "$client" "$endpoint" "$event" \ "$count" "$w_client" "$w_endpoint" - done <<< "$data" + done <<< "$resolved_data" printf "\n" } + function cmd::logs::show_merged() { local filter_ip="${1:-}" filter_name="${2:-}" filter_type="${3:-}" \ diff --git a/core/json_helper.py b/core/json_helper.py index ffbd626..b2528a3 100644 --- a/core/json_helper.py +++ b/core/json_helper.py @@ -154,14 +154,18 @@ def events_for(file, ip, limit): except: pass -def fw_events(file, filter_ip, filter_type, clients_dir, net_file, limit): +def fw_events(file, filter_ip, filter_type, clients_dir, net_file, limit, collapse='1'): """ Format firewall drop events with dedup, counts, and service annotation. + collapse='1' (default): hourly aggregation + collapse='0': show all deduplicated events (--detailed mode) Output per line: ts|client|dest_ip|dest_port|proto|service_name|count """ import glob from datetime import datetime + from collections import defaultdict proto_map = {1: 'icmp', 6: 'tcp', 17: 'udp'} + do_collapse = str(collapse) != '0' # Build ip->name map ip_to_name = {} @@ -176,7 +180,7 @@ def fw_events(file, filter_ip, filter_type, clients_dir, net_file, limit): except Exception: pass - # Load net services for reverse lookup — independent of rest of function + # Load net services for reverse lookup net_data = {} if net_file and os.path.exists(net_file): try: @@ -237,70 +241,104 @@ def fw_events(file, filter_ip, filter_type, clients_dir, net_file, limit): last_seen[key] = ts events.append(e) except Exception: - pass + continue except Exception: pass - # Second-pass dedup consecutive same events with count - deduped = [] - counts = [] - for e in events: - ts_str = e.get('timestamp', '') - try: - ts = datetime.fromisoformat(ts_str).timestamp() - except Exception: - ts = 0 + limit = int(limit) if limit else 50 - src = e.get('src_ip', '') - dst = e.get('dest_ip', '') - port = str(e.get('dest_port', '')) - proto_num = int(e.get('ip.protocol', 0)) - key = (src, dst, port, proto_num) + if do_collapse: + # Hourly aggregation: group by (client, dst, port, proto, date, hour) + hourly = defaultdict(int) + hourly_ts = {} # store first ts per hour bucket for output ordering + + for e in events: + src = e.get('src_ip', '') + dst = e.get('dest_ip', '') + port = str(e.get('dest_port', '')) + proto_num = int(e.get('ip.protocol', 0)) + proto = proto_map.get(proto_num, str(proto_num)) + ts_str = e.get('timestamp', '') + client = ip_to_name.get(src, src) + svc_name = reverse_lookup(dst, port, proto) - if deduped: - prev = deduped[-1] try: - prev_ts = datetime.fromisoformat(prev.get('timestamp', '')).timestamp() + dt = datetime.fromisoformat(ts_str) + hour_key = (client, dst, port, proto, svc_name, + dt.strftime('%Y-%m-%d %H')) + hourly[hour_key] += 1 + if hour_key not in hourly_ts: + hourly_ts[hour_key] = dt except Exception: - prev_ts = 0 - prev_key = ( - prev.get('src_ip', ''), - prev.get('dest_ip', ''), - str(prev.get('dest_port', '')), - int(prev.get('ip.protocol', 0)) - ) - if key == prev_key and (ts - prev_ts) < 300: - counts[-1] += 1 continue - deduped.append(e) - counts.append(1) + # Sort by timestamp and emit last N + sorted_buckets = sorted(hourly_ts.items(), key=lambda x: x[1]) + for hour_key, dt in sorted_buckets[-limit:]: + client, dst, port, proto, svc_name, _ = hour_key + count = hourly[hour_key] + ts_fmt = dt.strftime(DATETIME_FMT.replace('%M', '00')) + print(f"{ts_fmt}|{client}|{dst}|{port}|{proto}|{svc_name}|{count}") + else: + # Detailed mode — consecutive dedup only + deduped = [] + counts = [] + for e in events: + ts_str = e.get('timestamp', '') + src = e.get('src_ip', '') + dst = e.get('dest_ip', '') + port = str(e.get('dest_port', '')) + proto_num = int(e.get('ip.protocol', 0)) + key = (src, dst, port, proto_num) - limit = int(limit) if limit else 50 - for e, count in list(zip(deduped, counts))[-limit:]: - ts_str = e.get('timestamp', '') - src = e.get('src_ip', '') - dst = e.get('dest_ip', '') - port = str(e.get('dest_port', '')) - proto_num = int(e.get('ip.protocol', 0)) - proto = proto_map.get(proto_num, str(proto_num)) - client = ip_to_name.get(src, src) - svc_name = reverse_lookup(dst, port, proto) + if deduped: + prev = deduped[-1] + try: + prev_ts = datetime.fromisoformat( + prev.get('timestamp', '')).timestamp() + cur_ts = datetime.fromisoformat(ts_str).timestamp() + except Exception: + prev_ts = cur_ts = 0 + prev_key = ( + prev.get('src_ip', ''), + prev.get('dest_ip', ''), + str(prev.get('dest_port', '')), + int(prev.get('ip.protocol', 0)) + ) + if key == prev_key and (cur_ts - prev_ts) < 300: + counts[-1] += 1 + continue - try: - dt = datetime.fromisoformat(ts_str) - ts_fmt = dt.strftime(DATETIME_FMT) - except Exception: - ts_fmt = ts_str + deduped.append(e) + counts.append(1) - print(f"{ts_fmt}|{client}|{dst}|{port}|{proto}|{svc_name}|{count}") - -def wg_events(file, filter_client, filter_type, limit): + for e, count in list(zip(deduped, counts))[-limit:]: + ts_str = e.get('timestamp', '') + src = e.get('src_ip', '') + dst = e.get('dest_ip', '') + port = str(e.get('dest_port', '')) + proto_num = int(e.get('ip.protocol', 0)) + proto = proto_map.get(proto_num, str(proto_num)) + client = ip_to_name.get(src, src) + svc_name = reverse_lookup(dst, port, proto) + try: + dt = datetime.fromisoformat(ts_str) + ts_fmt = dt.strftime(DATETIME_FMT) + except Exception: + ts_fmt = ts_str + print(f"{ts_fmt}|{client}|{dst}|{port}|{proto}|{svc_name}|{count}") + + +def wg_events(file, filter_client, filter_type, limit, collapse='1'): """ Format WireGuard events with dedup and counts. + collapse='1' (default): hourly aggregation for attempt events + collapse='0': show all deduplicated events (--detailed mode) Output per line: ts|client|endpoint|event|count """ from datetime import datetime + from collections import defaultdict + do_collapse = str(collapse) != '0' events = [] try: @@ -321,52 +359,129 @@ def wg_events(file, filter_client, filter_type, limit): except Exception: pass - # Dedup consecutive same client+event+endpoint within 60s - deduped = [] - counts = [] - for e in events: - ts_str = e.get('timestamp', '') - try: - ts = datetime.fromisoformat(ts_str).timestamp() - except Exception: - ts = 0 - client = e.get('client', '') - event = e.get('event', '') - endpoint = e.get('endpoint', '') - key = (client, event, endpoint[:15]) - - if deduped: - prev = deduped[-1] - prev_ts_str = prev.get('timestamp', '') - try: - prev_ts = datetime.fromisoformat(prev_ts_str).timestamp() - except Exception: - prev_ts = 0 - prev_key = ( - prev.get('client', ''), - prev.get('event', ''), - prev.get('endpoint', '')[:15] - ) - if key == prev_key and (ts - prev_ts) < 300: - counts[-1] += 1 - continue - - deduped.append(e) - counts.append(1) - limit = int(limit) if limit else 50 - for e, count in list(zip(deduped, counts))[-limit:]: - ts_str = e.get('timestamp', '') - client = e.get('client', '') - endpoint = e.get('endpoint', '') - event = e.get('event', '') - try: - dt = datetime.fromisoformat(ts_str) - ts_fmt = dt.strftime('%d/%m %H:%M') - except Exception: - ts_fmt = ts_str - print(f"{ts_fmt}|{client}|{endpoint}|{event}|{count}") + if do_collapse: + # Hourly aggregation for attempts; individual for handshakes + hourly_attempts = defaultdict(int) + hourly_ts = {} + handshakes = [] + handshake_counts = [] + + for e in events: + ts_str = e.get('timestamp', '') + client = e.get('client', '') + endpoint = e.get('endpoint', '') + event = e.get('event', '') + + try: + dt = datetime.fromisoformat(ts_str) + ts = dt.timestamp() + except Exception: + dt = None + ts = 0 + + if event == 'attempt': + if dt: + hour_key = (client, endpoint, event, + dt.strftime('%Y-%m-%d %H')) + hourly_attempts[hour_key] += 1 + if hour_key not in hourly_ts: + hourly_ts[hour_key] = dt + else: + # Handshakes — consecutive dedup only + key = (client, event, endpoint[:15]) + if handshakes: + prev = handshakes[-1] + try: + prev_ts = datetime.fromisoformat( + prev.get('timestamp', '')).timestamp() + except Exception: + prev_ts = 0 + prev_key = ( + prev.get('client', ''), + prev.get('event', ''), + prev.get('endpoint', '')[:15] + ) + if key == prev_key and (ts - prev_ts) < 300: + handshake_counts[-1] += 1 + continue + handshakes.append(e) + handshake_counts.append(1) + + # Build output list: attempts (hourly) + handshakes, sorted by ts + output = [] + + for hour_key, dt in hourly_ts.items(): + client, endpoint, event, _ = hour_key + count = hourly_attempts[hour_key] + ts_fmt = dt.strftime(DATETIME_FMT.replace('%M', '00')) + output.append((dt.timestamp(), f"{ts_fmt}|{client}|{endpoint}|{event}|{count}")) + + for e, count in zip(handshakes, handshake_counts): + ts_str = e.get('timestamp', '') + client = e.get('client', '') + endpoint = e.get('endpoint', '') + event = e.get('event', '') + try: + dt = datetime.fromisoformat(ts_str) + ts_fmt = dt.strftime(DATETIME_FMT) + ts = dt.timestamp() + except Exception: + ts_fmt = ts_str + ts = 0 + output.append((ts, f"{ts_fmt}|{client}|{endpoint}|{event}|{count}")) + + output.sort(key=lambda x: x[0]) + for _, line in output[-limit:]: + print(line) + + else: + # Detailed mode — consecutive dedup only + deduped = [] + counts = [] + for e in events: + ts_str = e.get('timestamp', '') + client = e.get('client', '') + event = e.get('event', '') + endpoint = e.get('endpoint', '') + key = (client, event, endpoint[:15]) + + try: + ts = datetime.fromisoformat(ts_str).timestamp() + except Exception: + ts = 0 + + if deduped: + prev = deduped[-1] + try: + prev_ts = datetime.fromisoformat( + prev.get('timestamp', '')).timestamp() + except Exception: + prev_ts = 0 + prev_key = ( + prev.get('client', ''), + prev.get('event', ''), + prev.get('endpoint', '')[:15] + ) + if key == prev_key and (ts - prev_ts) < 300: + counts[-1] += 1 + continue + + deduped.append(e) + counts.append(1) + + for e, count in list(zip(deduped, counts))[-limit:]: + ts_str = e.get('timestamp', '') + client = e.get('client', '') + endpoint = e.get('endpoint', '') + event = e.get('event', '') + try: + dt = datetime.fromisoformat(ts_str) + ts_fmt = dt.strftime(DATETIME_FMT) + except Exception: + ts_fmt = ts_str + print(f"{ts_fmt}|{client}|{endpoint}|{event}|{count}") def format_fw_event(line, clients_dir): """Format a single fw_event line""" @@ -688,7 +803,7 @@ def rule_list_data(rules_dir, meta_dir): print(f"{r['name']}|{r['desc']}|{r['n_allows']}|{r['n_blocks']}|" f"{r['peer_count']}|{r['extends']}|{r['is_base']}|{r['group']}") -def group_list_data(groups_dir, blocks_dir): +def group_list_data(groups_dir, blocks_dir, clients_dir): """Return group summary data in one call""" import glob @@ -705,7 +820,10 @@ def group_list_data(groups_dir, blocks_dir): name = g.get('name', '') desc = g.get('desc', '') peers = [p for p in g.get('peers', []) if p] - total = len(peers) + valid_peers = [p for p in peers + if os.path.exists(os.path.join(clients_dir, f"{p}.conf"))] + + total = len(valid_peers) blocked = sum(1 for p in peers if p in blocked_peers) print(f"{name}|{desc}|{total}|{blocked}") except: @@ -2791,8 +2909,14 @@ commands = { 'filter_values': lambda args: filter_values(args[0], args[1], args[2]), 'last_event': lambda args: last_event(args[0], args[1], args[2], args[3]), 'events_for': lambda args: events_for(args[0], args[1], args[2]), - 'fw_events': lambda args: fw_events(args[0], args[1], args[2], args[3], args[4], args[5] if len(args) > 5 else '50'), - 'wg_events': lambda args: wg_events(args[0], args[1], args[2], args[3] if len(args) > 3 else '50'), + 'fw_events': lambda args: fw_events( + args[0], args[1], args[2], args[3], args[4], + args[5] if len(args) > 5 else '50', + args[6] if len(args) > 6 else '1'), + 'wg_events': lambda args: wg_events( + args[0], args[1], args[2], + args[3] if len(args) > 3 else '50', + args[4] if len(args) > 4 else '1'), 'format_fw_event': lambda args: format_fw_event(sys.stdin.read(), args[0]), 'format_wg_event': lambda args: format_wg_event(sys.stdin.read()), 'remove_events': lambda args: remove_events(args[0], args[1]), @@ -2804,7 +2928,7 @@ commands = { 'peer_data': lambda args: peer_data(args[0], args[1], args[2]), 'iso_to_ts': lambda args: iso_to_ts(args[0]), 'rule_list_data': lambda args: rule_list_data(args[0], args[1]), - 'group_list_data': lambda args: group_list_data(args[0], args[1]), + 'group_list_data': lambda args: group_list_data(args[0], args[1], args[2]), 'fmt_datetime': lambda args: fmt_datetime(args[0], args[1]), 'create_rule': lambda args: create_rule( args[0], args[1], args[2], args[3], args[4], args[5], args[6],