fix fw logs not showing, add hourly structuring to logs

commands/group.command.sh

@@ -115,7 +115,7 @@ function cmd::group::list() {
   fi
  
   local data
-  data=$(json::group_list_data "$groups_dir" "$(ctx::blocks)")
+  data=$(json::group_list_data "$groups_dir" "$(ctx::blocks)" "$(ctx::clients)")
   [[ -z "$data" ]] && log::wg "No groups configured" && return 0
  
   # Measure column widths

commands/inspect.command.sh

@@ -127,25 +127,6 @@ function cmd::inspect::_peer_info() {
   return 0
 }
 
-# function cmd::inspect::_rule_info() {
-#   local name="${1:-}"
-#   local rule
-#   rule=$(peers::get_meta "$name" "rule")
-#   [[ -z "$rule" ]] && return 0
-#   rule::exists "$rule" || return 0
-
-#   cmd::inspect::_section "Rule: ${rule}"
-
-#   if ui::rule::tree "$rule"; then
-#     # printf "\n"
-#     : # no-op
-#   else
-#     # No inheritance — flat view
-#     rule::render_flat "$rule"
-#   fi
-#   return 0
-# }
-
 function cmd::inspect::_rule_separator() {
   local line_width=20
   local total=$INSPECT_WIDTH

commands/logs.command.sh

@@ -17,6 +17,7 @@ function cmd::logs::on_load() {
   flag::register --force
   flag::register --days
   flag::register --raw
+  flag::register --detailed
 }
 
 function cmd::logs::help() {
@@ -86,7 +87,8 @@ function cmd::logs::run() {
 
 function cmd::logs::show() {
   local name="" type="" limit=50
-  local fw_only=false wg_only=false follow=false merged=false raw=false
+  local fw_only=false wg_only=false follow=false merged=false raw=false detailed=false
+  
 
   while [[ $# -gt 0 ]]; do
     case "$1" in
@@ -98,6 +100,7 @@ function cmd::logs::show() {
       --merged)    merged=true;     shift   ;;
       --follow|-f) follow=true;     shift   ;;
       --raw)       raw=true;        shift   ;;
+      --detailed)  detailed=true    shift   ;;
       --help)      cmd::logs::help; return  ;;
       *)
         log::error "Unknown flag: $1"
@@ -106,6 +109,10 @@ function cmd::logs::show() {
     esac
   done
 
+  local collapse=1
+  $detailed && collapse=0
+
+
   if [[ -n "$name" && -n "$type" ]]; then
     name=$(peers::resolve_and_require "$name" "$type") || return 1
   fi
@@ -132,19 +139,19 @@ function cmd::logs::show() {
     return
   fi
 
-  $wg_only || cmd::logs::show_fw_events  "$filter_ip" "$name" "$type" "$limit" "$net_file"
+  $wg_only || cmd::logs::show_fw_events "$filter_ip" "$name" "$type" "$limit" "$net_file" "$collapse"
-  $fw_only || cmd::logs::show_wg_events  "$filter_ip" "$name" "$type" "$limit"
+  $fw_only || cmd::logs::show_wg_events "$filter_ip" "$name" "$type" "$limit" "$collapse"
 }
 
 function cmd::logs::show_fw_events() {
   local filter_ip="${1:-}" filter_name="${2:-}" filter_type="${3:-}" \
-        limit="${4:-50}" net_file="${5:-}"
+        limit="${4:-50}" net_file="${5:-}" collapse="${6:-1}"
 
   [[ ! -f "$FW_EVENTS_LOG" ]] && return 0
 
   local data
   data=$(json::fw_events "$FW_EVENTS_LOG" "$filter_ip" "$filter_type" \
-    "$(ctx::clients)" "${net_file:-}" "$(ctx::hosts)" "$limit" 2>/dev/null)
+    "$(ctx::clients)" "${net_file:-}" "$limit" "$collapse" 2>/dev/null)
 
   [[ -z "$data" ]] && return 0
 
@@ -178,21 +185,27 @@ function cmd::logs::show_fw_events() {
 }
 
 function cmd::logs::show_wg_events() {
-  local filter_ip="${1:-}" filter_name="${2:-}" filter_type="${3:-}" limit="${4:-50}"
+  local filter_ip="${1:-}" filter_name="${2:-}" filter_type="${3:-}" \
+        limit="${4:-50}" collapse="${5:-1}"
  
   [[ ! -f "$WG_EVENTS_LOG" ]] && return 0
  
   local data
-  data=$(json::wg_events "$WG_EVENTS_LOG" "$filter_name" "$filter_type" "$limit" 2>/dev/null)
+  data=$(json::wg_events "$WG_EVENTS_LOG" "$filter_name" "$filter_type" "$limit" "$collapse" 2>/dev/null)
  
   [[ -z "$data" ]] && return 0
  
-  # Measure column widths
+  # Resolve endpoints and measure column widths
   local w_client=16 w_endpoint=16
+  local resolved_data=""
   while IFS='|' read -r ts client endpoint event count; do
     [[ -z "$ts" ]] && continue
-    (( ${#client}   > w_client   )) && w_client=${#client}
+    local endpoint_display
-    (( ${#endpoint} > w_endpoint )) && w_endpoint=${#endpoint}
+    endpoint_display=$(resolve::ip "$endpoint")
+    [[ -z "$endpoint_display" ]] && endpoint_display="$endpoint"
+    resolved_data+="${ts}|${client}|${endpoint_display}|${event}|${count}"$'\n'
+    (( ${#client}           > w_client   )) && w_client=${#client}
+    (( ${#endpoint_display} > w_endpoint )) && w_endpoint=${#endpoint_display}
   done <<< "$data"
   (( w_client   += 2 ))
   (( w_endpoint += 2 ))
@@ -202,10 +215,11 @@ function cmd::logs::show_wg_events() {
     [[ -z "$ts" ]] && continue
     ui::logs::wg_row "$ts" "$client" "$endpoint" "$event" \
       "$count" "$w_client" "$w_endpoint"
-  done <<< "$data"
+  done <<< "$resolved_data"
   printf "\n"
 }
  
+
 function cmd::logs::show_merged() {
   local filter_ip="${1:-}" filter_name="${2:-}" filter_type="${3:-}" \
         limit="${4:-50}" net_file="${5:-}"

core/json_helper.py

@@ -154,14 +154,18 @@ def events_for(file, ip, limit):
     except:
         pass
 
-def fw_events(file, filter_ip, filter_type, clients_dir, net_file, limit):
+def fw_events(file, filter_ip, filter_type, clients_dir, net_file, limit, collapse='1'):
     """
     Format firewall drop events with dedup, counts, and service annotation.
+    collapse='1' (default): hourly aggregation
+    collapse='0': show all deduplicated events (--detailed mode)
     Output per line: ts|client|dest_ip|dest_port|proto|service_name|count
     """
     import glob
     from datetime import datetime
+    from collections import defaultdict
     proto_map = {1: 'icmp', 6: 'tcp', 17: 'udp'}
+    do_collapse = str(collapse) != '0'
  
     # Build ip->name map
     ip_to_name = {}
@@ -176,7 +180,7 @@ def fw_events(file, filter_ip, filter_type, clients_dir, net_file, limit):
         except Exception:
             pass
  
-    # Load net services for reverse lookup — independent of rest of function
+    # Load net services for reverse lookup
     net_data = {}
     if net_file and os.path.exists(net_file):
         try:
@@ -237,70 +241,104 @@ def fw_events(file, filter_ip, filter_type, clients_dir, net_file, limit):
                     last_seen[key] = ts
                     events.append(e)
                 except Exception:
-                    pass
+                    continue
     except Exception:
         pass
  
-    # Second-pass dedup consecutive same events with count
+    limit = int(limit) if limit else 50
-    deduped = []
-    counts  = []
-    for e in events:
-        ts_str = e.get('timestamp', '')
-        try:
-            ts = datetime.fromisoformat(ts_str).timestamp()
-        except Exception:
-            ts = 0
  
-        src       = e.get('src_ip', '')
+    if do_collapse:
-        dst       = e.get('dest_ip', '')
+        # Hourly aggregation: group by (client, dst, port, proto, date, hour)
-        port      = str(e.get('dest_port', ''))
+        hourly = defaultdict(int)
-        proto_num = int(e.get('ip.protocol', 0))
+        hourly_ts = {}  # store first ts per hour bucket for output ordering
-        key       = (src, dst, port, proto_num)
+ 
+        for e in events:
+            src      = e.get('src_ip', '')
+            dst      = e.get('dest_ip', '')
+            port     = str(e.get('dest_port', ''))
+            proto_num = int(e.get('ip.protocol', 0))
+            proto    = proto_map.get(proto_num, str(proto_num))
+            ts_str   = e.get('timestamp', '')
+            client   = ip_to_name.get(src, src)
+            svc_name = reverse_lookup(dst, port, proto)
  
-        if deduped:
-            prev = deduped[-1]
             try:
-                prev_ts = datetime.fromisoformat(prev.get('timestamp', '')).timestamp()
+                dt = datetime.fromisoformat(ts_str)
+                hour_key = (client, dst, port, proto, svc_name,
+                            dt.strftime('%Y-%m-%d %H'))
+                hourly[hour_key] += 1
+                if hour_key not in hourly_ts:
+                    hourly_ts[hour_key] = dt
             except Exception:
-                prev_ts = 0
-            prev_key = (
-                prev.get('src_ip', ''),
-                prev.get('dest_ip', ''),
-                str(prev.get('dest_port', '')),
-                int(prev.get('ip.protocol', 0))
-            )
-            if key == prev_key and (ts - prev_ts) < 300:
-                counts[-1] += 1
                 continue
  
-        deduped.append(e)
+        # Sort by timestamp and emit last N
-        counts.append(1)
+        sorted_buckets = sorted(hourly_ts.items(), key=lambda x: x[1])
+        for hour_key, dt in sorted_buckets[-limit:]:
+            client, dst, port, proto, svc_name, _ = hour_key
+            count = hourly[hour_key]
+            ts_fmt = dt.strftime(DATETIME_FMT.replace('%M', '00'))
+            print(f"{ts_fmt}|{client}|{dst}|{port}|{proto}|{svc_name}|{count}")
+    else:
+        # Detailed mode — consecutive dedup only
+        deduped = []
+        counts  = []
+        for e in events:
+            ts_str    = e.get('timestamp', '')
+            src       = e.get('src_ip', '')
+            dst       = e.get('dest_ip', '')
+            port      = str(e.get('dest_port', ''))
+            proto_num = int(e.get('ip.protocol', 0))
+            key       = (src, dst, port, proto_num)
  
-    limit = int(limit) if limit else 50
+            if deduped:
-    for e, count in list(zip(deduped, counts))[-limit:]:
+                prev = deduped[-1]
-        ts_str    = e.get('timestamp', '')
+                try:
-        src       = e.get('src_ip', '')
+                    prev_ts = datetime.fromisoformat(
-        dst       = e.get('dest_ip', '')
+                        prev.get('timestamp', '')).timestamp()
-        port      = str(e.get('dest_port', ''))
+                    cur_ts  = datetime.fromisoformat(ts_str).timestamp()
-        proto_num = int(e.get('ip.protocol', 0))
+                except Exception:
-        proto     = proto_map.get(proto_num, str(proto_num))
+                    prev_ts = cur_ts = 0
-        client    = ip_to_name.get(src, src)
+                prev_key = (
-        svc_name  = reverse_lookup(dst, port, proto)
+                    prev.get('src_ip', ''),
+                    prev.get('dest_ip', ''),
+                    str(prev.get('dest_port', '')),
+                    int(prev.get('ip.protocol', 0))
+                )
+                if key == prev_key and (cur_ts - prev_ts) < 300:
+                    counts[-1] += 1
+                    continue
  
-        try:
+            deduped.append(e)
-            dt     = datetime.fromisoformat(ts_str)
+            counts.append(1)
-            ts_fmt = dt.strftime(DATETIME_FMT)
-        except Exception:
-            ts_fmt = ts_str
  
-        print(f"{ts_fmt}|{client}|{dst}|{port}|{proto}|{svc_name}|{count}")
+        for e, count in list(zip(deduped, counts))[-limit:]:
+            ts_str    = e.get('timestamp', '')
+            src       = e.get('src_ip', '')
+            dst       = e.get('dest_ip', '')
+            port      = str(e.get('dest_port', ''))
+            proto_num = int(e.get('ip.protocol', 0))
+            proto     = proto_map.get(proto_num, str(proto_num))
+            client    = ip_to_name.get(src, src)
+            svc_name  = reverse_lookup(dst, port, proto)
+            try:
+                dt     = datetime.fromisoformat(ts_str)
+                ts_fmt = dt.strftime(DATETIME_FMT)
+            except Exception:
+                ts_fmt = ts_str
+            print(f"{ts_fmt}|{client}|{dst}|{port}|{proto}|{svc_name}|{count}")
  
-def wg_events(file, filter_client, filter_type, limit):
+
+def wg_events(file, filter_client, filter_type, limit, collapse='1'):
     """
     Format WireGuard events with dedup and counts.
+    collapse='1' (default): hourly aggregation for attempt events
+    collapse='0': show all deduplicated events (--detailed mode)
     Output per line: ts|client|endpoint|event|count
     """
     from datetime import datetime
+    from collections import defaultdict
+    do_collapse = str(collapse) != '0'
  
     events = []
     try:
@@ -321,52 +359,129 @@ def wg_events(file, filter_client, filter_type, limit):
     except Exception:
         pass
  
-    # Dedup consecutive same client+event+endpoint within 60s
-    deduped = []
-    counts  = []
-    for e in events:
-        ts_str   = e.get('timestamp', '')
-        try:
-            ts = datetime.fromisoformat(ts_str).timestamp()
-        except Exception:
-            ts = 0
-        client   = e.get('client', '')
-        event    = e.get('event', '')
-        endpoint = e.get('endpoint', '')
-        key      = (client, event, endpoint[:15])
- 
-        if deduped:
-            prev     = deduped[-1]
-            prev_ts_str = prev.get('timestamp', '')
-            try:
-                prev_ts = datetime.fromisoformat(prev_ts_str).timestamp()
-            except Exception:
-                prev_ts = 0
-            prev_key = (
-                prev.get('client', ''),
-                prev.get('event', ''),
-                prev.get('endpoint', '')[:15]
-            )
-            if key == prev_key and (ts - prev_ts) < 300:
-                counts[-1] += 1
-                continue
- 
-        deduped.append(e)
-        counts.append(1)
- 
     limit = int(limit) if limit else 50
-    for e, count in list(zip(deduped, counts))[-limit:]:
-        ts_str   = e.get('timestamp', '')
-        client   = e.get('client', '')
-        endpoint = e.get('endpoint', '')
-        event    = e.get('event', '')
-        try:
-            dt = datetime.fromisoformat(ts_str)
-            ts_fmt = dt.strftime('%d/%m %H:%M')
-        except Exception:
-            ts_fmt = ts_str
  
-        print(f"{ts_fmt}|{client}|{endpoint}|{event}|{count}")
+    if do_collapse:
+        # Hourly aggregation for attempts; individual for handshakes
+        hourly_attempts = defaultdict(int)
+        hourly_ts = {}
+        handshakes = []
+        handshake_counts = []
+ 
+        for e in events:
+            ts_str   = e.get('timestamp', '')
+            client   = e.get('client', '')
+            endpoint = e.get('endpoint', '')
+            event    = e.get('event', '')
+ 
+            try:
+                dt = datetime.fromisoformat(ts_str)
+                ts = dt.timestamp()
+            except Exception:
+                dt = None
+                ts = 0
+ 
+            if event == 'attempt':
+                if dt:
+                    hour_key = (client, endpoint, event,
+                                dt.strftime('%Y-%m-%d %H'))
+                    hourly_attempts[hour_key] += 1
+                    if hour_key not in hourly_ts:
+                        hourly_ts[hour_key] = dt
+            else:
+                # Handshakes — consecutive dedup only
+                key = (client, event, endpoint[:15])
+                if handshakes:
+                    prev = handshakes[-1]
+                    try:
+                        prev_ts = datetime.fromisoformat(
+                            prev.get('timestamp', '')).timestamp()
+                    except Exception:
+                        prev_ts = 0
+                    prev_key = (
+                        prev.get('client', ''),
+                        prev.get('event', ''),
+                        prev.get('endpoint', '')[:15]
+                    )
+                    if key == prev_key and (ts - prev_ts) < 300:
+                        handshake_counts[-1] += 1
+                        continue
+                handshakes.append(e)
+                handshake_counts.append(1)
+ 
+        # Build output list: attempts (hourly) + handshakes, sorted by ts
+        output = []
+ 
+        for hour_key, dt in hourly_ts.items():
+            client, endpoint, event, _ = hour_key
+            count  = hourly_attempts[hour_key]
+            ts_fmt = dt.strftime(DATETIME_FMT.replace('%M', '00'))
+            output.append((dt.timestamp(), f"{ts_fmt}|{client}|{endpoint}|{event}|{count}"))
+ 
+        for e, count in zip(handshakes, handshake_counts):
+            ts_str = e.get('timestamp', '')
+            client   = e.get('client', '')
+            endpoint = e.get('endpoint', '')
+            event    = e.get('event', '')
+            try:
+                dt     = datetime.fromisoformat(ts_str)
+                ts_fmt = dt.strftime(DATETIME_FMT)
+                ts     = dt.timestamp()
+            except Exception:
+                ts_fmt = ts_str
+                ts     = 0
+            output.append((ts, f"{ts_fmt}|{client}|{endpoint}|{event}|{count}"))
+ 
+        output.sort(key=lambda x: x[0])
+        for _, line in output[-limit:]:
+            print(line)
+ 
+    else:
+        # Detailed mode — consecutive dedup only
+        deduped = []
+        counts  = []
+        for e in events:
+            ts_str   = e.get('timestamp', '')
+            client   = e.get('client', '')
+            event    = e.get('event', '')
+            endpoint = e.get('endpoint', '')
+            key      = (client, event, endpoint[:15])
+ 
+            try:
+                ts = datetime.fromisoformat(ts_str).timestamp()
+            except Exception:
+                ts = 0
+ 
+            if deduped:
+                prev = deduped[-1]
+                try:
+                    prev_ts = datetime.fromisoformat(
+                        prev.get('timestamp', '')).timestamp()
+                except Exception:
+                    prev_ts = 0
+                prev_key = (
+                    prev.get('client', ''),
+                    prev.get('event', ''),
+                    prev.get('endpoint', '')[:15]
+                )
+                if key == prev_key and (ts - prev_ts) < 300:
+                    counts[-1] += 1
+                    continue
+ 
+            deduped.append(e)
+            counts.append(1)
+ 
+        for e, count in list(zip(deduped, counts))[-limit:]:
+            ts_str   = e.get('timestamp', '')
+            client   = e.get('client', '')
+            endpoint = e.get('endpoint', '')
+            event    = e.get('event', '')
+            try:
+                dt     = datetime.fromisoformat(ts_str)
+                ts_fmt = dt.strftime(DATETIME_FMT)
+            except Exception:
+                ts_fmt = ts_str
+            print(f"{ts_fmt}|{client}|{endpoint}|{event}|{count}")
 
 def format_fw_event(line, clients_dir):
     """Format a single fw_event line"""
@@ -688,7 +803,7 @@ def rule_list_data(rules_dir, meta_dir):
         print(f"{r['name']}|{r['desc']}|{r['n_allows']}|{r['n_blocks']}|"
               f"{r['peer_count']}|{r['extends']}|{r['is_base']}|{r['group']}")
 
-def group_list_data(groups_dir, blocks_dir):
+def group_list_data(groups_dir, blocks_dir, clients_dir):
     """Return group summary data in one call"""
     import glob
     
@@ -705,7 +820,10 @@ def group_list_data(groups_dir, blocks_dir):
             name = g.get('name', '')
             desc = g.get('desc', '')
             peers = [p for p in g.get('peers', []) if p]
-            total = len(peers)
+            valid_peers = [p for p in peers 
+                if os.path.exists(os.path.join(clients_dir, f"{p}.conf"))]
+
+            total = len(valid_peers)
             blocked = sum(1 for p in peers if p in blocked_peers)
             print(f"{name}|{desc}|{total}|{blocked}")
         except:
@@ -2791,8 +2909,14 @@ commands = {
     'filter_values':        lambda args: filter_values(args[0], args[1], args[2]),
     'last_event':           lambda args: last_event(args[0], args[1], args[2], args[3]),
     'events_for':           lambda args: events_for(args[0], args[1], args[2]),
-    'fw_events':            lambda args: fw_events(args[0], args[1], args[2], args[3], args[4], args[5] if len(args) > 5 else '50'),
+    'fw_events':            lambda args: fw_events(
-    'wg_events':            lambda args: wg_events(args[0], args[1], args[2], args[3] if len(args) > 3 else '50'),
+                                args[0], args[1], args[2], args[3], args[4],
+                                args[5] if len(args) > 5 else '50',
+                                args[6] if len(args) > 6 else '1'),
+    'wg_events':            lambda args: wg_events(
+                                args[0], args[1], args[2],
+                                args[3] if len(args) > 3 else '50',
+                                args[4] if len(args) > 4 else '1'),
     'format_fw_event':      lambda args: format_fw_event(sys.stdin.read(), args[0]),
     'format_wg_event':      lambda args: format_wg_event(sys.stdin.read()),
     'remove_events':        lambda args: remove_events(args[0], args[1]),
@@ -2804,7 +2928,7 @@ commands = {
     'peer_data':            lambda args: peer_data(args[0], args[1], args[2]),
     'iso_to_ts':            lambda args: iso_to_ts(args[0]),
     'rule_list_data':       lambda args: rule_list_data(args[0], args[1]),
-    'group_list_data':      lambda args: group_list_data(args[0], args[1]),
+    'group_list_data':      lambda args: group_list_data(args[0], args[1], args[2]),
     'fmt_datetime':         lambda args: fmt_datetime(args[0], args[1]),
     'create_rule':          lambda args: create_rule(
         args[0], args[1], args[2], args[3], args[4], args[5], args[6],