metrics: add host multi-mount storage metrics, Prometheus exporter, and Grafana panels

core/metrics.py

@@ -1,15 +1,11 @@
 from prometheus_client import Counter, Histogram, Gauge, start_http_server
 import logging
+from typing import Optional
 
 logger = logging.getLogger(__name__)
 
 # Counters
-COLLECTED_POSTS_TOTAL = Counter(
-    "copykar_posts_collected_total",
-    "Total posts collected by the Telethon Userbot",
-    ["source_channel_id"]
-)
-
+# Ingest is counted once, by SOURCE_ACTIVITY_TOTAL, which also carries the channel title.
 SOURCE_ACTIVITY_TOTAL = Counter(
     "copykar_source_activity_total",
     "Total posts ingested per source channel",
@@ -40,10 +36,11 @@ ADMIN_ACTIONS_TOTAL = Counter(
     ["action"]
 )
 
-POSTS_PUBLISHED_TOTAL = Counter(
-    "copykar_posts_published_total",
-    "Total posts successfully published to target channels",
-    ["target_channel_id"]
+# Delivery is counted once, by TARGET_ACTIVITY_TOTAL.
+AUTO_ROUTED_POSTS_TOTAL = Counter(
+    "copykar_auto_routed_posts_total",
+    "Posts queued automatically by a source-to-target route",
+    ["source_channel_id", "target_title"]
 )
 
 ERRORS_TOTAL = Counter(
@@ -52,6 +49,12 @@ ERRORS_TOTAL = Counter(
     ["service", "error_type"]
 )
 
+ERRORS_RESOLVED_TOTAL = Counter(
+    "copykar_errors_resolved_total",
+    "Errors an admin has marked as fixed",
+    ["service", "error_type"]
+)
+
 # Histograms
 AI_LATENCY_SECONDS = Histogram(
     "copykar_ai_latency_seconds",
@@ -66,14 +69,136 @@ QUEUE_POSTS_GAUGE = Gauge(
     ["status"]
 )
 
-REDIS_QUEUE_SIZE_GAUGE = Gauge(
-    "copykar_redis_queue_size",
-    "Current number of posts waiting in Redis incoming queue"
+ERRORS_OPEN_GAUGE = Gauge(
+    "copykar_errors_open",
+    "Unresolved errors currently recorded, by service and exception type",
+    ["service", "error_type"]
 )
 
-def start_metrics_server(port: int = 8000):
+ERRORS_OPEN_TOTAL_GAUGE = Gauge(
+    "copykar_errors_open_total",
+    "Total unresolved errors across all services"
+)
+
+import shutil
+import asyncio
+
+# Disk Space Gauges per Mount Point
+DISK_TOTAL_BYTES = Gauge(
+    "copykar_disk_total_bytes",
+    "Total disk capacity in bytes",
+    ["mountpoint", "device"]
+)
+
+DISK_USED_BYTES = Gauge(
+    "copykar_disk_used_bytes",
+    "Used disk space in bytes",
+    ["mountpoint", "device"]
+)
+
+DISK_FREE_BYTES = Gauge(
+    "copykar_disk_free_bytes",
+    "Free/available disk space in bytes",
+    ["mountpoint", "device"]
+)
+
+DISK_FREE_PERCENT = Gauge(
+    "copykar_disk_free_percent",
+    "Percentage of free disk space",
+    ["mountpoint", "device"]
+)
+
+
+import os
+
+VOLUME_DEFINITIONS = [
+    {
+        "mountpoint": "/",
+        "device": "/dev/sda1",
+        "container_paths": ["/host_os/home", "/hostfs/host_mnt/home", "/home", "/host_os_disk", "/hostfs", "/"]
+    },
+    {
+        "mountpoint": "/projects",
+        "device": "/dev/sda2",
+        "container_paths": ["/host_os/projects", "/projects"]
+    },
+    {
+        "mountpoint": "/boot",
+        "device": "/dev/sda4",
+        "container_paths": ["/host_os/boot", "/boot"]
+    },
+    {
+        "mountpoint": "/boot/efi",
+        "device": "/dev/sda3",
+        "container_paths": ["/host_os/boot_efi", "/boot/efi"]
+    },
+    {
+        "mountpoint": "/run/media/mamad/WIN11_25H2_",
+        "device": "/dev/sdc1",
+        "container_paths": ["/host_os/media/mamad/WIN11_25H2_", "/run/media/mamad/WIN11_25H2_"]
+    }
+]
+
+
+def update_disk_metrics():
+    """Update Prometheus gauges with real host OS filesystem disk usage for each mounted volume."""
+    try:
+        recorded_mounts = set()
+        for v in VOLUME_DEFINITIONS:
+            mnt = v["mountpoint"]
+            dev = v["device"]
+            for path in v["container_paths"]:
+                if os.path.exists(path) and os.path.isdir(path):
+                    try:
+                        total, used, free = shutil.disk_usage(path)
+                        DISK_TOTAL_BYTES.labels(mountpoint=mnt, device=dev).set(total)
+                        DISK_USED_BYTES.labels(mountpoint=mnt, device=dev).set(used)
+                        DISK_FREE_BYTES.labels(mountpoint=mnt, device=dev).set(free)
+                        free_pct = (free / total * 100.0) if total > 0 else 0.0
+                        DISK_FREE_PERCENT.labels(mountpoint=mnt, device=dev).set(free_pct)
+                        recorded_mounts.add(mnt)
+                        break
+                    except Exception as err:
+                        logger.debug(f"Error measuring disk usage for {path}: {err}")
+
+        # If none of the specific mounts matched, fallback to root
+        if not recorded_mounts:
+            total, used, free = shutil.disk_usage("/")
+            DISK_TOTAL_BYTES.labels(mountpoint="/", device="default").set(total)
+            DISK_USED_BYTES.labels(mountpoint="/", device="default").set(used)
+            DISK_FREE_BYTES.labels(mountpoint="/", device="default").set(free)
+            free_pct = (free / total * 100.0) if total > 0 else 0.0
+            DISK_FREE_PERCENT.labels(mountpoint="/", device="default").set(free_pct)
+
+    except Exception as e:
+        logger.debug(f"Error updating host disk metrics: {e}")
+
+
+
+async def _disk_metrics_loop(interval: int = 15):
+    """Background task to keep disk metrics updated."""
+    while True:
+        try:
+            update_disk_metrics()
+        except Exception as e:
+            logger.debug(f"Disk metrics loop error: {e}")
+        await asyncio.sleep(interval)
+
+
+# The overall queue depth is sum(copykar_posts_queue_gauge) - no separate total series,
+# which previously made the dashboard report the queue twice.
+def start_metrics_server(port: int = 8008):
     try:
         start_http_server(port)
+        update_disk_metrics()
+        try:
+            loop = asyncio.get_event_loop()
+            if loop.is_running():
+                asyncio.create_task(_disk_metrics_loop())
+        except Exception:
+            pass
         logger.info(f"Prometheus metrics server running on port {port}")
     except Exception as e:
         logger.error(f"Failed to start Prometheus metrics server: {e}")
+
+