feat(scans): a sync that dies at login is now a failed run, on screen and by mail
A Wazuh API that refused every login for a day was visible only in the container log. Every scan-jobs row is per asset, and a sync that fails at authentication never reaches an asset — so /scans showed a clean history of COMPLETED runs, the manual button said "Sync Failed — check the logs", and nobody was told. Ledger: every sync execution (Wazuh, Nessus, Intune, vCenter, IGEL; button or scheduler) is one sync_runs row with start, end, status, stats and the error that stopped it, written from its own session so the record never depends on the job's transaction. A run where every agent failed the same way counts as failed, not as "0 triggered". Rows left running by a restart are closed at startup. Visible: Scan Jobs gets a Sync Health strip (per source: state, last run, last success, error) and a Sync Runs table; the dashboard shows a banner while a source is failed or stale (stale follows the schedule interval, a weekly Nessus is not stale after six days); the Sync Failed modal shows the real reason, and the sync endpoints return it instead of "check server logs". Mail: a failed run mails the sync-alert recipients (own setting, else the notification defaults) with the error and the last successful sync, i.e. how long the coverage gap already is — one mail per source per 24h while it keeps failing, logged as SYNC_FAILURE in the notification log.
This commit is contained in:
@@ -0,0 +1,50 @@
|
|||||||
|
"""Sync run ledger + SYNC_FAILURE notification type
|
||||||
|
|
||||||
|
Revision ID: 050
|
||||||
|
Revises: 049
|
||||||
|
Create Date: 2026-09-03 14:00:00.000000
|
||||||
|
|
||||||
|
A Wazuh API that refused every login for a day was visible only in the
|
||||||
|
container log. Every scan-jobs row is per asset, and a sync that fails at
|
||||||
|
authentication never reaches an asset — so /scans showed a clean history of
|
||||||
|
COMPLETED runs while nothing had been synced. This table records every sync
|
||||||
|
execution (any source, button or scheduler) with its outcome, and the new
|
||||||
|
notification type carries the failure mail in the notification log.
|
||||||
|
|
||||||
|
ALTER TYPE ... ADD VALUE cannot run inside a transaction block on older
|
||||||
|
Postgres → autocommit_block. Idempotent (IF NOT EXISTS).
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "050"
|
||||||
|
down_revision = "049"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"sync_runs",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column("source", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("trigger", sa.String(length=20), nullable=False, server_default="manual"),
|
||||||
|
sa.Column("status", sa.String(length=20), nullable=False, server_default="running"),
|
||||||
|
sa.Column("started_at", sa.DateTime(), nullable=False),
|
||||||
|
sa.Column("finished_at", sa.DateTime(), nullable=True),
|
||||||
|
sa.Column("error_message", sa.Text(), nullable=True),
|
||||||
|
sa.Column("stats", sa.Text(), nullable=True),
|
||||||
|
sa.Column("alerted_at", sa.DateTime(), nullable=True),
|
||||||
|
)
|
||||||
|
op.create_index("ix_sync_runs_source", "sync_runs", ["source"])
|
||||||
|
op.create_index("ix_sync_runs_status", "sync_runs", ["status"])
|
||||||
|
with op.get_context().autocommit_block():
|
||||||
|
op.execute("ALTER TYPE notificationtype ADD VALUE IF NOT EXISTS 'SYNC_FAILURE'")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_sync_runs_status", table_name="sync_runs")
|
||||||
|
op.drop_index("ix_sync_runs_source", table_name="sync_runs")
|
||||||
|
op.drop_table("sync_runs")
|
||||||
|
# Postgres cannot drop an enum label — see 043; deliberately left in place.
|
||||||
@@ -13,6 +13,7 @@ from app.models.policy import Policy
|
|||||||
from app.models.scan_schedule import ScanSchedule
|
from app.models.scan_schedule import ScanSchedule
|
||||||
from app.models.notification_log import NotificationLog
|
from app.models.notification_log import NotificationLog
|
||||||
from app.models.setting import Setting
|
from app.models.setting import Setting
|
||||||
|
from app.models.sync_run import SyncRun
|
||||||
from app.models.ai_report import AIReport
|
from app.models.ai_report import AIReport
|
||||||
from app.models.cve_remediation import CveRemediation
|
from app.models.cve_remediation import CveRemediation
|
||||||
from app.models.app_cve_cache import AppCveCache
|
from app.models.app_cve_cache import AppCveCache
|
||||||
@@ -33,6 +34,7 @@ __all__ = [
|
|||||||
"ScanSchedule",
|
"ScanSchedule",
|
||||||
"NotificationLog",
|
"NotificationLog",
|
||||||
"Setting",
|
"Setting",
|
||||||
|
"SyncRun",
|
||||||
"AIReport",
|
"AIReport",
|
||||||
"CveRemediation",
|
"CveRemediation",
|
||||||
"AppCveCache",
|
"AppCveCache",
|
||||||
|
|||||||
@@ -17,6 +17,8 @@ class NotificationType(str, Enum):
|
|||||||
# Actively exploited (CISA KEV / ENISA EUVD) AND present in our inventory —
|
# Actively exploited (CISA KEV / ENISA EUVD) AND present in our inventory —
|
||||||
# its own type so the log can be filtered for the alerts that meant "now".
|
# its own type so the log can be filtered for the alerts that meant "now".
|
||||||
KEV_ALERT = "kev_alert"
|
KEV_ALERT = "kev_alert"
|
||||||
|
# A sync job (Wazuh/Nessus/Intune/vCenter/IGEL) failed — coverage gap.
|
||||||
|
SYNC_FAILURE = "sync_failure"
|
||||||
MANUAL = "manual"
|
MANUAL = "manual"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
"""SyncRun — one row per sync job execution, whatever the source or trigger.
|
||||||
|
|
||||||
|
The `scans` table records per-asset results, so a sync that dies BEFORE it
|
||||||
|
reaches any asset (Wazuh authentication refused, Nessus unreachable, IGEL UMS
|
||||||
|
down) left no trace anywhere but the container log. This ledger is the
|
||||||
|
source-level record: started, finished, status, and the error that stopped it.
|
||||||
|
"""
|
||||||
|
from sqlalchemy import Column, Integer, String, DateTime, Text
|
||||||
|
from app.models.base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class SyncRun(Base):
|
||||||
|
__tablename__ = "sync_runs"
|
||||||
|
|
||||||
|
id = Column(Integer, primary_key=True, index=True)
|
||||||
|
# wazuh | nessus | intune | vcenter | igel
|
||||||
|
source = Column(String(30), nullable=False, index=True)
|
||||||
|
# manual | scheduled
|
||||||
|
trigger = Column(String(20), nullable=False, default="manual")
|
||||||
|
# running | completed | failed (plain strings: no Postgres enum to migrate)
|
||||||
|
status = Column(String(20), nullable=False, default="running", index=True)
|
||||||
|
started_at = Column(DateTime, nullable=False)
|
||||||
|
finished_at = Column(DateTime, nullable=True)
|
||||||
|
error_message = Column(Text, nullable=True)
|
||||||
|
# JSON: the service's stats dict (agents synced, created, errors[] ...)
|
||||||
|
stats = Column(Text, nullable=True)
|
||||||
|
# When a failure mail went out for this run — the cooldown anchor.
|
||||||
|
alerted_at = Column(DateTime, nullable=True)
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return f"<SyncRun(source='{self.source}', status='{self.status}')>"
|
||||||
+15
-1
@@ -1039,7 +1039,19 @@ def sync_wazuh_assets(
|
|||||||
"""
|
"""
|
||||||
Synchronizes assets from Wazuh Manager.
|
Synchronizes assets from Wazuh Manager.
|
||||||
fetch active agents -> update/create local assets.
|
fetch active agents -> update/create local assets.
|
||||||
|
|
||||||
|
Recorded in the sync-run ledger as the "assets" phase; when the
|
||||||
|
scheduled scan calls this, the record merges into that run.
|
||||||
"""
|
"""
|
||||||
|
from app.services.sync_run_service import record_sync_run
|
||||||
|
with record_sync_run("wazuh") as run:
|
||||||
|
run.stats.setdefault("phase", "assets")
|
||||||
|
stats = _sync_wazuh_assets(db)
|
||||||
|
run.stats.update(stats)
|
||||||
|
return stats
|
||||||
|
|
||||||
|
|
||||||
|
def _sync_wazuh_assets(db: Session) -> dict:
|
||||||
# 1. Get Wazuh Config (transparently decrypted)
|
# 1. Get Wazuh Config (transparently decrypted)
|
||||||
from app.auth.setting_crypto import read_setting_value
|
from app.auth.setting_crypto import read_setting_value
|
||||||
raw_wazuh = read_setting_value(db, "wazuh_config")
|
raw_wazuh = read_setting_value(db, "wazuh_config")
|
||||||
@@ -1276,7 +1288,9 @@ def sync_wazuh_assets(
|
|||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Wazuh Sync Failed: {e}")
|
logger.error(f"Wazuh Sync Failed: {e}")
|
||||||
raise HTTPException(status_code=500, detail="Wazuh sync failed. Check server logs for details.")
|
# The real reason goes to the caller (editors only): "check the
|
||||||
|
# server logs" left an operator with a failing Wazuh API for a day.
|
||||||
|
raise HTTPException(status_code=502, detail=f"Wazuh sync failed: {e}")
|
||||||
|
|
||||||
return stats
|
return stats
|
||||||
|
|
||||||
|
|||||||
+4
-1
@@ -17,6 +17,7 @@ from fastapi import APIRouter, Depends, HTTPException
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.auth.dependencies import RequireAdmin, RequireEditor
|
from app.auth.dependencies import RequireAdmin, RequireEditor
|
||||||
|
from app.services.sync_run_service import record_sync_run
|
||||||
from app.database import get_db, SessionLocal
|
from app.database import get_db, SessionLocal
|
||||||
from app.models.user import User
|
from app.models.user import User
|
||||||
from app.services.igel_service import (
|
from app.services.igel_service import (
|
||||||
@@ -61,7 +62,9 @@ async def test_igel(
|
|||||||
def _run_igel_sync_threaded() -> None:
|
def _run_igel_sync_threaded() -> None:
|
||||||
db = SessionLocal()
|
db = SessionLocal()
|
||||||
try:
|
try:
|
||||||
stats = run_igel_sync(db)
|
with record_sync_run("igel", "manual") as run:
|
||||||
|
stats = run_igel_sync(db)
|
||||||
|
run.stats.update(stats)
|
||||||
result = {k: v for k, v in stats.items() if k != "errors"}
|
result = {k: v for k, v in stats.items() if k != "errors"}
|
||||||
_IGEL_SYNC["result"] = result
|
_IGEL_SYNC["result"] = result
|
||||||
_IGEL_SYNC["error"] = None
|
_IGEL_SYNC["error"] = None
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ from fastapi import APIRouter, Depends, HTTPException
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.auth.dependencies import RequireAdmin, RequireEditor
|
from app.auth.dependencies import RequireAdmin, RequireEditor
|
||||||
|
from app.services.sync_run_service import record_sync_run
|
||||||
from app.database import get_db, SessionLocal
|
from app.database import get_db, SessionLocal
|
||||||
from app.models.user import User
|
from app.models.user import User
|
||||||
from app.services.intune_service import load_intune_config, run_intune_sync, _build_client
|
from app.services.intune_service import load_intune_config, run_intune_sync, _build_client
|
||||||
@@ -53,7 +54,9 @@ async def test_intune(
|
|||||||
def _run_intune_sync_threaded() -> None:
|
def _run_intune_sync_threaded() -> None:
|
||||||
db = SessionLocal()
|
db = SessionLocal()
|
||||||
try:
|
try:
|
||||||
stats = run_intune_sync(db)
|
with record_sync_run("intune", "manual") as run:
|
||||||
|
stats = run_intune_sync(db)
|
||||||
|
run.stats.update(stats)
|
||||||
result = {k: v for k, v in stats.items() if k != "errors"}
|
result = {k: v for k, v in stats.items() if k != "errors"}
|
||||||
_INTUNE_SYNC["result"] = result
|
_INTUNE_SYNC["result"] = result
|
||||||
_INTUNE_SYNC["error"] = None
|
_INTUNE_SYNC["error"] = None
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from pydantic import BaseModel, Field
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.auth.dependencies import RequireAdmin, RequireEditor
|
from app.auth.dependencies import RequireAdmin, RequireEditor
|
||||||
|
from app.services.sync_run_service import record_sync_run
|
||||||
from app.database import get_db
|
from app.database import get_db
|
||||||
from app.integrations.nessus_client import (
|
from app.integrations.nessus_client import (
|
||||||
NessusAPIError,
|
NessusAPIError,
|
||||||
@@ -152,7 +153,9 @@ def trigger_nessus_sync(
|
|||||||
"""
|
"""
|
||||||
import httpx as _httpx
|
import httpx as _httpx
|
||||||
try:
|
try:
|
||||||
stats = run_nessus_sync(db, scan_ids=payload.scan_ids)
|
with record_sync_run("nessus", "manual") as run:
|
||||||
|
stats = run_nessus_sync(db, scan_ids=payload.scan_ids)
|
||||||
|
run.stats.update(stats)
|
||||||
return {"message": "Nessus sync completed", **stats}
|
return {"message": "Nessus sync completed", **stats}
|
||||||
except RuntimeError as e:
|
except RuntimeError as e:
|
||||||
# config missing
|
# config missing
|
||||||
|
|||||||
+39
-1
@@ -235,7 +235,20 @@ def trigger_autoscan(
|
|||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Trigger automated scans for ALL assets connected to Wazuh.
|
Trigger automated scans for ALL assets connected to Wazuh.
|
||||||
|
Recorded in the sync-run ledger; a run where every agent failed
|
||||||
|
(authentication refused) counts as failed, not as 0 triggered.
|
||||||
"""
|
"""
|
||||||
|
from app.services.sync_run_service import record_sync_run, fail_if_nothing_synced
|
||||||
|
with record_sync_run("wazuh") as run:
|
||||||
|
run.stats["phase"] = "vulnerabilities"
|
||||||
|
result = _autoscan(db)
|
||||||
|
run.stats.update({"agents_synced": result.get("scans_triggered", 0),
|
||||||
|
"errors": result.get("errors", [])})
|
||||||
|
fail_if_nothing_synced(run, result.get("scans_triggered", 0), result.get("errors", []))
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _autoscan(db: Session) -> dict:
|
||||||
# 1. Get Wazuh Configuration from DB (transparently decrypted)
|
# 1. Get Wazuh Configuration from DB (transparently decrypted)
|
||||||
from app.auth.setting_crypto import read_setting_value
|
from app.auth.setting_crypto import read_setting_value
|
||||||
raw_wazuh = read_setting_value(db, "wazuh_config")
|
raw_wazuh = read_setting_value(db, "wazuh_config")
|
||||||
@@ -318,7 +331,7 @@ def trigger_autoscan(
|
|||||||
scan_obj.error_message = f"Wazuh connection failed: {str(e)}"
|
scan_obj.error_message = f"Wazuh connection failed: {str(e)}"
|
||||||
scan_obj.completed_at = datetime.now()
|
scan_obj.completed_at = datetime.now()
|
||||||
db.commit()
|
db.commit()
|
||||||
raise HTTPException(status_code=500, detail="Failed to connect to Wazuh. Check server logs for details.")
|
raise HTTPException(status_code=502, detail=f"Failed to connect to Wazuh: {e}")
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"message": f"Triggered scans for {triggered_count} assets.",
|
"message": f"Triggered scans for {triggered_count} assets.",
|
||||||
@@ -436,3 +449,28 @@ async def delete_schedule(
|
|||||||
|
|
||||||
db.delete(schedule)
|
db.delete(schedule)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
# --- Sync run ledger (every source, manual and scheduled) ---
|
||||||
|
|
||||||
|
@router.get("/sync-runs")
|
||||||
|
async def list_sync_runs(
|
||||||
|
limit: int = Query(30, ge=1, le=200),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
current_user: User = Depends(get_current_user),
|
||||||
|
):
|
||||||
|
"""Latest sync executions across all sources, newest first."""
|
||||||
|
from app.services.sync_run_service import list_runs
|
||||||
|
return list_runs(db, limit)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/sync-health")
|
||||||
|
async def get_sync_health(
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
current_user: User = Depends(get_current_user),
|
||||||
|
):
|
||||||
|
"""Per source: last run, last success, and a state the GUI can colour
|
||||||
|
(ok / failed / stale / running / unconfigured) — plus the failure-mail
|
||||||
|
settings so the panel can edit them in place."""
|
||||||
|
from app.services.sync_run_service import health_payload
|
||||||
|
return health_payload(db)
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ from fastapi import APIRouter, Depends, HTTPException
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.auth.dependencies import RequireAdmin, RequireEditor
|
from app.auth.dependencies import RequireAdmin, RequireEditor
|
||||||
|
from app.services.sync_run_service import record_sync_run
|
||||||
from app.database import get_db, SessionLocal
|
from app.database import get_db, SessionLocal
|
||||||
from app.models.user import User
|
from app.models.user import User
|
||||||
from app.services.vcenter_service import (
|
from app.services.vcenter_service import (
|
||||||
@@ -54,7 +55,9 @@ async def test_vcenter(
|
|||||||
def _run_vcenter_sync_threaded() -> None:
|
def _run_vcenter_sync_threaded() -> None:
|
||||||
db = SessionLocal()
|
db = SessionLocal()
|
||||||
try:
|
try:
|
||||||
stats = run_vcenter_sync(db)
|
with record_sync_run("vcenter", "manual") as run:
|
||||||
|
stats = run_vcenter_sync(db)
|
||||||
|
run.stats.update(stats)
|
||||||
result = {k: v for k, v in stats.items() if k != "errors"}
|
result = {k: v for k, v in stats.items() if k != "errors"}
|
||||||
_VCENTER_SYNC["result"] = result
|
_VCENTER_SYNC["result"] = result
|
||||||
_VCENTER_SYNC["error"] = None
|
_VCENTER_SYNC["error"] = None
|
||||||
|
|||||||
@@ -1788,12 +1788,24 @@ def sync_vulnerabilities_from_wazuh(
|
|||||||
try:
|
try:
|
||||||
result = run_wazuh_vulnerability_sync(db)
|
result = run_wazuh_vulnerability_sync(db)
|
||||||
return result
|
return result
|
||||||
|
except HTTPException:
|
||||||
|
raise # already carries the real reason and status
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise HTTPException(status_code=500, detail=str(e))
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
def run_wazuh_vulnerability_sync(db: Session) -> dict:
|
def run_wazuh_vulnerability_sync(db: Session) -> dict:
|
||||||
"""Synchronous Wazuh vulnerability sync with statistics return"""
|
"""Synchronous Wazuh vulnerability sync with statistics return.
|
||||||
|
Recorded in the sync-run ledger as the "vulnerabilities" phase."""
|
||||||
|
from app.services.sync_run_service import record_sync_run
|
||||||
|
with record_sync_run("wazuh") as run:
|
||||||
|
run.stats.setdefault("phase", "vulnerabilities")
|
||||||
|
result = _run_wazuh_vulnerability_sync(db)
|
||||||
|
run.stats.update({k: v for k, v in result.items() if k != "message"})
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _run_wazuh_vulnerability_sync(db: Session) -> dict:
|
||||||
|
|
||||||
# 1. Get Wazuh Config from DB (transparently decrypted)
|
# 1. Get Wazuh Config from DB (transparently decrypted)
|
||||||
from app.auth.setting_crypto import read_setting_value
|
from app.auth.setting_crypto import read_setting_value
|
||||||
@@ -2177,7 +2189,7 @@ def run_wazuh_vulnerability_sync(db: Session) -> dict:
|
|||||||
db.rollback()
|
db.rollback()
|
||||||
wazuh.close()
|
wazuh.close()
|
||||||
logger.error(f"Wazuh sync failed: {e}")
|
logger.error(f"Wazuh sync failed: {e}")
|
||||||
raise HTTPException(status_code=500, detail="Wazuh sync failed. Check server logs for details.")
|
raise HTTPException(status_code=502, detail=f"Wazuh sync failed: {e}")
|
||||||
|
|
||||||
|
|
||||||
async def sync_wazuh_vulnerabilities(db_session: Optional[Session] = None):
|
async def sync_wazuh_vulnerabilities(db_session: Optional[Session] = None):
|
||||||
|
|||||||
+105
-85
@@ -29,6 +29,9 @@ from app.models.policy import Policy
|
|||||||
from app.models.notification_log import NotificationLog, NotificationType, NotificationStatus
|
from app.models.notification_log import NotificationLog, NotificationType, NotificationStatus
|
||||||
from app.models.user import User
|
from app.models.user import User
|
||||||
from app.integrations.wazuh_client import WazuhClient
|
from app.integrations.wazuh_client import WazuhClient
|
||||||
|
from app.services.sync_run_service import (
|
||||||
|
record_sync_run, mark_interrupted_runs, fail_if_nothing_synced,
|
||||||
|
)
|
||||||
|
|
||||||
scheduler = AsyncIOScheduler() if HAS_APSCHEDULER else None
|
scheduler = AsyncIOScheduler() if HAS_APSCHEDULER else None
|
||||||
|
|
||||||
@@ -59,7 +62,9 @@ def execute_scheduled_scan(schedule_id: int):
|
|||||||
if scanner == "nessus":
|
if scanner == "nessus":
|
||||||
try:
|
try:
|
||||||
from app.services.nessus_sync import run_nessus_sync
|
from app.services.nessus_sync import run_nessus_sync
|
||||||
stats = run_nessus_sync(db)
|
with record_sync_run("nessus", "scheduled") as run:
|
||||||
|
stats = run_nessus_sync(db)
|
||||||
|
run.stats.update(stats)
|
||||||
logger.info(f"Scheduled Nessus sync done: {stats}")
|
logger.info(f"Scheduled Nessus sync done: {stats}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Scheduled Nessus sync failed: {e}")
|
logger.error(f"Scheduled Nessus sync failed: {e}")
|
||||||
@@ -85,97 +90,104 @@ def execute_scheduled_scan(schedule_id: int):
|
|||||||
db.commit()
|
db.commit()
|
||||||
return
|
return
|
||||||
|
|
||||||
# ---- Wazuh branch (default, existing behaviour) ----
|
with record_sync_run("wazuh", "scheduled") as run:
|
||||||
# Wazuh Config laden (transparently decrypted)
|
# ---- Wazuh branch (default, existing behaviour) ----
|
||||||
from app.auth.setting_crypto import read_setting_value
|
# Wazuh Config laden (transparently decrypted)
|
||||||
raw_wazuh = read_setting_value(db, "wazuh_config")
|
from app.auth.setting_crypto import read_setting_value
|
||||||
if not raw_wazuh:
|
raw_wazuh = read_setting_value(db, "wazuh_config")
|
||||||
logger.error("Scheduled scan aborted: Wazuh configuration missing")
|
if not raw_wazuh:
|
||||||
return
|
logger.error("Scheduled scan aborted: Wazuh configuration missing")
|
||||||
|
run.fail("Wazuh configuration missing")
|
||||||
|
return
|
||||||
|
|
||||||
config = json.loads(raw_wazuh)
|
config = json.loads(raw_wazuh)
|
||||||
|
|
||||||
# Asset sync first — the same call the manual "Sync Data (Wazuh)"
|
# Asset sync first — the same call the manual "Sync Data (Wazuh)"
|
||||||
# button makes before it syncs vulnerabilities. Without it this job
|
# button makes before it syncs vulnerabilities. Without it this job
|
||||||
# only ever sees the agent ids already in the DB: a host that
|
# only ever sees the agent ids already in the DB: a host that
|
||||||
# re-registers with a NEW agent id keeps its stale id forever, so
|
# re-registers with a NEW agent id keeps its stale id forever, so
|
||||||
# every CVE and EOL lookup asks Wazuh about an agent that no longer
|
# every CVE and EOL lookup asks Wazuh about an agent that no longer
|
||||||
# exists, and the asset never flips back from INACTIVE to ACTIVE
|
# exists, and the asset never flips back from INACTIVE to ACTIVE
|
||||||
# until someone clicks sync by hand.
|
# until someone clicks sync by hand.
|
||||||
# ponytail: the endpoint function is called directly (current_user is
|
# ponytail: the endpoint function is called directly (current_user is
|
||||||
# unused in its body) rather than extracted into a service — same
|
# unused in its body) rather than extracted into a service — same
|
||||||
# pattern as the sync_agent_vulnerabilities import below.
|
# pattern as the sync_agent_vulnerabilities import below.
|
||||||
try:
|
try:
|
||||||
from app.routers.assets import sync_wazuh_assets
|
from app.routers.assets import sync_wazuh_assets
|
||||||
logger.info(f"Scheduled Wazuh asset sync: {sync_wazuh_assets(db=db, current_user=None)}")
|
logger.info(f"Scheduled Wazuh asset sync: {sync_wazuh_assets(db=db, current_user=None)}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Scheduled Wazuh asset sync failed: {e}")
|
logger.error(f"Scheduled Wazuh asset sync failed: {e}")
|
||||||
|
run.stats.setdefault("errors", []).append(f"asset sync: {e}")
|
||||||
|
|
||||||
assets = db.query(Asset).filter(Asset.wazuh_agent_id.isnot(None)).all()
|
assets = db.query(Asset).filter(Asset.wazuh_agent_id.isnot(None)).all()
|
||||||
if not assets:
|
if not assets:
|
||||||
logger.info("No Wazuh assets found for scheduled scan")
|
logger.info("No Wazuh assets found for scheduled scan")
|
||||||
schedule.last_run = datetime.now()
|
schedule.last_run = datetime.now()
|
||||||
db.commit()
|
db.commit()
|
||||||
return
|
return
|
||||||
|
|
||||||
triggered = 0
|
triggered = 0
|
||||||
errors = []
|
errors = []
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with WazuhClient(
|
with WazuhClient(
|
||||||
base_url=config.get("api_url"),
|
base_url=config.get("api_url"),
|
||||||
username=config.get("username"),
|
username=config.get("username"),
|
||||||
password=config.get("password"),
|
password=config.get("password"),
|
||||||
indexer_url=config.get("indexer_url"),
|
indexer_url=config.get("indexer_url"),
|
||||||
indexer_username=config.get("indexer_username"),
|
indexer_username=config.get("indexer_username"),
|
||||||
indexer_password=config.get("indexer_password"),
|
indexer_password=config.get("indexer_password"),
|
||||||
verify_ssl=bool(config.get("verify_ssl", True))
|
verify_ssl=bool(config.get("verify_ssl", True))
|
||||||
) as client:
|
) as client:
|
||||||
# Sync vulnerabilities for each agent
|
# Sync vulnerabilities for each agent
|
||||||
from app.routers.vulnerabilities import (
|
from app.routers.vulnerabilities import (
|
||||||
sync_agent_vulnerabilities, reconcile_empty_agents,
|
sync_agent_vulnerabilities, reconcile_empty_agents,
|
||||||
)
|
)
|
||||||
# Shared across the run so an agent that returns nothing can be
|
# Shared across the run so an agent that returns nothing can be
|
||||||
# judged against the run as a whole (see reconcile_empty_agents).
|
# judged against the run as a whole (see reconcile_empty_agents).
|
||||||
run_stats: dict = {}
|
run_stats: dict = {}
|
||||||
for asset in assets:
|
for asset in assets:
|
||||||
|
try:
|
||||||
|
scan = Scan(
|
||||||
|
asset_id=asset.id,
|
||||||
|
scan_type=ScanType.WAZUH, # source-labelled (was FULL)
|
||||||
|
status=ScanStatus.RUNNING,
|
||||||
|
started_at=datetime.now()
|
||||||
|
)
|
||||||
|
db.add(scan)
|
||||||
|
|
||||||
|
sync_agent_vulnerabilities(db, client, asset.wazuh_agent_id,
|
||||||
|
asset, run_stats=run_stats)
|
||||||
|
|
||||||
|
scan.status = ScanStatus.COMPLETED
|
||||||
|
scan.completed_at = datetime.now()
|
||||||
|
triggered += 1
|
||||||
|
except Exception as e:
|
||||||
|
scan.status = ScanStatus.FAILED
|
||||||
|
scan.error_message = str(e)
|
||||||
|
errors.append(f"{asset.hostname}: {e}")
|
||||||
|
logger.error(f"Scheduled scan error for {asset.hostname}: {e}")
|
||||||
|
|
||||||
|
# Now that the run is done, agents that returned nothing can be
|
||||||
|
# judged: real emptiness if the API answered for anyone else.
|
||||||
try:
|
try:
|
||||||
scan = Scan(
|
reconcile_empty_agents(db, run_stats)
|
||||||
asset_id=asset.id,
|
|
||||||
scan_type=ScanType.WAZUH, # source-labelled (was FULL)
|
|
||||||
status=ScanStatus.RUNNING,
|
|
||||||
started_at=datetime.now()
|
|
||||||
)
|
|
||||||
db.add(scan)
|
|
||||||
|
|
||||||
sync_agent_vulnerabilities(db, client, asset.wazuh_agent_id,
|
|
||||||
asset, run_stats=run_stats)
|
|
||||||
|
|
||||||
scan.status = ScanStatus.COMPLETED
|
|
||||||
scan.completed_at = datetime.now()
|
|
||||||
triggered += 1
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
scan.status = ScanStatus.FAILED
|
logger.error(f"Wazuh empty-agent reconcile failed: {e}")
|
||||||
scan.error_message = str(e)
|
|
||||||
errors.append(f"{asset.hostname}: {e}")
|
|
||||||
logger.error(f"Scheduled scan error for {asset.hostname}: {e}")
|
|
||||||
|
|
||||||
# Now that the run is done, agents that returned nothing can be
|
except Exception as e:
|
||||||
# judged: real emptiness if the API answered for anyone else.
|
logger.error(f"Wazuh connection error during scheduled scan: {e}")
|
||||||
try:
|
run.fail(f"Wazuh connection failed: {e}")
|
||||||
reconcile_empty_agents(db, run_stats)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Wazuh empty-agent reconcile failed: {e}")
|
|
||||||
|
|
||||||
except Exception as e:
|
schedule.last_run = datetime.now()
|
||||||
logger.error(f"Wazuh connection error during scheduled scan: {e}")
|
interval_delta = INTERVAL_MAP.get(schedule.interval, {"days": 1})
|
||||||
|
schedule.next_run = datetime.now() + timedelta(**interval_delta)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
schedule.last_run = datetime.now()
|
logger.info(f"Scheduled scan '{schedule.name}' completed: {triggered} assets scanned, {len(errors)} errors")
|
||||||
interval_delta = INTERVAL_MAP.get(schedule.interval, {"days": 1})
|
run.stats.update({"phase": "assets+vulnerabilities", "agents_synced": triggered,
|
||||||
schedule.next_run = datetime.now() + timedelta(**interval_delta)
|
"errors": run.stats.get("errors", []) + errors})
|
||||||
db.commit()
|
fail_if_nothing_synced(run, triggered, errors)
|
||||||
|
|
||||||
logger.info(f"Scheduled scan '{schedule.name}' completed: {triggered} assets scanned, {len(errors)} errors")
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Scheduled scan error: {e}")
|
logger.error(f"Scheduled scan error: {e}")
|
||||||
@@ -592,7 +604,9 @@ def intune_sync_nightly():
|
|||||||
m365_service.fetch_security_data(db, force_refresh=True)
|
m365_service.fetch_security_data(db, force_refresh=True)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("M365 page refresh failed (non-fatal, cache kept): %s", e)
|
logger.warning("M365 page refresh failed (non-fatal, cache kept): %s", e)
|
||||||
stats = run_intune_sync(db)
|
with record_sync_run("intune", "scheduled") as run:
|
||||||
|
stats = run_intune_sync(db)
|
||||||
|
run.stats.update(stats)
|
||||||
logger.info("Intune nightly: %s", {k: v for k, v in stats.items() if k != "errors"})
|
logger.info("Intune nightly: %s", {k: v for k, v in stats.items() if k != "errors"})
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error("Intune nightly failed: %s", e)
|
logger.error("Intune nightly failed: %s", e)
|
||||||
@@ -618,7 +632,9 @@ def vcenter_sync_nightly():
|
|||||||
if not load_vcenter_config(db):
|
if not load_vcenter_config(db):
|
||||||
logger.info("vCenter sync skipped — vcenter_config not set")
|
logger.info("vCenter sync skipped — vcenter_config not set")
|
||||||
return
|
return
|
||||||
stats = run_vcenter_sync(db, refresh_catalog=True)
|
with record_sync_run("vcenter", "scheduled") as run:
|
||||||
|
stats = run_vcenter_sync(db, refresh_catalog=True)
|
||||||
|
run.stats.update(stats)
|
||||||
logger.info("vCenter nightly: %s", {k: v for k, v in stats.items() if k != "errors"})
|
logger.info("vCenter nightly: %s", {k: v for k, v in stats.items() if k != "errors"})
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error("vCenter nightly failed: %s", e)
|
logger.error("vCenter nightly failed: %s", e)
|
||||||
@@ -640,7 +656,9 @@ def igel_sync_nightly():
|
|||||||
if not load_igel_config(db):
|
if not load_igel_config(db):
|
||||||
logger.info("IGEL sync skipped — igel_config not set")
|
logger.info("IGEL sync skipped — igel_config not set")
|
||||||
return
|
return
|
||||||
stats = run_igel_sync(db)
|
with record_sync_run("igel", "scheduled") as run:
|
||||||
|
stats = run_igel_sync(db)
|
||||||
|
run.stats.update(stats)
|
||||||
logger.info("IGEL nightly: %s", {k: v for k, v in stats.items() if k != "errors"})
|
logger.info("IGEL nightly: %s", {k: v for k, v in stats.items() if k != "errors"})
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error("IGEL nightly failed: %s", e)
|
logger.error("IGEL nightly failed: %s", e)
|
||||||
@@ -1403,6 +1421,8 @@ def start_scheduler():
|
|||||||
replace_existing=True,
|
replace_existing=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Rows left "running" by a restart mid-sync would show a spinner forever.
|
||||||
|
mark_interrupted_runs()
|
||||||
scheduler.start()
|
scheduler.start()
|
||||||
logger.info(
|
logger.info(
|
||||||
"Background scheduler started (SLA Breach Checker + Threat Intel Refresh + KEV Alert Hourly + Vulnrichment Nightly + Compliance SCA Nightly + URS Nightly + Audit-Log Prune)"
|
"Background scheduler started (SLA Breach Checker + Threat Intel Refresh + KEV Alert Hourly + Vulnrichment Nightly + Compliance SCA Nightly + URS Nightly + Audit-Log Prune)"
|
||||||
|
|||||||
@@ -0,0 +1,429 @@
|
|||||||
|
"""Sync run ledger + failure alerting.
|
||||||
|
|
||||||
|
Every sync job — Wazuh, Nessus, Intune, vCenter, IGEL; button or scheduler —
|
||||||
|
runs inside `record_sync_run(source, trigger)`. That writes ONE `sync_runs`
|
||||||
|
row: started, finished, status, the stats dict, and the error that stopped it.
|
||||||
|
|
||||||
|
The ledger uses its OWN session. The job's session may be mid-transaction or
|
||||||
|
already rolled back when the job dies, and the record of the failure must not
|
||||||
|
depend on it.
|
||||||
|
|
||||||
|
A failed run sends one mail per source per ALERT_COOLDOWN (24h) — an hourly
|
||||||
|
schedule against a dead API must not send 24 mails — and states the last
|
||||||
|
successful sync, i.e. how long the coverage gap already is.
|
||||||
|
"""
|
||||||
|
import contextlib
|
||||||
|
import html
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import threading
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from typing import Dict, Iterable, List, Optional
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.database import SessionLocal
|
||||||
|
from app.models.sync_run import SyncRun
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
SOURCES = {
|
||||||
|
"wazuh": "Wazuh",
|
||||||
|
"nessus": "Tenable Nessus",
|
||||||
|
"intune": "Microsoft Intune",
|
||||||
|
"vcenter": "VMware vCenter",
|
||||||
|
"igel": "IGEL UMS",
|
||||||
|
}
|
||||||
|
# A source counts as configured when its config setting exists and is non-empty.
|
||||||
|
CONFIG_KEYS = {
|
||||||
|
"wazuh": "wazuh_config",
|
||||||
|
"nessus": "nessus_config",
|
||||||
|
"intune": "intune_config",
|
||||||
|
"vcenter": "vcenter_config",
|
||||||
|
"igel": "igel_config",
|
||||||
|
}
|
||||||
|
# Inventory syncs (Intune/vCenter/IGEL) are nightly: 36h means "missed one
|
||||||
|
# night" without flagging a slow one. Wazuh and Nessus follow their own
|
||||||
|
# ScanSchedule — see stale_thresholds(): twice the interval, at least 36h,
|
||||||
|
# and never stale without an enabled schedule (manual-only sources).
|
||||||
|
STALE_AFTER = timedelta(hours=36)
|
||||||
|
CRON_STALE_AFTER = timedelta(days=8) # a cron expression we do not parse
|
||||||
|
ALERT_COOLDOWN = timedelta(hours=24)
|
||||||
|
SETTING_ENABLED = "sync_alert_enabled"
|
||||||
|
SETTING_RECIPIENTS = "sync_alert_recipients"
|
||||||
|
MAX_ERRORS_KEPT = 50
|
||||||
|
|
||||||
|
_local = threading.local()
|
||||||
|
_alert_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
class SyncRunHandle:
|
||||||
|
"""What the job sees: a stats dict to fill and fail() for a run that
|
||||||
|
ended without an exception but did nothing useful."""
|
||||||
|
|
||||||
|
def __init__(self, run_id: Optional[int], source: str):
|
||||||
|
self.run_id = run_id
|
||||||
|
self.source = source
|
||||||
|
self.stats: dict = {}
|
||||||
|
self.error: Optional[str] = None
|
||||||
|
|
||||||
|
def fail(self, message: str) -> None:
|
||||||
|
self.error = str(message)[:2000]
|
||||||
|
|
||||||
|
|
||||||
|
def fail_if_nothing_synced(handle: SyncRunHandle, triggered: int, errors: list) -> None:
|
||||||
|
"""Every agent failed the same way (authentication refused, API down):
|
||||||
|
that is a failed run, not a completed one with N errors. Used by the
|
||||||
|
scheduled Wazuh scan and the manual autoscan alike."""
|
||||||
|
if triggered == 0 and errors:
|
||||||
|
handle.fail(f"all {len(errors)} agents failed — {errors[0]}")
|
||||||
|
|
||||||
|
|
||||||
|
def _error_text(e: BaseException) -> str:
|
||||||
|
detail = getattr(e, "detail", None) # HTTPException carries the message there
|
||||||
|
text = str(detail) if detail else (str(e) or type(e).__name__)
|
||||||
|
return text[:2000]
|
||||||
|
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def record_sync_run(source: str, trigger: str = "manual"):
|
||||||
|
"""Record one sync execution. Nested use (the scheduled Wazuh scan calls
|
||||||
|
the same asset-sync function the button calls) yields the OUTER handle so
|
||||||
|
a run is one row, not two, and one mail, not two."""
|
||||||
|
outer = getattr(_local, "active", None)
|
||||||
|
if outer is not None and outer.source == source:
|
||||||
|
yield outer
|
||||||
|
return
|
||||||
|
|
||||||
|
db = SessionLocal()
|
||||||
|
run = SyncRun(source=source, trigger=trigger, status="running",
|
||||||
|
started_at=datetime.now())
|
||||||
|
try:
|
||||||
|
db.add(run)
|
||||||
|
db.commit()
|
||||||
|
except Exception as e: # ledger unavailable: the job still has to run
|
||||||
|
logger.error("sync_runs: could not open run for %s: %s", source, e)
|
||||||
|
db.close()
|
||||||
|
yield SyncRunHandle(None, source)
|
||||||
|
return
|
||||||
|
|
||||||
|
handle = SyncRunHandle(run.id, source)
|
||||||
|
_local.active = handle
|
||||||
|
try:
|
||||||
|
yield handle
|
||||||
|
except BaseException as e:
|
||||||
|
handle.error = handle.error or _error_text(e)
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
_local.active = outer
|
||||||
|
try:
|
||||||
|
_finish(db, run, handle)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("sync_runs: could not close run %s: %s", run.id, e)
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _finish(db: Session, run: SyncRun, handle: SyncRunHandle) -> None:
|
||||||
|
stats = dict(handle.stats)
|
||||||
|
errs = stats.get("errors")
|
||||||
|
if isinstance(errs, list):
|
||||||
|
stats["error_count"] = len(errs)
|
||||||
|
stats["errors"] = [str(x)[:500] for x in errs[:MAX_ERRORS_KEPT]]
|
||||||
|
run.finished_at = datetime.now()
|
||||||
|
run.status = "failed" if handle.error else "completed"
|
||||||
|
run.error_message = handle.error
|
||||||
|
try:
|
||||||
|
run.stats = json.dumps(stats, default=str)
|
||||||
|
except Exception:
|
||||||
|
run.stats = None
|
||||||
|
db.commit()
|
||||||
|
level = logger.error if handle.error else logger.info
|
||||||
|
level("sync run #%s %s (%s): %s%s", run.id, run.source, run.trigger, run.status,
|
||||||
|
f" — {handle.error}" if handle.error else "")
|
||||||
|
if handle.error:
|
||||||
|
try:
|
||||||
|
notify_sync_failure(db, run)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("sync failure mail for %s failed: %s", run.source, e)
|
||||||
|
|
||||||
|
|
||||||
|
def mark_interrupted_runs() -> int:
|
||||||
|
"""Close rows left 'running' by a backend restart mid-sync, else the
|
||||||
|
health panel would show a spinner forever. Called from start_scheduler."""
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
rows = db.query(SyncRun).filter(SyncRun.status == "running").all()
|
||||||
|
for r in rows:
|
||||||
|
r.status = "failed"
|
||||||
|
r.finished_at = datetime.now()
|
||||||
|
r.error_message = "interrupted: backend restarted while the sync was running"
|
||||||
|
if rows:
|
||||||
|
db.commit()
|
||||||
|
logger.warning("sync_runs: closed %d interrupted run(s)", len(rows))
|
||||||
|
return len(rows)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("sync_runs: interrupted-run sweep failed: %s", e)
|
||||||
|
return 0
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
# ---- health ---------------------------------------------------------------
|
||||||
|
|
||||||
|
def _setting(db: Session, key: str) -> Optional[str]:
|
||||||
|
from app.models.setting import Setting
|
||||||
|
try:
|
||||||
|
row = db.query(Setting).filter(Setting.key == key).first()
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
return row.value if row and row.value is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
def is_alert_enabled(db: Session) -> bool:
|
||||||
|
raw = (_setting(db, SETTING_ENABLED) or "").strip().strip('"').lower()
|
||||||
|
return raw not in ("false", "0", "no", "off")
|
||||||
|
|
||||||
|
|
||||||
|
def get_recipients(db: Session) -> List[tuple]:
|
||||||
|
"""(user_id_or_None, email, display_name). Dedicated setting wins, else
|
||||||
|
the shared notification default (configured list, else all admins)."""
|
||||||
|
from app.models.user import User
|
||||||
|
from app.services.email_service import get_default_recipients
|
||||||
|
|
||||||
|
raw = _setting(db, SETTING_RECIPIENTS)
|
||||||
|
if raw and raw.strip():
|
||||||
|
out, seen = [], set()
|
||||||
|
for em in (e.strip() for e in re.split(r"[,;\s]+", raw) if e.strip()):
|
||||||
|
if em in seen:
|
||||||
|
continue
|
||||||
|
seen.add(em)
|
||||||
|
u = db.query(User).filter(User.email == em).first()
|
||||||
|
out.append(((u.id if u else None), em, (u.username if u else em)))
|
||||||
|
if out:
|
||||||
|
return out
|
||||||
|
return get_default_recipients(db)
|
||||||
|
|
||||||
|
|
||||||
|
def configured_sources(db: Session) -> set:
|
||||||
|
return {s for s, key in CONFIG_KEYS.items() if (_setting(db, key) or "").strip()}
|
||||||
|
|
||||||
|
|
||||||
|
def _serialize(run: Optional[SyncRun]) -> Optional[dict]:
|
||||||
|
if run is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
stats = json.loads(run.stats) if run.stats else {}
|
||||||
|
except Exception:
|
||||||
|
stats = {}
|
||||||
|
return {
|
||||||
|
"id": run.id,
|
||||||
|
"source": run.source,
|
||||||
|
"source_label": SOURCES.get(run.source, run.source),
|
||||||
|
"trigger": run.trigger,
|
||||||
|
"status": run.status,
|
||||||
|
"started_at": run.started_at,
|
||||||
|
"finished_at": run.finished_at,
|
||||||
|
"error_message": run.error_message,
|
||||||
|
"error_count": stats.get("error_count", 0) or 0,
|
||||||
|
"stats": stats,
|
||||||
|
"alerted_at": run.alerted_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def stale_thresholds(db: Session) -> Dict[str, Optional[timedelta]]:
|
||||||
|
"""How long without a success before a source counts as stale. None =
|
||||||
|
never (no enabled schedule drives it, so silence is not a failure)."""
|
||||||
|
from app.models.scan_schedule import ScanSchedule, ScheduleInterval
|
||||||
|
interval_of = {
|
||||||
|
ScheduleInterval.EVERY_HOUR: timedelta(hours=1),
|
||||||
|
ScheduleInterval.EVERY_6_HOURS: timedelta(hours=6),
|
||||||
|
ScheduleInterval.EVERY_12_HOURS: timedelta(hours=12),
|
||||||
|
ScheduleInterval.DAILY: timedelta(days=1),
|
||||||
|
ScheduleInterval.WEEKLY: timedelta(weeks=1),
|
||||||
|
}
|
||||||
|
out: Dict[str, Optional[timedelta]] = {s: STALE_AFTER for s in ("intune", "vcenter", "igel")}
|
||||||
|
out.update({"wazuh": None, "nessus": None})
|
||||||
|
try:
|
||||||
|
schedules = db.query(ScanSchedule).filter(ScanSchedule.enabled.is_(True)).all()
|
||||||
|
except Exception:
|
||||||
|
schedules = []
|
||||||
|
for sch in schedules:
|
||||||
|
source = (getattr(sch, "scanner_type", None) or "wazuh").lower()
|
||||||
|
if source not in ("wazuh", "nessus"):
|
||||||
|
continue
|
||||||
|
every = interval_of.get(sch.interval)
|
||||||
|
limit = max(STALE_AFTER, 2 * every) if every else CRON_STALE_AFTER
|
||||||
|
cur = out.get(source)
|
||||||
|
out[source] = limit if cur is None else min(cur, limit)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def summarize_health(latest: Dict[str, SyncRun], last_success: Dict[str, SyncRun],
|
||||||
|
configured: Iterable[str],
|
||||||
|
thresholds: Optional[Dict[str, Optional[timedelta]]] = None,
|
||||||
|
now: Optional[datetime] = None) -> List[dict]:
|
||||||
|
"""Pure: one entry per known source with a state the GUI can colour.
|
||||||
|
|
||||||
|
failed — the most recent run of this source failed
|
||||||
|
running — a run is in progress
|
||||||
|
stale — configured, but no successful run within its threshold
|
||||||
|
ok — last run completed and is recent enough
|
||||||
|
unconfigured — no config stored for this source
|
||||||
|
"""
|
||||||
|
now = now or datetime.now()
|
||||||
|
configured = set(configured)
|
||||||
|
thresholds = thresholds if thresholds is not None else {s: STALE_AFTER for s in SOURCES}
|
||||||
|
out = []
|
||||||
|
for source, label in SOURCES.items():
|
||||||
|
last = latest.get(source)
|
||||||
|
good = last_success.get(source)
|
||||||
|
good_at = good.finished_at if good else None
|
||||||
|
if source not in configured and last is None:
|
||||||
|
state = "unconfigured"
|
||||||
|
elif last is not None and last.status == "running":
|
||||||
|
state = "running"
|
||||||
|
elif last is not None and last.status == "failed":
|
||||||
|
state = "failed"
|
||||||
|
elif (limit := thresholds.get(source)) is not None and (good_at is None or now - good_at > limit):
|
||||||
|
state = "stale"
|
||||||
|
else:
|
||||||
|
state = "ok"
|
||||||
|
out.append({
|
||||||
|
"source": source,
|
||||||
|
"label": label,
|
||||||
|
"state": state,
|
||||||
|
"configured": source in configured,
|
||||||
|
"last_run": _serialize(last),
|
||||||
|
"last_success_at": good_at,
|
||||||
|
"error_message": last.error_message if last is not None else None,
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def sync_health(db: Session) -> List[dict]:
|
||||||
|
latest: Dict[str, SyncRun] = {}
|
||||||
|
last_success: Dict[str, SyncRun] = {}
|
||||||
|
for source in SOURCES:
|
||||||
|
latest[source] = (db.query(SyncRun).filter(SyncRun.source == source)
|
||||||
|
.order_by(SyncRun.started_at.desc()).first())
|
||||||
|
last_success[source] = (db.query(SyncRun)
|
||||||
|
.filter(SyncRun.source == source, SyncRun.status == "completed")
|
||||||
|
.order_by(SyncRun.finished_at.desc()).first())
|
||||||
|
return summarize_health(latest, last_success, configured_sources(db), stale_thresholds(db))
|
||||||
|
|
||||||
|
|
||||||
|
def health_payload(db: Session) -> dict:
|
||||||
|
"""What GET /scans/sync-health returns: per-source health plus the
|
||||||
|
failure-mail settings so the panel can edit them in place."""
|
||||||
|
return {
|
||||||
|
"sources": sync_health(db),
|
||||||
|
"alert_enabled": is_alert_enabled(db),
|
||||||
|
"recipients": [e for _, e, _ in get_recipients(db)],
|
||||||
|
"recipients_setting": _setting(db, SETTING_RECIPIENTS) or "",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def list_runs(db: Session, limit: int = 30) -> List[dict]:
|
||||||
|
rows = db.query(SyncRun).order_by(SyncRun.started_at.desc()).limit(limit).all()
|
||||||
|
return [_serialize(r) for r in rows]
|
||||||
|
|
||||||
|
|
||||||
|
# ---- failure mail ---------------------------------------------------------
|
||||||
|
|
||||||
|
def in_cooldown(last_alerted: Optional[SyncRun], now: Optional[datetime] = None) -> bool:
|
||||||
|
if last_alerted is None or last_alerted.alerted_at is None:
|
||||||
|
return False
|
||||||
|
return (now or datetime.now()) - last_alerted.alerted_at < ALERT_COOLDOWN
|
||||||
|
|
||||||
|
|
||||||
|
def _fmt(dt: Optional[datetime]) -> str:
|
||||||
|
return dt.strftime("%Y-%m-%d %H:%M") if dt else "never"
|
||||||
|
|
||||||
|
|
||||||
|
def notify_sync_failure(db: Session, run: SyncRun) -> dict:
|
||||||
|
"""Mail the failure to the sync-alert recipients, once per source per
|
||||||
|
cooldown window, and log it in notification_logs."""
|
||||||
|
from app.models.notification_log import (NotificationLog, NotificationStatus,
|
||||||
|
NotificationType)
|
||||||
|
from app.services.email_service import send_email
|
||||||
|
|
||||||
|
stats = {"sent": 0, "failed": 0, "skipped": None}
|
||||||
|
if not is_alert_enabled(db):
|
||||||
|
stats["skipped"] = "disabled"
|
||||||
|
return stats
|
||||||
|
with _alert_lock:
|
||||||
|
return _notify_locked(db, run, stats)
|
||||||
|
|
||||||
|
|
||||||
|
def _notify_locked(db: Session, run: SyncRun, stats: dict) -> dict:
|
||||||
|
from app.models.notification_log import (NotificationLog, NotificationStatus,
|
||||||
|
NotificationType)
|
||||||
|
from app.services.email_service import send_email
|
||||||
|
|
||||||
|
last_alerted = (db.query(SyncRun)
|
||||||
|
.filter(SyncRun.source == run.source, SyncRun.alerted_at.isnot(None))
|
||||||
|
.order_by(SyncRun.alerted_at.desc()).first())
|
||||||
|
if in_cooldown(last_alerted):
|
||||||
|
stats["skipped"] = "cooldown"
|
||||||
|
logger.info("sync failure mail for %s suppressed (cooldown until %s)",
|
||||||
|
run.source, last_alerted.alerted_at + ALERT_COOLDOWN)
|
||||||
|
return stats
|
||||||
|
|
||||||
|
recipients = get_recipients(db)
|
||||||
|
if not recipients:
|
||||||
|
stats["skipped"] = "no recipients"
|
||||||
|
return stats
|
||||||
|
|
||||||
|
good = (db.query(SyncRun)
|
||||||
|
.filter(SyncRun.source == run.source, SyncRun.status == "completed")
|
||||||
|
.order_by(SyncRun.finished_at.desc()).first())
|
||||||
|
label = SOURCES.get(run.source, run.source)
|
||||||
|
base = os.getenv("DASHBOARD_URL", "http://localhost:3000").rstrip("/")
|
||||||
|
subject = f"[TRUEVULN] Sync failed: {label} ({run.trigger}) — vulnerability coverage at risk"
|
||||||
|
gap = ("No successful sync of this source has ever been recorded."
|
||||||
|
if not (good and good.finished_at) else
|
||||||
|
f"Last successful sync: {_fmt(good.finished_at)} "
|
||||||
|
f"({(datetime.now() - good.finished_at).days} day(s) ago).")
|
||||||
|
body = f"""<!DOCTYPE html><html><body style="font-family:Arial,sans-serif;color:#111">
|
||||||
|
<h2 style="color:#b91c1c">Sync failed: {html.escape(label)}</h2>
|
||||||
|
<table cellpadding="6" style="border-collapse:collapse">
|
||||||
|
<tr><td><b>Source</b></td><td>{html.escape(label)}</td></tr>
|
||||||
|
<tr><td><b>Trigger</b></td><td>{html.escape(run.trigger)}</td></tr>
|
||||||
|
<tr><td><b>Started</b></td><td>{_fmt(run.started_at)}</td></tr>
|
||||||
|
<tr><td><b>Failed</b></td><td>{_fmt(run.finished_at)}</td></tr>
|
||||||
|
<tr><td><b>Error</b></td><td><code>{html.escape(run.error_message or "unknown")}</code></td></tr>
|
||||||
|
</table>
|
||||||
|
<p>{html.escape(gap)}</p>
|
||||||
|
<p><b>Until the next successful sync, new vulnerabilities and inventory changes
|
||||||
|
from this source are not detected.</b> Check the connection and credentials in
|
||||||
|
Settings, then run the sync again from Scan Jobs.</p>
|
||||||
|
<p><a href="{html.escape(base)}/scans">Open Scan Jobs</a></p>
|
||||||
|
<p style="color:#6b7280;font-size:12px">You receive at most one mail per source
|
||||||
|
per 24 hours while it keeps failing.</p>
|
||||||
|
</body></html>"""
|
||||||
|
|
||||||
|
for user_id, email, _name in recipients:
|
||||||
|
ok, err = send_email(db, email, subject, body)
|
||||||
|
db.add(NotificationLog(
|
||||||
|
user_id=user_id,
|
||||||
|
notification_type=NotificationType.SYNC_FAILURE,
|
||||||
|
sent_at=datetime.now(),
|
||||||
|
subject=subject[:500],
|
||||||
|
recipient_email=email,
|
||||||
|
status=NotificationStatus.SENT if ok else NotificationStatus.FAILED,
|
||||||
|
message_body=f"{label} sync ({run.trigger}) failed: {run.error_message}",
|
||||||
|
error_message=None if ok else err,
|
||||||
|
))
|
||||||
|
stats["sent" if ok else "failed"] += 1
|
||||||
|
|
||||||
|
# Only a mail somebody actually got starts the cooldown — a broken SMTP
|
||||||
|
# must not silence the alert for a day.
|
||||||
|
if stats["sent"]:
|
||||||
|
run.alerted_at = datetime.now()
|
||||||
|
db.commit()
|
||||||
|
logger.info("sync failure mail for %s: %d sent, %d failed",
|
||||||
|
run.source, stats["sent"], stats["failed"])
|
||||||
|
return stats
|
||||||
@@ -10,6 +10,7 @@ import {
|
|||||||
} from '@heroicons/react/24/outline';
|
} from '@heroicons/react/24/outline';
|
||||||
import Link from 'next/link';
|
import Link from 'next/link';
|
||||||
import api from '../lib/api';
|
import api from '../lib/api';
|
||||||
|
import { SyncHealthBanner } from '../components/shared/SyncHealth';
|
||||||
import { kevBadge } from '../lib/kevBadge';
|
import { kevBadge } from '../lib/kevBadge';
|
||||||
import { DashboardStats, Vulnerability, AIPriorityResponse } from '../types';
|
import { DashboardStats, Vulnerability, AIPriorityResponse } from '../types';
|
||||||
import AIRecommendations from '@/components/AIRecommendations';
|
import AIRecommendations from '@/components/AIRecommendations';
|
||||||
@@ -437,6 +438,7 @@ export default function Dashboard() {
|
|||||||
// Full available width (minus AppShell padding) — the empty left/right
|
// Full available width (minus AppShell padding) — the empty left/right
|
||||||
// gutters should be used on wide monitors. No max-width cap.
|
// gutters should be used on wide monitors. No max-width cap.
|
||||||
<div className="w-full">
|
<div className="w-full">
|
||||||
|
<SyncHealthBanner />
|
||||||
{/* Header Section */}
|
{/* Header Section */}
|
||||||
<div className="md:flex md:items-center md:justify-between mb-8">
|
<div className="md:flex md:items-center md:justify-between mb-8">
|
||||||
<div className="min-w-0 flex-1">
|
<div className="min-w-0 flex-1">
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { Fragment, useEffect, useState } from 'react';
|
|||||||
import api from '../../lib/api';
|
import api from '../../lib/api';
|
||||||
import { Asset, ScanRunSummary } from '../../types';
|
import { Asset, ScanRunSummary } from '../../types';
|
||||||
import { PlayIcon, ArrowPathIcon, ClockIcon, TrashIcon, ChevronDownIcon, ChevronRightIcon, PencilIcon } from '@heroicons/react/24/outline';
|
import { PlayIcon, ArrowPathIcon, ClockIcon, TrashIcon, ChevronDownIcon, ChevronRightIcon, PencilIcon } from '@heroicons/react/24/outline';
|
||||||
|
import { SyncHealthPanel } from '../../components/shared/SyncHealth';
|
||||||
|
|
||||||
interface ScanSchedule {
|
interface ScanSchedule {
|
||||||
id: number;
|
id: number;
|
||||||
@@ -36,6 +37,11 @@ export default function ScansPage() {
|
|||||||
const [newSchedule, setNewSchedule] = useState({ name: '', interval: 'daily', cron_expression: '', scanner_type: 'wazuh' });
|
const [newSchedule, setNewSchedule] = useState({ name: '', interval: 'daily', cron_expression: '', scanner_type: 'wazuh' });
|
||||||
const [editingScheduleId, setEditingScheduleId] = useState<number | null>(null);
|
const [editingScheduleId, setEditingScheduleId] = useState<number | null>(null);
|
||||||
const [toast, setToast] = useState<{ message: string; type: 'success' | 'error' | 'info' } | null>(null);
|
const [toast, setToast] = useState<{ message: string; type: 'success' | 'error' | 'info' } | null>(null);
|
||||||
|
// Bumped after every fetch/sync so the sync-health panel reloads with the rest.
|
||||||
|
const [healthKey, setHealthKey] = useState(0);
|
||||||
|
const [userRole, setUserRole] = useState('');
|
||||||
|
// The real reason a manual Wazuh sync failed — was "check the logs" before.
|
||||||
|
const [syncError, setSyncError] = useState<string | null>(null);
|
||||||
|
|
||||||
// Autoscan Modal State
|
// Autoscan Modal State
|
||||||
const [isAutoscanModalOpen, setIsAutoscanModalOpen] = useState(false);
|
const [isAutoscanModalOpen, setIsAutoscanModalOpen] = useState(false);
|
||||||
@@ -68,11 +74,13 @@ export default function ScansPage() {
|
|||||||
console.error("Failed to fetch data:", error);
|
console.error("Failed to fetch data:", error);
|
||||||
} finally {
|
} finally {
|
||||||
setLoading(false);
|
setLoading(false);
|
||||||
|
setHealthKey((k) => k + 1);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
fetchData();
|
fetchData();
|
||||||
|
api.get('/auth/me').then((r) => setUserRole(r.data?.role || '')).catch(() => {});
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const handleAutoscan = async () => {
|
const handleAutoscan = async () => {
|
||||||
@@ -121,6 +129,7 @@ export default function ScansPage() {
|
|||||||
setSyncStatus('running');
|
setSyncStatus('running');
|
||||||
setSyncStep('assets');
|
setSyncStep('assets');
|
||||||
setSyncResult(null);
|
setSyncResult(null);
|
||||||
|
setSyncError(null);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Sync assets first
|
// Sync assets first
|
||||||
@@ -139,8 +148,11 @@ export default function ScansPage() {
|
|||||||
fetchData();
|
fetchData();
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
console.error("Sync failed:", error);
|
console.error("Sync failed:", error);
|
||||||
setToast({ message: error.response?.data?.detail || "Failed to sync with Wazuh.", type: 'error' });
|
const detail = error.response?.data?.detail || error.message || "Failed to sync with Wazuh.";
|
||||||
|
setToast({ message: detail, type: 'error' });
|
||||||
|
setSyncError(detail);
|
||||||
setSyncStatus('error');
|
setSyncStatus('error');
|
||||||
|
fetchData(); // the failed run is in the ledger now — show it
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -301,6 +313,8 @@ export default function ScansPage() {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<SyncHealthPanel refreshKey={healthKey} canEdit={userRole === 'admin'} />
|
||||||
|
|
||||||
{/* Scan Runs Summary Table */}
|
{/* Scan Runs Summary Table */}
|
||||||
<div className="bg-white border border-gray-200 shadow-sm rounded-sm overflow-hidden">
|
<div className="bg-white border border-gray-200 shadow-sm rounded-sm overflow-hidden">
|
||||||
{scanRuns.length > 0 && (
|
{scanRuns.length > 0 && (
|
||||||
@@ -734,7 +748,9 @@ export default function ScansPage() {
|
|||||||
</svg>
|
</svg>
|
||||||
</div>
|
</div>
|
||||||
<h4 className="text-base font-semibold leading-6 text-gray-900 font-mono">Sync Failed</h4>
|
<h4 className="text-base font-semibold leading-6 text-gray-900 font-mono">Sync Failed</h4>
|
||||||
<p className="text-sm text-gray-500 mt-2 font-mono">Check the logs or try again later.</p>
|
<p className="text-sm text-gray-500 mt-2 font-mono">Step {syncStep === 'assets' ? '1 (Importing Assets)' : '2 (Syncing Vulnerabilities)'} failed:</p>
|
||||||
|
<p className="text-sm text-red-700 mt-2 font-mono text-left bg-red-50 p-3 rounded-md break-words">{syncError || 'Unknown error'}</p>
|
||||||
|
<p className="text-xs text-gray-500 mt-2 font-mono">Recorded under Sync Runs. Check the Wazuh connection in Settings and try again.</p>
|
||||||
<div className="mt-6">
|
<div className="mt-6">
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
|
|||||||
@@ -0,0 +1,229 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import Link from 'next/link';
|
||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
import api from '../../lib/api';
|
||||||
|
import { SyncHealthResponse, SyncHealthSource, SyncRun } from '../../types';
|
||||||
|
import { ExclamationTriangleIcon } from '@heroicons/react/24/outline';
|
||||||
|
|
||||||
|
const STATE_STYLE: Record<SyncHealthSource['state'], { badge: string; label: string }> = {
|
||||||
|
ok: { badge: 'bg-green-100 text-green-800', label: 'OK' },
|
||||||
|
failed: { badge: 'bg-red-100 text-red-800', label: 'FAILED' },
|
||||||
|
stale: { badge: 'bg-amber-100 text-amber-800', label: 'STALE' },
|
||||||
|
running: { badge: 'bg-blue-100 text-blue-800', label: 'RUNNING' },
|
||||||
|
unconfigured: { badge: 'bg-gray-100 text-gray-500', label: 'NOT CONFIGURED' },
|
||||||
|
};
|
||||||
|
|
||||||
|
const fmt = (iso: string | null | undefined) => (iso ? new Date(iso).toLocaleString() : '—');
|
||||||
|
|
||||||
|
const ago = (iso: string | null | undefined) => {
|
||||||
|
if (!iso) return 'never';
|
||||||
|
const h = Math.round((Date.now() - new Date(iso).getTime()) / 36e5);
|
||||||
|
if (h < 1) return 'less than an hour ago';
|
||||||
|
if (h < 48) return `${h}h ago`;
|
||||||
|
return `${Math.round(h / 24)}d ago`;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Sources needing attention: the most recent run failed, or nothing has
|
||||||
|
// succeeded within the stale window. Shared by the banner and the panel.
|
||||||
|
const problems = (h: SyncHealthResponse | null) =>
|
||||||
|
(h?.sources || []).filter((s) => s.state === 'failed' || s.state === 'stale');
|
||||||
|
|
||||||
|
/** Dashboard strip: only rendered when a sync source is failed or stale. */
|
||||||
|
export function SyncHealthBanner() {
|
||||||
|
const [health, setHealth] = useState<SyncHealthResponse | null>(null);
|
||||||
|
useEffect(() => {
|
||||||
|
api.get('/api/v1/scans/sync-health').then((r) => setHealth(r.data)).catch(() => {});
|
||||||
|
}, []);
|
||||||
|
const bad = problems(health);
|
||||||
|
if (bad.length === 0) return null;
|
||||||
|
return (
|
||||||
|
<div className="mb-6 rounded-md border border-red-300 bg-red-50 p-4">
|
||||||
|
<div className="flex items-start gap-3">
|
||||||
|
<ExclamationTriangleIcon className="h-6 w-6 text-red-600 flex-shrink-0" />
|
||||||
|
<div className="min-w-0 flex-1">
|
||||||
|
<p className="text-sm font-bold text-red-800 font-mono">
|
||||||
|
Vulnerability coverage at risk: {bad.length} sync source{bad.length > 1 ? 's' : ''} not delivering data
|
||||||
|
</p>
|
||||||
|
<ul className="mt-1 text-sm text-red-700 font-mono space-y-0.5">
|
||||||
|
{bad.map((s) => (
|
||||||
|
<li key={s.source} className="truncate">
|
||||||
|
<b>{s.label}</b>: {s.state === 'failed'
|
||||||
|
? `last run failed ${ago(s.last_run?.finished_at)} — ${s.error_message || 'unknown error'}`
|
||||||
|
: `no successful sync since ${s.last_success_at ? fmt(s.last_success_at) : 'ever'}`}
|
||||||
|
{' · '}last success {ago(s.last_success_at)}
|
||||||
|
</li>
|
||||||
|
))}
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<Link href="/scans" className="text-sm font-semibold text-red-800 underline font-mono whitespace-nowrap">
|
||||||
|
Scan Jobs →
|
||||||
|
</Link>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Scan Jobs panel: per-source health, the run ledger, and the failure-mail settings. */
|
||||||
|
export function SyncHealthPanel({ refreshKey, canEdit }: { refreshKey: number; canEdit: boolean }) {
|
||||||
|
const [health, setHealth] = useState<SyncHealthResponse | null>(null);
|
||||||
|
const [runs, setRuns] = useState<SyncRun[]>([]);
|
||||||
|
const [expanded, setExpanded] = useState<number | null>(null);
|
||||||
|
const [cfg, setCfg] = useState({ enabled: true, recipients: '' });
|
||||||
|
// An unsaved edit must survive the reloads other actions on the page trigger.
|
||||||
|
const [cfgDirty, setCfgDirty] = useState(false);
|
||||||
|
const [saving, setSaving] = useState(false);
|
||||||
|
const [saveMsg, setSaveMsg] = useState<string | null>(null);
|
||||||
|
|
||||||
|
const load = async () => {
|
||||||
|
try {
|
||||||
|
const [h, r] = await Promise.all([
|
||||||
|
api.get('/api/v1/scans/sync-health'),
|
||||||
|
api.get('/api/v1/scans/sync-runs', { params: { limit: 30 } }),
|
||||||
|
]);
|
||||||
|
setHealth(h.data);
|
||||||
|
setRuns(r.data || []);
|
||||||
|
if (!cfgDirty) setCfg({ enabled: h.data?.alert_enabled !== false, recipients: h.data?.recipients_setting || '' });
|
||||||
|
} catch (e) {
|
||||||
|
console.error('sync health load failed', e);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
useEffect(() => { load(); }, [refreshKey]); // eslint-disable-line react-hooks/exhaustive-deps
|
||||||
|
|
||||||
|
const saveCfg = async () => {
|
||||||
|
setSaving(true);
|
||||||
|
setSaveMsg(null);
|
||||||
|
try {
|
||||||
|
await api.put('/api/v1/settings/sync_alert_enabled', { value: cfg.enabled ? 'true' : 'false' });
|
||||||
|
await api.put('/api/v1/settings/sync_alert_recipients', { value: cfg.recipients.trim() });
|
||||||
|
setSaveMsg('Saved.');
|
||||||
|
setCfgDirty(false);
|
||||||
|
load();
|
||||||
|
} catch (e) {
|
||||||
|
const detail = (e as { response?: { data?: { detail?: string } } })?.response?.data?.detail;
|
||||||
|
setSaveMsg(detail || 'Save failed (admin only).');
|
||||||
|
} finally {
|
||||||
|
setSaving(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const bad = problems(health);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="mb-6 space-y-4">
|
||||||
|
{/* Per-source health cards */}
|
||||||
|
<div className="bg-white border border-gray-200 shadow-sm rounded-sm">
|
||||||
|
<div className="px-6 py-3 bg-gray-50 border-b border-gray-200 flex items-center justify-between">
|
||||||
|
<span className="text-sm font-bold text-gray-900 font-mono">Sync Health</span>
|
||||||
|
{bad.length > 0 && (
|
||||||
|
<span className="text-xs font-mono text-red-700">{bad.length} source{bad.length > 1 ? 's' : ''} need attention</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
<div className="grid grid-cols-1 md:grid-cols-5 divide-y md:divide-y-0 md:divide-x divide-gray-200">
|
||||||
|
{(health?.sources || []).map((s) => {
|
||||||
|
const st = STATE_STYLE[s.state] || STATE_STYLE.unconfigured;
|
||||||
|
return (
|
||||||
|
<div key={s.source} className="p-4 font-mono text-xs space-y-1">
|
||||||
|
<div className="flex items-center justify-between">
|
||||||
|
<span className="font-bold text-sm text-gray-900">{s.label}</span>
|
||||||
|
<span className={`px-2 py-0.5 rounded-full text-[10px] font-bold ${st.badge}`}>{st.label}</span>
|
||||||
|
</div>
|
||||||
|
{s.state !== 'unconfigured' && (
|
||||||
|
<>
|
||||||
|
<div className="text-gray-500">Last run: {fmt(s.last_run?.started_at)}{s.last_run ? ` (${s.last_run.trigger})` : ''}</div>
|
||||||
|
<div className="text-gray-500">Last success: {s.last_success_at ? `${fmt(s.last_success_at)} · ${ago(s.last_success_at)}` : 'never'}</div>
|
||||||
|
{s.error_message && (
|
||||||
|
<div className="text-red-700 break-words" title={s.error_message}>{s.error_message}</div>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Run ledger */}
|
||||||
|
<div className="bg-white border border-gray-200 shadow-sm rounded-sm overflow-hidden">
|
||||||
|
<div className="px-6 py-3 bg-gray-50 border-b border-gray-200">
|
||||||
|
<span className="text-sm font-bold text-gray-900 font-mono">Sync Runs</span>
|
||||||
|
<span className="ml-2 text-xs text-gray-500 font-mono">every source, manual and scheduled — including runs that never reached an asset</span>
|
||||||
|
</div>
|
||||||
|
{runs.length === 0 ? (
|
||||||
|
<p className="p-6 text-sm text-gray-500 font-mono">No sync runs recorded yet.</p>
|
||||||
|
) : (
|
||||||
|
<div className="overflow-x-auto">
|
||||||
|
<table className="min-w-full divide-y divide-gray-200">
|
||||||
|
<thead className="bg-gray-50">
|
||||||
|
<tr>
|
||||||
|
{['Started', 'Source', 'Trigger', 'Status', 'Duration', 'Result'].map((h) => (
|
||||||
|
<th key={h} className="px-6 py-3 text-left text-xs font-mono font-medium text-gray-500 uppercase tracking-wider">{h}</th>
|
||||||
|
))}
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody className="bg-white divide-y divide-gray-200 font-mono text-sm">
|
||||||
|
{runs.map((r) => {
|
||||||
|
const dur = r.finished_at
|
||||||
|
? `${Math.max(0, Math.round((new Date(r.finished_at).getTime() - new Date(r.started_at).getTime()) / 1000))}s`
|
||||||
|
: '…';
|
||||||
|
const badge = STATE_STYLE[r.status === 'failed' ? 'failed' : r.status === 'running' ? 'running' : r.error_count > 0 ? 'stale' : 'ok'].badge;
|
||||||
|
const label = r.status === 'completed' && r.error_count > 0 ? `COMPLETED · ${r.error_count} ERR` : r.status.toUpperCase();
|
||||||
|
const isOpen = expanded === r.id;
|
||||||
|
const errs: string[] = Array.isArray(r.stats?.errors) ? r.stats.errors : [];
|
||||||
|
const summary = Object.entries(r.stats || {})
|
||||||
|
.filter(([k, v]) => k !== 'errors' && k !== 'error_count' && k !== 'phase' && (typeof v === 'number' || typeof v === 'string'))
|
||||||
|
.map(([k, v]) => `${k}=${v}`).join(' ');
|
||||||
|
return (
|
||||||
|
<tr key={r.id} className={`${r.status === 'failed' ? 'bg-red-50/40' : ''} ${errs.length || r.error_message ? 'cursor-pointer hover:bg-gray-50' : ''}`}
|
||||||
|
onClick={() => (errs.length || r.error_message) && setExpanded(isOpen ? null : r.id)}>
|
||||||
|
<td className="px-6 py-3 whitespace-nowrap text-gray-900">{fmt(r.started_at)}</td>
|
||||||
|
<td className="px-6 py-3 whitespace-nowrap text-gray-900">{r.source_label}{r.stats?.phase ? <span className="text-gray-400"> · {r.stats.phase}</span> : null}</td>
|
||||||
|
<td className="px-6 py-3 whitespace-nowrap text-gray-500">{r.trigger}</td>
|
||||||
|
<td className="px-6 py-3 whitespace-nowrap"><span className={`px-2 py-0.5 rounded-full text-[10px] font-bold ${badge}`}>{label}</span></td>
|
||||||
|
<td className="px-6 py-3 whitespace-nowrap text-gray-500">{dur}</td>
|
||||||
|
<td className="px-6 py-3 text-xs max-w-xl">
|
||||||
|
{r.error_message ? <span className="text-red-700">{r.error_message}</span> : <span className="text-gray-500">{summary}</span>}
|
||||||
|
{isOpen && errs.length > 0 && (
|
||||||
|
<ul className="mt-2 space-y-0.5 text-red-700 border-l-2 border-red-200 pl-2">
|
||||||
|
{errs.map((e, i) => <li key={i}>{e}</li>)}
|
||||||
|
</ul>
|
||||||
|
)}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Failure-mail settings */}
|
||||||
|
<div className="bg-white border border-gray-200 shadow-sm rounded-sm p-4 font-mono text-sm">
|
||||||
|
<div className="flex flex-wrap items-center gap-4">
|
||||||
|
<label className="flex items-center gap-2">
|
||||||
|
<input type="checkbox" checked={cfg.enabled} disabled={!canEdit}
|
||||||
|
onChange={(e) => { setCfg({ ...cfg, enabled: e.target.checked }); setCfgDirty(true); }}
|
||||||
|
className="h-4 w-4 rounded border-gray-300 text-truevuln-blue" />
|
||||||
|
<span className="font-bold text-gray-900">E-mail on sync failure</span>
|
||||||
|
</label>
|
||||||
|
<input type="text" value={cfg.recipients} disabled={!canEdit}
|
||||||
|
onChange={(e) => { setCfg({ ...cfg, recipients: e.target.value }); setCfgDirty(true); }}
|
||||||
|
placeholder={`recipients (empty = ${(health?.recipients || []).join(', ') || 'notification defaults / admins'})`}
|
||||||
|
className="flex-1 min-w-[16rem] rounded border border-gray-300 px-2 py-1 text-xs" />
|
||||||
|
{canEdit && (
|
||||||
|
<button onClick={saveCfg} disabled={saving}
|
||||||
|
className="rounded-sm bg-truevuln-blue px-3 py-1.5 text-xs font-semibold text-white hover:bg-blue-600 disabled:opacity-50">
|
||||||
|
{saving ? 'Saving…' : 'Save'}
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
{saveMsg && <span className="text-xs text-gray-500">{saveMsg}</span>}
|
||||||
|
</div>
|
||||||
|
<p className="mt-2 text-xs text-gray-500">
|
||||||
|
One mail per source per 24h while it keeps failing, stating the error and the last successful sync. Needs SMTP in Settings.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -230,3 +230,35 @@ export interface AIPriorityResponse {
|
|||||||
global_strategy: string;
|
global_strategy: string;
|
||||||
error?: string;
|
error?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- Sync run ledger (GET /api/v1/scans/sync-runs, /sync-health) ---
|
||||||
|
export interface SyncRun {
|
||||||
|
id: number;
|
||||||
|
source: string;
|
||||||
|
source_label: string;
|
||||||
|
trigger: 'manual' | 'scheduled' | string;
|
||||||
|
status: 'running' | 'completed' | 'failed' | string;
|
||||||
|
started_at: string;
|
||||||
|
finished_at: string | null;
|
||||||
|
error_message: string | null;
|
||||||
|
error_count: number;
|
||||||
|
stats: Record<string, any>;
|
||||||
|
alerted_at: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SyncHealthSource {
|
||||||
|
source: string;
|
||||||
|
label: string;
|
||||||
|
state: 'ok' | 'failed' | 'stale' | 'running' | 'unconfigured';
|
||||||
|
configured: boolean;
|
||||||
|
last_run: SyncRun | null;
|
||||||
|
last_success_at: string | null;
|
||||||
|
error_message: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SyncHealthResponse {
|
||||||
|
sources: SyncHealthSource[];
|
||||||
|
alert_enabled: boolean;
|
||||||
|
recipients: string[];
|
||||||
|
recipients_setting: string;
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,349 @@
|
|||||||
|
"""Sync run ledger + failure alert — run: python -m pytest tests/test_sync_runs.py
|
||||||
|
|
||||||
|
A Wazuh API that answered every login with 500 for a day was visible only in
|
||||||
|
the container log: the scan-jobs page is per asset, and a sync that fails at
|
||||||
|
authentication never reaches an asset. So it showed a clean history of
|
||||||
|
COMPLETED runs while nothing had been synced, and nobody was told.
|
||||||
|
|
||||||
|
The ledger records every sync execution with its outcome, whatever the source
|
||||||
|
and whether a button or the scheduler started it; a failure sends ONE mail
|
||||||
|
per source per cooldown window, and a nested recorder (the scheduled Wazuh
|
||||||
|
scan calling the asset sync) merges into the outer run instead of writing a
|
||||||
|
second row.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
from app.models.sync_run import SyncRun # noqa: E402
|
||||||
|
from app.services import sync_run_service as svc # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
class _Q:
|
||||||
|
def __init__(self, rows):
|
||||||
|
self._rows = rows
|
||||||
|
|
||||||
|
def filter(self, *a, **k):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def order_by(self, *a, **k):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def limit(self, *a, **k):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def first(self):
|
||||||
|
return self._rows[0] if self._rows else None
|
||||||
|
|
||||||
|
def all(self):
|
||||||
|
return list(self._rows)
|
||||||
|
|
||||||
|
|
||||||
|
class _DB:
|
||||||
|
"""Enough Session for the ledger: add() assigns ids, query(SyncRun)
|
||||||
|
returns what was added, newest first."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.rows = []
|
||||||
|
self.commits = 0
|
||||||
|
self.closed = False
|
||||||
|
|
||||||
|
def add(self, obj):
|
||||||
|
if isinstance(obj, SyncRun) and obj.id is None:
|
||||||
|
obj.id = len(self.rows) + 1
|
||||||
|
self.rows.append(obj)
|
||||||
|
|
||||||
|
def commit(self):
|
||||||
|
self.commits += 1
|
||||||
|
|
||||||
|
def rollback(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.closed = True
|
||||||
|
|
||||||
|
def query(self, model):
|
||||||
|
return _Q([r for r in reversed(self.rows) if isinstance(r, model)])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def db(monkeypatch):
|
||||||
|
fake = _DB()
|
||||||
|
monkeypatch.setattr(svc, "SessionLocal", lambda: fake)
|
||||||
|
monkeypatch.setattr(svc, "notify_sync_failure", lambda db, run: None)
|
||||||
|
return fake
|
||||||
|
|
||||||
|
|
||||||
|
def _runs(db):
|
||||||
|
return [r for r in db.rows if isinstance(r, SyncRun)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_success_is_recorded_with_stats(db):
|
||||||
|
with svc.record_sync_run("wazuh", "manual") as run:
|
||||||
|
run.stats.update({"agents_synced": 3})
|
||||||
|
(row,) = _runs(db)
|
||||||
|
assert row.status == "completed"
|
||||||
|
assert row.finished_at is not None
|
||||||
|
assert row.error_message is None
|
||||||
|
assert '"agents_synced": 3' in row.stats
|
||||||
|
assert db.closed
|
||||||
|
|
||||||
|
|
||||||
|
def test_exception_marks_run_failed_and_reraises(db):
|
||||||
|
with pytest.raises(RuntimeError):
|
||||||
|
with svc.record_sync_run("wazuh", "scheduled"):
|
||||||
|
raise RuntimeError("Authentication failed: Server error '500'")
|
||||||
|
(row,) = _runs(db)
|
||||||
|
assert row.status == "failed"
|
||||||
|
assert "Authentication failed" in row.error_message
|
||||||
|
assert row.finished_at is not None
|
||||||
|
|
||||||
|
|
||||||
|
def test_http_exception_detail_is_the_error(db):
|
||||||
|
from fastapi import HTTPException
|
||||||
|
with pytest.raises(HTTPException):
|
||||||
|
with svc.record_sync_run("wazuh"):
|
||||||
|
raise HTTPException(status_code=500, detail="Wazuh sync failed: 500 daemons not ready")
|
||||||
|
assert _runs(db)[0].error_message == "Wazuh sync failed: 500 daemons not ready"
|
||||||
|
|
||||||
|
|
||||||
|
def test_explicit_fail_without_exception(db):
|
||||||
|
with svc.record_sync_run("wazuh", "scheduled") as run:
|
||||||
|
run.fail("all 14 agents failed: Authentication failed")
|
||||||
|
assert _runs(db)[0].status == "failed"
|
||||||
|
|
||||||
|
|
||||||
|
def test_failure_triggers_notification(db, monkeypatch):
|
||||||
|
seen = []
|
||||||
|
monkeypatch.setattr(svc, "notify_sync_failure", lambda db, run: seen.append(run.source))
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
with svc.record_sync_run("igel"):
|
||||||
|
raise ValueError("boom")
|
||||||
|
assert seen == ["igel"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_notification_crash_never_breaks_the_job(db, monkeypatch):
|
||||||
|
def _explode(db, run):
|
||||||
|
raise RuntimeError("smtp down")
|
||||||
|
monkeypatch.setattr(svc, "notify_sync_failure", _explode)
|
||||||
|
with pytest.raises(ValueError): # the job's own error, not smtp's
|
||||||
|
with svc.record_sync_run("igel"):
|
||||||
|
raise ValueError("boom")
|
||||||
|
assert _runs(db)[0].status == "failed"
|
||||||
|
|
||||||
|
|
||||||
|
def test_nested_recorder_merges_into_outer_run(db):
|
||||||
|
with svc.record_sync_run("wazuh", "scheduled") as outer:
|
||||||
|
with svc.record_sync_run("wazuh", "manual") as inner:
|
||||||
|
inner.stats["created"] = 2
|
||||||
|
assert inner is outer
|
||||||
|
rows = _runs(db)
|
||||||
|
assert len(rows) == 1
|
||||||
|
assert rows[0].trigger == "scheduled"
|
||||||
|
assert '"created": 2' in rows[0].stats
|
||||||
|
|
||||||
|
|
||||||
|
def test_nested_exception_does_not_finish_outer_early(db):
|
||||||
|
with svc.record_sync_run("wazuh", "scheduled") as outer:
|
||||||
|
try:
|
||||||
|
with svc.record_sync_run("wazuh"):
|
||||||
|
raise RuntimeError("asset sync failed")
|
||||||
|
except RuntimeError as e:
|
||||||
|
outer.stats.setdefault("errors", []).append(str(e))
|
||||||
|
outer.stats["agents_synced"] = 5
|
||||||
|
(row,) = _runs(db)
|
||||||
|
assert row.status == "completed"
|
||||||
|
assert '"agents_synced": 5' in row.stats
|
||||||
|
|
||||||
|
|
||||||
|
def test_stale_running_rows_are_closed_on_startup(db):
|
||||||
|
db.add(SyncRun(source="nessus", trigger="scheduled", status="running",
|
||||||
|
started_at=datetime.now() - timedelta(hours=3)))
|
||||||
|
svc.mark_interrupted_runs()
|
||||||
|
row = _runs(db)[0]
|
||||||
|
assert row.status == "failed"
|
||||||
|
assert "interrupted" in row.error_message
|
||||||
|
|
||||||
|
|
||||||
|
# ---- alert cooldown ------------------------------------------------------
|
||||||
|
|
||||||
|
def test_cooldown_suppresses_repeat_mail_for_same_source():
|
||||||
|
now = datetime.now()
|
||||||
|
recent = SyncRun(source="wazuh", status="failed", started_at=now,
|
||||||
|
alerted_at=now - timedelta(hours=2))
|
||||||
|
old = SyncRun(source="wazuh", status="failed", started_at=now,
|
||||||
|
alerted_at=now - timedelta(hours=30))
|
||||||
|
assert svc.in_cooldown(recent, now) is True
|
||||||
|
assert svc.in_cooldown(old, now) is False
|
||||||
|
assert svc.in_cooldown(None, now) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_health_flags_failed_and_stale_sources():
|
||||||
|
now = datetime.now()
|
||||||
|
ok = SyncRun(source="wazuh", status="completed", started_at=now,
|
||||||
|
finished_at=now - timedelta(hours=1))
|
||||||
|
failed = SyncRun(source="igel", status="failed", started_at=now,
|
||||||
|
finished_at=now, error_message="UMS login refused")
|
||||||
|
stale = SyncRun(source="nessus", status="completed", started_at=now,
|
||||||
|
finished_at=now - timedelta(days=3))
|
||||||
|
out = svc.summarize_health(
|
||||||
|
latest={"wazuh": ok, "igel": failed, "nessus": stale},
|
||||||
|
last_success={"wazuh": ok, "nessus": stale},
|
||||||
|
configured={"wazuh", "igel", "nessus"},
|
||||||
|
now=now,
|
||||||
|
)
|
||||||
|
by = {h["source"]: h for h in out}
|
||||||
|
assert by["wazuh"]["state"] == "ok"
|
||||||
|
assert by["igel"]["state"] == "failed"
|
||||||
|
assert by["igel"]["error_message"] == "UMS login refused"
|
||||||
|
assert by["igel"]["last_success_at"] is None
|
||||||
|
assert by["nessus"]["state"] == "stale"
|
||||||
|
assert by["intune"]["state"] == "unconfigured"
|
||||||
|
|
||||||
|
|
||||||
|
# ---- against real SQL (SQLite in memory): the queries, the mail, the log ---
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def sql(monkeypatch):
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
from sqlalchemy.pool import StaticPool
|
||||||
|
import app.models # noqa: F401 — registers every table on Base
|
||||||
|
from app.models.base import Base
|
||||||
|
|
||||||
|
engine = create_engine("sqlite://", connect_args={"check_same_thread": False},
|
||||||
|
poolclass=StaticPool)
|
||||||
|
Base.metadata.create_all(engine)
|
||||||
|
Local = sessionmaker(bind=engine)
|
||||||
|
monkeypatch.setattr(svc, "SessionLocal", Local)
|
||||||
|
yield Local
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_run_mails_once_and_logs_it(sql, monkeypatch):
|
||||||
|
from app.models.notification_log import NotificationLog, NotificationType
|
||||||
|
from app.models.setting import Setting
|
||||||
|
from app.services import email_service
|
||||||
|
|
||||||
|
sent = []
|
||||||
|
monkeypatch.setattr(email_service, "send_email",
|
||||||
|
lambda db, to, subject, body: (sent.append((to, subject, body)) or (True, "OK")))
|
||||||
|
s = sql()
|
||||||
|
s.add(Setting(key=svc.SETTING_RECIPIENTS, value="ops@example.com"))
|
||||||
|
s.add(Setting(key="wazuh_config", value='{"api_url": "https://w"}'))
|
||||||
|
s.commit()
|
||||||
|
s.close()
|
||||||
|
|
||||||
|
for _ in range(2): # hourly schedule, API still down
|
||||||
|
with pytest.raises(RuntimeError):
|
||||||
|
with svc.record_sync_run("wazuh", "scheduled"):
|
||||||
|
raise RuntimeError("Authentication failed: Server error '500'")
|
||||||
|
|
||||||
|
assert len(sent) == 1 # second failure is inside the cooldown
|
||||||
|
to, subject, body = sent[0]
|
||||||
|
assert to == "ops@example.com"
|
||||||
|
assert "Wazuh" in subject and "scheduled" in subject
|
||||||
|
assert "Authentication failed" in body
|
||||||
|
assert "No successful sync" in body # the coverage gap, spelled out
|
||||||
|
|
||||||
|
s = sql()
|
||||||
|
logs = s.query(NotificationLog).all()
|
||||||
|
assert len(logs) == 1
|
||||||
|
assert logs[0].notification_type == NotificationType.SYNC_FAILURE
|
||||||
|
assert logs[0].recipient_email == "ops@example.com"
|
||||||
|
|
||||||
|
runs = svc.list_runs(s, 10)
|
||||||
|
assert [r["status"] for r in runs] == ["failed", "failed"]
|
||||||
|
assert runs[1]["alerted_at"] is not None and runs[0]["alerted_at"] is None
|
||||||
|
|
||||||
|
health = {h["source"]: h for h in svc.sync_health(s)}
|
||||||
|
assert health["wazuh"]["state"] == "failed"
|
||||||
|
assert health["wazuh"]["configured"] is True
|
||||||
|
assert "Authentication failed" in health["wazuh"]["error_message"]
|
||||||
|
assert health["nessus"]["state"] == "unconfigured"
|
||||||
|
s.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_success_after_failure_shows_ok(sql, monkeypatch):
|
||||||
|
monkeypatch.setattr(svc, "notify_sync_failure", lambda db, run: None)
|
||||||
|
s = sql()
|
||||||
|
s.add(__import__("app.models.setting", fromlist=["Setting"]).Setting(
|
||||||
|
key="igel_config", value='{"host": "ums"}'))
|
||||||
|
s.commit()
|
||||||
|
with pytest.raises(RuntimeError):
|
||||||
|
with svc.record_sync_run("igel"):
|
||||||
|
raise RuntimeError("UMS down")
|
||||||
|
with svc.record_sync_run("igel", "scheduled") as run:
|
||||||
|
run.stats.update({"devices": 12, "errors": []})
|
||||||
|
health = {h["source"]: h for h in svc.sync_health(s)}
|
||||||
|
assert health["igel"]["state"] == "ok"
|
||||||
|
assert health["igel"]["last_success_at"] is not None
|
||||||
|
assert health["igel"]["last_run"]["stats"]["devices"] == 12
|
||||||
|
s.close()
|
||||||
|
|
||||||
|
|
||||||
|
# ---- review follow-ups -----------------------------------------------------
|
||||||
|
|
||||||
|
def test_nested_different_source_gets_its_own_row(db):
|
||||||
|
with svc.record_sync_run("wazuh", "scheduled") as outer:
|
||||||
|
with svc.record_sync_run("igel", "manual") as inner:
|
||||||
|
assert inner is not outer
|
||||||
|
with svc.record_sync_run("wazuh") as again: # outer restored after inner
|
||||||
|
assert again is outer
|
||||||
|
rows = _runs(db)
|
||||||
|
assert sorted(r.source for r in rows) == ["igel", "wazuh"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_stale_threshold_follows_the_schedule():
|
||||||
|
from app.models.scan_schedule import ScanSchedule, ScheduleInterval
|
||||||
|
|
||||||
|
class _SchedDB:
|
||||||
|
def __init__(self, rows):
|
||||||
|
self._rows = rows
|
||||||
|
|
||||||
|
def query(self, model):
|
||||||
|
return _Q(self._rows)
|
||||||
|
|
||||||
|
weekly = ScanSchedule(name="w", interval=ScheduleInterval.WEEKLY, scanner_type="nessus", enabled=True)
|
||||||
|
hourly = ScanSchedule(name="h", interval=ScheduleInterval.EVERY_HOUR, scanner_type="wazuh", enabled=True)
|
||||||
|
t = svc.stale_thresholds(_SchedDB([weekly, hourly]))
|
||||||
|
assert t["nessus"] == timedelta(weeks=2) # twice the interval
|
||||||
|
assert t["wazuh"] == svc.STALE_AFTER # never below the floor
|
||||||
|
assert t["igel"] == svc.STALE_AFTER # nightly job
|
||||||
|
assert svc.stale_thresholds(_SchedDB([]))["nessus"] is None # manual-only: never stale
|
||||||
|
|
||||||
|
now = datetime.now()
|
||||||
|
six_days = SyncRun(source="nessus", status="completed", started_at=now,
|
||||||
|
finished_at=now - timedelta(days=6))
|
||||||
|
by = {h["source"]: h for h in svc.summarize_health(
|
||||||
|
{"nessus": six_days}, {"nessus": six_days}, {"nessus"},
|
||||||
|
thresholds={"nessus": timedelta(weeks=2)}, now=now)}
|
||||||
|
assert by["nessus"]["state"] == "ok" # a weekly Nessus is not stale after 6 days
|
||||||
|
by = {h["source"]: h for h in svc.summarize_health(
|
||||||
|
{"nessus": six_days}, {"nessus": six_days}, {"nessus"},
|
||||||
|
thresholds={"nessus": None}, now=now)}
|
||||||
|
assert by["nessus"]["state"] == "ok"
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_agent_failed_is_a_failed_run(db):
|
||||||
|
"""Authentication refused → each agent raises, 0 triggered, N errors.
|
||||||
|
That is the failure the user could not see anywhere. One agent through
|
||||||
|
is a completed run with errors — the per-asset rows carry those."""
|
||||||
|
errs = ["srv01: Authentication failed: Server error '500'",
|
||||||
|
"srv02: Authentication failed: Server error '500'"]
|
||||||
|
with svc.record_sync_run("wazuh", "scheduled") as run:
|
||||||
|
svc.fail_if_nothing_synced(run, 0, errs)
|
||||||
|
with svc.record_sync_run("wazuh", "manual") as run:
|
||||||
|
svc.fail_if_nothing_synced(run, 1, errs)
|
||||||
|
with svc.record_sync_run("wazuh", "manual") as run:
|
||||||
|
svc.fail_if_nothing_synced(run, 0, []) # no assets at all: not a failure
|
||||||
|
dead, partial, empty = _runs(db)
|
||||||
|
assert dead.status == "failed"
|
||||||
|
assert "all 2 agents failed" in dead.error_message
|
||||||
|
assert "Authentication failed" in dead.error_message
|
||||||
|
assert partial.status == "completed"
|
||||||
|
assert empty.status == "completed"
|
||||||
Reference in New Issue
Block a user