vulncheck/app/services/msrc_scan_service.py

"""
MSRC-driven OS CVE detection — patch-level accurate, ahead of the Wazuh CTI feed.

Why this exists (and why NVD/cvelistV5 can't do it): Microsoft does not express
fixes as version ranges. NVD lists MS OS entries as rangeless CPEs
(`cpe:2.3:o:microsoft:windows_server_2016:-:*`, versionEndExcluding=null) and
cvelistV5 MS records use `lessThan: "publication"` — neither says which BUILD
carries the fix, so neither can tell a patched host from an unpatched one.
MSRC's CVRF does: each Type-2 remediation carries a `FixedBuild` plus the
`ProductID`s it applies to, e.g.

    CVE-2026-33834  FixedBuild 10.0.14393.9140  ProductID ['10816','10855']  KB5087537
    ProductTree:    10816 -> "Windows Server 2016"
                    10855 -> "Windows Server 2016 (Server Core installation)"

So: index (product → [cve, fixed_build]) from the monthly CVRF docs, then compare
an asset's installed OS build against the fixed build of its own servicing
branch. installed < fixed → affected. Same curated-and-precise contract as the
other scanners: only products we can map to an asset are indexed, nothing is
guessed, and a host that is patched is never flagged.

Scope: Windows Server only for now — its OS string names the product outright.
Client Windows (10/11) needs a build→release table before it can join.
"""
from __future__ import annotations

import json
import logging
import re
from datetime import datetime, timedelta
from typing import Dict, List, Optional

from sqlalchemy.orm import Session

logger = logging.getLogger(__name__)

SOURCE_NAME = "msrc"
_INDEX_SETTING = "msrc_product_index_v1"
_INDEX_TTL = timedelta(hours=26)  # rebuilt nightly; a missed night still serves

# Curated map: asset side (`match_re`) → MSRC ProductTree name (`msrc_re`).
#
# `kind`   os  = matched against asset.operating_system + asset.os_version
#          pkg = matched against an installed-software name + its version
# `branch` True  → a fix only speaks to hosts sharing its build prefix, because
#                  the 3rd segment IS the release (Windows Server 2016 = 14393;
#                  SharePoint 2019 = 10417 — verified stable across releases).
#          False → the 3rd segment moves with every CU, so prefix-matching would
#                  silently never hit; identity comes from the NAME instead and
#                  the compare is a plain installed < fixed. Verified: SharePoint
#                  2016 shows 5535/5539/5543/5552/5556 over six months.
#
# `msrc_re` is anchored so "Microsoft .NET Framework 4.8 on Windows Server 2016"
# (a *different* product that merely mentions the OS) can't match the OS family.
# Order matters — R2 must precede its base year.
#
# SharePoint 2013 is deliberately absent: it is EOL (2023-04-11) and Microsoft
# publishes no fixes for it, so there is no FixedBuild to compare — no amount of
# fix data can flag it. The EOL finding is the signal there (see eol_service).
_PRODUCTS: List[dict] = [
    {"key": "ws2012r2", "kind": "os", "branch": True, "match_re": r"windows server\s*2012\s*r2",
     "msrc_re": r"^windows server 2012 r2\b", "label": "Microsoft Windows Server 2012 R2"},
    {"key": "ws2012", "kind": "os", "branch": True, "match_re": r"windows server\s*2012(?!\s*r2)",
     "msrc_re": r"^windows server 2012\b(?!\s*r2)", "label": "Microsoft Windows Server 2012"},
    {"key": "ws2016", "kind": "os", "branch": True, "match_re": r"windows server\s*2016",
     "msrc_re": r"^windows server 2016\b", "label": "Microsoft Windows Server 2016"},
    {"key": "ws2019", "kind": "os", "branch": True, "match_re": r"windows server\s*2019",
     "msrc_re": r"^windows server 2019\b", "label": "Microsoft Windows Server 2019"},
    {"key": "ws2022", "kind": "os", "branch": True, "match_re": r"windows server\s*2022",
     "msrc_re": r"^windows server 2022\b", "label": "Microsoft Windows Server 2022"},
    {"key": "ws2025", "kind": "os", "branch": True, "match_re": r"windows server\s*2025",
     "msrc_re": r"^windows server 2025\b", "label": "Microsoft Windows Server 2025"},
    # SharePoint — 2016, 2019 and Subscription Edition ALL report 16.0.x, so the
    # year in the name is the only thing that tells the releases apart.
    {"key": "sp2016", "kind": "pkg", "branch": False, "match_re": r"sharepoint.*\b2016\b",
     "msrc_re": r"^microsoft sharepoint (enterprise )?server 2016\b",
     "label": "Microsoft SharePoint Server 2016"},
    {"key": "sp2019", "kind": "pkg", "branch": False, "match_re": r"sharepoint.*\b2019\b",
     "msrc_re": r"^microsoft sharepoint server 2019\b",
     "label": "Microsoft SharePoint Server 2019"},
    {"key": "spse", "kind": "pkg", "branch": False, "match_re": r"sharepoint.*subscription",
     "msrc_re": r"^microsoft sharepoint server subscription edition\b",
     "label": "Microsoft SharePoint Server Subscription Edition"},
]
_OS_COMPILED = [(re.compile(p["match_re"], re.I), p) for p in _PRODUCTS if p["kind"] == "os"]
_PKG_COMPILED = [(re.compile(p["match_re"], re.I), p) for p in _PRODUCTS if p["kind"] == "pkg"]
_MSRC_COMPILED = [(re.compile(p["msrc_re"], re.I), p) for p in _PRODUCTS]

_BUILD_RE = re.compile(r"^\d+(\.\d+)+$")


def resolve_os(os_name: str) -> Optional[dict]:
    """Asset OS string → curated product entry (None = not ours to scan)."""
    n = (os_name or "").strip().lower()
    if not n:
        return None
    for rx, p in _OS_COMPILED:
        if rx.search(n):
            return p
    return None


def _resolve_msrc_product(name: str) -> Optional[dict]:
    n = (name or "").strip().lower()
    for rx, p in _MSRC_COMPILED:
        if rx.search(n):
            return p
    return None


def _btuple(b: str) -> Optional[tuple]:
    if not b or not _BUILD_RE.match(b):
        return None
    try:
        return tuple(int(x) for x in b.split("."))
    except ValueError:
        return None


def _branch(b: str) -> Optional[tuple]:
    """Servicing branch = build minus its revision, e.g.
    10.0.14393.9140 → (10, 0, 14393). A fix only speaks to hosts on its own
    branch: Server 2016 (14393) says nothing about Server 2019 (17763)."""
    t = _btuple(b)
    return t[:3] if t and len(t) >= 3 else None


# ---------- index ----------

def build_product_index(db: Session, months_back: Optional[int] = None) -> dict:
    """Walk the recent monthly CVRF docs → {product_key: [{cve, build, kb}]}."""
    import httpx
    from app.services.msrc_service import (
        MSRC_BASE, HTTP_TIMEOUT, DEFAULT_MONTHS_BACK, SETTING_MONTHS_BACK, _setting_int,
    )

    months = months_back or _setting_int(db, SETTING_MONTHS_BACK, DEFAULT_MONTHS_BACK)
    index: Dict[str, List[dict]] = {}
    seen: set = set()
    docs_done = 0

    with httpx.Client(timeout=HTTP_TIMEOUT, headers={"Accept": "application/json"}) as client:
        r = client.get(f"{MSRC_BASE}/updates")
        r.raise_for_status()
        doc_ids = [v["ID"] for v in (r.json().get("value") or [])][-months:]

        for doc_id in doc_ids:
            try:
                resp = client.get(f"{MSRC_BASE}/cvrf/{doc_id}")
                resp.raise_for_status()
                doc = resp.json()
            except Exception as e:
                logger.warning("msrc-scan: doc %s failed: %s", doc_id, e)
                continue

            # ProductID → curated product key (only the ones we can map).
            pid_key: Dict[str, str] = {}
            for fp in (doc.get("ProductTree", {}) or {}).get("FullProductName", []) or []:
                p = _resolve_msrc_product(fp.get("Value") or "")
                if p and fp.get("ProductID"):
                    pid_key[str(fp["ProductID"])] = p["key"]
            if not pid_key:
                continue

            for v in doc.get("Vulnerability", []) or []:
                cve = (v.get("CVE") or "").strip().upper()
                if not cve.startswith("CVE-"):
                    continue
                for rem in v.get("Remediations", []) or []:
                    if rem.get("Type") != 2:
                        continue
                    build = (rem.get("FixedBuild") or "").strip()
                    if not _btuple(build):
                        continue  # no usable build → tells us nothing about patch state
                    desc = str((rem.get("Description") or {}).get("Value", "") or "").strip()
                    kb = desc if desc.isdigit() else None
                    for pid in rem.get("ProductID") or []:
                        key = pid_key.get(str(pid))
                        if not key:
                            continue
                        sig = (key, cve, build)
                        if sig in seen:
                            continue
                        seen.add(sig)
                        index.setdefault(key, []).append({"cve": cve, "build": build, "kb": kb})
            docs_done += 1

    _store_index(db, index)
    logger.info("msrc-scan: index built (%d docs) → %d products, %d fix entries",
                docs_done, len(index), sum(len(v) for v in index.values()))
    return index


def _store_index(db: Session, index: dict) -> None:
    from app.models.setting import Setting
    payload = json.dumps({"built_at": datetime.now().isoformat(), "index": index})
    row = db.query(Setting).filter(Setting.key == _INDEX_SETTING).first()
    if row:
        row.value = payload
    else:
        db.add(Setting(key=_INDEX_SETTING, value=payload,
                       description="MSRC product→(CVE, FixedBuild) index (curated)"))
    db.commit()


def load_index(db: Session, allow_stale: bool = True) -> Optional[dict]:
    from app.models.setting import Setting
    row = db.query(Setting).filter(Setting.key == _INDEX_SETTING).first()
    if not row or not row.value:
        return None
    try:
        blob = json.loads(row.value)
        built = datetime.fromisoformat(blob.get("built_at"))
    except Exception:
        return None
    if not allow_stale and datetime.now() - built > _INDEX_TTL:
        return None
    return blob.get("index") or {}


# ---------- scan ----------

def affected_cves(entries: List[dict], installed: str, branch_match: bool = True) -> List[dict]:
    """CVEs whose fix is newer than the installed build. Per CVE the NEWEST
    fixed build wins — that's the one that actually has to be on the box.

    branch_match: see _PRODUCTS. True → only fixes on the host's own build
    prefix count (the prefix is the release). False → the prefix moves with
    every CU, so identity already came from the product name and every fix for
    that product applies (MS servicing is cumulative, so installed < fixed is
    exactly the right test)."""
    inst_t = _btuple(installed)
    if not inst_t:
        return []
    inst_b = _branch(installed)
    if branch_match and not inst_b:
        return []
    newest: Dict[str, dict] = {}
    for e in entries:
        bt = _btuple(e.get("build") or "")
        if not bt:
            continue
        if branch_match and _branch(e["build"]) != inst_b:
            continue  # different servicing branch → says nothing about this host
        cur = newest.get(e["cve"])
        if cur is None or bt > _btuple(cur["build"]):
            newest[e["cve"]] = e
    return [e for e in newest.values() if inst_t < _btuple(e["build"])]


def resolve_package(name: str) -> Optional[dict]:
    """Installed-software name → curated MSRC product (None = not ours)."""
    n = (name or "").strip().lower()
    if not n:
        return None
    for rx, p in _PKG_COMPILED:
        if rx.search(n):
            return p
    return None


def scan_asset(db: Session, asset, index: dict, new_ids: list,
               touched: Optional[set] = None) -> int:
    """Flag MS OS CVEs whose FixedBuild is ahead of this host's build."""
    if not index:
        return 0
    prod = resolve_os(asset.operating_system or "")
    if not prod:
        return 0
    entries = index.get(prod["key"]) or []
    if not entries:
        return 0
    installed = (asset.os_version or "").strip()
    return _flag(db, asset, prod, installed, entries, new_ids, touched)


def scan_asset_packages(db: Session, asset, packages: list, index: dict,
                        new_ids: list, touched: Optional[set] = None) -> int:
    """Same fixed-build compare for installed MS software (SharePoint today).
    Called from the app-CVE scan, which already has the inventory in hand."""
    if not index:
        return 0
    count = 0
    seen: set = set()
    for pkg in packages or []:
        name = (pkg.get("name") or "").strip()
        version = (pkg.get("version") or "").strip()
        if not name or not version:
            continue
        prod = resolve_package(name)
        if not prod:
            continue
        # One product reports several components (Core / Lang Pack / SQL
        # Express) all carrying the same build — scan the product once.
        dedup = (prod["key"], version)
        if dedup in seen:
            continue
        seen.add(dedup)
        entries = index.get(prod["key"]) or []
        if entries:
            count += _flag(db, asset, prod, version, entries, new_ids, touched)
    return count


def _flag(db: Session, asset, prod: dict, installed: str, entries: List[dict],
          new_ids: list, touched: Optional[set]) -> int:
    hits = affected_cves(entries, installed, branch_match=prod.get("branch", True))
    count = 0
    for h in hits:
        if touched is not None:
            touched.add(h["cve"])
        try:
            if _upsert(db, asset, prod["label"], installed, h, new_ids):
                count += 1
        except Exception as e:
            logger.debug("msrc-scan upsert failed (%s on %s): %s", h["cve"], asset.id, e)
    return count


def _upsert(db: Session, asset, product: str, installed: str, hit: dict, new_ids: list) -> bool:
    from app.models.vulnerability import Vulnerability, VulnerabilityStatus
    cve_id = hit["cve"]
    fixed = hit["build"]
    kb = hit.get("kb")
    existing = (db.query(Vulnerability)
                  .filter(Vulnerability.cve_id == cve_id, Vulnerability.asset_id == asset.id)
                  .first())
    if existing:
        existing.add_source(SOURCE_NAME)
        if not existing.package_name:
            existing.package_name = product[:255]
        if not existing.package_version:
            existing.package_version = installed[:100]
        if not existing.fixed_version:
            existing.fixed_version = fixed
        if existing.status == VulnerabilityStatus.patched:
            existing.status = VulnerabilityStatus.open
            existing.patched_at = None
        try:
            existing.refresh_scores()
        except Exception:
            pass
        return False

    title = f"{product} — {cve_id}" + (f" (KB{kb})" if kb else "")
    row = Vulnerability(
        cve_id=cve_id, asset_id=asset.id,
        status=VulnerabilityStatus.open,
        title=title[:500],
        description=(f"MSRC reports {product} is fixed in build {fixed}"
                     + (f" via KB{kb}" if kb else "")
                     + f"; this host reports {installed}."),
        package_name=product[:255], package_version=installed[:100],
        fixed_version=fixed,
        detected_at=datetime.now(),
        sources=json.dumps([SOURCE_NAME]), first_detected_by=SOURCE_NAME,
    )
    db.add(row)
    db.flush()
    try:
        row.refresh_scores()
    except Exception:
        pass
    new_ids.append(row.id)
    return True


def _resolve_stale(db: Session, asset, touched: set, considered: set) -> int:
    """A msrc-only finding no longer reported means the host caught up past the
    FixedBuild → patched. Same contract as the app-scan reconcile: never touch
    findings another scanner also reports.

    `considered` = the product labels THIS pass actually evaluated. Without it
    the OS pass would close every SharePoint finding it never looked at (and
    vice versa), since neither pass touches the other's CVEs."""
    from app.models.vulnerability import Vulnerability, VulnerabilityStatus
    if not considered:
        return 0
    rows = (db.query(Vulnerability)
              .filter(Vulnerability.asset_id == asset.id,
                      Vulnerability.status == VulnerabilityStatus.open,
                      Vulnerability.sources.contains('"msrc"'),
                      Vulnerability.package_name.in_(list(considered)))
              .all())
    resolved = 0
    for v in rows:
        if v.cve_id in touched:
            continue
        # Drop OUR source; close only when nobody else still reports it (the
        # skip-if-cross-confirmed rule deadlocked across reconciles).
        v.remove_source(SOURCE_NAME)
        if v.source_list:
            continue
        old = v.status
        v.status = VulnerabilityStatus.patched
        v.patched_at = datetime.now()
        resolved += 1
        try:
            from app.routers.vulnerabilities import log_vulnerability_change
            log_vulnerability_change(
                db, None, v.id, old, v.status,
                reason=f"MSRC scan: {asset.hostname} is now at or past the fixed build",
                cve_id=v.cve_id, source="msrc_scan",
            )
        except Exception as e:
            logger.warning("audit log for msrc auto-resolve failed (vuln_id=%s): %s", v.id, e)
    return resolved


def resolve_stale_packages(db: Session, asset, touched: set) -> int:
    """Reconcile the package (pkg-kind) MSRC findings after a package scan.
    Considers ALL pkg product labels — we just saw the full inventory, so a
    product that vanished (uninstalled) should resolve too."""
    labels = {p["label"] for p in _PRODUCTS if p["kind"] == "pkg"}
    return _resolve_stale(db, asset, touched, labels)


def run_msrc_scan(db: Session, asset_id: Optional[int] = None) -> dict:
    """Scan Windows-Server assets against the MSRC fixed-build index."""
    from app.models.asset import Asset

    stats = {"assets": 0, "findings": 0, "new": 0, "resolved": 0, "errors": []}
    index = load_index(db)
    if not index:
        logger.info("msrc-scan: index missing — building now (one-time, then nightly)")
        try:
            index = build_product_index(db) or {}
        except Exception as e:
            stats["errors"].append(f"index build failed: {e}")
            return stats
    if not index:
        return stats

    new_ids: list = []
    q = db.query(Asset)
    if asset_id is not None:
        q = q.filter(Asset.id == asset_id)
    for asset in q.all():
        prod = resolve_os(asset.operating_system or "")
        if not prod:
            continue
        if not (asset.os_version or "").strip():
            continue  # no build → nothing to compare
        touched: set = set()
        try:
            stats["findings"] += scan_asset(db, asset, index, new_ids, touched=touched)
            # Only this asset's OS product — the package pass owns its own labels.
            stats["resolved"] += _resolve_stale(db, asset, touched, {prod["label"]})
            stats["assets"] += 1
            db.commit()
        except Exception as e:
            db.rollback()
            stats["errors"].append(f"asset {asset.id}: {e}")

    stats["new"] = len(new_ids)
    if new_ids:
        try:
            from app.services.audit_events import audit_new_vulnerabilities
            audit_new_vulnerabilities(db, new_ids, source=SOURCE_NAME)
            db.commit()
        except Exception as e:
            logger.debug("msrc-scan detected-audit failed: %s", e)
        try:
            from app.models.vulnerability import Vulnerability
            from app.services.enrichment_service import enrich_vulnerabilities
            from app.services.email_service import dispatch_new_vuln_notifications
            fresh = db.query(Vulnerability).filter(Vulnerability.id.in_(new_ids)).all()
            if fresh:
                enrich_vulnerabilities(db, fresh)
                stats["notifications"] = dispatch_new_vuln_notifications(db, fresh)
        except Exception as e:
            logger.debug("msrc-scan enrichment/notify failed: %s", e)

    logger.info("MSRC scan: %d assets, %d findings (%d new, %d auto-resolved)",
                stats["assets"], stats["findings"], stats["new"], stats["resolved"])
    return stats