vulncheck/app/services/msrc_service.py

"""
Microsoft Security Response Center (MSRC) CVRF enrichment.

Microsoft's per-CVE shortcut endpoint 404s; the stable source is the
monthly CVRF document (https://api.msrc.microsoft.com/cvrf/v3.0/cvrf/
{YYYY-Mon}, ~4-5 MB, ~1000 CVEs each). So we pull the last N monthly
documents in a background job, extract per-CVE remediations, and upsert
them into cve_remediations(source='msrc'). The CVE detail page then reads
them from the DB (no live 4 MB fetch per click).

Covers Windows OS **and** Microsoft products (Office/365, .NET, SQL,
Exchange, Visual Studio, ...), not just OS CVEs.

CVRF shapes we use, per Vulnerability:
  Remediations[]:
    Type 2 "Security Update" → KB in Description.Value (digits) + FixedBuild
                               + download URL                → kind=fix
    Type 3                   → support.microsoft.com/help/{KB} link (merged
                               into the matching fix row by KB)
  Notes[]:
    Title "Workarounds"      → kind=workaround (HTML → text)
    Title "Mitigations"      → kind=mitigation (HTML → text)
"""
import logging
import re
from datetime import datetime, timedelta
from typing import Dict, List, Optional

import httpx
from sqlalchemy.orm import Session

from app.models.cve_remediation import CveRemediation
from app.models.setting import Setting

logger = logging.getLogger(__name__)

MSRC_BASE = "https://api.msrc.microsoft.com/cvrf/v3.0"
HTTP_TIMEOUT = 90.0
# How many recent monthly documents to ingest per run. ~18 months covers
# the CVEs realistically present on managed estates. Override via setting
# `msrc_months_back`.
DEFAULT_MONTHS_BACK = 18
SETTING_MONTHS_BACK = "msrc_months_back"

MSRC_LAST_REFRESH_KEY = "msrc_last_refresh_at"

_TAG_RE = re.compile(r"<[^>]+>")
_WS_RE = re.compile(r"[ \t]*\n[ \t]*")


def _html_to_text(html: Optional[str]) -> Optional[str]:
    if not html:
        return None
    txt = _TAG_RE.sub(" ", html)
    txt = (txt.replace("&nbsp;", " ").replace("&amp;", "&")
              .replace("&lt;", "<").replace("&gt;", ">").replace("&quot;", '"'))
    txt = re.sub(r"[ \t]{2,}", " ", txt).strip()
    return txt or None


def _setting_int(db: Session, key: str, default: int) -> int:
    s = db.query(Setting).filter(Setting.key == key).first()
    if s and s.value and str(s.value).strip().isdigit():
        return int(s.value)
    return default


# ============================================================
# parse one CVRF Vulnerability into remediation rows
# ============================================================

def parse_vulnerability(v: dict) -> List[dict]:
    """Return remediation dicts for one CVRF Vulnerability entry."""
    out: List[dict] = []

    # Fixes — Type 2 carries the KB (digit Description) + FixedBuild + URL.
    # Type 3 is the support.microsoft.com link for the same KB; merge it in.
    fixes: Dict[str, dict] = {}
    support_links: Dict[str, str] = {}
    for r in v.get("Remediations", []) or []:
        rtype = r.get("Type")
        desc = str((r.get("Description") or {}).get("Value", "") or "").strip()
        url = (r.get("URL") or "").strip() or None
        build = (r.get("FixedBuild") or "").strip() or None
        sub = (r.get("SubType") or "").strip() or None
        if rtype == 2:
            kb = desc if desc.isdigit() else None
            key = kb or build or url or (sub or "fix")
            row = fixes.setdefault(key, {"kb": kb, "fixed_build": build, "url": url, "sub": sub})
            # keep first non-empty values
            row["kb"] = row.get("kb") or kb
            row["fixed_build"] = row.get("fixed_build") or build
            row["url"] = row.get("url") or url
            row["sub"] = row.get("sub") or sub
        elif rtype == 3:
            kb3 = (sub if (sub or "").isdigit() else None) or (desc if desc.isdigit() else None)
            if kb3 and url:
                support_links[kb3] = url

    for key, row in fixes.items():
        kb = row.get("kb")
        url = row.get("url") or (support_links.get(kb) if kb else None)
        bits = []
        if kb:
            bits.append(f"KB{kb}")
        if row.get("fixed_build"):
            bits.append(f"build {row['fixed_build']}")
        title = " · ".join(bits) or (row.get("sub") or "Security Update")
        out.append({
            "kind": "fix",
            "title": title[:300],
            "detail": None,
            "kb": kb,
            "fixed_build": row.get("fixed_build"),
            "url": url,
        })

    # Workarounds / Mitigations from Notes (HTML → text).
    for n in v.get("Notes", []) or []:
        title = (n.get("Title") or "").strip()
        if title not in ("Workarounds", "Mitigations"):
            continue
        text = _html_to_text(str(n.get("Value", "")))
        if not text:
            continue
        out.append({
            "kind": "workaround" if title == "Workarounds" else "mitigation",
            "title": title[:300],
            "detail": text[:4000],
            "kb": None,
            "fixed_build": None,
            "url": None,
        })
    return out


# ============================================================
# fetch + ingest
# ============================================================

def _list_recent_docs(client: "httpx.Client", months_back: int) -> List[str]:
    """Return the last `months_back` monthly CVRF document IDs (YYYY-Mon)."""
    r = client.get(f"{MSRC_BASE}/updates", headers={"Accept": "application/json"})
    r.raise_for_status()
    ids = [
        u.get("ID") for u in (r.json().get("value") or [])
        if u.get("ID") and re.match(r"^\d{4}-[A-Za-z]{3}$", u["ID"])
    ]
    # The index is roughly chronological but not guaranteed; sort by the
    # CurrentReleaseDate when present, else keep order, then take the tail.
    return ids[-months_back:]


def _upsert_cve(db: Session, cve_id: str, rows: List[dict]) -> None:
    """Replace all msrc rows for a cve_id with the freshly parsed set."""
    db.query(CveRemediation).filter(
        CveRemediation.cve_id == cve_id,
        CveRemediation.source == "msrc",
    ).delete(synchronize_session=False)
    now = datetime.now()
    for r in rows:
        db.add(CveRemediation(
            cve_id=cve_id, source="msrc", kind=r["kind"],
            title=r.get("title"), detail=r.get("detail"),
            kb=r.get("kb"), fixed_build=r.get("fixed_build"),
            url=r.get("url"), fetched_at=now,
        ))


def refresh_msrc(db: Session, months_back: Optional[int] = None,
                 only_known_cves: bool = True) -> dict:
    """Ingest the last N monthly MSRC documents into cve_remediations.

    only_known_cves: when True (default) we only store remediations for CVE
    ids already present in our vulnerabilities table — keeps the table
    relevant + small instead of mirroring ~18k MS CVEs.
    """
    if months_back is None:
        months_back = _setting_int(db, SETTING_MONTHS_BACK, DEFAULT_MONTHS_BACK)

    known: Optional[set] = None
    if only_known_cves:
        from app.models.vulnerability import Vulnerability
        known = {
            c for (c,) in db.query(Vulnerability.cve_id).distinct().all()
            if c and c.upper().startswith("CVE-")
        }

    stats = {"docs": 0, "cves_seen": 0, "cves_stored": 0, "rows": 0, "errors": []}
    with httpx.Client(timeout=HTTP_TIMEOUT, follow_redirects=True,
                      headers={"User-Agent": "VulnCheck/1.0"}) as client:
        try:
            doc_ids = _list_recent_docs(client, months_back)
        except httpx.HTTPError as e:
            raise RuntimeError(f"MSRC updates index fetch failed: {e}") from e

        for doc_id in doc_ids:
            try:
                r = client.get(f"{MSRC_BASE}/cvrf/{doc_id}",
                               headers={"Accept": "application/json"})
                if r.status_code != 200:
                    stats["errors"].append(f"{doc_id}: HTTP {r.status_code}")
                    continue
                doc = r.json()
            except (httpx.HTTPError, ValueError) as e:
                stats["errors"].append(f"{doc_id}: {e}")
                continue
            stats["docs"] += 1
            for v in doc.get("Vulnerability", []) or []:
                cve = (v.get("CVE") or "").strip().upper()
                if not cve.startswith("CVE-"):
                    continue
                stats["cves_seen"] += 1
                if known is not None and cve not in known:
                    continue
                rows = parse_vulnerability(v)
                if not rows:
                    continue
                _upsert_cve(db, cve, rows)
                stats["cves_stored"] += 1
                stats["rows"] += len(rows)
            db.commit()
            logger.info("MSRC: ingested %s (%d CVEs stored so far)", doc_id, stats["cves_stored"])

    _set_last_refresh(db)
    logger.info("MSRC refresh done: %s", {k: v for k, v in stats.items() if k != "errors"})
    return stats


def _set_last_refresh(db: Session) -> None:
    s = db.query(Setting).filter(Setting.key == MSRC_LAST_REFRESH_KEY).first()
    if s:
        s.value = datetime.now().isoformat()
    else:
        db.add(Setting(key=MSRC_LAST_REFRESH_KEY, value=datetime.now().isoformat(),
                       description="Timestamp of last MSRC CVRF refresh"))
    db.commit()