""" MSRC-driven OS CVE detection — patch-level accurate, ahead of the Wazuh CTI feed. Why this exists (and why NVD/cvelistV5 can't do it): Microsoft does not express fixes as version ranges. NVD lists MS OS entries as rangeless CPEs (`cpe:2.3:o:microsoft:windows_server_2016:-:*`, versionEndExcluding=null) and cvelistV5 MS records use `lessThan: "publication"` — neither says which BUILD carries the fix, so neither can tell a patched host from an unpatched one. MSRC's CVRF does: each Type-2 remediation carries a `FixedBuild` plus the `ProductID`s it applies to, e.g. CVE-2026-33834 FixedBuild 10.0.14393.9140 ProductID ['10816','10855'] KB5087537 ProductTree: 10816 -> "Windows Server 2016" 10855 -> "Windows Server 2016 (Server Core installation)" So: index (product → [cve, fixed_build]) from the monthly CVRF docs, then compare an asset's installed OS build against the fixed build of its own servicing branch. installed < fixed → affected. Same curated-and-precise contract as the other scanners: only products we can map to an asset are indexed, nothing is guessed, and a host that is patched is never flagged. Scope: Windows Server only for now — its OS string names the product outright. Client Windows (10/11) needs a build→release table before it can join. """ from __future__ import annotations import json import logging import re from datetime import datetime, timedelta from typing import Dict, List, Optional from sqlalchemy.orm import Session logger = logging.getLogger(__name__) SOURCE_NAME = "msrc" _INDEX_SETTING = "msrc_product_index_v1" _INDEX_TTL = timedelta(hours=26) # rebuilt nightly; a missed night still serves # Curated map: asset side (`match_re`) → MSRC ProductTree name (`msrc_re`). # # `kind` os = matched against asset.operating_system + asset.os_version # pkg = matched against an installed-software name + its version # `branch` True → a fix only speaks to hosts sharing its build prefix, because # the 3rd segment IS the release (Windows Server 2016 = 14393; # SharePoint 2019 = 10417 — verified stable across releases). # False → the 3rd segment moves with every CU, so prefix-matching would # silently never hit; identity comes from the NAME instead and # the compare is a plain installed < fixed. Verified: SharePoint # 2016 shows 5535/5539/5543/5552/5556 over six months. # # `msrc_re` is anchored so "Microsoft .NET Framework 4.8 on Windows Server 2016" # (a *different* product that merely mentions the OS) can't match the OS family. # Order matters — R2 must precede its base year. # # SharePoint 2013 is deliberately absent: it is EOL (2023-04-11) and Microsoft # publishes no fixes for it, so there is no FixedBuild to compare — no amount of # fix data can flag it. The EOL finding is the signal there (see eol_service). _PRODUCTS: List[dict] = [ {"key": "ws2012r2", "kind": "os", "branch": True, "match_re": r"windows server\s*2012\s*r2", "msrc_re": r"^windows server 2012 r2\b", "label": "Microsoft Windows Server 2012 R2"}, {"key": "ws2012", "kind": "os", "branch": True, "match_re": r"windows server\s*2012(?!\s*r2)", "msrc_re": r"^windows server 2012\b(?!\s*r2)", "label": "Microsoft Windows Server 2012"}, {"key": "ws2016", "kind": "os", "branch": True, "match_re": r"windows server\s*2016", "msrc_re": r"^windows server 2016\b", "label": "Microsoft Windows Server 2016"}, {"key": "ws2019", "kind": "os", "branch": True, "match_re": r"windows server\s*2019", "msrc_re": r"^windows server 2019\b", "label": "Microsoft Windows Server 2019"}, {"key": "ws2022", "kind": "os", "branch": True, "match_re": r"windows server\s*2022", "msrc_re": r"^windows server 2022\b", "label": "Microsoft Windows Server 2022"}, {"key": "ws2025", "kind": "os", "branch": True, "match_re": r"windows server\s*2025", "msrc_re": r"^windows server 2025\b", "label": "Microsoft Windows Server 2025"}, # SharePoint — 2016, 2019 and Subscription Edition ALL report 16.0.x, so the # year in the name is the only thing that tells the releases apart. {"key": "sp2016", "kind": "pkg", "branch": False, "match_re": r"sharepoint.*\b2016\b", "msrc_re": r"^microsoft sharepoint (enterprise )?server 2016\b", "label": "Microsoft SharePoint Server 2016"}, {"key": "sp2019", "kind": "pkg", "branch": False, "match_re": r"sharepoint.*\b2019\b", "msrc_re": r"^microsoft sharepoint server 2019\b", "label": "Microsoft SharePoint Server 2019"}, {"key": "spse", "kind": "pkg", "branch": False, "match_re": r"sharepoint.*subscription", "msrc_re": r"^microsoft sharepoint server subscription edition\b", "label": "Microsoft SharePoint Server Subscription Edition"}, ] _OS_COMPILED = [(re.compile(p["match_re"], re.I), p) for p in _PRODUCTS if p["kind"] == "os"] _PKG_COMPILED = [(re.compile(p["match_re"], re.I), p) for p in _PRODUCTS if p["kind"] == "pkg"] _MSRC_COMPILED = [(re.compile(p["msrc_re"], re.I), p) for p in _PRODUCTS] _BUILD_RE = re.compile(r"^\d+(\.\d+)+$") def resolve_os(os_name: str) -> Optional[dict]: """Asset OS string → curated product entry (None = not ours to scan).""" n = (os_name or "").strip().lower() if not n: return None for rx, p in _OS_COMPILED: if rx.search(n): return p return None def _resolve_msrc_product(name: str) -> Optional[dict]: n = (name or "").strip().lower() for rx, p in _MSRC_COMPILED: if rx.search(n): return p return None def _btuple(b: str) -> Optional[tuple]: if not b or not _BUILD_RE.match(b): return None try: return tuple(int(x) for x in b.split(".")) except ValueError: return None def _branch(b: str) -> Optional[tuple]: """Servicing branch = build minus its revision, e.g. 10.0.14393.9140 → (10, 0, 14393). A fix only speaks to hosts on its own branch: Server 2016 (14393) says nothing about Server 2019 (17763).""" t = _btuple(b) return t[:3] if t and len(t) >= 3 else None # ---------- index ---------- def build_product_index(db: Session, months_back: Optional[int] = None) -> dict: """Walk the recent monthly CVRF docs → {product_key: [{cve, build, kb}]}.""" import httpx from app.services.msrc_service import ( MSRC_BASE, HTTP_TIMEOUT, DEFAULT_MONTHS_BACK, SETTING_MONTHS_BACK, _setting_int, ) months = months_back or _setting_int(db, SETTING_MONTHS_BACK, DEFAULT_MONTHS_BACK) index: Dict[str, List[dict]] = {} seen: set = set() docs_done = 0 with httpx.Client(timeout=HTTP_TIMEOUT, headers={"Accept": "application/json"}) as client: r = client.get(f"{MSRC_BASE}/updates") r.raise_for_status() doc_ids = [v["ID"] for v in (r.json().get("value") or [])][-months:] for doc_id in doc_ids: try: resp = client.get(f"{MSRC_BASE}/cvrf/{doc_id}") resp.raise_for_status() doc = resp.json() except Exception as e: logger.warning("msrc-scan: doc %s failed: %s", doc_id, e) continue # ProductID → curated product key (only the ones we can map). pid_key: Dict[str, str] = {} for fp in (doc.get("ProductTree", {}) or {}).get("FullProductName", []) or []: p = _resolve_msrc_product(fp.get("Value") or "") if p and fp.get("ProductID"): pid_key[str(fp["ProductID"])] = p["key"] if not pid_key: continue for v in doc.get("Vulnerability", []) or []: cve = (v.get("CVE") or "").strip().upper() if not cve.startswith("CVE-"): continue for rem in v.get("Remediations", []) or []: if rem.get("Type") != 2: continue build = (rem.get("FixedBuild") or "").strip() if not _btuple(build): continue # no usable build → tells us nothing about patch state desc = str((rem.get("Description") or {}).get("Value", "") or "").strip() kb = desc if desc.isdigit() else None for pid in rem.get("ProductID") or []: key = pid_key.get(str(pid)) if not key: continue sig = (key, cve, build) if sig in seen: continue seen.add(sig) index.setdefault(key, []).append({"cve": cve, "build": build, "kb": kb}) docs_done += 1 _store_index(db, index) logger.info("msrc-scan: index built (%d docs) → %d products, %d fix entries", docs_done, len(index), sum(len(v) for v in index.values())) return index def _store_index(db: Session, index: dict) -> None: from app.models.setting import Setting payload = json.dumps({"built_at": datetime.now().isoformat(), "index": index}) row = db.query(Setting).filter(Setting.key == _INDEX_SETTING).first() if row: row.value = payload else: db.add(Setting(key=_INDEX_SETTING, value=payload, description="MSRC product→(CVE, FixedBuild) index (curated)")) db.commit() def load_index(db: Session, allow_stale: bool = True) -> Optional[dict]: from app.models.setting import Setting row = db.query(Setting).filter(Setting.key == _INDEX_SETTING).first() if not row or not row.value: return None try: blob = json.loads(row.value) built = datetime.fromisoformat(blob.get("built_at")) except Exception: return None if not allow_stale and datetime.now() - built > _INDEX_TTL: return None return blob.get("index") or {} # ---------- scan ---------- def affected_cves(entries: List[dict], installed: str, branch_match: bool = True) -> List[dict]: """CVEs whose fix is newer than the installed build. Per CVE the NEWEST fixed build wins — that's the one that actually has to be on the box. branch_match: see _PRODUCTS. True → only fixes on the host's own build prefix count (the prefix is the release). False → the prefix moves with every CU, so identity already came from the product name and every fix for that product applies (MS servicing is cumulative, so installed < fixed is exactly the right test).""" inst_t = _btuple(installed) if not inst_t: return [] inst_b = _branch(installed) if branch_match and not inst_b: return [] newest: Dict[str, dict] = {} for e in entries: bt = _btuple(e.get("build") or "") if not bt: continue if branch_match and _branch(e["build"]) != inst_b: continue # different servicing branch → says nothing about this host cur = newest.get(e["cve"]) if cur is None or bt > _btuple(cur["build"]): newest[e["cve"]] = e return [e for e in newest.values() if inst_t < _btuple(e["build"])] def resolve_package(name: str) -> Optional[dict]: """Installed-software name → curated MSRC product (None = not ours).""" n = (name or "").strip().lower() if not n: return None for rx, p in _PKG_COMPILED: if rx.search(n): return p return None def scan_asset(db: Session, asset, index: dict, new_ids: list, touched: Optional[set] = None) -> int: """Flag MS OS CVEs whose FixedBuild is ahead of this host's build.""" if not index: return 0 prod = resolve_os(asset.operating_system or "") if not prod: return 0 entries = index.get(prod["key"]) or [] if not entries: return 0 installed = (asset.os_version or "").strip() return _flag(db, asset, prod, installed, entries, new_ids, touched) def scan_asset_packages(db: Session, asset, packages: list, index: dict, new_ids: list, touched: Optional[set] = None) -> int: """Same fixed-build compare for installed MS software (SharePoint today). Called from the app-CVE scan, which already has the inventory in hand.""" if not index: return 0 count = 0 seen: set = set() for pkg in packages or []: name = (pkg.get("name") or "").strip() version = (pkg.get("version") or "").strip() if not name or not version: continue prod = resolve_package(name) if not prod: continue # One product reports several components (Core / Lang Pack / SQL # Express) all carrying the same build — scan the product once. dedup = (prod["key"], version) if dedup in seen: continue seen.add(dedup) entries = index.get(prod["key"]) or [] if entries: count += _flag(db, asset, prod, version, entries, new_ids, touched) return count def _flag(db: Session, asset, prod: dict, installed: str, entries: List[dict], new_ids: list, touched: Optional[set]) -> int: hits = affected_cves(entries, installed, branch_match=prod.get("branch", True)) count = 0 for h in hits: if touched is not None: touched.add(h["cve"]) try: if _upsert(db, asset, prod["label"], installed, h, new_ids): count += 1 except Exception as e: logger.debug("msrc-scan upsert failed (%s on %s): %s", h["cve"], asset.id, e) return count def _upsert(db: Session, asset, product: str, installed: str, hit: dict, new_ids: list) -> bool: from app.models.vulnerability import Vulnerability, VulnerabilityStatus cve_id = hit["cve"] fixed = hit["build"] kb = hit.get("kb") existing = (db.query(Vulnerability) .filter(Vulnerability.cve_id == cve_id, Vulnerability.asset_id == asset.id) .first()) if existing: existing.add_source(SOURCE_NAME) if not existing.package_name: existing.package_name = product[:255] if not existing.package_version: existing.package_version = installed[:100] if not existing.fixed_version: existing.fixed_version = fixed if existing.status == VulnerabilityStatus.patched: existing.status = VulnerabilityStatus.open existing.patched_at = None try: existing.refresh_scores() except Exception: pass return False title = f"{product} — {cve_id}" + (f" (KB{kb})" if kb else "") row = Vulnerability( cve_id=cve_id, asset_id=asset.id, status=VulnerabilityStatus.open, title=title[:500], description=(f"MSRC reports {product} is fixed in build {fixed}" + (f" via KB{kb}" if kb else "") + f"; this host reports {installed}."), package_name=product[:255], package_version=installed[:100], fixed_version=fixed, detected_at=datetime.now(), sources=json.dumps([SOURCE_NAME]), first_detected_by=SOURCE_NAME, ) db.add(row) db.flush() try: row.refresh_scores() except Exception: pass new_ids.append(row.id) return True def _resolve_stale(db: Session, asset, touched: set, considered: set) -> int: """A msrc-only finding no longer reported means the host caught up past the FixedBuild → patched. Same contract as the app-scan reconcile: never touch findings another scanner also reports. `considered` = the product labels THIS pass actually evaluated. Without it the OS pass would close every SharePoint finding it never looked at (and vice versa), since neither pass touches the other's CVEs.""" from app.models.vulnerability import Vulnerability, VulnerabilityStatus if not considered: return 0 rows = (db.query(Vulnerability) .filter(Vulnerability.asset_id == asset.id, Vulnerability.status == VulnerabilityStatus.open, Vulnerability.sources.contains('"msrc"'), Vulnerability.package_name.in_(list(considered))) .all()) resolved = 0 for v in rows: if v.cve_id in touched: continue # Drop OUR source; close only when nobody else still reports it (the # skip-if-cross-confirmed rule deadlocked across reconciles). v.remove_source(SOURCE_NAME) if v.source_list: continue old = v.status v.status = VulnerabilityStatus.patched v.patched_at = datetime.now() resolved += 1 try: from app.routers.vulnerabilities import log_vulnerability_change log_vulnerability_change( db, None, v.id, old, v.status, reason=f"MSRC scan: {asset.hostname} is now at or past the fixed build", cve_id=v.cve_id, source="msrc_scan", ) except Exception as e: logger.warning("audit log for msrc auto-resolve failed (vuln_id=%s): %s", v.id, e) return resolved def resolve_stale_packages(db: Session, asset, touched: set) -> int: """Reconcile the package (pkg-kind) MSRC findings after a package scan. Considers ALL pkg product labels — we just saw the full inventory, so a product that vanished (uninstalled) should resolve too.""" labels = {p["label"] for p in _PRODUCTS if p["kind"] == "pkg"} return _resolve_stale(db, asset, touched, labels) def run_msrc_scan(db: Session, asset_id: Optional[int] = None) -> dict: """Scan Windows-Server assets against the MSRC fixed-build index.""" from app.models.asset import Asset stats = {"assets": 0, "findings": 0, "new": 0, "resolved": 0, "errors": []} index = load_index(db) if not index: logger.info("msrc-scan: index missing — building now (one-time, then nightly)") try: index = build_product_index(db) or {} except Exception as e: stats["errors"].append(f"index build failed: {e}") return stats if not index: return stats new_ids: list = [] q = db.query(Asset) if asset_id is not None: q = q.filter(Asset.id == asset_id) for asset in q.all(): prod = resolve_os(asset.operating_system or "") if not prod: continue if not (asset.os_version or "").strip(): continue # no build → nothing to compare touched: set = set() try: stats["findings"] += scan_asset(db, asset, index, new_ids, touched=touched) # Only this asset's OS product — the package pass owns its own labels. stats["resolved"] += _resolve_stale(db, asset, touched, {prod["label"]}) stats["assets"] += 1 db.commit() except Exception as e: db.rollback() stats["errors"].append(f"asset {asset.id}: {e}") stats["new"] = len(new_ids) if new_ids: try: from app.services.audit_events import audit_new_vulnerabilities audit_new_vulnerabilities(db, new_ids, source=SOURCE_NAME) db.commit() except Exception as e: logger.debug("msrc-scan detected-audit failed: %s", e) try: from app.models.vulnerability import Vulnerability from app.services.enrichment_service import enrich_vulnerabilities from app.services.email_service import dispatch_new_vuln_notifications fresh = db.query(Vulnerability).filter(Vulnerability.id.in_(new_ids)).all() if fresh: enrich_vulnerabilities(db, fresh) stats["notifications"] = dispatch_new_vuln_notifications(db, fresh) except Exception as e: logger.debug("msrc-scan enrichment/notify failed: %s", e) logger.info("MSRC scan: %d assets, %d findings (%d new, %d auto-resolved)", stats["assets"], stats["findings"], stats["new"], stats["resolved"]) return stats