vulncheck
444bcd0d1d
Tester's CSV uses 'cisecurity.org/recommendation' as the id header — never matched any exact candidate, all 45 rows skipped. Two-tier matching now: 1) exact candidate list (cis_id, recommendation #, section, …, plus the new 'cisecurity.org/recommendation' explicitly) 2) substring fallback — any header containing 'recommendation', 'subsection' or 'section' is treated as cis_id when no exact match was found. Picks up exotic shapes like 'CIS Subsection ID', 'Workbench Recommendation', URL-style 'cisecurity.org/...' etc. Diagnose label flags substring matches explicitly so the operator can verify which header was used, e.g.: ID column used: cisecurity.org/recommendation (matched by substring)
287 lines
9.6 KiB
Python
287 lines
9.6 KiB
Python
"""
|
||
CSV importer for ComplianceImpact rows.
|
||
|
||
Accepts the colleague's CIS-Benchmark exports and any similar shape.
|
||
Auto-detects column names so we don't lock the operator into one
|
||
specific layout — the only HARD requirement is a column that maps to
|
||
the CIS subsection id (e.g. "2.3.4.1") and one that maps to impact.
|
||
|
||
Column heuristics (case-insensitive, first match wins):
|
||
cis_id ← cis_id | recommendation | section | id | control
|
||
title ← title | description | recommendation | name
|
||
impact ← impact | score | weight | risk
|
||
level ← level | profile | tier
|
||
benchmark← benchmark | policy | os (else from filename)
|
||
|
||
Benchmark fallback: when no benchmark column exists, use the file's
|
||
basename without extension. That matches the colleague's filenames
|
||
like `windows_server_2025_level1_member_server.csv` directly.
|
||
|
||
Impact normalisation:
|
||
- integer 0..100 → kept
|
||
- integer 0..10 → ×10 (auto-rescale)
|
||
- float → round(x)
|
||
- missing/garbage → 50 (neutral fallback)
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import csv
|
||
import io
|
||
import logging
|
||
import os
|
||
import re
|
||
from typing import Dict, Iterable, List, Optional, Tuple
|
||
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.models.compliance import ComplianceImpact
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
_CIS_ID_CANDIDATES = (
|
||
"cis_id", "cis id", "cis-id",
|
||
"cisecurity.org/recommendation", # CIS Workbench CSV export shape
|
||
"recommendation #", "recommendation#", "recommendation no",
|
||
"recommendation", "recommendation id",
|
||
"section", "section #", "subsection",
|
||
"id", "control", "control id", "ref", "#",
|
||
)
|
||
# Fallback: any header whose name CONTAINS one of these substrings is
|
||
# treated as the cis_id column when none of the exact candidates match.
|
||
_CIS_ID_SUBSTRINGS = ("recommendation", "subsection", "section")
|
||
_TITLE_CANDIDATES = (
|
||
"title", "description", "name", "recommendation_title",
|
||
"recommendation title", "control title", "control name",
|
||
)
|
||
_IMPACT_CANDIDATES = (
|
||
"impact", "score", "weight", "risk_score", "risk score", "risk",
|
||
"severity",
|
||
)
|
||
_LEVEL_CANDIDATES = ("level", "profile", "tier")
|
||
_BENCHMARK_CANDIDATES = ("benchmark", "policy", "os", "policy_name")
|
||
|
||
# Strict CIS subsection pattern (1, 1.1, 1.1.1, …).
|
||
_CIS_PATTERN = re.compile(r"^\d+(\.\d+)*$")
|
||
# Loose pattern — extracts the first numeric-dotted token from a value
|
||
# like "Section 1.1.1" or "L1 - 2.3.4.1 Ensure ..." so prefixed labels
|
||
# still match. Returns None if nothing CIS-shaped is present.
|
||
_CIS_EXTRACT = re.compile(r"\b(\d+(?:\.\d+){0,5})\b")
|
||
|
||
|
||
def _pick(row: Dict[str, str], candidates: Iterable[str]) -> Optional[str]:
|
||
"""Case-insensitive header lookup; returns the first non-empty value."""
|
||
lower = {k.lower().strip(): v for k, v in row.items() if k}
|
||
for c in candidates:
|
||
v = lower.get(c)
|
||
if v is not None and str(v).strip():
|
||
return str(v).strip()
|
||
return None
|
||
|
||
|
||
def _pick_by_substring(
|
||
row: Dict[str, str], substrings: Iterable[str],
|
||
) -> Optional[str]:
|
||
"""Fallback for non-standard CSV header shapes (e.g.
|
||
'cisecurity.org/recommendation', 'CIS Subsection ID', etc.) —
|
||
matches when ANY substring appears in a header. First hit wins."""
|
||
for k, v in row.items():
|
||
if not k:
|
||
continue
|
||
kl = k.lower()
|
||
for s in substrings:
|
||
if s in kl and v is not None and str(v).strip():
|
||
return str(v).strip()
|
||
return None
|
||
|
||
|
||
def _normalise_impact(raw: Optional[str]) -> int:
|
||
if raw is None:
|
||
return 50
|
||
raw = raw.strip().replace(",", ".").rstrip("%")
|
||
if not raw:
|
||
return 50
|
||
try:
|
||
f = float(raw)
|
||
except ValueError:
|
||
return 50
|
||
if f < 0:
|
||
return 0
|
||
# auto-rescale 0-10 → 0-100 (CIS-Workbench sometimes uses 1-10 risk)
|
||
if 0 < f <= 10 and f != int(f):
|
||
return int(round(f * 10))
|
||
if 0 <= f <= 10:
|
||
return int(round(f * 10))
|
||
if f > 100:
|
||
return 100
|
||
return int(round(f))
|
||
|
||
|
||
def _looks_like_cis_id(value: str) -> bool:
|
||
return bool(_CIS_PATTERN.match(value or ""))
|
||
|
||
|
||
def _extract_cis_id(value: str) -> Optional[str]:
|
||
"""Pull the first '1.2.3.4'-style token out of a free-form cell.
|
||
Lets us accept 'Section 2.3.4.1' or 'L1 1.2 Ensure ...' as cis_id."""
|
||
if not value:
|
||
return None
|
||
m = _CIS_EXTRACT.search(value)
|
||
return m.group(1) if m else None
|
||
|
||
|
||
def _benchmark_from_filename(filename: str) -> str:
|
||
base = os.path.basename(filename or "").rsplit(".", 1)[0]
|
||
return base.strip() or "unknown"
|
||
|
||
|
||
def import_impact_csv(
|
||
db: Session,
|
||
file_content: bytes,
|
||
filename: str,
|
||
benchmark_override: Optional[str] = None,
|
||
) -> Dict[str, int]:
|
||
"""
|
||
Parse one CSV blob, upsert ComplianceImpact rows.
|
||
|
||
Returns stats: {parsed, imported, skipped, errors[]}
|
||
"""
|
||
stats: Dict[str, int] = {
|
||
"parsed": 0,
|
||
"imported": 0,
|
||
"skipped_no_id": 0,
|
||
"skipped_not_cis": 0,
|
||
"errors": [],
|
||
"detected_headers": [],
|
||
"id_column_used": None,
|
||
"preview": [],
|
||
}
|
||
|
||
# Decode — try utf-8, fall back to cp1252 (Excel-on-Windows export).
|
||
try:
|
||
text = file_content.decode("utf-8-sig")
|
||
except UnicodeDecodeError:
|
||
try:
|
||
text = file_content.decode("cp1252")
|
||
except Exception as e:
|
||
stats["errors"].append(f"decode failed: {e}")
|
||
return stats
|
||
|
||
# Sniff delimiter — Excel-on-DE often uses ';'.
|
||
sample = text[:4096]
|
||
try:
|
||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t|")
|
||
except csv.Error:
|
||
class _D(csv.excel):
|
||
delimiter = ","
|
||
dialect = _D
|
||
|
||
reader = csv.DictReader(io.StringIO(text), dialect=dialect)
|
||
rows = list(reader)
|
||
stats["parsed"] = len(rows)
|
||
stats["detected_headers"] = list(reader.fieldnames or [])
|
||
|
||
benchmark = benchmark_override or _benchmark_from_filename(filename)
|
||
|
||
# Diagnose which header maps to id/impact for the operator.
|
||
if rows:
|
||
lower_headers = {(h or "").lower().strip(): h for h in (reader.fieldnames or [])}
|
||
for c in _CIS_ID_CANDIDATES:
|
||
if c in lower_headers:
|
||
stats["id_column_used"] = lower_headers[c]
|
||
break
|
||
# Substring fallback for the diagnose label too.
|
||
if not stats["id_column_used"]:
|
||
for h_lower, h_orig in lower_headers.items():
|
||
if any(s in h_lower for s in _CIS_ID_SUBSTRINGS):
|
||
stats["id_column_used"] = f"{h_orig} (matched by substring)"
|
||
break
|
||
for c in _IMPACT_CANDIDATES:
|
||
if c in lower_headers:
|
||
stats["impact_column_used"] = lower_headers[c]
|
||
break
|
||
|
||
for idx, row in enumerate(rows):
|
||
cis_id_raw = _pick(row, _CIS_ID_CANDIDATES)
|
||
if not cis_id_raw:
|
||
# Substring fallback — picks up exotic headers like
|
||
# 'cisecurity.org/recommendation', 'CIS Subsection ID' etc.
|
||
cis_id_raw = _pick_by_substring(row, _CIS_ID_SUBSTRINGS)
|
||
if not cis_id_raw:
|
||
stats["skipped_no_id"] += 1
|
||
if len(stats["preview"]) < 3:
|
||
stats["preview"].append({"row": idx + 2, "reason": "no recognised id column", "sample": dict(list(row.items())[:4])})
|
||
continue
|
||
# Try strict match first, then loose extract (e.g. 'Section 1.1.1').
|
||
if _looks_like_cis_id(cis_id_raw):
|
||
cis_id = cis_id_raw
|
||
else:
|
||
extracted = _extract_cis_id(cis_id_raw)
|
||
if not extracted:
|
||
stats["skipped_not_cis"] += 1
|
||
if len(stats["preview"]) < 3:
|
||
stats["preview"].append({"row": idx + 2, "reason": "id not CIS-shaped", "value": cis_id_raw[:80]})
|
||
continue
|
||
cis_id = extracted
|
||
|
||
impact = _normalise_impact(_pick(row, _IMPACT_CANDIDATES))
|
||
title = _pick(row, _TITLE_CANDIDATES)
|
||
level = _pick(row, _LEVEL_CANDIDATES)
|
||
row_benchmark = _pick(row, _BENCHMARK_CANDIDATES) or benchmark
|
||
|
||
existing = (
|
||
db.query(ComplianceImpact)
|
||
.filter(
|
||
ComplianceImpact.cis_id == cis_id,
|
||
ComplianceImpact.benchmark == row_benchmark,
|
||
)
|
||
.first()
|
||
)
|
||
if existing is None:
|
||
db.add(ComplianceImpact(
|
||
cis_id=cis_id,
|
||
benchmark=row_benchmark[:128],
|
||
level=(level or None) and level[:16],
|
||
title=(title or None) and title[:500],
|
||
impact=impact,
|
||
))
|
||
else:
|
||
existing.impact = impact
|
||
if title:
|
||
existing.title = title[:500]
|
||
if level:
|
||
existing.level = level[:16]
|
||
stats["imported"] += 1
|
||
|
||
try:
|
||
db.commit()
|
||
except Exception as e:
|
||
db.rollback()
|
||
stats["errors"].append(f"db commit: {e}")
|
||
logger.exception("compliance impact import commit failed")
|
||
return stats
|
||
|
||
|
||
def import_impact_csv_files(
|
||
db: Session,
|
||
files: List[Tuple[str, bytes]],
|
||
) -> Dict[str, int]:
|
||
"""Batch helper — multiple uploads in one call."""
|
||
overall: Dict[str, int] = {
|
||
"files": len(files),
|
||
"parsed": 0,
|
||
"imported": 0,
|
||
"skipped_no_id": 0,
|
||
"skipped_not_cis": 0,
|
||
"errors": [],
|
||
"per_file": {},
|
||
}
|
||
for fname, blob in files:
|
||
r = import_impact_csv(db, blob, fname)
|
||
overall["per_file"][fname] = r
|
||
overall["parsed"] += r["parsed"]
|
||
overall["imported"] += r["imported"]
|
||
overall["skipped_no_id"] += r["skipped_no_id"]
|
||
overall["skipped_not_cis"] += r["skipped_not_cis"]
|
||
overall["errors"].extend(r["errors"])
|
||
return overall
|