diff --git a/audit/README.md b/audit/README.md index cd8d0640..93ec8883 100644 --- a/audit/README.md +++ b/audit/README.md @@ -107,12 +107,17 @@ python audit/aggregate/report.py --selftest python audit/static/run_static.py --selftest # full pipeline end-to-end on fixtures ``` +`run_static.py` writes all four report formats to its `--out` directory: +`report.md`, `report.json`, `report.sarif` (upload to GitHub code scanning), and +`report.html` (a self-contained heatmap page). + ## Status -- **Done (this slice):** static build-free runners, normalization + taxonomy - (incl. the OWN001 `[resource:]` split and OWN014 region-escape labelling), - DevExpress baseline-suppress, cross-tool agreement scoring, the pain heatmap, - markdown + json reports, the analyzer-injection props/targets, and selftests. -- **Deferred:** HTML + merged-SARIF renderers (more views over the same model); - the runtime layer (FlaUI + ClrMD leak-harness, duplicate-immutable detector); - the AI-reviewer layer; feeding confirmed findings back into the OwnLang corpus. +- **Done:** static build-free runners, normalization + taxonomy (incl. the OWN001 + `[resource:]` split, OWN014 region-escape labelling, and OWN050 routed to the + coverage ledger), DevExpress baseline-suppress, cross-tool agreement scoring, the + pain heatmap, **all four renderers (markdown / json / merged SARIF / HTML)**, the + analyzer-injection props/targets, and selftests. +- **Deferred:** the runtime layer (FlaUI + ClrMD leak-harness, duplicate-immutable + detector); the AI-reviewer layer; feeding confirmed findings back into the + OwnLang corpus. diff --git a/audit/aggregate/report.py b/audit/aggregate/report.py index 4ec72c06..6a903ae9 100755 --- a/audit/aggregate/report.py +++ b/audit/aggregate/report.py @@ -6,9 +6,11 @@ * **Markdown** — for humans / a GitHub run summary (as oracle/mine do today). * **JSON** — machine-readable, for the downstream AI layer and regression diffs. + * **merged SARIF** — one combined SARIF 2.1.0 log (audit categories as rules) so + the findings surface in GitHub code scanning / the Security tab. + * **HTML** — a self-contained heatmap page for browsing. -HTML and merged-SARIF renderers are deferred (Plan.md §3.5) — they are additional -views over the same scored model, not new analysis. +All four are views over the same scored model, not new analysis. The coverage section is load-bearing: it states which tiers ran, which categories are NO-TOOL / deferred-to-runtime, how many DevExpress findings were suppressed, @@ -19,6 +21,7 @@ from __future__ import annotations import argparse +import html import json import sys from pathlib import Path @@ -29,6 +32,9 @@ from score import score from score import to_json as score_to_json +# Audit P0..P3 severities -> SARIF result levels (merged-SARIF renderer). +SEV_TO_LEVEL = {"P0": "error", "P1": "error", "P2": "warning", "P3": "note"} + # Human labels for the Plan.md §2 categories, used in the coverage section. CATEGORY_LABELS = { 1: "IDisposable leak", 2: "event/subscription leak & region escape", @@ -115,6 +121,12 @@ def _coverage_section(meta: dict[str, Any], cov: dict[str, Any], if cov.get("suppressed_by"): for reason, n in sorted(cov["suppressed_by"].items()): out.append(f" - suppressed — {reason}: {n}") + skipped = cov.get("analysis_skipped", 0) + if skipped: + by = cov.get("analysis_skipped_by") or {} + detail = ", ".join(f"{r} x{n}" for r, n in sorted(by.items())) + out.append(f"- analysis-skipped (coverage notes, not scored): {skipped}" + + (f" — {detail}" if detail else "")) no_tool = meta.get("no_tool_static") or [] if no_tool: labels = ", ".join(f"{c} ({CATEGORY_LABELS.get(c, '?')})" for c in no_tool) @@ -137,12 +149,143 @@ def render_json(meta: dict[str, Any], cov: dict[str, Any], return {"meta": meta, "coverage": cov, **score_to_json(scored)} +def render_sarif(meta: dict[str, Any], cov: dict[str, Any], + scored: dict[str, Any]) -> dict[str, Any]: + """One merged SARIF 2.1.0 log for GitHub code scanning (Plan.md §3.5). + + Audit categories become the SARIF rules (so the Security tab groups by our + taxonomy, not the underlying tool ids); each scored cluster is one result, with + the contributing tools, confidence and pain carried in properties. Suppressed + and analysis-skipped counts ride in the run properties so the honesty ledger + travels with the log.""" + rules: dict[str, dict[str, Any]] = {} + results: list[dict[str, Any]] = [] + for c in scored["clusters"]: + rule_id = f"own-audit/{c.category_name}" + level = SEV_TO_LEVEL.get(c.severity, "warning") + if rule_id not in rules: + rules[rule_id] = { + "id": rule_id, + "name": c.category_name, + "shortDescription": { + "text": f"{CATEGORY_LABELS.get(c.category, c.category_name)} " + f"(category {c.category})"}, + "defaultConfiguration": {"level": level}, + "properties": {"category": c.category}, + } + # A genuinely file-level finding (parse_sarif records line 0 when the + # upstream result has no region) must STAY file-level — omit region rather + # than fabricate a line-1 region that mis-pins the code-scanning alert. + phys: dict[str, Any] = {"artifactLocation": {"uri": c.path}} + if c.line >= 1: + phys["region"] = {"startLine": c.line} + results.append({ + "ruleId": rule_id, + "level": level, + "message": {"text": f"[{c.severity} · {c.confidence}] {c.category_name} — " + + "; ".join(f"{f.tool}:{f.rule}" for f in c.findings)}, + "locations": [{"physicalLocation": phys}], + "properties": {"category": c.category, "categoryName": c.category_name, + "confidence": c.confidence, "severity": c.severity, + "pain": c.pain, "tools": c.tools}, + }) + return { + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + "version": "2.1.0", + "runs": [{ + "tool": {"driver": { + "name": "Own.NET Audit", + "informationUri": "https://github.com/PhysShell/Own.NET", + "rules": list(rules.values()), + }}, + "results": results, + "properties": { + "target": meta.get("target", ""), + "commit": meta.get("commit", ""), + "suppressed": cov.get("suppressed", 0), + "analysisSkipped": cov.get("analysis_skipped", 0), + }, + }], + } + + +def render_html(meta: dict[str, Any], cov: dict[str, Any], + scored: dict[str, Any], max_list: int = 200) -> str: + """A self-contained HTML heatmap page (Plan.md §3.5) — no external assets.""" + e = html.escape + clusters = scored["clusters"] + high = [c for c in clusters if c.confidence == "high"] + sev_class = {"P0": "p0", "P1": "p1", "P2": "p2", "P3": "p3"} + + rows = "\n".join( + f"{e(r['module'])}{r['pain']}" + f"{r['findings']}{r['high_confidence']}" + f"{e(r['top_category'])}" + for r in scored["heatmap"][:max_list]) or \ + "no findings" + + def finding_li(c: Any) -> str: + cls = sev_class.get(c.severity, "p3") + return (f"
  • {e(c.severity)} " + f"{e(c.path)}:{c.line} " + f"{e(c.category_name)} " + f"{e(', '.join(c.tools))}
    • ") + + high_li = "\n".join(finding_li(c) for c in high[:max_list]) or "
  • none
    • " + cand = [c for c in clusters if c.confidence == "candidate"] + cand_li = "\n".join(finding_li(c) for c in cand[:max_list]) or "
  • none
    • " + cand_more = (f"
  • +{len(cand) - max_list} more
    • " + if len(cand) > max_list else "") + cov_rows = "".join(f"
  • {e(line.strip().removeprefix('- '))}
    • " + for line in _coverage_section(meta, cov, scored) + if line.strip() and not line.startswith("#")) + + return f""" + +Own.NET Audit — {e(str(meta.get('target', '?')))} + +

    Own.NET Audit — health report — {e(str(meta.get('target', '?')))}

    +

    commit {e(str(meta.get('commit', '?')))} · + generated {e(str(meta.get('generated', '?')))} · + profile {e(str(meta.get('profile', '?')))} · + tools {e(', '.join(cov.get('tools') or []) or '(none)')}

    +

    {scored['totals']['clusters']} findings + ({scored['totals']['high_confidence']} high-confidence, + {scored['totals']['candidates']} candidate)

    +

    Where it hurts most

    + + +{rows} +
    modulepainfindingshigh-conftop category
    +

    High-confidence findings — {len(high)}

    + +

    Candidates — {len(cand)}

    + +

    Coverage / honesty

    + + +""" + + def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(description="Render the audit health report (markdown/json).") ap.add_argument("--findings", help="normalize.py --json output") ap.add_argument("--taxonomy", default=str( Path(__file__).resolve().parents[1] / "static" / "taxonomy" / "categories.yml")) - ap.add_argument("--format", choices=["markdown", "json"], default="markdown") + ap.add_argument("--format", choices=["markdown", "json", "sarif", "html"], + default="markdown") ap.add_argument("--target", default="") ap.add_argument("--commit", default="") ap.add_argument("--line-tol", type=int, default=3) @@ -166,6 +309,10 @@ def main(argv: list[str] | None = None) -> int: meta = {"target": args.target, "commit": args.commit, "line_tol": args.line_tol} if args.format == "json": print(json.dumps(render_json(meta, cov, scored), indent=2)) + elif args.format == "sarif": + print(json.dumps(render_sarif(meta, cov, scored), indent=2)) + elif args.format == "html": + print(render_html(meta, cov, scored)) else: print(render_markdown(meta, cov, scored)) return 0 @@ -193,6 +340,8 @@ def check(ok: bool, msg: str) -> None: # total derives from the call count AuditFinding("codeql", "DevExpress.Xpf/G.cs", 4, "cs/empty-block", "x", 0, "uncategorized", suppressed=True, suppress_reason="third-party: DevExpress."), AuditFinding("codeql", "src/Util/Misc.cs", 3, "FOO999", "y", 0, "uncategorized"), + AuditFinding("own-check", "src/Io/Weird.cs", 5, "OWN050", "cannot verify", 0, + "uncategorized", note=True), ] cov = coverage(findings) scored = score(findings, tax) @@ -207,6 +356,8 @@ def check(ok: bool, msg: str) -> None: # total derives from the call count check(needle in md, f"markdown missing section/marker: {needle!r}") check("third-party: DevExpress." in md, "coverage must report the suppressed DevExpress count") check("`FOO999`" in md, "coverage must surface the unmapped FOO999 rule") + check("analysis-skipped" in md and "OWN050" in md, + "coverage must surface analysis-skipped notes (OWN050)") check("src/Util" in md, "heatmap must list the worst module (src/Util)") # the agreed leak (high-confidence) must be the worst module, ahead of src/Vm util_pos, vm_pos = md.find("`src/Util`"), md.find("`src/Vm`") @@ -220,6 +371,42 @@ def check(ok: bool, msg: str) -> None: # total derives from the call count check(bool(js["clusters"]) and "evidence" in js["clusters"][0], "json clusters missing evidence") + # merged SARIF: 3 scored clusters -> 3 results; categories become rules; the + # suppressed DevExpress finding is excluded from results but counted in run props. + sa = render_sarif(meta, cov, scored) + run = sa["runs"][0] + check(sa["version"] == "2.1.0", "sarif version must be 2.1.0") + check(len(run["results"]) == 3, f"sarif must have 3 results, got {len(run['results'])}") + check(all(r["ruleId"].startswith("own-audit/") for r in run["results"]), + "sarif results must use own-audit/ rule ids") + check(all(loc["region"]["startLine"] >= 1 + for r in run["results"] + for loc in (r["locations"][0]["physicalLocation"],) if "region" in loc), + "sarif region startLine (when present) must be >= 1") + # file-level findings (parse_sarif records line 0) must stay file-level — no + # fabricated line-1 region that mis-pins the alert (Codex review on #101). + fl = score([AuditFinding("codeql", "src/App/Asm.cs", 0, "cs/assembly", "x", 14, + "general-quality")], tax) + fl_phys = render_sarif(meta, cov, fl)["runs"][0]["results"][0]["locations"][0][ + "physicalLocation"] + check("region" not in fl_phys, "file-level finding (line 0) must omit SARIF region") + levels = {r["ruleId"]: r["level"] for r in run["results"]} + check(levels.get("own-audit/idisposable-leak") == "error", + "P1 leak must map to SARIF level error") + check(levels.get("own-audit/uncategorized") == "note", + "P3 uncategorized must map to SARIF level note") + check(run["properties"]["suppressed"] == 1, "sarif run props must carry the suppressed count") + rule_ids = {ru["id"] for ru in run["tool"]["driver"]["rules"]} + check("own-audit/idisposable-leak" in rule_ids, "sarif driver must declare the category rules") + + # HTML: self-contained heatmap page with the worst module and the coverage ledger. + h = render_html(meta, cov, scored) + for needle in ("", "", " dict[str, Any]: def aggregate(sarif_inputs: list[tuple[str, str]], out_dir: Path, meta: dict[str, Any], taxonomy: Path = DEFAULT_TAXONOMY, line_tol: int = 3) -> dict[str, Any]: - """Normalize → score → render the SARIFs in ``sarif_inputs`` and write the - markdown + json reports to ``out_dir``. Returns the scored totals.""" + """Normalize → score → render the SARIFs in ``sarif_inputs``, writing all four + report artifacts to ``out_dir``: ``report.md``, ``report.json``, + ``report.sarif`` (merged SARIF for code scanning) and ``report.html``. Returns + a dict with ``totals``, ``coverage`` and the four ``report_{md,json,sarif,html}`` + paths.""" out_dir.mkdir(parents=True, exist_ok=True) tax = load_taxonomy(taxonomy) raw: list[Any] = [] @@ -73,9 +76,14 @@ def aggregate(sarif_inputs: list[tuple[str, str]], out_dir: Path, meta: dict[str (out_dir / "report.md").write_text(render_markdown(meta, cov, scored), encoding="utf-8") (out_dir / "report.json").write_text( json.dumps(render_json(meta, cov, scored), indent=2), encoding="utf-8") + (out_dir / "report.sarif").write_text( + json.dumps(render_sarif(meta, cov, scored), indent=2), encoding="utf-8") + (out_dir / "report.html").write_text(render_html(meta, cov, scored), encoding="utf-8") return {"totals": scored["totals"], "coverage": cov, "report_md": str(out_dir / "report.md"), - "report_json": str(out_dir / "report.json")} + "report_json": str(out_dir / "report.json"), + "report_sarif": str(out_dir / "report.sarif"), + "report_html": str(out_dir / "report.html")} def _run_codeql(target: str, out_dir: Path) -> dict[str, Any]: @@ -235,7 +243,8 @@ def check(ok: bool, msg: str) -> None: # total derives from the call count js = json.loads(Path(result["report_json"]).read_text(encoding="utf-8")) check(js["coverage"]["suppressed"] == 1, "report.json coverage lost the suppressed count") check(js["meta"]["target"] == "acme/legacy", "report.json lost meta") - check((tmp / "report" / "report.md").exists(), "aggregate did not write report.md to disk") + for art in ("report.md", "report.json", "report.sarif", "report.html"): + check((tmp / "report" / art).exists(), f"aggregate did not write {art}") # Roslyn build-required tier writes one SARIF PER PROJECT under roslyn/; run() # must glob the directory, not a single fixed filename (Codex review on #100).