diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f11f03d1..13d0ed60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,7 @@ jobs: python audit/aggregate/score.py --selftest python audit/aggregate/report.py --selftest python audit/static/run_static.py --selftest + python audit/runtime/ingest.py --selftest tests: name: tests (py${{ matrix.python-version }}) diff --git a/audit/README.md b/audit/README.md index 93ec8883..8f21ad68 100644 --- a/audit/README.md +++ b/audit/README.md @@ -52,6 +52,10 @@ audit/ inject/ # OwnAudit.Directory.Build.props/.targets (analyzer injection, gated) taxonomy/ categories.yml # rule-id -> category knowledge base (Plan.md §2/§3.4) + runtime/ # runtime layer (Plan.md §4) — see runtime/README.md + ingest.py # leak-harness JSON -> SARIF -> the unified pipeline (PURE PYTHON, CI-gated) + scenarios/ # declarative leak-harness scenarios (+ schema) + LeakHarness/ # C# harness (FlaUI + procdump + ClrMD), Windows/build-required, not CI-gated config/profiles/ desktop-wpf.yml # which packs / severity floor for the net472 WPF target requirements.txt # PyYAML (audit-scoped) @@ -113,11 +117,15 @@ python audit/static/run_static.py --selftest # full pipeline end-to-end on fix ## Status -- **Done:** static build-free runners, normalization + taxonomy (incl. the OWN001 - `[resource:]` split, OWN014 region-escape labelling, and OWN050 routed to the - coverage ledger), DevExpress baseline-suppress, cross-tool agreement scoring, the - pain heatmap, **all four renderers (markdown / json / merged SARIF / HTML)**, the - analyzer-injection props/targets, and selftests. -- **Deferred:** the runtime layer (FlaUI + ClrMD leak-harness, duplicate-immutable - detector); the AI-reviewer layer; feeding confirmed findings back into the - OwnLang corpus. +- **Static (Phase 1) — done:** build-free runners, normalization + taxonomy (incl. + the OWN001 `[resource:]` split, OWN014 region-escape labelling, and OWN050 routed + to the coverage ledger), DevExpress baseline-suppress, cross-tool agreement + scoring, the pain heatmap, **all four renderers (markdown / json / merged SARIF / + HTML)**, the analyzer-injection props/targets, and selftests. +- **Runtime (Phase 2) — started:** the runtime→pipeline bridge (`runtime/ingest.py`, + CI-gated), the leak-harness scenario schema + one scenario, runtime rule mappings + in the taxonomy (categories 2/3/4/11), and the C# leak-harness skeleton. See + `runtime/README.md`. +- **Deferred:** the ClrMD duplicate-immutable detector and PropertyChanged-storm + profiler; the AI-reviewer layer; feeding confirmed findings back into the OwnLang + corpus. diff --git a/audit/runtime/LeakHarness/HeapCounter.cs b/audit/runtime/LeakHarness/HeapCounter.cs new file mode 100644 index 00000000..61d10bce --- /dev/null +++ b/audit/runtime/LeakHarness/HeapCounter.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Diagnostics.Runtime; + +namespace OwnNet.Audit.Runtime +{ + /// + /// Snapshots the TARGET process heap and counts live instances of suspect types. + /// The harness drives the app out-of-process (FlaUI), so the only way to read the + /// target's managed heap is a full dump (procdump) + ClrMD — dotnet-gcdump / + /// dotnet-counters do not attach to .NET Framework, only CoreCLR (Plan.md §4). + /// + internal sealed class HeapCounter + { + private readonly string _procdump; + private readonly string _scratch; + + public HeapCounter(string procdumpPath, string scratchDir) + { + _procdump = procdumpPath; + _scratch = scratchDir; + Directory.CreateDirectory(_scratch); + } + + /// + /// Full-dump the process and return { type -> live instance count } for the + /// requested types. A full dump captures the heap as the GC last left it, so + /// request a GC in the target (SematixTrace) before calling this. + /// + public Dictionary CountLiveInstances(int pid, IEnumerable types) + { + var wanted = new HashSet(types); + var counts = new Dictionary(); + foreach (var t in wanted) + { + counts[t] = 0; + } + + var dump = Path.Combine(_scratch, $"target-{pid}-{Stopwatch.GetTimestamp()}.dmp"); + RunProcdump(pid, dump); + try + { + using var dataTarget = DataTarget.LoadDump(dump); + var clr = dataTarget.ClrVersions.FirstOrDefault() + ?? throw new InvalidOperationException( + $"dump for pid {pid} contains no CLR — is the target a managed process?"); + using var runtime = clr.CreateRuntime(); + foreach (var obj in runtime.Heap.EnumerateObjects()) + { + var name = obj.Type?.Name; + if (name != null && wanted.Contains(name)) + { + counts[name]++; + } + } + } + finally + { + File.Delete(dump); // dumps are large; the counts are the artifact, not the dump + } + return counts; + } + + private void RunProcdump(int pid, string dumpPath) + { + // -ma = full dump (managed heap included), -accepteula for unattended runs. + var psi = new ProcessStartInfo(_procdump, $"-accepteula -ma {pid} \"{dumpPath}\"") + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + using var p = Process.Start(psi)!; + // Drain both pipes asynchronously BEFORE waiting: if procdump fills a pipe + // buffer while the harness blocks in WaitForExit(), both deadlock. Bound the + // wait so a stuck procdump can't hang the harness forever, and check the + // exit code, not just whether a file appeared. + var stdout = p.StandardOutput.ReadToEndAsync(); + var stderr = p.StandardError.ReadToEndAsync(); + if (!p.WaitForExit(120_000)) + { + try { p.Kill(); } catch { /* best effort */ } + throw new IOException($"procdump timed out (>120s) for pid {pid}"); + } + Task.WaitAll(stdout, stderr); + if (p.ExitCode != 0 || !File.Exists(dumpPath)) + { + throw new IOException( + $"procdump failed for pid {pid} (exit {p.ExitCode}): {stderr.Result}"); + } + } + } +} diff --git a/audit/runtime/LeakHarness/LeakHarness.csproj b/audit/runtime/LeakHarness/LeakHarness.csproj new file mode 100644 index 00000000..017aa74c --- /dev/null +++ b/audit/runtime/LeakHarness/LeakHarness.csproj @@ -0,0 +1,27 @@ + + + + Exe + net472 + latest + enable + LeakHarness + OwnNet.Audit.Runtime + x64 + + x64 + + + + + + + + diff --git a/audit/runtime/LeakHarness/Program.cs b/audit/runtime/LeakHarness/Program.cs new file mode 100644 index 00000000..0e6a1d74 --- /dev/null +++ b/audit/runtime/LeakHarness/Program.cs @@ -0,0 +1,255 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using FlaUI.Core; +using FlaUI.Core.AutomationElements; +using FlaUI.UIA3; +using Newtonsoft.Json; + +namespace OwnNet.Audit.Runtime +{ + /// + /// Deterministic leak-harness (Plan.md §4.1): drive the target through a YAML + /// scenario N times, force a GC + heap snapshot each cycle, and assert that the + /// retained instance count of each suspect type does NOT grow ~linearly with the + /// iteration count. Emits a JSON result that audit/runtime/ingest.py converts to + /// SARIF for the unified report. + /// + /// Windows / build-required: needs a running target, procdump (Sysinternals) and + /// ClrMD. Not part of the Linux CI gate — the ingest bridge is what CI tests. + /// + internal static class Program + { + private static int Main(string[] args) + { + var opts = CliOptions.Parse(args); + if (opts == null) + { + return 2; + } + try + { + return Run(opts); + } + catch (ScenarioException ex) + { + // A broken scenario must NOT be reported as a clean pass — exit 2 + // (distinct from clean=0 and leak-found=1) and write no result. + Console.Error.WriteLine($"leak-harness: scenario failed — {ex.Message}"); + return 2; + } + } + + private static int Run(CliOptions opts) + { + var scenario = Scenario.Load(opts.ScenarioPath); + var counter = new HeapCounter(opts.ProcdumpPath, opts.ScratchDir); + var types = scenario.Suspects.Select(s => s.Type).ToList(); + + using var automation = new UIA3Automation(); + var app = Application.Launch(scenario.App); + try + { + var window = app.GetMainWindow(automation, TimeSpan.FromSeconds(30)) + ?? throw new InvalidOperationException("main window did not appear"); + + // Warm-up cycle BEFORE the baseline: the first open/close JITs methods + // and lazily initialises caches, so taking the baseline after one cycle + // avoids counting one-time allocations as a leak. + RunCycle(window, scenario); + var baseline = Snapshot(counter, app.ProcessId, types); + + for (var i = 0; i < scenario.Iterations; i++) + { + RunCycle(window, scenario); + } + + var final = Snapshot(counter, app.ProcessId, types); + var findings = BuildFindings(scenario, baseline, final); + var result = new Dictionary + { + ["tool"] = "leak-harness", + ["scenario"] = scenario.Name, + ["target"] = opts.Target, + ["commit"] = opts.Commit, + ["iterations"] = scenario.Iterations, + ["findings"] = findings, + }; + + Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(opts.OutPath))!); + File.WriteAllText(opts.OutPath, JsonConvert.SerializeObject(result, Formatting.Indented)); + + var leaked = findings.Count(f => (bool)f["leaked"]); + Console.WriteLine( + $"leak-harness: {leaked} leak(s) over {scenario.Iterations} cycles -> {opts.OutPath}"); + return leaked > 0 ? 1 : 0; + } + finally + { + // The target may already have exited/crashed; a teardown throw must not + // mask the real exception flowing out of the try. + try { app.Close(); } catch { /* target may already be gone */ } + app.Dispose(); + } + } + + /// + /// One scenario cycle: replay the declared steps (open the screen, interact, + /// close it). Determinism comes from the asserts, not the UI timing. + /// + /// A step that cannot run — control not found (typo, or the screen has not + /// loaded yet) or an unknown action — THROWS rather than being silently + /// skipped. Skipping would baseline/snapshot the wrong screen and report a + /// clean result: a broken scenario masquerading as "no leak" (Codex review). + /// + private static void RunCycle(Window window, Scenario scenario) + { + foreach (var step in scenario.Steps) + { + switch (step.Action) + { + case "open": + case "click": + case "close": + var element = window.FindFirstDescendant(cf => cf.ByAutomationId(step.Target)) + ?? throw new ScenarioException( + $"step '{step.Action}' target '{step.Target}' not found"); + // Invoke() requires the control to support the InvokePattern; + // if it does not, FlaUI throws — which fails the scenario too. + element.AsButton().Invoke(); + break; + case "wait": + Thread.Sleep(step.Ms); + break; + default: + throw new ScenarioException($"unknown step action '{step.Action}'"); + } + } + } + + /// + /// Request a GC in the target (SematixTrace, diagnostic build) so the dump + /// reflects only retained objects, then snapshot the heap. + /// + private static Dictionary Snapshot(HeapCounter counter, int pid, List types) + { + GcRequester.RequestCollect(pid); // best-effort; no-op if the target has no SematixTrace + Thread.Sleep(500); // let finalizers run before the dump + return counter.CountLiveInstances(pid, types); + } + + private static List> BuildFindings( + Scenario scenario, Dictionary baseline, Dictionary final) + { + var findings = new List>(); + foreach (var s in scenario.Suspects) + { + var b = baseline.TryGetValue(s.Type, out var bv) ? bv : 0; + var f = final.TryGetValue(s.Type, out var fv) ? fv : 0; + var growth = (double)(f - b) / Math.Max(1, scenario.Iterations); + findings.Add(new Dictionary + { + ["rule"] = s.Rule, + ["type"] = s.Type, + ["location"] = s.Location, + ["line"] = s.Line, + ["baseline"] = b, + ["final"] = f, + ["growthPerIteration"] = growth, + ["threshold"] = scenario.Threshold, + ["leaked"] = growth > scenario.Threshold, + ["message"] = + $"retained instances of {s.Type} grew {b}->{f} over {scenario.Iterations} cycles", + }); + } + return findings; + } + } + + /// + /// A scenario could not be executed (missing control, unknown action). Distinct + /// from a leak: it means the audit did not actually run, so the result must NOT + /// be treated as "clean". + /// + internal sealed class ScenarioException : Exception + { + public ScenarioException(string message) : base(message) + { + } + } + + /// Best-effort GC trigger in the target via a SematixTrace named event. + internal static class GcRequester + { + public static void RequestCollect(int pid) + { + var name = $"OwnNet.Sematix.RequestGc.{pid}"; + try + { + if (EventWaitHandle.TryOpenExisting(name, out var handle)) + { + using (handle) + { + handle.Set(); + } + } + } + catch + { + // The target has no SematixTrace GC hook (release build) — fall back to + // the settle delay in Snapshot(). A missing GC request inflates counts + // uniformly across baseline and final, so the GROWTH signal still holds. + } + } + } + + /// Command-line options for the harness. + internal sealed class CliOptions + { + public string ScenarioPath { get; private set; } = ""; + public string OutPath { get; private set; } = "artifacts/own-audit/leak-harness.json"; + public string ProcdumpPath { get; private set; } = "procdump.exe"; + public string ScratchDir { get; private set; } = "artifacts/own-audit/dumps"; + public string Target { get; private set; } = ""; + public string Commit { get; private set; } = ""; + + public static CliOptions? Parse(string[] args) + { + var o = new CliOptions(); + for (var i = 0; i < args.Length; i++) + { + switch (args[i]) + { + case "--scenario": o.ScenarioPath = Next(args, ref i); break; + case "--out": o.OutPath = Next(args, ref i); break; + case "--procdump": o.ProcdumpPath = Next(args, ref i); break; + case "--scratch": o.ScratchDir = Next(args, ref i); break; + case "--target": o.Target = Next(args, ref i); break; + case "--commit": o.Commit = Next(args, ref i); break; + default: + Console.Error.WriteLine($"LeakHarness: unknown arg {args[i]}"); + return null; + } + } + if (string.IsNullOrEmpty(o.ScenarioPath)) + { + Console.Error.WriteLine( + "Usage: LeakHarness --scenario [--out json] [--procdump path] " + + "[--scratch dir] [--target owner/repo] [--commit sha]"); + return null; + } + return o; + } + + private static string Next(string[] args, ref int i) + { + if (i + 1 >= args.Length) + { + throw new ArgumentException($"{args[i]} requires a value"); + } + return args[++i]; + } + } +} diff --git a/audit/runtime/LeakHarness/Scenario.cs b/audit/runtime/LeakHarness/Scenario.cs new file mode 100644 index 00000000..76f0aa4c --- /dev/null +++ b/audit/runtime/LeakHarness/Scenario.cs @@ -0,0 +1,68 @@ +using System; +using System.Collections.Generic; +using System.IO; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConvention; + +namespace OwnNet.Audit.Runtime +{ + /// + /// POCO mirror of a leak-harness scenario YAML (see audit/runtime/scenarios/). + /// The scenario is the part an AI drafts (which screen, how to open/close it); + /// the asserts are deterministic — retained instance growth vs. a threshold. + /// + public sealed class Scenario + { + public string Name { get; set; } = ""; + public string App { get; set; } = ""; // path to the target .exe + public int Iterations { get; set; } = 10; + public double Threshold { get; set; } = 0.5; // allowed retained growth / iteration + public List Steps { get; set; } = new(); + public List Suspects { get; set; } = new(); + + public static Scenario Load(string path) + { + var deserializer = new DeserializerBuilder() + .WithNamingConvention(HyphenatedNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + var scenario = deserializer.Deserialize(File.ReadAllText(path)); + // Fail fast on a malformed scenario: a missing `rule`/`type` would otherwise + // emit a finding under the wrong taxonomy rather than be caught here. + foreach (var s in scenario.Suspects) + { + if (string.IsNullOrWhiteSpace(s.Type)) + { + throw new ArgumentException($"scenario '{scenario.Name}': a suspect is missing 'type'"); + } + if (string.IsNullOrWhiteSpace(s.Rule)) + { + throw new ArgumentException( + $"scenario '{scenario.Name}': suspect '{s.Type}' is missing 'rule'"); + } + } + return scenario; + } + } + + /// One UI action in a scenario cycle (open a screen, click, close, wait). + public sealed class Step + { + public string Action { get; set; } = ""; // open | click | close | wait + public string Target { get; set; } = ""; // automation id / name of the control + public int Ms { get; set; } // dwell time for `wait` + } + + /// + /// A type whose retained-instance count the harness watches across cycles, plus + /// the source location and audit rule it maps to so a confirmed leak correlates + /// with the matching static finding (Plan.md §3.5). + /// + public sealed class Suspect + { + public string Type { get; set; } = ""; // fully-qualified CLR type + public string Rule { get; set; } = ""; // required — validated on load + public string Location { get; set; } = ""; // source file, for correlation + public int Line { get; set; } // 0 = file-level + } +} diff --git a/audit/runtime/README.md b/audit/runtime/README.md new file mode 100644 index 00000000..b05ab5a8 --- /dev/null +++ b/audit/runtime/README.md @@ -0,0 +1,84 @@ +# Own.NET Audit — runtime layer (Plan.md §4) + +The static layer answers "where might it hurt"; the runtime layer answers "where +does it *actually* hurt", and **confirms** static findings by observing the running +app. Its findings flow through the *same* `normalize → score → report` pipeline as +the static tiers (via `ingest.py`), so a runtime-confirmed leak in the same file as +a static finding clusters with it → **high confidence** (Plan.md §3.5). + +This layer covers the categories static analysis honestly can't (Plan.md §2): +event/subscription & timer leaks confirmed under load (cat. 2/3), the +`DependencyPropertyDescriptor.AddValueChanged` leak (cat. 4), and the +**duplicated-immutable-data** detector — the project's "gold" (cat. 11). For these, +the runtime layer is the *only* tool, so they were `NO-TOOL` until now. + +## Stack (Windows / build-required — Plan.md §4) + +net472 / WPF / DevExpress precision beats fashion, so the stack is ETW + dump, not +the CoreCLR-only `dotnet-*` tools: + +| Role | Tool | +|---|---| +| UI driver (deterministic scenarios, not clicks) | **FlaUI** (UIA3) | +| Scenario ↔ snapshot breadcrumbs + GC trigger | **SematixTrace** (diagnostic build) | +| GC / alloc / CPU / WPF-render telemetry | **PerfView** (ETW) | +| Heap snapshot / full dump | **procdump** (`-ma`) | +| Heap analysis (retained, duplicates, retention paths) | **ClrMD** | + +## Layout + +```text +audit/runtime/ + ingest.py # leak-harness JSON -> SARIF -> the unified pipeline (PURE PYTHON, CI-gated) + scenarios/ + open-close-declaration.yml # one deterministic leak-harness scenario (+ schema docs) + LeakHarness/ # C# harness — Windows/build-required, NOT CI-gated + LeakHarness.csproj # net472; FlaUI.UIA3 + Microsoft.Diagnostics.Runtime + YamlDotNet + Program.cs # GC+snapshot loop, growth assertion, JSON result + Scenario.cs # YAML model + HeapCounter.cs # procdump + ClrMD: count live instances of suspect types +``` + +## How the leak-harness works (Plan.md §4.1) + +Deterministic loop, run on the local Windows machine against the target: + +1. Launch the target (FlaUI), run the scenario once to warm up (JIT + lazy caches), + take the **baseline** retained-instance count of each suspect type. +2. Replay the scenario `iterations` times; each cycle requests a GC in the target + (SematixTrace) and the loop ends with a **final** snapshot. +3. A suspect **leaks** when `(final − baseline) / iterations > threshold` — retained + instances grow ~linearly with the open/close count. A clean loop is *not* a + finding (it's evidence of no leak). + +```bash +# on Windows, against a built/running target: +LeakHarness.exe --scenario audit/runtime/scenarios/open-close-declaration.yml \ + --procdump procdump.exe --out artifacts/own-audit/leak-harness.json \ + --target acme/LegacyApp --commit "$COMMIT" + +# then, anywhere (this is what CI exercises): +python audit/runtime/ingest.py --leak-harness artifacts/own-audit/leak-harness.json \ + --out artifacts/own-audit/leak-harness.sarif +# -> drop leak-harness.sarif next to the static SARIFs; run_static aggregation +# folds it in and a confirmed leak clusters with its static OWN014/OWN001. +``` + +## Selftest + +`ingest.py` carries embedded-fixture selftests (no harness, no Windows needed) and +gates on Linux CI — including the end-to-end check that a static OWN014 plus a +runtime leak in the same file form one high-confidence cluster: + +```bash +python audit/runtime/ingest.py --selftest +``` + +## Status + +- **Done:** the runtime→pipeline bridge (`ingest.py`, CI-gated), the leak-harness + scenario schema + one scenario, runtime rule mappings in the taxonomy (categories + 2/3/4/11), and the C# leak-harness skeleton. +- **Deferred:** the ClrMD duplicate-immutable detector and PropertyChanged-storm + profiler (more C# over the same dump/ETW), PerfView/SematixTrace wiring, and a + scenario corpus for the top-N screens. diff --git a/audit/runtime/ingest.py b/audit/runtime/ingest.py new file mode 100644 index 00000000..ad8485fb --- /dev/null +++ b/audit/runtime/ingest.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +""" +Own.NET Audit — runtime ingest bridge (Plan.md §4 → §3.5). + +Converts the leak-harness's JSON result into SARIF so runtime findings flow through +the *same* ``normalize → score → report`` pipeline as the static tiers — runtime is +not a separate report, it's more tools feeding one model. The pay-off: a +runtime-confirmed leak that lands in the same file as a static finding clusters +with it → **high confidence** (the static→runtime confirmation in Plan.md §3.5, +e.g. own-check OWN014 + leak-harness on ``VideoSource.xaml.cs:123``). + +The C# leak-harness (``audit/runtime/LeakHarness/``, Windows / build-required) runs +the deterministic GC+snapshot loop and writes the JSON; this bridge is pure Python +and gates on Linux CI, like the aggregation selftests. + +Usage: + ingest.py --leak-harness result.json --out artifacts/own-audit/leak-harness.sarif + ingest.py --selftest +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +_AGG = Path(__file__).resolve().parent.parent / "aggregate" +sys.path.insert(0, str(_AGG)) +from normalize import load_taxonomy, normalize_results, parse_sarif # noqa: E402 +from score import score # noqa: E402 + +DEFAULT_TAXONOMY = (Path(__file__).resolve().parent.parent + / "static" / "taxonomy" / "categories.yml") + + +def leak_harness_to_sarif(result: dict[str, Any]) -> dict[str, Any]: + """Turn one leak-harness JSON result into a SARIF 2.1.0 log. Only findings with + ``leaked: true`` (the deterministic growth assertion tripped) become results; a + clean iteration loop is evidence of *no* leak, not a finding.""" + tool = result.get("tool", "leak-harness") + rules: dict[str, dict[str, Any]] = {} + results: list[dict[str, Any]] = [] + for f in result.get("findings", []): + if not f.get("leaked"): + continue + rule_id = f.get("rule", "RUNTIME-LEAK") + rules.setdefault(rule_id, {"id": rule_id, "name": rule_id, + "defaultConfiguration": {"level": "error"}}) + # A type-based finding (e.g. duplicate-immutable) may have no source line; + # keep it file-level (no region) rather than fabricate line 1. parse_sarif + # drops results with NO location, so always carry at least an artifactLocation + # (the source file when known, else the type as a synthetic uri). + uri = f.get("location") or f.get("type") or "runtime" + phys: dict[str, Any] = {"artifactLocation": {"uri": uri}} + line = int(f.get("line", 0) or 0) + if line >= 1: + phys["region"] = {"startLine": line} + msg = (f.get("message", "") + + f" [scenario: {result.get('scenario', '?')}, " + + f"x{result.get('iterations', '?')} iterations]") + results.append({ + "ruleId": rule_id, + "level": "error", + "message": {"text": msg}, + "locations": [{"physicalLocation": phys}], + "properties": {k: f[k] for k in + ("type", "baseline", "final", "growthPerIteration", "threshold") + if k in f}, + }) + return { + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + "version": "2.1.0", + "runs": [{ + "tool": {"driver": { + "name": tool, + "informationUri": "https://github.com/PhysShell/Own.NET", + "rules": list(rules.values()), + }}, + "results": results, + "properties": { + "target": result.get("target", ""), + "commit": result.get("commit", ""), + "scenario": result.get("scenario", ""), + "iterations": result.get("iterations", 0), + }, + }], + } + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="Convert a leak-harness JSON result to SARIF.") + ap.add_argument("--leak-harness", help="leak-harness result JSON") + ap.add_argument("--out", default="artifacts/own-audit/leak-harness.sarif") + ap.add_argument("--selftest", action="store_true") + args = ap.parse_args(argv) + + if args.selftest: + return _selftest() + if not args.leak_harness: + ap.error("--leak-harness is required (or use --selftest)") + + result = json.loads(Path(args.leak_harness).read_text(encoding="utf-8")) + sarif = leak_harness_to_sarif(result) + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(sarif, indent=2), encoding="utf-8") + n = len(sarif["runs"][0]["results"]) + print(json.dumps({"out": str(out), "leaked_findings": n}, indent=2)) + return 0 + + +# --------------------------------------------------------------------------- # +# Selftest — embedded fixtures; proves runtime findings flow into the pipeline # +# and confirm static findings (Plan.md §3.5). Pure Python, gates on Linux CI. # +# --------------------------------------------------------------------------- # + +def _sample_result() -> dict[str, Any]: + return { + "tool": "leak-harness", "scenario": "open-close-declaration", + "target": "acme/LegacyApp", "commit": "deadbeef", "iterations": 10, + "findings": [ + # a confirmed subscription leak, correlated to its source line + {"rule": "RUNTIME-LEAK-SUBSCRIPTION", "type": "Acme.Vm.DeclarationViewModel", + "location": "src/Vm/DeclarationViewModel.cs", "line": 123, + "baseline": 1, "final": 11, "growthPerIteration": 1.0, "threshold": 0.5, + "leaked": True, "message": "retained instances grew 1->11 over 10 cycles"}, + # a type-based duplicate-immutable finding — no source line (file-level) + {"rule": "RUNTIME-DUP-IMMUTABLE", "type": "System.String", + "location": "Acme.Ref.Country", "line": 0, + "baseline": 0, "final": 48211, "leaked": True, + "message": "48211 duplicate 'Country' strings on the heap"}, + # a clean loop — NOT a finding, must be dropped + {"rule": "RUNTIME-LEAK-TIMER", "type": "Acme.Vm.CleanTimerViewModel", + "location": "src/Vm/CleanTimerViewModel.cs", "line": 40, + "baseline": 1, "final": 1, "growthPerIteration": 0.0, "threshold": 0.5, + "leaked": False, "message": "stable across 10 cycles"}, + ], + } + + +def _selftest() -> int: + tax = load_taxonomy(DEFAULT_TAXONOMY) + checks: list[str] = [] + + def check(ok: bool, msg: str) -> None: + checks.append("" if ok else msg) + + sarif = leak_harness_to_sarif(_sample_result()) + run = sarif["runs"][0] + results = run["results"] + + check(sarif["version"] == "2.1.0", "sarif version must be 2.1.0") + check(run["tool"]["driver"]["name"] == "leak-harness", "driver name must be leak-harness") + check(len(results) == 2, f"only leaked findings become results: expected 2, got {len(results)}") + rule_ids = {r["ruleId"] for r in results} + check(rule_ids == {"RUNTIME-LEAK-SUBSCRIPTION", "RUNTIME-DUP-IMMUTABLE"}, + f"unexpected rule ids: {rule_ids}") + # source-located finding keeps its region; the type-based one stays file-level + by_rule = {r["ruleId"]: r for r in results} + sub_loc = by_rule["RUNTIME-LEAK-SUBSCRIPTION"]["locations"][0]["physicalLocation"] + dup_loc = by_rule["RUNTIME-DUP-IMMUTABLE"]["locations"][0]["physicalLocation"] + check(sub_loc.get("region", {}).get("startLine") == 123, + "subscription leak must keep its source line") + check("region" not in dup_loc, "type-based duplicate finding must stay file-level (no region)") + check(run["properties"]["scenario"] == "open-close-declaration", "scenario lost from run props") + + # runtime findings flow through the static taxonomy: correct categories. + findings = normalize_results(parse_sarif(json.dumps(sarif), "leak-harness", []), tax) + cat = {f.rule: (f.category, f.category_name) for f in findings} + check(cat.get("RUNTIME-LEAK-SUBSCRIPTION") == (2, "subscription-leak"), + f"runtime subscription leak -> category 2, got {cat.get('RUNTIME-LEAK-SUBSCRIPTION')}") + check(cat.get("RUNTIME-DUP-IMMUTABLE") == (11, "duplicate-immutable"), + f"runtime duplicate-immutable -> category 11, got {cat.get('RUNTIME-DUP-IMMUTABLE')}") + + # Plan.md §3.5: a runtime-confirmed leak in the same file as a static finding + # clusters with it -> high confidence (static->runtime confirmation). + static_sarif = json.dumps({"version": "2.1.0", + "runs": [{"tool": {"driver": {"name": "Own.NET"}}, + "results": [{"ruleId": "OWN014", "level": "warning", + "message": {"text": "region escape: subscribes to AppEventBus, no -="}, + "locations": [{"physicalLocation": { + "artifactLocation": {"uri": "src/Vm/DeclarationViewModel.cs"}, + "region": {"startLine": 122}}}]}]}]}) + both = (normalize_results(parse_sarif(static_sarif, "own-check", []), tax) + + normalize_results(parse_sarif(json.dumps(sarif), "leak-harness", []), tax)) + scored = score(both, tax, line_tol=3) + confirmed = [c for c in scored["clusters"] + if c.confidence == "high" and set(c.tools) == {"own-check", "leak-harness"}] + check(len(confirmed) == 1, + f"static OWN014 + runtime leak in the same file must form 1 high-confidence cluster, " + f"got {[(c.module, c.tools, c.confidence) for c in scored['clusters']]}") + + fails = [c for c in checks if c] + for f in fails: + print(f"INGEST SELFTEST FAIL: {f}") + print(f"ingest selftest: {len(checks) - len(fails)}/{len(checks)} checks passed") + return 1 if fails else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/audit/runtime/scenarios/open-close-declaration.yml b/audit/runtime/scenarios/open-close-declaration.yml new file mode 100644 index 00000000..9126f611 --- /dev/null +++ b/audit/runtime/scenarios/open-close-declaration.yml @@ -0,0 +1,47 @@ +# Own.NET Audit — leak-harness scenario (Plan.md §4.1). +# +# A scenario is the part an AI/dev drafts (which screen, how to open and close it); +# the asserts are deterministic — the harness opens the screen `iterations` times, +# forces a GC + heap snapshot each cycle, and flags a leak when a suspect type's +# retained-instance count grows faster than `threshold` per iteration. +# +# Schema: +# name: report id for the scenario +# app: path to the target .exe (FlaUI launches it) +# iterations: open/close cycles after a warm-up cycle (baseline taken post-warm-up) +# threshold: max allowed retained-instance growth PER ITERATION before it's a leak +# (0.5 ⇒ "should not retain ~1 instance every 2 cycles") +# steps: the cycle, replayed each iteration. action ∈ {open, click, close, wait} +# - open/click/close: `target` is the control's AutomationId +# - wait: `ms` dwell +# suspects: types whose retention is watched. Each maps to a source location and an +# audit rule so a confirmed leak correlates with the matching STATIC +# finding (own-check OWN001/OWN014) → high-confidence cluster (Plan.md §3.5). +# rule ∈ RUNTIME-LEAK-SUBSCRIPTION | RUNTIME-LEAK-TIMER | +# RUNTIME-DPD-ADDVALUECHANGED | RUNTIME-DUP-IMMUTABLE + +name: open-close-declaration +app: C:\Targets\LegacyApp\LegacyApp.exe +iterations: 10 +threshold: 0.5 + +steps: + - { action: open, target: OpenDeclarationButton } + - { action: wait, ms: 800 } + - { action: click, target: GraphTab47 } # exercise the heavy "графа 47" view + - { action: wait, ms: 500 } + - { action: close, target: CloseDeclarationButton } + - { action: wait, ms: 500 } + +suspects: + # The declaration view-model: if it subscribes to a long-lived bus without `-=`, + # each open retains one more instance — own-check flags this statically as OWN014. + - type: Acme.Vm.DeclarationViewModel + rule: RUNTIME-LEAK-SUBSCRIPTION + location: src/Vm/DeclarationViewModel.cs + line: 123 + # A DispatcherTimer started on open and never stopped retains the view that owns it. + - type: Acme.Views.DeclarationView + rule: RUNTIME-LEAK-TIMER + location: src/Views/DeclarationView.xaml.cs + line: 0 diff --git a/audit/static/run_static.py b/audit/static/run_static.py index 5cfa566f..4ce77d7d 100755 --- a/audit/static/run_static.py +++ b/audit/static/run_static.py @@ -141,6 +141,16 @@ def run(target: str, profile: dict[str, Any], out_dir: Path, target_name: str = tiers.append({"tool": "infersharp", "tier": "build-required", "available": True, "sarif": str(infer), "reason": ""}) + # Runtime tier (Plan.md §4): the leak-harness SARIF (produced by + # audit/runtime/ingest.py from the harness JSON) is folded in here, so a + # runtime-confirmed leak clusters with its static OWN014/OWN001 -> high + # confidence (§3.5) through the orchestrator, not only the lower-level aggregate(). + leak = out_dir / "leak-harness.sarif" + if leak.exists(): + sarif_inputs.append(("leak-harness", str(leak))) + tiers.append({"tool": "leak-harness", "tier": "runtime", "available": True, + "sarif": str(leak), "reason": ""}) + meta = { "target": target_name or target, "commit": commit, "generated": f"{datetime.now(UTC):%Y-%m-%d %H:%M UTC}", @@ -255,11 +265,22 @@ def check(ok: bool, msg: str) -> None: # total derives from the call count "locations": [{"physicalLocation": {"artifactLocation": {"uri": "src/App/Svc.cs"}, "region": {"startLine": 5}}}]}]}]} (out2 / "roslyn" / "ProjA.sarif").write_text(json.dumps(rosl), encoding="utf-8") + # a runtime leak-harness SARIF in the SAME file -> must be folded in by run() + # and cluster with the static finding -> high confidence (Plan.md §3.5 / #102). + leak = {"runs": [{"tool": {"driver": {"name": "leak-harness"}}, "results": [ + {"ruleId": "RUNTIME-LEAK-SUBSCRIPTION", "level": "error", + "message": {"text": "retained Svc grew 1->11"}, + "locations": [{"physicalLocation": {"artifactLocation": {"uri": "src/App/Svc.cs"}, + "region": {"startLine": 6}}}]}]}]} + (out2 / "leak-harness.sarif").write_text(json.dumps(leak), encoding="utf-8") profile = {"name": "t", "severity_floor": "warning", "tiers": {"build_free": []}} res2 = run("/nonexistent-target", profile, out2, target_name="t/p") check(any(t["tool"] == "roslyn-pack" for t in res2["tiers"]), "roslyn per-project SARIF under roslyn/ must be picked up") - check(res2["totals"]["clusters"] >= 1, "roslyn finding must reach the aggregated totals") + check(any(t["tool"] == "leak-harness" for t in res2["tiers"]), + "runtime leak-harness.sarif must be folded in by run()") + check(res2["totals"]["high_confidence"] >= 1, + "runtime leak + static finding in one file must form a high-confidence cluster") fails = [c for c in checks if c] for f in fails: diff --git a/audit/static/taxonomy/categories.yml b/audit/static/taxonomy/categories.yml index d51c03ae..4190f5b6 100644 --- a/audit/static/taxonomy/categories.yml +++ b/audit/static/taxonomy/categories.yml @@ -55,14 +55,26 @@ rules: # ── Category 15: architecture metrics / hotspots ───────────────────────────── "RCS1*": {category: 15, name: architecture} # Roslynator maintainability subset + # ── RUNTIME tier (Plan.md §4) — emitted by audit/runtime/ (leak-harness, ClrMD + # duplicate detector). These categories are NO-TOOL for static analysis, so the + # runtime layer is the ONLY tool that covers them. A runtime finding that lands + # in the same file/category as a static one clusters with it -> high confidence + # (the static→runtime confirmation in Plan.md §3.5). + "RUNTIME-LEAK-SUBSCRIPTION": {category: 2, name: subscription-leak} + "RUNTIME-LEAK-TIMER": {category: 3, name: timer-leak} + "RUNTIME-DPD-ADDVALUECHANGED": {category: 4, name: dpd-addvaluechanged-leak} + "RUNTIME-DUP-IMMUTABLE": {category: 11, name: duplicate-immutable} + # Baseline P-level per category (Plan.md §3.5.2). Confidence (cross-tool # agreement) is a SEPARATE axis handled by score.py; this is severity only. category_severity: 1: P1 # leaks — the high-value classes 2: P1 3: P1 + 4: P1 # DependencyPropertyDescriptor.AddValueChanged leak (runtime-confirmed) 5: P2 9: P2 + 11: P2 # duplicated immutable data — memory bloat (the project's "gold"), perf-tier 14: P2 15: P3 0: P3 # uncategorized