diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2c5a91..f2e71bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,7 @@ jobs: PYTHONPATH=. python3 report/tests/test_baseline.py PYTHONPATH=. python3 arch/tests/test_arch.py PYTHONPATH=. python3 oracle/fixtures/test_oracle_arch.py + PYTHONPATH=. python3 oracle/fixtures/test_oracle_runtime.py PYTHONPATH=. python3 runtime/tests/test_runtime.py - name: re-run under -O (assert-stripping safety) @@ -50,6 +51,7 @@ jobs: PYTHONPATH=. python3 -O report/tests/test_baseline.py PYTHONPATH=. python3 -O arch/tests/test_arch.py PYTHONPATH=. python3 -O oracle/fixtures/test_oracle_arch.py + PYTHONPATH=. python3 -O oracle/fixtures/test_oracle_runtime.py PYTHONPATH=. python3 -O runtime/tests/test_runtime.py audit-report: diff --git a/oracle/LeakyOracle/LeakScenario.cs b/oracle/LeakyOracle/LeakScenario.cs index f090e0f..82cfbde 100644 --- a/oracle/LeakyOracle/LeakScenario.cs +++ b/oracle/LeakyOracle/LeakScenario.cs @@ -7,15 +7,17 @@ namespace LeakyOracle; /// /// A headless, self-validating proof that the oracle leaks — no display, so it runs in CI. It mimics -/// the scenario the runtime audit drives ("open and close a screen N times") for BOTH view-models: +/// the scenario the runtime audit drives ("open and close a screen N times") for TWO independent leak +/// kinds, each with its corrected counterpart run through the SAME WeakReference harness: /// -/// • the leaky (subscribes, never detaches) must ALL survive GC; -/// • the corrected (detaches on Dispose) must ALL be collected. +/// • subscription leak — leaky (subscribes, never detaches) must ALL +/// survive GC; (detaches on Dispose) must ALL be collected; +/// • timer leak — leaky (undisposed Timer) must ALL survive GC; +/// (disposes the Timer) must ALL be collected. /// -/// Checking both with the same WeakReference harness proves the harness isn't rigged: if it were, -/// the fixed batch would look alive too. Exit code 0 means the oracle leaked exactly where it should -/// and nowhere it shouldn't — this is a TARGET, not a test of our auditor; if it stops behaving we -/// fail loudly because the oracle (not the auditor) is broken. +/// Checking the fixed batches too proves the harness isn't rigged: if it were, they'd look alive too. +/// Exit 0 means the oracle leaked exactly where it should and nowhere it shouldn't — this is a TARGET, +/// not a test of our auditor; if it stops behaving we fail loudly because the oracle is broken. /// public static class LeakScenario { @@ -23,21 +25,21 @@ public static int Run(int screens = 50) { var service = new MarketDataService(); - var leakedAlive = OpenAndDrop(screens, () => new WatchlistViewModel(service)); - var fixedAlive = OpenAndDrop(screens, () => - { - var vm = new FixedWatchlistViewModel(service); - return vm; - }, dispose: vm => ((FixedWatchlistViewModel)vm).Dispose()); + var subLeaked = OpenAndDrop(screens, () => new WatchlistViewModel(service)); + var subFixed = OpenAndDrop(screens, () => new FixedWatchlistViewModel(service), + dispose: vm => ((FixedWatchlistViewModel)vm).Dispose()); + var timerLeaked = OpenAndDrop(screens, () => new TickerViewModel()); + var timerFixed = OpenAndDrop(screens, () => new FixedTickerViewModel(), + dispose: vm => ((FixedTickerViewModel)vm).Dispose()); - var leaksWhereItShould = leakedAlive == screens; - var cleanWhereItShould = fixedAlive == 0; - var ok = leaksWhereItShould && cleanWhereItShould; + var ok = subLeaked == screens && subFixed == 0 && timerLeaked == screens && timerFixed == 0; - Console.WriteLine($"screens opened+closed : {screens}"); - Console.WriteLine($"leaky still alive (GC) : {leakedAlive,3} (expect {screens} — rooted by MarketDataService.QuoteReceived)"); - Console.WriteLine($"fixed still alive (GC) : {fixedAlive,3} (expect 0 — detached on Dispose)"); - Console.WriteLine($"verdict : {(ok ? "LEAK confirmed and isolated to the un-detached subscription" : "UNEXPECTED")}"); + Console.WriteLine($"screens opened+closed : {screens}"); + Console.WriteLine($"subscription leaky alive (GC): {subLeaked,3} (expect {screens} — rooted by MarketDataService.QuoteReceived)"); + Console.WriteLine($"subscription fixed alive (GC): {subFixed,3} (expect 0 — detached on Dispose)"); + Console.WriteLine($"timer leaky alive (GC): {timerLeaked,3} (expect {screens} — rooted by the TimerQueue)"); + Console.WriteLine($"timer fixed alive (GC): {timerFixed,3} (expect 0 — Timer disposed)"); + Console.WriteLine($"verdict : {(ok ? "BOTH leaks confirmed, each isolated to its un-released resource" : "UNEXPECTED")}"); Console.WriteLine(ok ? "ORACLE OK: leaks as designed — a valid target for the heap/lifetime audit." : "ORACLE BROKEN: leak signature is wrong; fix the oracle, not the auditor."); diff --git a/oracle/LeakyOracle/ViewModels/FixedTickerViewModel.cs b/oracle/LeakyOracle/ViewModels/FixedTickerViewModel.cs new file mode 100644 index 0000000..8c83260 --- /dev/null +++ b/oracle/LeakyOracle/ViewModels/FixedTickerViewModel.cs @@ -0,0 +1,25 @@ +using System; +using System.Threading; + +namespace LeakyOracle.ViewModels; + +/// +/// The corrected counterpart of : it owns the and +/// disposes it. Disposing unlinks the timer from the runtime's TimerQueue, so once disposed this +/// view-model is no longer rooted and collects normally — the control case that keeps the timer-leak +/// proof honest (same WeakReference harness; correct code collects, leaky code doesn't). +/// +public sealed class FixedTickerViewModel : IDisposable +{ + private readonly Timer _timer; + private int _ticks; + + public FixedTickerViewModel() + { + _timer = new Timer(OnTick, null, 300_000, 300_000); + } + + private void OnTick(object? state) => _ticks++; + + public void Dispose() => _timer.Dispose(); // the fix: unlink from the TimerQueue +} diff --git a/oracle/LeakyOracle/ViewModels/TickerViewModel.cs b/oracle/LeakyOracle/ViewModels/TickerViewModel.cs new file mode 100644 index 0000000..de6df85 --- /dev/null +++ b/oracle/LeakyOracle/ViewModels/TickerViewModel.cs @@ -0,0 +1,31 @@ +using System.Threading; + +namespace LeakyOracle.ViewModels; + +/// +/// A per-screen view-model that drives a live "ticker" off a . DELIBERATELY LEAKY: +/// it starts a recurring in the constructor and never disposes it. +/// +/// An active timer is registered in the runtime's (static) TimerQueue, which holds the timer's callback +/// delegate — and the delegate's target is this view-model. So every TickerViewModel ever created stays +/// rooted by the timer queue until is called: the timer-lifetime leak +/// (own-check small rule; docs/wpf-audit-coverage.md, "Timers": DispatcherTimer/Timers.Timer/ +/// Threading.Timer with no Stop/Dispose). The framework-agnostic core: identical on WPF and Avalonia. +/// +/// The `_timer` field is deliberate — without it the public Timer wrapper would be finalized and stop +/// the timer; holding it keeps the timer alive (and the leak real), exactly as leaky code does. +/// +public sealed class TickerViewModel +{ + private readonly Timer _timer; + private int _ticks; + + public TickerViewModel() + { + // recurring + far-future due time: registered (so it roots `this`) but won't actually fire + // during a sub-second scenario. Never disposed -> never unlinked from the TimerQueue. + _timer = new Timer(OnTick, null, 300_000, 300_000); + } + + private void OnTick(object? state) => _ticks++; +} diff --git a/oracle/README.md b/oracle/README.md index d7949d3..b779a99 100644 --- a/oracle/README.md +++ b/oracle/README.md @@ -23,11 +23,12 @@ and deliberately absent here (`docs/wpf-audit-coverage.md`, "Avalonia mappabilit | Smell | Where | Rule it exercises | |---|---|---| | **Subscription leak** — `+=` to an app-scoped service, never `-=` | `ViewModels/WatchlistViewModel.cs` | OWN001 (event lifetime) + phase-5 heap confirm | +| **Timer leak** — undisposed recurring `System.Threading.Timer` (rooted by the TimerQueue) | `ViewModels/TickerViewModel.cs` | OWN-TIMER (timer no Stop/Dispose) + phase-5 heap confirm | | **Duplicated strings** — `new string(...)` per row, identical content | `ViewModels/WatchlistViewModel.cs` | string-canonicalization (`docs/string-canonicalization.md`) | | **Virtualization killed** — `ListBox` `ItemsPanel` swapped to a plain `StackPanel` | `Views/MainWindow.axaml` | XAML107 (`VirtualizationExplicitlyDisabled`) + heap confirm | -`ViewModels/FixedWatchlistViewModel.cs` is the corrected counterpart (detaches on `Dispose`) — the -control case that keeps the leak proof honest and gives the fix-arm a before/after target. +`ViewModels/Fixed*.cs` are the corrected counterparts (detach on `Dispose` / dispose the timer) — the +control cases that keep the leak proof honest and give the fix-arm a before/after target. ## Build & run @@ -48,14 +49,16 @@ dotnet run -c Release ### What the leak proof shows ``` -screens opened+closed : 50 -leaky still alive (GC) : 50 (expect 50 — rooted by MarketDataService.QuoteReceived) -fixed still alive (GC) : 0 (expect 0 — detached on Dispose) -verdict : LEAK confirmed and isolated to the un-detached subscription +screens opened+closed : 50 +subscription leaky alive (GC): 50 (expect 50 — rooted by MarketDataService.QuoteReceived) +subscription fixed alive (GC): 0 (expect 0 — detached on Dispose) +timer leaky alive (GC): 50 (expect 50 — rooted by the TimerQueue) +timer fixed alive (GC): 0 (expect 0 — Timer disposed) +verdict : BOTH leaks confirmed, each isolated to its un-released resource ORACLE OK: leaks as designed — a valid target for the heap/lifetime audit. ``` -Both view-models go through the **same** WeakReference harness: the leaky one survives a full GC -(rooted by the service event), the fixed one is collected. Checking both proves the harness isn't -rigged — a correct app collects, the oracle does not. If it ever stops leaking, the proof fails loudly -(exit 1): the oracle is broken, not the auditor. +Each leak's leaky and fixed view-models go through the **same** WeakReference harness: the leaky ones +survive a full GC (rooted by the service event / the TimerQueue), the fixed ones are collected. +Checking the fixed batches proves the harness isn't rigged — a correct app collects, the oracle does +not. If it ever stops leaking, the proof fails loudly (exit 1): the oracle is broken, not the auditor. diff --git a/oracle/fixtures/README.md b/oracle/fixtures/README.md index d86b429..3b28249 100644 --- a/oracle/fixtures/README.md +++ b/oracle/fixtures/README.md @@ -43,13 +43,37 @@ back to a view (`WatchlistViewModel → MainWindow`) — lights up three distinc The test keeps the degraded graph in-memory (clean graph + one documented edge) so there's a single source of truth and the regression can't silently drift from the contract. +## Runtime correlation (phase 5) + +The same idea for the runtime side: pin the `runtime.json` contract the слой-2 ClrMD/gcdump collector +must emit for the oracle, and exercise `runtime/correlate.py` on it now. + +- **`findings.json`** — the static leak SUSPECTS own-check would emit for the oracle's two intentional + lifetime leaks (OWN001 subscription on `WatchlistViewModel`, OWN-TIMER on `TickerViewModel`). + `resource` is a *description*, so correlation keys on the source-file stem (the owning class). +- **`runtime.json`** — the heap evidence, faithful to the headless leak proof: 50 of each leaky + view-model survive GC (`expected` 0), plus the 250 000 `QuoteRow` they transitively retain. +- **`test_oracle_runtime.py`** — runs `correlate.py` and asserts the three-way split: + +| bucket | result | +|---|---| +| **confirmed** | `WatchlistViewModel` ×50 + `TickerViewModel` ×50 — both **high**, each naming the leaked CLR type and the static rule it corroborates | +| **static-only** | _(none)_ — both suspects were retained | +| **runtime-only** | `QuoteRow` ×250 000 — a static blind spot (the rows are retained *through* the leaked VMs; the static pass flags the VM, not the pile) | + +The `high` gate fails on the two confirmed leaks — the highest-signal finding the auditor produces. +A drift guard ties every retained type to a real `oracle/LeakyOracle/ViewModels/*.cs`. + ## Run ```bash PYTHONPATH=. python3 oracle/fixtures/test_oracle_arch.py -# or the pass directly: +PYTHONPATH=. python3 oracle/fixtures/test_oracle_runtime.py +# or the passes directly: PYTHONPATH=. python3 -m arch.cli --graph oracle/fixtures/graph.json --rules oracle/fixtures/rules.json # → architecture pass: 0 findings (clean) +PYTHONPATH=. python3 -m runtime.cli --findings oracle/fixtures/findings.json --runtime oracle/fixtures/runtime.json +# → 2 high confirmed · 0 static-only · 1 runtime-only ``` When слой 2 lands in Own.NET, its extractor run over LeakyOracle should reproduce the **internal** diff --git a/oracle/fixtures/findings.json b/oracle/fixtures/findings.json new file mode 100644 index 0000000..266b965 --- /dev/null +++ b/oracle/fixtures/findings.json @@ -0,0 +1,25 @@ +{ + "_comment": "Static leak SUSPECTS for the LeakyOracle app — the findings.json a static pass (own-check) would emit for the oracle's two intentional lifetime leaks. Fed to runtime/correlate.py together with runtime.json to confirm them against heap evidence (test_oracle_runtime.py). `resource` is a DESCRIPTION (has spaces) not a type, so correlate keys on the source-file stem (the owning class) — WatchlistViewModel / TickerViewModel.", + "findings": [ + { + "tool": "own-check", + "rule": "OWN001", + "category_name": "subscription-leak", + "resource": "QuoteReceived subscription", + "path": "oracle/LeakyOracle/ViewModels/WatchlistViewModel.cs", + "line": 32, + "message": "subscribes to MarketDataService.QuoteReceived with no matching unsubscribe", + "suppressed": false + }, + { + "tool": "own-check", + "rule": "OWN-TIMER", + "category_name": "idisposable-leak", + "resource": "Timer never disposed", + "path": "oracle/LeakyOracle/ViewModels/TickerViewModel.cs", + "line": 27, + "message": "System.Threading.Timer created but never disposed (Stop/Dispose missing)", + "suppressed": false + } + ] +} diff --git a/oracle/fixtures/graph.json b/oracle/fixtures/graph.json index b7161cd..4fe7720 100644 --- a/oracle/fixtures/graph.json +++ b/oracle/fixtures/graph.json @@ -9,6 +9,8 @@ { "id": "T:LeakyOracle.ViewModels.QuoteRow", "kind": "type", "name": "QuoteRow", "namespace": "LeakyOracle.ViewModels", "assembly": "LeakyOracle", "internal": true, "loc": { "file": "oracle/LeakyOracle/ViewModels/QuoteRow.cs", "line": 6 }, "metrics": { "methods": 0, "fields": 1, "loc": 8 } }, { "id": "T:LeakyOracle.ViewModels.WatchlistViewModel", "kind": "type", "name": "WatchlistViewModel", "namespace": "LeakyOracle.ViewModels", "assembly": "LeakyOracle", "internal": true, "loc": { "file": "oracle/LeakyOracle/ViewModels/WatchlistViewModel.cs", "line": 19 }, "metrics": { "methods": 2, "fields": 3, "loc": 45 } }, { "id": "T:LeakyOracle.ViewModels.FixedWatchlistViewModel", "kind": "type", "name": "FixedWatchlistViewModel", "namespace": "LeakyOracle.ViewModels", "assembly": "LeakyOracle", "internal": true, "loc": { "file": "oracle/LeakyOracle/ViewModels/FixedWatchlistViewModel.cs", "line": 16 }, "metrics": { "methods": 3, "fields": 3, "loc": 40 } }, + { "id": "T:LeakyOracle.ViewModels.TickerViewModel", "kind": "type", "name": "TickerViewModel", "namespace": "LeakyOracle.ViewModels", "assembly": "LeakyOracle", "internal": true, "loc": { "file": "oracle/LeakyOracle/ViewModels/TickerViewModel.cs", "line": 18 }, "metrics": { "methods": 2, "fields": 2, "loc": 20 } }, + { "id": "T:LeakyOracle.ViewModels.FixedTickerViewModel", "kind": "type", "name": "FixedTickerViewModel", "namespace": "LeakyOracle.ViewModels", "assembly": "LeakyOracle", "internal": true, "loc": { "file": "oracle/LeakyOracle/ViewModels/FixedTickerViewModel.cs", "line": 12 }, "metrics": { "methods": 3, "fields": 2, "loc": 22 } }, { "id": "T:LeakyOracle.Views.MainWindow", "kind": "type", "name": "MainWindow", "namespace": "LeakyOracle.Views", "assembly": "LeakyOracle", "internal": true, "loc": { "file": "oracle/LeakyOracle/Views/MainWindow.axaml.cs", "line": 5 }, "metrics": { "methods": 1, "fields": 0, "loc": 8 } }, { "id": "T:Avalonia.Application", "kind": "type", "name": "Application", "namespace": "Avalonia", "assembly": "Avalonia.Base", "internal": false }, @@ -21,7 +23,8 @@ { "id": "T:System.Collections.Generic.List`1", "kind": "type", "name": "List", "namespace": "System.Collections.Generic", "assembly": "System.Private.CoreLib", "internal": false }, { "id": "T:System.Func`1", "kind": "type", "name": "Func", "namespace": "System", "assembly": "System.Private.CoreLib", "internal": false }, { "id": "T:System.Action`1", "kind": "type", "name": "Action", "namespace": "System", "assembly": "System.Private.CoreLib", "internal": false }, - { "id": "T:System.String", "kind": "type", "name": "String", "namespace": "System", "assembly": "System.Private.CoreLib", "internal": false } + { "id": "T:System.String", "kind": "type", "name": "String", "namespace": "System", "assembly": "System.Private.CoreLib", "internal": false }, + { "id": "T:System.Threading.Timer", "kind": "type", "name": "Timer", "namespace": "System.Threading", "assembly": "System.Private.CoreLib", "internal": false } ], "edges": [ { "from": "T:LeakyOracle.Program", "to": "T:LeakyOracle.App", "kind": "depends" }, @@ -62,6 +65,12 @@ { "from": "T:LeakyOracle.ViewModels.QuoteRow", "to": "T:System.String", "kind": "depends" }, { "from": "T:LeakyOracle.ViewModels.WatchlistViewModel", "to": "T:System.String", "kind": "depends" }, { "from": "T:LeakyOracle.ViewModels.FixedWatchlistViewModel", "to": "T:System.String", "kind": "depends" }, - { "from": "T:LeakyOracle.LeakScenario", "to": "T:System.String", "kind": "depends" } + { "from": "T:LeakyOracle.LeakScenario", "to": "T:System.String", "kind": "depends" }, + + { "from": "T:LeakyOracle.LeakScenario", "to": "T:LeakyOracle.ViewModels.TickerViewModel", "kind": "depends" }, + { "from": "T:LeakyOracle.LeakScenario", "to": "T:LeakyOracle.ViewModels.FixedTickerViewModel", "kind": "depends" }, + { "from": "T:LeakyOracle.ViewModels.TickerViewModel", "to": "T:System.Threading.Timer", "kind": "depends" }, + { "from": "T:LeakyOracle.ViewModels.FixedTickerViewModel", "to": "T:System.Threading.Timer", "kind": "depends" }, + { "from": "T:LeakyOracle.ViewModels.FixedTickerViewModel", "to": "T:System.IDisposable", "kind": "depends" } ] } diff --git a/oracle/fixtures/runtime.json b/oracle/fixtures/runtime.json new file mode 100644 index 0000000..9accdd1 --- /dev/null +++ b/oracle/fixtures/runtime.json @@ -0,0 +1,37 @@ +{ + "_comment": "Golden heap-retention artifact for the LeakyOracle app — what a stand-side ClrMD/gcdump collector would emit after running `LeakyOracle --leak-scenario` (open+close 50 screens). Faithful to the headless leak proof: 50 of each leaky view-model survive GC (expected 0), plus the 250k QuoteRow they transitively retain. Fed to runtime/correlate.py with findings.json to exercise all three buckets (test_oracle_runtime.py). The слой-2 ClrMD collector in Own.NET must emit this shape; here it's hand-authored from the proven leak.", + "schema": "ownAudit/runtime/v1", + "scenario": "open+close 50 watchlist/ticker screens (LeakyOracle --leak-scenario)", + "iterations": 50, + "retained": [ + { + "type": "LeakyOracle.ViewModels.WatchlistViewModel", + "count": 50, + "expected": 0, + "bytes": 11534336, + "roots": [ + { "kind": "static-field", "holder": "Avalonia.Application", "member": "Current", + "via": "App._service (MarketDataService).QuoteReceived delegate -> this" } + ] + }, + { + "type": "LeakyOracle.ViewModels.TickerViewModel", + "count": 50, + "expected": 0, + "roots": [ + { "kind": "timer", "holder": "System.Threading.TimerQueue", + "via": "undisposed Timer callback -> this" } + ] + }, + { + "type": "LeakyOracle.ViewModels.QuoteRow", + "count": 250000, + "expected": 0, + "bytes": 10485760, + "roots": [ + { "kind": "static-field", "holder": "Avalonia.Application", "member": "Current", + "via": "retained transitively through the 50 leaked WatchlistViewModel.Rows" } + ] + } + ] +} diff --git a/oracle/fixtures/test_oracle_arch.py b/oracle/fixtures/test_oracle_arch.py index 6d6b98d..39ce747 100644 --- a/oracle/fixtures/test_oracle_arch.py +++ b/oracle/fixtures/test_oracle_arch.py @@ -11,6 +11,7 @@ This pins the graph.json contract the слой-2 Roslyn extractor must emit for the oracle, and anchors arch/ against a realistic regression. -O-safe (explicit raises). """ +import glob import json import os import sys @@ -18,6 +19,7 @@ HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.dirname(os.path.dirname(HERE)) sys.path.insert(0, ROOT) +APP_DIR = os.path.join(ROOT, "oracle", "LeakyOracle") from arch.graph import Graph # noqa: E402 from arch import rules as R # noqa: E402 @@ -45,16 +47,33 @@ def test_clean_oracle_graph_has_no_arch_findings(): f"{[f['rule'] for f in findings]}") -def test_graph_matches_the_real_oracle_types(): - # guard the contract: every internal node must point at a file that exists under oracle/, - # so this golden can't silently drift from the app it claims to describe. +def _source_type_stems(): + # the oracle is one public type per .cs file; strip .axaml.cs / .cs to the type name. + stems = set() + for cs in glob.glob(os.path.join(APP_DIR, "**", "*.cs"), recursive=True): + if f"{os.sep}obj{os.sep}" in cs or f"{os.sep}bin{os.sep}" in cs: + continue + base = os.path.basename(cs) + stems.add(base[:-len(".axaml.cs")] if base.endswith(".axaml.cs") else base[:-len(".cs")]) + return stems + + +def test_graph_internal_nodes_track_the_oracle_sources_exactly(): + # Strong drift guard, both directions: the internal node set must equal the set of oracle + # source types. A new view-model with no graph node (or a node for a deleted type) fails here + # — not just "every node points at a real file", which a stale-but-incomplete graph passes. g = Graph(json.load(open(GRAPH, encoding="utf-8"))) - internal = g.type_ids() - _expect(len(internal) == 8, f"expected 8 internal oracle types, got {len(internal)}") - for nid in internal: + node_names = {g.name(nid) for nid in g.type_ids()} + sources = _source_type_stems() + _expect(node_names == sources, + f"graph internal types and oracle sources diverged:\n" + f" in graph not in src: {sorted(node_names - sources)}\n" + f" in src not in graph: {sorted(sources - node_names)}") + # and every internal node still anchors at a real file + for nid in g.type_ids(): path = g.node(nid).get("loc", {}).get("file", "") _expect(os.path.isfile(os.path.join(ROOT, path)), - f"node {nid} points at missing file {path!r} — graph drifted from the app") + f"node {nid} points at missing file {path!r}") def test_planted_mvvm_inversion_lights_up_layering_and_both_cycles(): diff --git a/oracle/fixtures/test_oracle_runtime.py b/oracle/fixtures/test_oracle_runtime.py new file mode 100644 index 0000000..893ffa5 --- /dev/null +++ b/oracle/fixtures/test_oracle_runtime.py @@ -0,0 +1,120 @@ +"""Runs the runtime correlation on the LeakyOracle golden fixtures. Bare python3 or pytest: + + PYTHONPATH=. python3 oracle/fixtures/test_oracle_runtime.py + +Exercises phase 5 on real, oracle-shaped data: the static suspects (findings.json) the own-check +pass would emit for the oracle's two intentional leaks, correlated against the heap evidence +(runtime.json) that matches the headless leak proof. Asserts the three-way split: + * both leaks CONFIRMED high (subscription + timer), each naming the leaked CLR type; + * nothing static-only (both suspects retained); + * the 250k transitively-retained QuoteRow surfaces as RUNTIME-ONLY (a static blind spot); + * the high gate fails on the two confirmed leaks. + +Pins the runtime.json contract the слой-2 ClrMD collector must emit for the oracle, and guards it +against drift (every retained type maps to a real oracle source file). -O-safe (explicit raises). +""" +import json +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +sys.path.insert(0, ROOT) + +from runtime import correlate as C # noqa: E402 + +FINDINGS = os.path.join(HERE, "findings.json") +RUNTIME = os.path.join(HERE, "runtime.json") +VM_DIR = os.path.join(ROOT, "oracle", "LeakyOracle", "ViewModels") + +# Pinned config so this golden depends only on its own fixtures — unrelated tuning in the shared +# runtime/config.json must not silently shift the oracle's confirmed/runtime-only split or downgrade +# the expected `high` confidence. (Mirrors the default config; same convention as test_runtime.py.) +ORACLE_CFG = { + "leak_categories": ["subscription-leak", "idisposable-leak", "region-escape"], + "default_expected": 1, "min_count": 2, "high_count": 10, +} + + +def _expect(cond, msg): + if not cond: + raise AssertionError(msg) + + +def _correlate(): + findings = json.load(open(FINDINGS, encoding="utf-8"))["findings"] + dump = json.load(open(RUNTIME, encoding="utf-8")) + return C.correlate(findings, dump, ORACLE_CFG), dump + + +def test_both_oracle_leaks_are_confirmed_high(): + res, _ = _correlate() + by_res = {f["resource"]: f for f in res["confirmed"]} + _expect(set(by_res) == {"LeakyOracle.ViewModels.WatchlistViewModel", + "LeakyOracle.ViewModels.TickerViewModel"}, + f"expected both leaks confirmed, got {sorted(by_res)}") + for t, f in by_res.items(): + _expect(f["confidence"] == "high", f"{t} should be high, got {f['confidence']}") + _expect(f["retained"] == 50 and f["expected"] == 0, f) + # the subscription leak confirms OWN001; the timer leak confirms OWN-TIMER + _expect(by_res["LeakyOracle.ViewModels.WatchlistViewModel"]["static_rule"] == "OWN001", by_res) + _expect(by_res["LeakyOracle.ViewModels.TickerViewModel"]["static_rule"] == "OWN-TIMER", by_res) + + +def test_nothing_is_static_only(): + res, _ = _correlate() + _expect(res["static_only"] == [], f"both suspects should be retained, got " + f"{[f['rule'] for f in res['static_only']]}") + + +def test_transitively_retained_rows_are_a_runtime_only_blind_spot(): + res, _ = _correlate() + _expect(len(res["runtime_only"]) == 1, f"expected one blind-spot type, got {len(res['runtime_only'])}") + ro = res["runtime_only"][0] + _expect(ro["resource"] == "LeakyOracle.ViewModels.QuoteRow" and ro["retained"] == 250000, ro) + _expect(ro["confidence"] == "high", ro) # 250000 >> high_count + + +def test_high_gate_fails_on_the_two_confirmed_leaks(): + res, _ = _correlate() + passed, blocking = C.gate(res, "high") + _expect(not passed and len(blocking) == 2, f"high gate should block 2, got {len(blocking)}") + + +def test_confirmed_are_in_findings_json_shape(): + res, _ = _correlate() + base = {"tool", "rule", "category_name", "resource", "path", "line", "message", "suppressed"} + for f in res["confirmed"]: + _expect(base <= set(f), f"confirmed finding missing base keys: {sorted(f)}") + _expect(f["tool"] == "own-runtime" and f["category_name"] == "runtime-confirmed-leak", f) + + +def test_runtime_types_map_to_real_oracle_files(): + # guard the contract against drift: every retained type must be a real oracle view-model. + _, dump = _correlate() + for rec in dump["retained"]: + short = rec["type"].rsplit(".", 1)[-1] + _expect(os.path.isfile(os.path.join(VM_DIR, short + ".cs")), + f"retained type {rec['type']} has no source {short}.cs under oracle ViewModels") + + +def _main() -> int: + tests = [v for k, v in sorted(globals().items()) + if k.startswith("test_") and callable(v)] + failed = 0 + for t in tests: + try: + t() + print(f"PASS {t.__name__}") + except AssertionError as e: + failed += 1 + print(f"FAIL {t.__name__}: {e}") + except Exception as e: # noqa: BLE001 + failed += 1 + print(f"ERROR {t.__name__}: {type(e).__name__}: {e}") + print(f"\n{len(tests) - failed}/{len(tests)} passed") + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(_main())