diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..6d9c2347 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +*.pyc +*.ownreport.json diff --git a/README.md b/README.md index 49d5e53b..54243790 100644 --- a/README.md +++ b/README.md @@ -46,11 +46,12 @@ C# (шаблоны emit_* → реальный .NET; try/finally на straight- ```bash cd ownlang -python -m ownlang check examples/ok_extern_calls.own # проверка -python -m ownlang emit examples/golden_arraypool/buffer.own # проверка + печать C# -python -m ownlang cfg examples/bad_maybe_release.own # дамп CFG +python -m ownlang check examples/ok_extern_calls.own # проверка +python -m ownlang emit examples/golden_arraypool/buffer.own # проверка + печать C# +python -m ownlang cfg examples/bad_maybe_release.own # дамп CFG +python -m ownlang report examples/buffer_scratch.own # buffer-отчёт + .ownreport.json -python tests/run_tests.py # 42 кейса + codegen + golden smoke +python tests/run_tests.py # кейсы + codegen + golden + buffer smoke ``` `check` возвращает ненулевой код при наличии ошибок — годится для CI. @@ -216,6 +217,17 @@ fn process(size: int) { | **OWN012** | shared borrow при живом `borrow_mut` | | **OWN013** | прямое обращение к владельцу, пока он `borrow_mut` | +### Буферы: storage policies + +| Код | Что ловит | +|-----|-----------| +| OWN015 | stack-backed буфер (`stack`/`scratch`/`inline`) пытается убежать из функции (`return`) | +| OWN016 | stack-backed буфер отдан в `consume`-вызов (move в более долгоживущего владельца) | +| OWN017 | movable-буфер (`pooled`/`native`) escape'ит — модель это разрешает, но PoC-codegen пока не умеет честно лоуэрить escape (см. ниже) | +| OWN019 | inline-ёмкость слишком велика для stack-backed политики (выше потолка стека) | +| OWN021 | `stack`/`inline` динамического размера без статической границы (нет `max =`) | +| OWN023 | `scratch` с `fallback = forbidden`, но размер может превысить inline-лимит | + ### Неподдерживаемое / структурное / граница | Код | Что ловит | @@ -299,6 +311,179 @@ control-flow в `finally` — настоящая работа, она в roadmap --- +## Буферы: storage policies + логирование + +`stackalloc` — это не оптимизация сама по себе. Это **storage strategy с жёстким +lifetime-контрактом**. Поэтому буфер в OwnLang — это owned-ресурс (release ровно +один раз, escape-проверки, конфликты borrow'ов — всё как обычно), но с явной +**политикой хранения**. Модель: *пользователь задаёт intent → checker проверяет +lifetime/ownership → backend выбирает или строго соблюдает storage → codegen +генерит безопасный C# → логи показывают фактический выбор → benchmark доказывает +выигрыш*. Не «компилятор молча решил за тебя» — а «ты задал политику, компилятор +её соблюл, runtime показал, что реально выбралось». + +### Режимы + +``` +let a = Buffer.stack(256); // только stackalloc, fallback запрещён +let b = Buffer.stack(size, max = 1024); // динамика, но с забором (guard) +let c = Buffer.scratch(size, inline = 1024, fallback = pool); // стек, иначе ArrayPool +let d = Buffer.pooled(size); // только ArrayPool; movable, Return обязателен +let e = Buffer.native(size); // NativeMemory; unsafe, Free обязателен +let f = Buffer.inline(128); // фиксированный compile-time стековый буфер +``` + +Главное правило: **`stack` никогда не падает в heap**; **`scratch` может**, потому +что пользователь явно разрешил fallback. API, который врёт про память, — это не +абстракция. `stack`/`scratch`/`inline` — stack-backed → не могут escape (OWN015/016). + +Буфер можно `move` внутри функции — владение и storage-политика переходят на +нового владельца, и `release` нового имени освобождает исходный backing. Namespace +обязан быть `Buffer`: `Foo.stack(...)` (опечатка/чужой идентификатор) — это +**OWN030**, а не тихая аллокация. + +`pooled`/`native` в **ownership-модели** movable (теоретически их можно +`return`/`consume`). Но **deliverable здесь — checker, а codegen лишь доказывает, +что модель лоуэрится в настоящий .NET**, и не раздувается в самоцель. Честно +лоуэрить *escaping* буфер нечем: значение внутри функции — это `Span`, а +отдавать наружу надо handle (`byte[]`/`byte*`+длина), которым вызывающий сделает +`Return`/`Free`. Поэтому PoC **отвергает** escape movable-буфера (**OWN017**), а не +шипает C#, который течёт или не компилируется. Локально `pooled`/`native` работают +полноценно (rent→borrow→release с реальным `ArrayPool.Return`/`NativeMemory.Free`). +Полноценный movable-lowering (через `byte[]`-handle или обёртку +`IMemoryOwner`) — **roadmap**. + +### `scratch` лоуэрится так (это и есть golden buffer-пример) + +```csharp +byte[]? tmp_rented = null; +Span tmp_backing = stackalloc byte[1024]; +Span tmp; +if (size <= 1024) +{ + OwnTrace.ScratchSelected("parse", "tmp", size, 1024, "stackalloc"); + OwnCounters.StackHit(); + tmp = tmp_backing[..size]; +} +else +{ + OwnTrace.ScratchSelected("parse", "tmp", size, 1024, "ArrayPool"); + OwnCounters.PoolFallback(size); + tmp_rented = ArrayPool.Shared.Rent(size); + tmp = tmp_rented.AsSpan(0, size); +} +try { /* ... */ } +finally +{ + OwnCounters.Release(); + if (tmp_rented is not null) + ArrayPool.Shared.Return(tmp_rented); +} +``` + +### Логирование — обязательная часть, а не опция + +Без логов `scratch` стал бы той самой «умной» абстракцией, которая молча выбрала +pool, а ты три часа смотришь на GC-график. Поэтому логи — в трёх местах: + +1. **Compile-time report** (`python -m ownlang report file.own`): что checker/codegen + решил по каждому буферу — mode, inline-лимит, fallback, escape-policy, clear, + сгенерированные ветки и какие проверки прошли. Выводится текстом и пишется в + `file.ownreport.json` (удобно для ревью/CI). + +2. **Runtime trace** — хук `OwnTrace.*` в сгенерированном C#: какой backend реально + выбран при конкретном `size`. Под `[Conditional("OWNSHARP_TRACE")]` — в обычном + Release вызовы вырезаются, логирование не становится новым bottleneck'ом. + +3. **Runtime counters** — `OwnCounters` (`ScratchStackHits`, `ScratchPoolFallbacks`, + `ScratchPoolBytesRented`, `ScratchReleaseCount`) под `[Conditional("OWNSHARP_COUNTERS")]`. + Отвечают на главный вопрос: мы реально часто попадаем в стек, или inline-лимит + подобран мимо? + +### Политики + +`policy`-блок — это переиспользуемый набор дефолтов; буфер ссылается на него через +`policy =`, инлайновые опции перекрывают: + +``` +policy SensitiveScratch { + inline_bytes = 512; + fallback = pool; + counters = true; + clear_on_release = true; // занулить байты перед возвратом в пул +} + +fn handle(size: int) { + let secret = Buffer.scratch(size, policy = SensitiveScratch); + borrow_mut secret as m { Fill(m); } + release secret; // codegen: secret.Clear(); затем Return +} +``` + +### Runnable golden + +`buffer_scratch_program.cs.txt` — запускаемый пример: метод `parse` и классы +`OwnTrace`/`OwnCounters` вклеены **дословно** из `python -m ownlang emit +buffer_scratch.own`, а `Fill`/`Hash`/`Main` — host-код. Доказывает, что +buffer-модель лоуэрится в настоящий .NET с реальным `ArrayPool.Rent/Return`: + +```bash +dotnet run -p:DefineConstants="OWNSHARP_TRACE;OWNSHARP_COUNTERS" +# parse(64) -> stackalloc-ветка (heap не трогаем) +# parse(4096) -> ArrayPool-ветка (реальные Rent/Return), trace + counters в выводе +``` + +### Где это жульничает + +Элемент буфера зафиксирован как `byte` (как во всех примерах). В straight-line +функции (без `if`/`move`/owned-return) буферы и обычные ресурсы лоуэрятся в порядке +исходника, каждый в свой exception-safe `try/finally` со split'ом по точке +`release` — **но только если времена жизни laminar** (любая пара вложена или +раздельна) **и каждый `release` на верхнем уровне**: непересекающиеся остаются +раздельными (a возвращается до аренды b), вложенные — нестятся (LIFO). Частичное +пересечение (`let a; let b; release a; … release b;`), `release` во вложенном +`borrow`/`if`-блоке, или ресурс, consume'нутый вызовом, hoist'ить нельзя без +искажения lifetime'а / двойной очистки, поэтому такие функции лоуэрятся +faithful-inline (release ровно там, где написан; без `try/finally`). +`scratch`/`stack`/`native` динамического размера guard'ят некорректный (в т.ч. +отрицательный) запрос **до** любого trace/counter, чтобы битый ввод не портил +метрики. Размер буфера обязан быть целым числом — `Buffer.pooled(flag: bool)`, owned-ресурс +или plain неизвестного типа (например копия borrow'а) как размер это **OWN018**; +а `inline` требует compile-time литерала — `Buffer.inline(n, max = …)` это +**OWN021** (для динамики есть `stack`). Plain-локал, объявленный в теле буфера и +использованный после release, не оборачивается в hoist'нутый `try` (иначе вышел бы +из C#-scope) — такой буфер лоуэрится inline. +Булевы настройки (`clear_on_release`, `counters`) и `trace` валидируются: опечатка +вроде `clear_on_release = ture` — **OWN030**, а не тихое отключение clear на +sensitive-буфере. `native` хранит `byte*` (backing, освобождается на release), но наружу +отдаёт `Span`-view — borrow/call видят тот же логический тип, что и +pooled/stack/scratch. Borrow-параметр типа `Buffer` (и в `extern`, и в **локальной** +`fn`) рендерится как `Span`/`ReadOnlySpan`, так что один +`fn helper(x: &mut Buffer)` лоуэрится в одну C#-сигнатуру для всех storage-режимов, +а вызов `helper(b)` компилируется. Отчёт атрибутирует диагностики по идентичности +буфера (`name#line:col`, переносится через `move`-алиасы), а не по имени в тексте — +два одноимённых буфера в соседних скоупах не путаются. +В ветвистой функции (есть `if`/`move`/owned-return) используется inline-режим: +буфер с чистым вложением получает `try/finally`, а перекрывающиеся времена жизни, +ветвистый release и moved-алиасы — inline-release (реальный cleanup в местах +release'ов, без подъёма в `finally`; обычные ресурсы там тоже inline — подъём из +произвольного control-flow это roadmap). `native` динамического размера +guard'ит отрицательный запрос перед `NativeMemory.Alloc`. Escaping movable-буферы +отвергаются (OWN017), полноценный movable-lowering — roadmap. Неизвестные значения +**и имена** настроек (mode, namespace, policy, fallback, а также сами имена опций +буфера и ключей policy-блока) ловятся как **OWN030** — опечатка в +`fallback = forbidden`, `fallback = 0` или `fallbak = forbidden` не «протечёт» в +heap, а будет отвергнута. Повторённая опция/ключ (`fallback = forbidden, +fallback = pool`) — тоже **OWN030**: конфликтующее обещание не разрешается +правилом «последний выигрывает». Бенчмарк-матрица из дизайн-дока +(safe vs unsafe, stack vs pool на размерах 32 B … 1 MB) — **следующий слой**: +правило «unsafe-backend разрешён только при выигрыше ≥ 10-15 % с disassembly- +обоснованием» задаёт дисциплину, но прогон бенчей вне песочницы. Unsafe-контракты +(`UNS0xx`) пока не реализованы: `native` лоуэрится в `NativeMemory.Alloc/Free` в +`unsafe`-блоке, но pointer-escape проверки — roadmap. + +--- + ## Changelog: перенумерация кодов Коды переразложены в связную схему. Если ты смотришь вывод прошлой версии: @@ -379,13 +564,15 @@ control-flow в `finally` — настоящая работа, она в roadmap ownlang/ ownlang/ lexer.py # токенизатор; цикл/async лексятся как REJECTED; строки для emit_* - ast_nodes.py # dataclass-узлы AST (resource, extern, call, эффекты) + ast_nodes.py # dataclass-узлы AST (resource, extern, call, эффекты, buffer, policy) parser.py # recursive descent; грамматика в docstring + buffers.py # storage policies: режимы, резолв policy+intent, валидация cfg.py # resolver (Symbol/Kind) + collect_signatures + lowering, Invoke analysis.py # flow-sensitive dataflow: var-states + active loans + permissions diagnostics.py # коды OWN0xx в одном месте - codegen.py # C# codegen (emit_* шаблоны, try/finally hoist + inline) - __main__.py # CLI: check / emit / cfg + codegen.py # C# codegen (emit_* шаблоны, try/finally hoist + inline, буферы) + report.py # compile-time buffer report -> stdout + .ownreport.json + __main__.py # CLI: check / emit / cfg / report examples/ ok_*.own # проходят bad_*.own # падают с конкретным кодом diff --git a/__main__.py b/__main__.py index 901657f7..634aa8e6 100644 --- a/__main__.py +++ b/__main__.py @@ -4,6 +4,7 @@ python -m ownlang check file.own # report ownership diagnostics python -m ownlang emit file.own # check, then print generated C# python -m ownlang cfg file.own # dump the control-flow graph + python -m ownlang report file.own # buffer storage report + .ownreport.json Exit code is non-zero if any error-level diagnostic was produced. """ @@ -14,9 +15,11 @@ from .parser import parse, ParseError from .lexer import LexError -from .cfg import build_cfg, collect_signatures, CFG +from .cfg import build_cfg, collect_signatures, collect_policies, CFG from .analysis import analyze from .codegen import generate +from .buffers import validate_policies +from .report import build_report, render_report from .diagnostics import Diagnostic, Severity @@ -28,9 +31,10 @@ def _collect(src: str) -> tuple[list[Diagnostic], object | None]: return [Diagnostic("OWN020", str(e).split(": ", 1)[-1], line)], None rnames = {r.name for r in mod.resources} sigs = collect_signatures(mod) - diags: list[Diagnostic] = [] + pols = collect_policies(mod) + diags: list[Diagnostic] = list(validate_policies(pols)) for fn in mod.functions: - cfg, d1 = build_cfg(fn, rnames, sigs) + cfg, d1 = build_cfg(fn, rnames, sigs, pols) d2 = analyze(cfg) diags.extend(d1) diags.extend(d2) @@ -73,12 +77,38 @@ def cmd_cfg(path: str) -> int: return 1 rnames = {r.name for r in mod.resources} sigs = collect_signatures(mod) + pols = collect_policies(mod) for fn in mod.functions: - cfg, _ = build_cfg(fn, rnames, sigs) + cfg, _ = build_cfg(fn, rnames, sigs, pols) _print_cfg(cfg) return 0 +def cmd_report(path: str) -> int: + """Emit the compile-time buffer report: what storage policy the checker and + codegen settled on for every buffer, and which checks passed. Prints a + human summary to stdout and writes the machine-readable .ownreport.json.""" + src = _read(path) + diags, mod = _collect(src) + if mod is None: + for d in diags: + print(d.render(path), file=sys.stderr) + return 1 + # surface diagnostics (e.g. a mistyped buffer mode) without crashing the + # report; the report still covers every well-formed buffer. + errors = [d for d in diags if d.severity == Severity.ERROR] + for d in diags: + print(d.render(path), file=sys.stderr) + report = build_report(mod, diags) # type: ignore[arg-type] + print(render_report(report)) + out_path = path.rsplit(".", 1)[0] + ".ownreport.json" + import json + with open(out_path, "w", encoding="utf-8") as f: + json.dump(report, f, indent=2) + print(f"\nwrote {out_path}") + return 1 if errors else 0 + + def _print_cfg(cfg: CFG) -> None: print(f"fn {cfg.fn_name} (entry: B{cfg.entry}, " f"params: {[p.name for p in cfg.params]})") @@ -91,10 +121,15 @@ def _print_cfg(cfg: CFG) -> None: def _fmt_instr(ins) -> str: - from .cfg import (Acquire, MoveInto, Release, Use, Invoke, + from .cfg import (Acquire, AcquireBuffer, MoveInto, Release, Use, Invoke, BorrowStart, BorrowEnd, Return) if isinstance(ins, Acquire): return f"acquire {ins.sym.name} : {ins.resource}" + if isinstance(ins, AcquireBuffer): + i = ins.info + size = i.size_const if i.size_is_const else (i.size_var or "?") + return (f"buffer {ins.sym.name} : {i.mode.value}(size={size}, " + f"inline={i.inline_bytes}, fallback={'pool' if i.fallback_pool else 'none'})") if isinstance(ins, MoveInto): return f"move {ins.src.name} -> {ins.dst.name}" if isinstance(ins, Release): @@ -122,11 +157,12 @@ def _read(path: str) -> str: def main(argv: list[str]) -> int: - if len(argv) < 2 or argv[0] not in {"check", "emit", "cfg"}: + if len(argv) < 2 or argv[0] not in {"check", "emit", "cfg", "report"}: print(__doc__) return 2 cmd, path = argv[0], argv[1] - return {"check": cmd_check, "emit": cmd_emit, "cfg": cmd_cfg}[cmd](path) + return {"check": cmd_check, "emit": cmd_emit, "cfg": cmd_cfg, + "report": cmd_report}[cmd](path) if __name__ == "__main__": diff --git a/analysis.py b/analysis.py index 2ab46e47..8e6f2cc1 100644 --- a/analysis.py +++ b/analysis.py @@ -38,7 +38,8 @@ from .cfg import ( CFG, Block, Symbol, Kind, - Acquire, MoveInto, Release, Use, Invoke, BorrowStart, BorrowEnd, Return, + Acquire, AcquireBuffer, MoveInto, Release, Use, Invoke, + BorrowStart, BorrowEnd, Return, ) from .ast_nodes import Effect from .diagnostics import Diagnostic @@ -103,8 +104,9 @@ def initial_state(self) -> State: s.var[id(p)] = {VarState.OWNED} return s - def err(self, code: str, msg: str, line: int) -> None: - self.diags.append(Diagnostic(code, msg, line)) + def err(self, code: str, msg: str, line: int, + subject: str | None = None) -> None: + self.diags.append(Diagnostic(code, msg, line, subject=subject)) # -- loan / permission helpers ----------------------------------------- @@ -129,24 +131,28 @@ def binding_live(self, st: State, sym: Symbol) -> bool: # gone / maybe-gone cases shared by use/move/release/borrow/consume. def _state_problem(self, st: State, sym: Symbol, verb: str, line: int) -> bool: S = st.var.get(id(sym), {VarState.OWNED}) + subj = sym.origin if VarState.OWNED not in S: if VarState.MOVED in S: - self.err("OWN005", f"{verb} '{sym.name}' after it was moved", line) + self.err("OWN005", f"{verb} '{sym.name}' after it was moved", + line, subject=subj) elif VarState.ESCAPED in S and VarState.RELEASED not in S: self.err("OWN002", - f"{verb} '{sym.name}' after it was consumed", line) + f"{verb} '{sym.name}' after it was consumed", line, + subject=subj) else: - self.err("OWN002", f"{verb} '{sym.name}' after it was released", line) + self.err("OWN002", f"{verb} '{sym.name}' after it was released", + line, subject=subj) return True if S & {VarState.RELEASED, VarState.ESCAPED}: self.err("OWN009", f"{verb} '{sym.name}', which may have been released on some " - f"path", line) + f"path", line, subject=subj) return True if VarState.MOVED in S: self.err("OWN010", f"{verb} '{sym.name}', which may have been moved on some " - f"path", line) + f"path", line, subject=subj) return True return False @@ -230,7 +236,8 @@ def leak_check(self, st: State, at_line: int, context: str, name = sym.name if sym else f"#{symid}" self.err("OWN001", f"'{name}' is owned but not released {context} " - f"(leaks on at least one path)", at_line) + f"(leaks on at least one path)", at_line, + subject=(sym.origin if sym else None)) def _sym_by_id(self, symid: int) -> Symbol | None: if not hasattr(self, "_symindex"): @@ -263,6 +270,10 @@ def step(self, ins, st: State) -> None: st.var[id(ins.sym)] = {VarState.OWNED} return + if isinstance(ins, AcquireBuffer): + st.var[id(ins.sym)] = {VarState.OWNED} + return + if isinstance(ins, MoveInto): self._consume_like(st, ins.src, "move", ins.line, code_borrowed="OWN007") st.var[id(ins.src)] = {VarState.MOVED} @@ -270,19 +281,21 @@ def step(self, ins, st: State) -> None: return if isinstance(ins, Release): + subj = ins.sym.origin S = st.var.get(id(ins.sym), {VarState.OWNED}) if S == {VarState.RELEASED}: - self.err("OWN003", f"'{ins.sym.name}' is released twice", ins.line) + self.err("OWN003", f"'{ins.sym.name}' is released twice", + ins.line, subject=subj) elif VarState.RELEASED in S: self.err("OWN003", f"'{ins.sym.name}' may already be released on some path " - f"before this release", ins.line) + f"before this release", ins.line, subject=subj) elif not self._state_problem(st, ins.sym, "release", ins.line): shared, mut = self.loans_on(st, ins.sym) if shared or mut: self.err("OWN008", f"cannot release '{ins.sym.name}' while it is borrowed", - ins.line) + ins.line, subject=subj) st.var[id(ins.sym)] = {VarState.RELEASED} return @@ -327,16 +340,29 @@ def step(self, ins, st: State) -> None: self.leak_check(st, at_line=ins.line, context="before return", exclude=ins.sym) if ins.sym is not None: + subj = ins.sym.origin S = st.var.get(id(ins.sym), {VarState.OWNED}) if VarState.OWNED not in S: if VarState.MOVED in S: self.err("OWN005", f"'{ins.sym.name}' returned after it was moved", - ins.line) + ins.line, subject=subj) else: self.err("OWN002", f"'{ins.sym.name}' returned after it was released", - ins.line) + ins.line, subject=subj) + elif ins.sym.buffer is not None and ins.sym.buffer.stack_backed: + self.err("OWN015", + f"'{ins.sym.name}' is a {ins.sym.buffer.mode.value} " + f"buffer and may be stack-backed; it cannot escape " + f"the current function", ins.line, subject=subj) + elif ins.sym.buffer is not None: + self.err("OWN017", + f"'{ins.sym.name}' is a {ins.sym.buffer.mode.value} " + f"buffer; the PoC code generator cannot lower an " + f"escaping buffer to faithful .NET (the caller gets " + f"no handle to Return/Free), so returning it is " + f"rejected", ins.line, subject=subj) st.var[id(ins.sym)] = {VarState.ESCAPED} return @@ -381,6 +407,18 @@ def _apply_effect(self, st: State, sym: Symbol, eff: Effect, if sym.kind == Kind.OWNED: if eff == Effect.CONSUME: self._consume_like(st, sym, "consume", line, code_borrowed="OWN007") + if sym.buffer is not None and sym.buffer.stack_backed: + self.err("OWN016", + f"'{sym.name}' is a {sym.buffer.mode.value} buffer " + f"and may be stack-backed; it cannot be moved to a " + f"longer-lived owner by consuming it in '{callee}'", + line, subject=sym.origin) + elif sym.buffer is not None: + self.err("OWN017", + f"'{sym.name}' is a {sym.buffer.mode.value} buffer; " + f"the PoC code generator cannot lower an escaping " + f"buffer, so consuming it in '{callee}' is rejected", + line, subject=sym.origin) st.var[id(sym)] = {VarState.ESCAPED} elif eff == Effect.BORROW_MUT: self._check_mut_borrowable(st, sym, line) diff --git a/ast_nodes.py b/ast_nodes.py index db839df1..33110275 100644 --- a/ast_nodes.py +++ b/ast_nodes.py @@ -60,7 +60,23 @@ class Move: line: int -Expr = IntLit | VarRef | Acquire | Move +@dataclass +class BufferIntent: + """Buffer.(size, name = value, ...) -> Owned with a storage + policy. `mode` is one of stack/scratch/pooled/native/inline. `size` is the + single positional argument (an IntLit or VarRef), or None. `options` maps a + named option (inline, max, fallback, clear, trace, counters, policy) to its + value expression. `ns` is the namespace as written (must be "Buffer").""" + mode: str + size: "Expr | None" + options: dict[str, "Expr"] + line: int + ns: str = "Buffer" + col: int = 0 + dups: tuple = () # option names that appeared more than once + + +Expr = IntLit | VarRef | Acquire | Move | BufferIntent # ---- statements ----------------------------------------------------------- @@ -177,9 +193,20 @@ class FnDecl: line: int +@dataclass +class PolicyDecl: + """policy Name { key = value; ... } — a named bundle of buffer defaults + (inline_bytes, max_bytes, mode, fallback, trace, counters, clear_on_release).""" + name: str + settings: dict[str, object] + line: int + dups: tuple = () # setting keys that appeared more than once + + @dataclass class Module: name: str resources: list[ResourceDecl] = field(default_factory=list) externs: list[ExternDecl] = field(default_factory=list) functions: list[FnDecl] = field(default_factory=list) + policies: list[PolicyDecl] = field(default_factory=list) diff --git a/bad_scratch_escape.own.txt b/bad_scratch_escape.own.txt new file mode 100644 index 00000000..3ea1f305 --- /dev/null +++ b/bad_scratch_escape.own.txt @@ -0,0 +1,10 @@ +module BadScratchEscape + +// OWN015: a scratch buffer may be stack-backed, so returning it would hand back +// a span into a frame that is about to pop. If you need to return a buffer, use +// Buffer.pooled(...) — a movable owned resource whose Return the checker enforces. + +fn leak(size: int) -> Buffer { + let tmp = Buffer.scratch(size, inline = 1024); + return tmp; +} diff --git a/buffer_scratch.own.txt b/buffer_scratch.own.txt new file mode 100644 index 00000000..8417de87 --- /dev/null +++ b/buffer_scratch.own.txt @@ -0,0 +1,32 @@ +module ScratchDemo + +// A scratch buffer: prefer the stack, fall back to ArrayPool when the request +// exceeds the inline limit. The user states the intent; the checker proves it +// stays local; codegen emits both branches; and OwnTrace/OwnCounters log which +// backend was actually chosen at runtime. +// +// python -m ownlang check buffer_scratch.own # ownership is clean +// python -m ownlang emit buffer_scratch.own # stack-first + pool fallback +// python -m ownlang report buffer_scratch.own # what the compiler decided + +policy DefaultScratch { + inline_bytes = 1024; + fallback = pool; + trace = debug; + counters = true; + clear_on_release = false; +} + +extern fn Fill(borrow_mut Buffer); +extern fn Hash(borrow Buffer); + +fn parse(size: int) { + let tmp = Buffer.scratch(size, inline = 1024, fallback = pool); + borrow_mut tmp as bytes { + Fill(bytes); + } + borrow tmp as view { + Hash(view); + } + release tmp; +} diff --git a/buffer_scratch_program.cs.txt b/buffer_scratch_program.cs.txt new file mode 100644 index 00000000..da0f9bcb --- /dev/null +++ b/buffer_scratch_program.cs.txt @@ -0,0 +1,132 @@ +// Runnable golden for the OwnLang buffer line — a scratch buffer that prefers +// the stack and falls back to System.Buffers.ArrayPool. +// +// The ScratchDemo class body (parse) and the OwnTrace/OwnCounters classes are +// EMITTED VERBATIM by: +// python -m ownlang emit buffer_scratch.own +// Fill/Hash/Main are hand-written host code. This proves the buffer model lowers +// to real, runnable .NET — it is not executed in the PoC sandbox (no .NET SDK). +// +// Build & run (see the runtime trace + counters): +// dotnet run -p:DefineConstants="OWNSHARP_TRACE;OWNSHARP_COUNTERS" +using System; +using System.Buffers; + +public static class ScratchDemo +{ + public static void parse(int size) + { + if (size < 0) + throw new ArgumentOutOfRangeException(nameof(size)); + byte[]? tmp_rented = null; + Span tmp_backing = stackalloc byte[1024]; + Span tmp; + if (size <= 1024) + { + OwnTrace.ScratchSelected("parse", "tmp", size, 1024, "stackalloc"); + OwnCounters.StackHit(); + tmp = tmp_backing[..size]; + } + else + { + OwnTrace.ScratchSelected("parse", "tmp", size, 1024, "ArrayPool"); + OwnCounters.PoolFallback(size); + tmp_rented = ArrayPool.Shared.Rent(size); + tmp = tmp_rented.AsSpan(0, size); + } + try + { + { // mutable borrow of tmp as bytes + var bytes = tmp; + Fill(bytes); + } + { // shared borrow of tmp as view + var view = tmp; + Hash(view); + } + } + finally + { + OwnCounters.Release(); + if (tmp_rented is not null) + ArrayPool.Shared.Return(tmp_rented); + } + } + + // ---- hand-written host (an `extern fn` is a promise the host keeps) ---- + // Borrow params are noescape: these must not retain the span past the call. + static void Fill(Span b) + { + for (int i = 0; i < b.Length; i++) b[i] = (byte)(i & 0xFF); + } + + static void Hash(ReadOnlySpan b) + { + int h = 17; + foreach (var x in b) h = unchecked(h * 31 + x); + Console.WriteLine($"hash={h} len={b.Length}"); + } + + public static void Main() + { + parse(64); // <= inline limit -> stackalloc arm (no heap) + parse(4096); // > inline limit -> ArrayPool arm (real Rent/Return) + // Visible only when built with -p:DefineConstants="OWNSHARP_COUNTERS". + Console.WriteLine( + $"stackHits={OwnCounters.ScratchStackHits} " + + $"poolFallbacks={OwnCounters.ScratchPoolFallbacks} " + + $"poolBytesRented={OwnCounters.ScratchPoolBytesRented}"); + Console.WriteLine("ok"); + } +} + +internal static class OwnTrace +{ + [System.Diagnostics.Conditional("OWNSHARP_TRACE")] + public static void ScratchSelected(string function, string buffer, + int requestedBytes, int inlineLimit, string backend) + => System.Diagnostics.Trace.WriteLine( + $"[OwnSharp] {function}.{buffer}: requested={requestedBytes}, " + + $"inline={inlineLimit}, backend={backend}"); + + [System.Diagnostics.Conditional("OWNSHARP_TRACE")] + public static void StackSelected(string function, string buffer, + int requestedBytes, int inlineLimit) + => System.Diagnostics.Trace.WriteLine( + $"[OwnSharp] {function}.{buffer}: requested={requestedBytes}, " + + $"inline={inlineLimit}, backend=stackalloc"); + + [System.Diagnostics.Conditional("OWNSHARP_TRACE")] + public static void PooledSelected(string function, string buffer, int requestedBytes) + => System.Diagnostics.Trace.WriteLine( + $"[OwnSharp] {function}.{buffer}: requested={requestedBytes}, backend=ArrayPool"); + + [System.Diagnostics.Conditional("OWNSHARP_TRACE")] + public static void NativeSelected(string function, string buffer, int requestedBytes) + => System.Diagnostics.Trace.WriteLine( + $"[OwnSharp] {function}.{buffer}: requested={requestedBytes}, backend=NativeMemory"); +} + +internal static class OwnCounters +{ + public static long ScratchStackHits; + public static long ScratchPoolFallbacks; + public static long ScratchPoolBytesRented; + public static long ScratchReleaseCount; + + [System.Diagnostics.Conditional("OWNSHARP_COUNTERS")] + public static void StackHit() + => System.Threading.Interlocked.Increment(ref ScratchStackHits); + + [System.Diagnostics.Conditional("OWNSHARP_COUNTERS")] + public static void PoolFallback(int bytes) + { + System.Threading.Interlocked.Increment(ref ScratchPoolFallbacks); + System.Threading.Interlocked.Add(ref ScratchPoolBytesRented, bytes); + } + + [System.Diagnostics.Conditional("OWNSHARP_COUNTERS")] + public static void Release() + => System.Threading.Interlocked.Increment(ref ScratchReleaseCount); +} + diff --git a/buffer_stack.own.txt b/buffer_stack.own.txt new file mode 100644 index 00000000..9b8388ae --- /dev/null +++ b/buffer_stack.own.txt @@ -0,0 +1,26 @@ +module StackDemo + +// Strict stack mode: stackalloc only, the heap fallback is FORBIDDEN, the buffer +// cannot escape. A constant bound lowers to `stackalloc byte[N]`; a dynamic size +// requires a `max =` guard so the real stack frame is bounded — "we put up a +// fence", not "the user promised". + +extern fn Fill(borrow_mut Buffer); + +fn fixed_buf() { + let line = Buffer.stack(256); // -> Span line = stackalloc byte[256]; + borrow_mut line as m { + Fill(m); + } + release line; +} + +fn bounded(size: int) +{ + // dynamic size, but bounded: codegen emits a guard then slices the backing. + let buf = Buffer.stack(size, max = 1024); + borrow_mut buf as m { + Fill(m); + } + release buf; +} diff --git a/buffers.py b/buffers.py new file mode 100644 index 00000000..04757192 --- /dev/null +++ b/buffers.py @@ -0,0 +1,452 @@ +""" +Buffer storage policies — the stackalloc / scratch / pool / native line. + +A *buffer* is an owned resource (it is checked for release-exactly-once, escape, +and borrow conflicts just like any `acquire`d resource), but it additionally +carries an explicit **storage policy**. The policy is something the user states +as an intent; the checker proves the lifetime/ownership rules; the backend either +chooses or strictly honours the storage; codegen emits safe C#; and — the part +that matters — the choice is *logged* so nobody has to guess whether a "stack" +buffer quietly went to the heap. + +Modes (intent the user writes as `Buffer.(...)`): + + stack(N) / stack(size, max = M) + stack only. fallback to the heap is FORBIDDEN. cannot escape. the bound + must be statically known (a literal, or a `max =` guard for a dynamic size). + + scratch(size, inline = L, fallback = pool) + prefer the stack; fall back to ArrayPool when the request exceeds the + inline limit. local-only (may be stack-backed, so it cannot escape). + + pooled(size) + ArrayPool only. a movable owned resource (it may escape via consume/return). + Return is mandatory — enforced by the ownership checker. + + native(size) + unmanaged memory via NativeMemory. an unsafe owned resource. Free is + mandatory. movable. + + inline(N) + a fixed compile-time stack buffer. the most predictable mode. cannot escape. + +The single rule the whole design rests on: `stack` never falls into the heap; +`scratch` may, because the user explicitly allowed it. An API that lies about +where memory lives is not an abstraction. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum + +from . import ast_nodes as A +from .diagnostics import Diagnostic + + +class BufferMode(Enum): + STACK = "stack" + SCRATCH = "scratch" + POOLED = "pooled" + NATIVE = "native" + INLINE = "inline" + + +MODE_NAMES = {m.value for m in BufferMode} + +# Recognized named options on a buffer intent, and recognized keys in a policy +# block. An unknown name (a typo like `fallbak`) must be rejected, not silently +# ignored — otherwise a misspelled `fallback = forbidden` quietly defaults to the +# heap, defeating the explicit storage guarantee. +VALID_OPTIONS = frozenset({ + "policy", "inline", "inline_bytes", "max", "max_bytes", + "fallback", "clear", "trace", "counters", +}) +VALID_POLICY_KEYS = frozenset({ + "inline_bytes", "max_bytes", "fallback", "clear_on_release", + "trace", "counters", +}) + +# Modes whose backing storage may live on the stack. A stack-backed buffer must +# not escape the function: it would be a dangling span the instant the frame +# pops. `scratch` is included because at runtime it *might* be the stack arm. +STACK_BACKED = {BufferMode.STACK, BufferMode.SCRATCH, BufferMode.INLINE} + +# Default inline limit for scratch when the user omits `inline =`. +DEFAULT_INLINE_BYTES = 1024 + +# Hard ceiling on how many bytes a stack-backed mode may reserve on the frame. +# Past this you are not optimising, you are inviting a stack overflow. Tunable. +MAX_STACK_BYTES = 4096 + + +@dataclass +class BufferInfo: + """Resolved, validated metadata for one buffer intent. Attached to the + owning Symbol and to the AcquireBuffer instruction, and read by codegen and + the report writer. Resolution is total: by the time this exists, any policy + contradiction has already been turned into a diagnostic.""" + mode: BufferMode + elem: str # element type, e.g. "byte" + # size: exactly one of const / var is set (or both None for a no-arg mode) + size_const: int | None + size_var: str | None + inline_bytes: int # inline capacity / stack bound, in elements + fallback_pool: bool # scratch: heap fallback allowed + fallback_forbidden: bool # stack: heap fallback explicitly forbidden + clear_on_release: bool # zero the bytes before returning/releasing + trace: bool # emit OwnTrace hooks + counters: bool # emit OwnCounters hooks + policy_name: str | None + line: int + + @property + def stack_backed(self) -> bool: + return self.mode in STACK_BACKED + + @property + def size_is_const(self) -> bool: + return self.size_const is not None + + @property + def escape_policy(self) -> str: + return "local-only" if self.stack_backed else "movable" + + def branches(self) -> list[dict]: + """The runtime backend branches, for the compile-time report.""" + if self.mode == BufferMode.SCRATCH and self.fallback_pool: + return [ + {"condition": f"size <= {self.inline_bytes}", "backend": "stackalloc"}, + {"condition": f"size > {self.inline_bytes}", "backend": "ArrayPool"}, + ] + if self.mode in (BufferMode.STACK, BufferMode.INLINE) or ( + self.mode == BufferMode.SCRATCH and not self.fallback_pool): + # a scratch that forbids the heap fallback is, at runtime, stack-only; + # the report must not advertise an ArrayPool branch that cannot occur. + return [{"condition": "always", "backend": "stackalloc"}] + if self.mode == BufferMode.POOLED: + return [{"condition": "always", "backend": "ArrayPool"}] + return [{"condition": "always", "backend": "NativeMemory"}] + + +@dataclass +class Policy: + """A named, reusable `policy { ... }` block of defaults.""" + name: str + settings: dict[str, object] = field(default_factory=dict) + line: int = 0 + dups: tuple = () # setting keys that appeared more than once + + +# -------------------------------------------------------------------------- +# Option parsing helpers (turn raw AST option values into Python values) +# -------------------------------------------------------------------------- + + +def _as_int(expr) -> int | None: + return expr.value if isinstance(expr, A.IntLit) else None + + +def _as_ident(expr) -> str | None: + return expr.name if isinstance(expr, A.VarRef) else None + + +# -------------------------------------------------------------------------- +# Resolution: intent + policies -> validated BufferInfo + diagnostics +# -------------------------------------------------------------------------- + + +def validate_policies(policies: dict[str, Policy]) -> list[Diagnostic]: + """Reject unknown keys in `policy` blocks (a typo like `fallbak` must not be + silently ignored). Reported once, at the policy's declaration.""" + diags: list[Diagnostic] = [] + for pol in policies.values(): + for key in pol.settings: + if key not in VALID_POLICY_KEYS: + diags.append(Diagnostic( + "OWN030", + f"unknown policy setting '{key}' in policy '{pol.name}'; " + f"expected one of {', '.join(sorted(VALID_POLICY_KEYS))}", + pol.line)) + for key in pol.dups: + diags.append(Diagnostic( + "OWN030", + f"duplicate policy setting '{key}' in policy '{pol.name}'", + pol.line)) + return diags + + +def resolve(intent: "A.BufferIntent", policies: dict[str, Policy] + ) -> tuple[BufferInfo, list[Diagnostic]]: + """Resolve one buffer intent against the available policies. Returns the + metadata plus any policy/bound diagnostics (OWN019/021/023). Always returns + a usable BufferInfo so later stages have something to lower.""" + diags: list[Diagnostic] = [] + line = intent.line + + mode = BufferMode(intent.mode) + opts = dict(intent.options) + + # reject misspelled / unknown option names before any defaults are applied, + # so e.g. `fallbak = forbidden` cannot silently fall back to the pool. + for key in intent.options: + if key not in VALID_OPTIONS: + diags.append(Diagnostic( + "OWN030", + f"unknown buffer option '{key}'; expected one of " + f"{', '.join(sorted(VALID_OPTIONS))}", line)) + # a repeated option (e.g. fallback = forbidden, fallback = pool) is a + # conflicting storage promise; reject it instead of letting the last win. + for key in intent.dups: + diags.append(Diagnostic( + "OWN030", f"duplicate buffer option '{key}'", line)) + + # Start from a referenced policy's defaults, then let inline options win. + base: dict[str, object] = {} + pol_name: str | None = None + pol_expr = opts.pop("policy", None) + if pol_expr is not None: + pol_name = _as_ident(pol_expr) + if pol_name is None: + # present but not a policy name (e.g. policy = 0): never fall through + # to defaults silently — that could bypass an intended policy. + diags.append(Diagnostic( + "OWN030", + f"invalid policy reference '{_fallback_token(pol_expr)}'; " + f"expected a policy name", line)) + elif pol_name in policies: + base = dict(policies[pol_name].settings) + else: + diags.append(Diagnostic( + "OWN030", f"undefined policy '{pol_name}'", line)) + + def opt_int(name: str, default: int) -> int: + # present-but-not-an-integer (e.g. inline = bogus) must fail safe and + # diagnose, not silently fall through to the default and change the policy. + if name in opts: + v = _as_int(opts[name]) + if v is not None: + return v + diags.append(Diagnostic( + "OWN030", + f"invalid '{name}' value '{_fallback_token(opts[name])}'; " + f"expected an integer", line)) + return default + if name in base: + bv = base[name] + if isinstance(bv, int) and not isinstance(bv, bool): + return bv + diags.append(Diagnostic( + "OWN030", + f"invalid '{name}' value '{bv}' in policy; expected an integer", + line)) + return default + return default + + def first_int(sources: list[tuple[bool, str]], label: str, + default: int) -> int: + """First *present* source, in priority order, as an integer. Only that + source is validated/diagnosed — lower-priority sources are ignored once a + higher-priority one is present, so an inline option overrides a policy + default even when the (now-irrelevant) policy value is malformed.""" + for from_opts, key in sources: + container = opts if from_opts else base + if key not in container: + continue + raw = container[key] + v = (_as_int(raw) if from_opts + else (raw if isinstance(raw, int) and not isinstance(raw, bool) + else None)) + if v is not None: + return v + where = "" if from_opts else " in policy" + diags.append(Diagnostic( + "OWN030", + f"invalid '{label}' value '{_fallback_token(raw)}'{where}; " + f"expected an integer", line)) + return default + return default + + # ---- size -------------------------------------------------------------- + size_const = _as_int(intent.size) if intent.size is not None else None + size_var = _as_ident(intent.size) if intent.size is not None else None + + # ---- per-mode resolution ---------------------------------------------- + inline_bytes = DEFAULT_INLINE_BYTES + fallback_pool = False + fallback_forbidden = False + + if mode in (BufferMode.STACK, BufferMode.INLINE): + fallback_forbidden = True + if size_const is not None: + inline_bytes = size_const + elif mode == BufferMode.INLINE: + # inline is a fixed compile-time stack buffer: the size MUST be an + # integer literal. A dynamic size (even with `max`) is `stack`, not + # `inline`. + diags.append(Diagnostic( + "OWN021", + "'inline' buffer requires a compile-time integer literal size; " + "use 'stack' with a 'max =' bound for a dynamic size", line)) + inline_bytes = MAX_STACK_BYTES + else: + # dynamic size: needs an explicit, integer max bound. + inline_bytes = MAX_STACK_BYTES # safe default while we validate + if "max" in opts and _as_int(opts["max"]) is None: + diags.append(Diagnostic( + "OWN030", + f"invalid 'max' value '{_fallback_token(opts['max'])}'; " + f"expected an integer", line)) + else: + mx_val = (_as_int(opts["max"]) if "max" in opts + else opt_int("max_bytes", -1)) + if mx_val < 0: + diags.append(Diagnostic( + "OWN021", + f"'{mode.value}' allocation of a dynamic size requires a " + f"statically known bound (add 'max = N')", line)) + else: + inline_bytes = mx_val + + elif mode == BufferMode.SCRATCH: + inline_bytes = first_int( + [(True, "inline"), (True, "inline_bytes"), (False, "inline_bytes")], + "inline", DEFAULT_INLINE_BYTES) + # distinguish "absent" (default to pool) from "present but malformed". + # A present-but-malformed value — a string typo (`forbiden`) OR a + # non-identifier (`fallback = 0`) — must fail safe and diagnose, never + # silently fall through to enabling the heap. + fb_present = "fallback" in opts or "fallback" in base + if fb_present: + raw = opts["fallback"] if "fallback" in opts else base["fallback"] + fb = _fallback_token(raw) + else: + fb = "pool" # scratch defaults to a pool fallback + fb_valid = fb in ("pool", "forbidden") + if fb_present and not fb_valid: + diags.append(Diagnostic( + "OWN030", + f"invalid fallback '{fb}' for scratch buffer; expected 'pool' " + f"or 'forbidden'", line)) + if fb == "pool": + fallback_pool = True + else: + # 'forbidden', or an invalid value handled fail-safe (no heap) + fallback_forbidden = True + # scratch with no heap fallback and a size that may exceed the inline + # limit cannot honour the "stack only" promise. + if fb_valid and (size_const is None or size_const > inline_bytes): + diags.append(Diagnostic( + "OWN023", + f"scratch buffer forbids a heap fallback but its size may " + f"exceed the inline limit of {inline_bytes}; use 'stack' " + f"with a 'max =' bound instead", line)) + + elif mode == BufferMode.POOLED: + fallback_pool = True + + # native: heap-via-unmanaged, nothing extra to resolve here. + + # ---- shared bound check on stack-backed modes ------------------------- + if mode in STACK_BACKED and inline_bytes > MAX_STACK_BYTES: + diags.append(Diagnostic( + "OWN019", + f"inline capacity {inline_bytes} bytes is too large for a " + f"stack-backed buffer (limit {MAX_STACK_BYTES}); use 'pooled' or " + f"raise the policy ceiling deliberately", line)) + + # ---- flags from options / policy -------------------------------------- + clear = _bool_flag(opts.get("clear"), base.get("clear_on_release"), + False, "clear_on_release", diags, line) + trace = _trace_flag(opts.get("trace"), base.get("trace"), True, diags, line) + counters = _bool_flag(opts.get("counters"), base.get("counters"), + True, "counters", diags, line) + + info = BufferInfo( + mode=mode, + elem="byte", + size_const=size_const, + size_var=size_var, + inline_bytes=inline_bytes, + fallback_pool=fallback_pool, + fallback_forbidden=fallback_forbidden, + clear_on_release=clear, + trace=trace, + counters=counters, + policy_name=pol_name, + line=line, + ) + return info, diags + + +def _fallback_token(v) -> str: + """Render a fallback value (an AST expr from an inline option, or a Python + value from a policy) as a display token for validation/diagnostics.""" + if isinstance(v, A.IntLit): + return str(v.value) + if isinstance(v, A.VarRef): + return v.name + if isinstance(v, bool): + return "true" if v else "false" + return str(v) + + +def _bool_flag(opt_expr, policy_val, default: bool, label: str, + diags, line: int) -> bool: + # a malformed boolean (a typo like `ture`, or any non-bool) must be rejected, + # not silently treated as the default — for a sensitive buffer that would + # quietly turn off clear-on-release. + if opt_expr is not None: + name = _as_ident(opt_expr) + if name == "true": + return True + if name == "false": + return False + diags.append(Diagnostic( + "OWN030", + f"invalid '{label}' value '{_fallback_token(opt_expr)}'; " + f"expected true or false", line)) + return default + if policy_val is not None: + if isinstance(policy_val, bool): + return policy_val + diags.append(Diagnostic( + "OWN030", + f"invalid '{label}' value '{policy_val}' in policy; expected true " + f"or false", line)) + return default + return default + + +_TRACE_ON = ("debug", "on", "true") +_TRACE_OFF = ("off", "none", "false") + + +def _trace_flag(opt_expr, policy_val, default: bool, diags, line: int) -> bool: + # `trace = debug` / `trace = off` / `trace = false`; on/off/none/true/false + # toggle the (Conditional) hooks. A malformed value is rejected, not assumed + # on. An inline option wins over the policy value. + if opt_expr is not None: + name = _as_ident(opt_expr) + if name in _TRACE_ON: + return True + if name in _TRACE_OFF: + return False + diags.append(Diagnostic( + "OWN030", + f"invalid 'trace' value '{_fallback_token(opt_expr)}'; expected one " + f"of debug/on/off/none/true/false", line)) + return default + if policy_val is not None: + if isinstance(policy_val, bool): + return policy_val + if policy_val in _TRACE_ON: + return True + if policy_val in _TRACE_OFF: + return False + diags.append(Diagnostic( + "OWN030", + f"invalid 'trace' value '{policy_val}' in policy; expected one of " + f"debug/on/off/none/true/false", line)) + return default + return default diff --git a/cfg.py b/cfg.py index 751db25e..6577938e 100644 --- a/cfg.py +++ b/cfg.py @@ -26,6 +26,7 @@ from . import ast_nodes as A from .ast_nodes import Effect from .diagnostics import Diagnostic +from .buffers import BufferInfo, Policy, resolve as resolve_buffer, MODE_NAMES # --------------------------------------------------------------------------- @@ -51,6 +52,18 @@ class Symbol: # for BORROW symbols: True if it is a mutable borrow (&mut / borrow_mut), # False if shared (&T / borrow). None for non-borrow symbols. borrow_is_mut: bool | None = None + # for OWNED buffer symbols: the resolved storage policy. None for ordinary + # `acquire`d resources. A stack-backed buffer (info.stack_backed) must not + # escape the function. + buffer: BufferInfo | None = None + # the declared/inferred type name (e.g. "int", "bool", a resource name), so a + # buffer size can be required to be an integer. None when unknown. + type_name: str | None = None + # a stable identity for the originating buffer (name#line). Set when a buffer + # is acquired and inherited across `move`, so a diagnostic about any alias can + # be attributed to the right buffer in the report (distinct from a same-named + # buffer in a sibling scope). + origin: str | None = None def __repr__(self) -> str: return f"<{self.name}:{self.kind.name}>" @@ -68,6 +81,16 @@ class Acquire: line: int +@dataclass +class AcquireBuffer: + """Acquire an owned buffer with an explicit storage policy. Carries the + resolved BufferInfo so analysis can apply escape rules and codegen can emit + the right backend (stackalloc / ArrayPool / NativeMemory) plus its logging.""" + sym: Symbol + info: BufferInfo + line: int + + @dataclass class MoveInto: dst: Symbol @@ -118,7 +141,8 @@ class Return: line: int -Instr = Acquire | MoveInto | Release | Use | Invoke | BorrowStart | BorrowEnd | Return +Instr = (Acquire | AcquireBuffer | MoveInto | Release | Use | Invoke + | BorrowStart | BorrowEnd | Return) # --------------------------------------------------------------------------- @@ -187,10 +211,12 @@ def collect_signatures(mod: A.Module) -> dict[str, Signature]: class _Builder: def __init__(self, fn: A.FnDecl, resource_names: set[str], - signatures: dict[str, Signature]): + signatures: dict[str, Signature], + policies: dict[str, Policy] | None = None): self.fn = fn self.resource_names = resource_names self.signatures = signatures + self.policies = policies or {} self.diags: list[Diagnostic] = [] self.blocks: list[Block] = [] self.scopes: list[dict[str, Symbol]] = [] @@ -242,6 +268,7 @@ def build(self) -> CFG: sym = self.declare(p.name, Kind.OWNED, p.line) else: sym = self.declare(p.name, Kind.PLAIN, p.line) + sym.type_name = p.type.name self.params.append(sym) entry = self.new_block("entry") @@ -306,6 +333,8 @@ def lower_let(self, st: A.Let, cur: Block) -> Block: sym = self.declare(st.name, Kind.OWNED, st.line) cur.instrs.append(Acquire(sym, rhs.resource, st.line)) return cur + if isinstance(rhs, A.BufferIntent): + return self.lower_buffer(st, rhs, cur) if isinstance(rhs, A.Move): src = self.lookup(rhs.var, rhs.line) if src is not None and src.kind != Kind.OWNED: @@ -315,6 +344,12 @@ def lower_let(self, st: A.Let, cur: Block) -> Block: src = None dst = self.declare(st.name, Kind.OWNED, st.line) if src is not None: + # a moved buffer keeps its storage policy AND its origin identity: + # a stack-backed buffer is still stack-backed after `move` (escape + # rules carry over), and diagnostics on the alias attribute to the + # original buffer in the report. + dst.buffer = src.buffer + dst.origin = src.origin cur.instrs.append(MoveInto(dst, src, st.line)) return cur if isinstance(rhs, A.VarRef): @@ -324,13 +359,61 @@ def lower_let(self, st: A.Let, cur: Block) -> Block: "OWN032", f"cannot copy owned resource '{src.name}' into '{st.name}'; " f"use 'move {src.name}' to transfer ownership", st.line)) - self.declare(st.name, Kind.PLAIN, st.line) + dst = self.declare(st.name, Kind.PLAIN, st.line) + if src is not None and src.kind == Kind.PLAIN: + dst.type_name = src.type_name # a copy keeps the value's type return cur if isinstance(rhs, A.IntLit): - self.declare(st.name, Kind.PLAIN, st.line) + dst = self.declare(st.name, Kind.PLAIN, st.line) + dst.type_name = "int" return cur raise AssertionError(f"unknown rhs {rhs!r}") + def lower_buffer(self, st: A.Let, rhs: A.BufferIntent, cur: Block) -> Block: + if rhs.ns != "Buffer": + self.diags.append(Diagnostic( + "OWN030", + f"unknown buffer namespace '{rhs.ns}'; buffer intents are " + f"written 'Buffer.(...)'", rhs.line)) + self.declare(st.name, Kind.OWNED, st.line) + return cur + if rhs.mode not in MODE_NAMES: + self.diags.append(Diagnostic( + "OWN030", + f"unknown buffer mode '{rhs.mode}'; expected one of " + f"{', '.join(sorted(MODE_NAMES))}", rhs.line)) + self.declare(st.name, Kind.OWNED, st.line) + return cur + # the size must resolve to an integer: an IntLit, or a plain `int` value. + # A bool/other plain, a borrow, or an owned resource as the size would + # lower to uncompilable C# (Rent(flag) / AsSpan(0, flag)). + if rhs.size is None: + self.diags.append(Diagnostic( + "OWN018", f"buffer '{st.name}' requires a size", rhs.line)) + elif isinstance(rhs.size, A.VarRef): + ssym = self.lookup(rhs.size.name, rhs.size.line) + if ssym is not None and ssym.kind != Kind.PLAIN: + self.diags.append(Diagnostic( + "OWN018", + f"buffer size '{rhs.size.name}' must be an integer, not " + f"{ssym.kind.name.lower()}", rhs.size.line)) + elif ssym is not None and ssym.type_name != "int": + # an unknown-typed plain (e.g. a copy of a borrow) is NOT an int; + # accepting it would lower to Rent(span)/AsSpan(0, span). + what = (f"it is '{ssym.type_name}'" if ssym.type_name + else "its type cannot be determined") + self.diags.append(Diagnostic( + "OWN018", + f"buffer size '{rhs.size.name}' must be an integer " + f"({what})", rhs.size.line)) + info, bdiags = resolve_buffer(rhs, self.policies) + self.diags.extend(bdiags) + sym = self.declare(st.name, Kind.OWNED, st.line) + sym.buffer = info + sym.origin = f"{st.name}#{rhs.line}:{rhs.col}" + cur.instrs.append(AcquireBuffer(sym, info, st.line)) + return cur + def lower_call(self, st: A.Call, cur: Block) -> Block: sig = self.signatures.get(st.callee) if sig is None: @@ -425,8 +508,15 @@ def lower_return(self, st: A.Return, cur: Block) -> Block | None: return None +def collect_policies(mod: A.Module) -> dict[str, Policy]: + return {p.name: Policy(p.name, dict(p.settings), p.line, p.dups) + for p in mod.policies} + + def build_cfg(fn: A.FnDecl, resource_names: set[str], - signatures: dict[str, Signature]) -> tuple[CFG, list[Diagnostic]]: - b = _Builder(fn, resource_names, signatures) + signatures: dict[str, Signature], + policies: dict[str, Policy] | None = None + ) -> tuple[CFG, list[Diagnostic]]: + b = _Builder(fn, resource_names, signatures, policies) cfg = b.build() return cfg, b.diags diff --git a/codegen.py b/codegen.py index 591f385f..09ad6356 100644 --- a/codegen.py +++ b/codegen.py @@ -29,6 +29,7 @@ from __future__ import annotations from . import ast_nodes as A +from .buffers import BufferMode, Policy, resolve as resolve_buffer class CodegenError(Exception): @@ -36,6 +37,11 @@ class CodegenError(Exception): def _csharp_type(t: A.TypeRef) -> str: + # a borrowed Buffer is the buffer's view type: Span / ReadOnlySpan + # (the same view a buffer intent and `emit_borrow` produce), so a local helper + # `fn h(x: &mut Buffer)` lowers to one that accepts a buffer value by value. + if t.name == "Buffer" and t.borrowed: + return "Span" if t.mutable else "ReadOnlySpan" base = {"int": "int", "bool": "bool"}.get(t.name, t.name) if t.borrowed: return ("ref " if t.mutable else "ref readonly ") + base @@ -47,9 +53,17 @@ def __init__(self, mod: A.Module, fn: A.FnDecl): self.mod = mod self.fn = fn self.res = {r.name: r for r in mod.resources} + self.policies: dict[str, Policy] = { + p.name: Policy(p.name, dict(p.settings), p.line) for p in mod.policies + } self.owned_resource: dict[str, str] = {} # borrow binding name -> owner resource type (so calls know the C# view) self.binding_owner_res: dict[str, str] = {} + # buffer local name -> resolved BufferInfo (so borrows/args render right) + self.buffer_vars: dict[str, object] = {} + # buffer local name -> cleanup lines, for buffers whose `release` is + # nested in branches (emitted at each release site, not in a finally) + self.buffer_cleanup: dict[str, list[str]] = {} for p in fn.params: if not p.type.borrowed and p.type.name in self.res: self.owned_resource[p.name] = p.type.name @@ -57,14 +71,35 @@ def __init__(self, mod: A.Module, fn: A.FnDecl): # -- shape detection ---------------------------------------------------- def _is_simple(self) -> bool: - return not _contains_branch_or_transfer(self.fn.body) + # Straight-line functions (no branch / move / owned-return) use the + # try/finally hoist, which nests buffers AND ordinary resources so each + # gets its own exception-safe finally. The hoist is only safe when: + # * every scope has a TOP-LEVEL `release` (a release nested in a borrow/if + # block, or a resource consumed by a call, cannot be hoisted into a + # finally without double-cleaning or emitting a stray release), and + # * scope lifetimes are LAMINAR (every pair nested or disjoint). + # Otherwise it falls back to faithful inline, which emits releases exactly + # where the source put them. + if _contains_branch_or_transfer(self.fn.body): + return False + if not _scopes_release_top_level(self.fn.body): + return False + if _scope_body_has_plain_let(self.fn.body): + # a plain local in a scope's body, used after the release, would be + # trapped in the hoisted try; emit faithfully inline instead. + return False + return _laminar_scopes(self.fn.body) # -- emit --------------------------------------------------------------- def emit(self) -> str: ret = _csharp_type(self.fn.ret) if self.fn.ret else "void" params = ", ".join(f"{_csharp_type(p.type)} {p.name}" for p in self.fn.params) - head = f"public static {ret} {self.fn.name}({params})" + # a function that allocates a native buffer needs pointers, so the whole + # method is `unsafe` (cleaner than scoping each pointer in its own block, + # and it lets native buffers follow the same lifetime shapes as the rest). + unsafe = "unsafe " if _fn_has_native(self.fn.body) else "" + head = f"public static {unsafe}{ret} {self.fn.name}({params})" if self._is_simple(): body = self._emit_simple(self.fn.body) else: @@ -74,63 +109,295 @@ def emit(self) -> str: # -- simple (try/finally hoist) ---------------------------------------- def _emit_simple(self, stmts: list[A.Stmt]) -> str: - acquired: list[tuple[str, str, str]] = [] # (var, resource, args_csv) - other: list[A.Stmt] = [] - for st in stmts: - if isinstance(st, A.Let) and isinstance(st.rhs, A.Acquire): - rt = st.rhs.resource - self.owned_resource[st.name] = rt - args_csv = ", ".join(self._arg(x) for x in st.rhs.args) - acquired.append((st.name, rt, args_csv)) - elif isinstance(st, A.Release): - pass # consumed by the finally - else: - other.append(st) - - lines: list[str] = [] + return "".join(l + "\n" for l in self._emit_hoist(stmts, " ")) + + def _emit_hoist(self, stmts: list[A.Stmt], base: str) -> list[str]: + """Emit a straight-line sequence, nesting each openable resource (an + ordinary acquire or a buffer) in its own try/finally. Statements are + emitted in SOURCE ORDER, and each scope is split at its `release`: the + statements during its lifetime go inside the try, the statements after + the release are emitted after the finally (as siblings). So disjoint + sequential resources stay disjoint (a is returned before b is rented), + while overlapping ones nest (releases run LIFO). A scope with no cleanup + adds no try block.""" ind = " " - - def open_resource(idx: int, base: str) -> None: - var, rt, args_csv = acquired[idx] - lines.append(f"{base}{self._local_type(rt)} {var} = {self._acquire_expr(rt, args_csv)};") - lines.append(f"{base}try") - lines.append(f"{base}{{") - inner = base + ind - if idx + 1 < len(acquired): - open_resource(idx + 1, inner) - else: - for st in other: - lines.extend(self._stmt_inline(st, inner)) - lines.append(f"{base}}}") - lines.append(f"{base}finally") - lines.append(f"{base}{{") - lines.append(f"{base}{ind}{self._release_stmt(var, rt)}") - lines.append(f"{base}}}") - - if acquired: - open_resource(0, ind) - else: - for st in other: - lines.extend(self._stmt_inline(st, ind)) - return "".join(l + "\n" for l in lines) + out: list[str] = [] + i = 0 + while i < len(stmts): + st = stmts[i] + scope = self._scope_lowering(st) + if scope is not None: + prelude, fin = scope + out.extend(base + p for p in prelude) + after = stmts[i + 1:] + rel = self._find_release(after, 0, st.name) + body = after if rel is None else after[:rel] + tail = [] if rel is None else after[rel + 1:] + if fin: + out.append(f"{base}try") + out.append(f"{base}{{") + out.extend(self._emit_hoist(body, base + ind)) + out.append(f"{base}}}") + out.append(f"{base}finally") + out.append(f"{base}{{") + out.extend(base + ind + f for f in fin) + out.append(f"{base}}}") + else: + out.extend(self._emit_hoist(body, base)) + # statements after this scope's release are siblings, emitted + # after its finally so its lifetime ends at the source release. + out.extend(self._emit_hoist(tail, base)) + return out + if isinstance(st, A.Release): + i += 1 # stray release (already consumed by its scope), skip + continue + out.extend(self._stmt_inline(st, base)) + i += 1 + return out + + def _scope_lowering(self, st: A.Stmt) -> tuple[list[str], list[str]] | None: + """If `st` opens a resource (an `acquire` let or a buffer let), return its + (prelude, cleanup) lines; otherwise None.""" + if isinstance(st, A.Let) and isinstance(st.rhs, A.Acquire): + rt = st.rhs.resource + self.owned_resource[st.name] = rt + args_csv = ", ".join(self._arg(x) for x in st.rhs.args) + return ([f"{self._local_type(rt)} {st.name} = " + f"{self._acquire_expr(rt, args_csv)};"], + [self._release_stmt(st.name, rt)]) + if isinstance(st, A.Let) and isinstance(st.rhs, A.BufferIntent): + return self._buffer_lowering(st.name, st.rhs) + return None # -- faithful inline ---------------------------------------------------- def _emit_inline(self, stmts: list[A.Stmt], indent: int) -> str: ind = " " * indent + out = self._emit_block(stmts, ind) + return "".join(l + "\n" for l in out) + + def _emit_block(self, stmts: list[A.Stmt], ind: str) -> list[str]: + """Emit a statement list, lowering each buffer let by its lifetime shape. + + A buffer gets the exception-safe try/finally only when it nests cleanly: + its `release` is straight-line at this level AND no other buffer is + acquired inside its body (which would make the lifetimes overlap in + non-LIFO order). Otherwise — overlapping lifetimes, or a release nested in + branches — it uses inline-release: the prelude here, the real cleanup + emitted at each `release` site (no hoist into finally; the same trade-off + the codegen already makes for branchy ordinary resources).""" out: list[str] = [] - for st in stmts: + i = 0 + while i < len(stmts): + st = stmts[i] + if isinstance(st, A.Let) and isinstance(st.rhs, A.BufferIntent): + rest = stmts[i + 1:] + name = st.name + j = self._find_release(rest, 0, name) + if (j is not None and not _fn_has_buffer(rest[:j]) + and not _body_has_plain_let(rest[:j])): + out.extend(self._emit_buffer_scoped(name, st.rhs, rest[:j], ind)) + i += 1 + j + 1 # consume the let, its body, and its release + continue + if not _buffer_released(rest, name): + # leak or escape — both rejected by the checker first + # (OWN001 / OWN015 / OWN016 / OWN017); unreachable cleanly. + raise CodegenError( + f"buffer '{name}' is never released and does not escape " + f"cleanly; the checker should have rejected this") + out.extend(self._emit_buffer_inline(name, st.rhs, ind)) + i += 1 + continue out.extend(self._stmt_inline(st, ind)) - return "".join(l + "\n" for l in out) + i += 1 + return out + + @staticmethod + def _find_release(stmts: list[A.Stmt], start: int, name: str) -> int | None: + for k in range(start, len(stmts)): + st = stmts[k] + if isinstance(st, A.Release) and st.var == name: + return k + return None + + # -- buffer lowering (the stackalloc / scratch / pool / native line) ----- + + def _buffer_lowering(self, name: str, intent: A.BufferIntent + ) -> tuple[list[str], list[str]]: + """Lower one buffer to its (prelude, cleanup) C# lines. The prelude holds + the allocation + trace/counter hooks; the cleanup is the pool Return / + native Free / clear. How they are placed (try/finally vs inline) is the + caller's decision.""" + info, _ = resolve_buffer(intent, self.policies) + self.buffer_vars[name] = info + # a fresh declaration of this name shadows any stale moved-alias cleanup + # left over from an earlier (now out-of-scope) buffer of the same name. + self.buffer_cleanup.pop(name, None) + fn = self.fn.name + size = self._size_expr(info) + L = info.inline_bytes + pre: list[str] = [] # declarations + trace/counters, before the try + fin: list[str] = [] # cleanup, inside the finally / at release sites + scratch_pool = info.mode == BufferMode.SCRATCH and info.fallback_pool + # the OwnCounters are Scratch.* metrics — they answer "do scratch requests + # hit the stack?". Only scratch buffers touch them; pooled/native/stack + # allocations must not pollute the scratch hit/fallback counts. + sc = info.counters and info.mode == BufferMode.SCRATCH + + if info.mode in (BufferMode.STACK, BufferMode.INLINE) or ( + info.mode == BufferMode.SCRATCH and not info.fallback_pool): + if info.size_is_const: + if info.trace: + pre.append(f'OwnTrace.StackSelected("{fn}", "{name}", {size}, {L});') + if sc: + pre.append("OwnCounters.StackHit();") + if info.size_const == L: + pre.append(f"Span {name} = stackalloc byte[{L}];") + else: + # reserve the inline capacity but expose only the requested + # length (e.g. scratch(64, inline = 1024, fallback = forbidden)). + pre.append(f"Span {name}_backing = stackalloc byte[{L}];") + pre.append(f"Span {name} = {name}_backing[..{size}];") + else: + pre.append(f"if ((uint){size} > {L})") + pre.append(f" throw new ArgumentOutOfRangeException(nameof({size}));") + if info.trace: + pre.append(f'OwnTrace.StackSelected("{fn}", "{name}", {size}, {L});') + if sc: + pre.append("OwnCounters.StackHit();") + pre.append(f"Span {name}_backing = stackalloc byte[{L}];") + pre.append(f"Span {name} = {name}_backing[..{size}];") + if sc: + fin.append("OwnCounters.Release();") + if info.clear_on_release: + fin.append(f"{name}.Clear();") + + elif scratch_pool: + if not info.size_is_const: + # reject a negative size before any trace/counter runs: otherwise + # `size <= limit` takes the stack arm, logs a hit, and the [..size] + # slice throws before the try can balance it — corrupting metrics. + pre.append(f"if ({size} < 0)") + pre.append(f" throw new ArgumentOutOfRangeException(nameof({size}));") + pre.append(f"byte[]? {name}_rented = null;") + pre.append(f"Span {name}_backing = stackalloc byte[{L}];") + pre.append(f"Span {name};") + pre.append(f"if ({size} <= {L})") + pre.append("{") + if info.trace: + pre.append(f' OwnTrace.ScratchSelected("{fn}", "{name}", {size}, {L}, "stackalloc");') + if sc: + pre.append(" OwnCounters.StackHit();") + pre.append(f" {name} = {name}_backing[..{size}];") + pre.append("}") + pre.append("else") + pre.append("{") + if info.trace: + pre.append(f' OwnTrace.ScratchSelected("{fn}", "{name}", {size}, {L}, "ArrayPool");') + if sc: + pre.append(f" OwnCounters.PoolFallback({size});") + pre.append(f" {name}_rented = ArrayPool.Shared.Rent({size});") + pre.append(f" {name} = {name}_rented.AsSpan(0, {size});") + pre.append("}") + if sc: + fin.append("OwnCounters.Release();") + if info.clear_on_release: + fin.append(f"{name}.Clear();") + fin.append(f"if ({name}_rented is not null)") + fin.append(f" ArrayPool.Shared.Return({name}_rented);") + + elif info.mode == BufferMode.POOLED: + # pooled is not scratch: trace it, but do not touch the Scratch.* + # counters (that would report normal pool rents as scratch misses). + if info.trace: + pre.append(f'OwnTrace.PooledSelected("{fn}", "{name}", {size});') + pre.append(f"byte[] {name}_array = ArrayPool.Shared.Rent({size});") + pre.append(f"Span {name} = {name}_array.AsSpan(0, {size});") + if info.clear_on_release: + fin.append(f"{name}.Clear();") + fin.append(f"ArrayPool.Shared.Return({name}_array);") + + elif info.mode == BufferMode.NATIVE: + # the method is emitted `unsafe`, so the pointer needs no local block. + # The pointer is the backing (freed on release); the buffer is exposed + # as a Span view, so borrows/calls see the SAME logical type as + # pooled/stack/scratch — an `extern fn Fill(borrow_mut Buffer)` lowers + # to one C# signature (Span) regardless of storage mode. + if not info.size_is_const: + # a negative size would wrap when cast to nuint and request an + # enormous allocation; reject it before Alloc (and before clear). + pre.append(f"if ({size} < 0)") + pre.append(f" throw new ArgumentOutOfRangeException(nameof({size}));") + if info.trace: + pre.append(f'OwnTrace.NativeSelected("{fn}", "{name}", {size});') + pre.append(f"byte* {name}_ptr = (byte*)System.Runtime.InteropServices." + f"NativeMemory.Alloc((nuint){size});") + pre.append(f"Span {name} = new Span({name}_ptr, {size});") + if info.clear_on_release: + fin.append(f"{name}.Clear();") + fin.append(f"System.Runtime.InteropServices.NativeMemory.Free({name}_ptr);") + + return pre, fin + + def _emit_buffer_scoped(self, name: str, intent: A.BufferIntent, + body: list[A.Stmt], ind: str) -> list[str]: + """The buffer nests cleanly: wrap its body in an exception-safe + try/finally (this is the golden scratch shape).""" + pre, fin = self._buffer_lowering(name, intent) + lines = [ind + p for p in pre] + body_lines = self._emit_block(body, ind + " ") + if fin: + lines.append(f"{ind}try") + lines.append(f"{ind}{{") + lines.extend(body_lines) + lines.append(f"{ind}}}") + lines.append(f"{ind}finally") + lines.append(f"{ind}{{") + for f in fin: + lines.append(ind + " " + f) + lines.append(f"{ind}}}") + else: + # nothing to clean up (e.g. a stack buffer with no clear and no + # counters): the body runs straight, the frame reclaims the bytes. + for bl in body_lines: + lines.append(bl[4:] if bl.startswith(" ") else bl) + return lines + + def _emit_buffer_inline(self, name: str, intent: A.BufferIntent, + ind: str) -> list[str]: + """Overlapping or branchy lifetime: emit the prelude here and attach the + cleanup to each `release` site (handled by _stmt_inline). No try/finally — + the same exception-safety trade-off as branchy ordinary resources.""" + pre, fin = self._buffer_lowering(name, intent) + self.buffer_cleanup[name] = fin + return [ind + p for p in pre] + + def _size_expr(self, info) -> str: + if info.size_is_const: + return str(info.size_const) + if info.size_var: + return info.size_var + return "0" def _stmt_inline(self, st: A.Stmt, ind: str) -> list[str]: if isinstance(st, A.Let): + # a (re)declaration of this name shadows any stale buffer-cleanup + # alias from an earlier same-named buffer in another scope. + self.buffer_cleanup.pop(st.name, None) if isinstance(st.rhs, A.Acquire): rt = st.rhs.resource self.owned_resource[st.name] = rt args_csv = ", ".join(self._arg(x) for x in st.rhs.args) return [f"{ind}{self._local_type(rt)} {st.name} = {self._acquire_expr(rt, args_csv)};"] if isinstance(st.rhs, A.Move): + # a moved buffer carries its identity to the new owner: copy the + # pending cleanup to the new name (do NOT remove the original — + # sibling branches that did not move still need it, and releasing + # the original after a move is use-after-move, rejected upstream). + if st.rhs.var in self.buffer_cleanup: + self.buffer_cleanup[st.name] = self.buffer_cleanup[st.rhs.var] + if st.rhs.var in self.buffer_vars: + self.buffer_vars[st.name] = self.buffer_vars[st.rhs.var] self.owned_resource[st.name] = self.owned_resource.get(st.rhs.var, "") return [f"{ind}var {st.name} = {st.rhs.var}; " f"// ownership moved from {st.rhs.var}"] @@ -139,6 +406,10 @@ def _stmt_inline(self, st: A.Stmt, ind: str) -> list[str]: if isinstance(st.rhs, A.VarRef): return [f"{ind}var {st.name} = {st.rhs.name};"] if isinstance(st, A.Release): + if st.var in self.buffer_cleanup: + # a branchy buffer release: emit this buffer's real cleanup + # (pool Return / native Free / clear), not a generic Dispose. + return [ind + line for line in self.buffer_cleanup[st.var]] rt = self.owned_resource.get(st.var, "") return [f"{ind}{self._release_stmt(st.var, rt)}"] if isinstance(st, A.Use): @@ -151,20 +422,16 @@ def _stmt_inline(self, st: A.Stmt, ind: str) -> list[str]: self.binding_owner_res[st.binding] = rt head = [f"{ind}{{ // {kind} borrow of {st.owner} as {st.binding}", f"{ind} var {st.binding} = {self._borrow_expr(st.owner, rt)};"] - body: list[str] = [] - for inner in st.body: - body.extend(self._stmt_inline(inner, ind + " ")) + body = self._emit_block(st.body, ind + " ") return head + body + [f"{ind}}}"] if isinstance(st, A.If): out = [f"{ind}if ({st.cond_text or 'cond'})", f"{ind}{{"] - for s in st.then_body: - out.extend(self._stmt_inline(s, ind + " ")) + out.extend(self._emit_block(st.then_body, ind + " ")) out.append(f"{ind}}}") if st.else_body: out.append(f"{ind}else") out.append(f"{ind}{{") - for s in st.else_body: - out.extend(self._stmt_inline(s, ind + " ")) + out.extend(self._emit_block(st.else_body, ind + " ")) out.append(f"{ind}}}") return out if isinstance(st, A.Return): @@ -228,17 +495,213 @@ def _contains_branch_or_transfer(stmts: list[A.Stmt]) -> bool: return False +def _iter_stmts(stmts: list[A.Stmt]): + """Yield every statement in the tree, descending into if-branches and + borrow blocks.""" + for st in stmts: + yield st + if isinstance(st, A.If): + yield from _iter_stmts(st.then_body) + yield from _iter_stmts(st.else_body) + elif isinstance(st, A.BorrowBlock): + yield from _iter_stmts(st.body) + + +def _move_aliases(stmts: list[A.Stmt], name: str) -> set[str]: + """The set of names a buffer flows into via `let X = move `, starting + from its declaration name (transitive).""" + aliases = {name} + changed = True + while changed: + changed = False + for st in _iter_stmts(stmts): + if (isinstance(st, A.Let) and isinstance(st.rhs, A.Move) + and st.rhs.var in aliases and st.name not in aliases): + aliases.add(st.name) + changed = True + return aliases + + +def _buffer_released(stmts: list[A.Stmt], name: str) -> bool: + """Is the buffer released somewhere — possibly through a moved alias?""" + aliases = _move_aliases(stmts, name) + return any(isinstance(st, A.Release) and st.var in aliases + for st in _iter_stmts(stmts)) + + +def _fn_has_buffer(stmts: list[A.Stmt]) -> bool: + for st in stmts: + if isinstance(st, A.Let) and isinstance(st.rhs, A.BufferIntent): + return True + if isinstance(st, A.If): + if _fn_has_buffer(st.then_body) or _fn_has_buffer(st.else_body): + return True + if isinstance(st, A.BorrowBlock): + if _fn_has_buffer(st.body): + return True + return False + + +def _body_has_plain_let(stmts: list[A.Stmt]) -> bool: + """True if a plain local (`let x = `) is declared at this level. Such + a local declared inside a hoisted try would go out of scope after the finally, + so a scope whose body contains one is emitted inline (no try) instead.""" + return any(isinstance(s, A.Let) and isinstance(s.rhs, (A.IntLit, A.VarRef)) + for s in stmts) + + +def _scope_body_has_plain_let(stmts: list[A.Stmt]) -> bool: + """True if any top-level scope (acquire/buffer let) has a plain local declared + between its acquire and its release.""" + for i, st in enumerate(stmts): + if isinstance(st, A.Let) and isinstance(st.rhs, (A.Acquire, A.BufferIntent)): + rel = None + for k in range(i + 1, len(stmts)): + if isinstance(stmts[k], A.Release) and stmts[k].var == st.name: + rel = k + break + body = stmts[i + 1:rel] if rel is not None else stmts[i + 1:] + if _body_has_plain_let(body): + return True + return False + + +def _scopes_release_top_level(stmts: list[A.Stmt]) -> bool: + """True if every top-level owned scope (an acquire or buffer let) has a + matching top-level `release`. A scope whose release is nested in a borrow/if + block — or which is consumed by a call instead of released — cannot be safely + hoisted into a finally, so such functions use faithful inline instead.""" + released = {st.var for st in stmts if isinstance(st, A.Release)} + for st in stmts: + if isinstance(st, A.Let) and isinstance(st.rhs, (A.Acquire, A.BufferIntent)): + if st.name not in released: + return False + return True + + +def _laminar_scopes(stmts: list[A.Stmt]) -> bool: + """True if every pair of top-level resource lifetimes is nested or disjoint + (never partially overlapping). Only then is the try/finally hoist safe: a + partial overlap `let a; let b; release a; ... release b;` cannot be nested + without forcing b's release before its source point.""" + intervals: list[tuple[int, int]] = [] + for i, st in enumerate(stmts): + if isinstance(st, A.Let) and isinstance(st.rhs, (A.Acquire, A.BufferIntent)): + for k in range(i + 1, len(stmts)): + s2 = stmts[k] + if isinstance(s2, A.Release) and s2.var == st.name: + intervals.append((i, k)) + break + for a1, a2 in intervals: + for b1, b2 in intervals: + if a1 < b1 < a2 < b2: # partial overlap + return False + return True + + +def _fn_has_native(stmts: list[A.Stmt]) -> bool: + for st in stmts: + if (isinstance(st, A.Let) and isinstance(st.rhs, A.BufferIntent) + and st.rhs.mode == "native"): + return True + if isinstance(st, A.If): + if _fn_has_native(st.then_body) or _fn_has_native(st.else_body): + return True + if isinstance(st, A.BorrowBlock): + if _fn_has_native(st.body): + return True + return False + + +def _buffer_modes(mod: A.Module) -> set[str]: + modes: set[str] = set() + + def walk(stmts): + for st in stmts: + if isinstance(st, A.Let) and isinstance(st.rhs, A.BufferIntent): + modes.add(st.rhs.mode) + elif isinstance(st, A.If): + walk(st.then_body) + walk(st.else_body) + elif isinstance(st, A.BorrowBlock): + walk(st.body) + + for fn in mod.functions: + walk(fn.body) + return modes + + def _usings(mod: A.Module) -> list[str]: out = ["using System;"] blob = " ".join( (r.emit_acquire or "") + (r.emit_release or "") + (r.emit_type or "") for r in mod.resources ) - if "ArrayPool" in blob: + modes = _buffer_modes(mod) + if "ArrayPool" in blob or (modes & {"scratch", "pooled"}): out.append("using System.Buffers;") return out +# Runtime logging support emitted alongside any module that uses buffers. The +# two hooks are the runtime half of the design: a text trace of which backend a +# scratch/stack request actually selected, and counters answering "how often do +# we really hit the stack?". Both are [Conditional]: a normal Release build that +# defines neither symbol pays nothing — you do not want logging to become the +# new bottleneck on a hot path. +_RUNTIME_SUPPORT = '''\ +internal static class OwnTrace +{ + [System.Diagnostics.Conditional("OWNSHARP_TRACE")] + public static void ScratchSelected(string function, string buffer, + int requestedBytes, int inlineLimit, string backend) + => System.Diagnostics.Trace.WriteLine( + $"[OwnSharp] {function}.{buffer}: requested={requestedBytes}, " + + $"inline={inlineLimit}, backend={backend}"); + + [System.Diagnostics.Conditional("OWNSHARP_TRACE")] + public static void StackSelected(string function, string buffer, + int requestedBytes, int inlineLimit) + => System.Diagnostics.Trace.WriteLine( + $"[OwnSharp] {function}.{buffer}: requested={requestedBytes}, " + + $"inline={inlineLimit}, backend=stackalloc"); + + [System.Diagnostics.Conditional("OWNSHARP_TRACE")] + public static void PooledSelected(string function, string buffer, int requestedBytes) + => System.Diagnostics.Trace.WriteLine( + $"[OwnSharp] {function}.{buffer}: requested={requestedBytes}, backend=ArrayPool"); + + [System.Diagnostics.Conditional("OWNSHARP_TRACE")] + public static void NativeSelected(string function, string buffer, int requestedBytes) + => System.Diagnostics.Trace.WriteLine( + $"[OwnSharp] {function}.{buffer}: requested={requestedBytes}, backend=NativeMemory"); +} + +internal static class OwnCounters +{ + public static long ScratchStackHits; + public static long ScratchPoolFallbacks; + public static long ScratchPoolBytesRented; + public static long ScratchReleaseCount; + + [System.Diagnostics.Conditional("OWNSHARP_COUNTERS")] + public static void StackHit() + => System.Threading.Interlocked.Increment(ref ScratchStackHits); + + [System.Diagnostics.Conditional("OWNSHARP_COUNTERS")] + public static void PoolFallback(int bytes) + { + System.Threading.Interlocked.Increment(ref ScratchPoolFallbacks); + System.Threading.Interlocked.Add(ref ScratchPoolBytesRented, bytes); + } + + [System.Diagnostics.Conditional("OWNSHARP_COUNTERS")] + public static void Release() + => System.Threading.Interlocked.Increment(ref ScratchReleaseCount); +} +''' + + def generate(mod: A.Module) -> str: parts = [ "// by OwnLang PoC. Ownership was checked at the .own", @@ -255,4 +718,7 @@ def generate(mod: A.Module) -> str: for line in b.splitlines())) parts.append("\n\n".join(indented)) parts.append("}") + if _buffer_modes(mod): + parts.append("") + parts.append(_RUNTIME_SUPPORT) return "\n".join(parts) + "\n" diff --git a/diagnostics.py b/diagnostics.py index b7cb3277..0a91355f 100644 --- a/diagnostics.py +++ b/diagnostics.py @@ -40,6 +40,14 @@ class Severity(Enum): "OWN011": "mutable borrow while another mutable borrow is live", "OWN012": "shared borrow while a mutable borrow is live", "OWN013": "owner accessed while it is mutably borrowed", + # ---- buffer storage policies (stackalloc / scratch / pool / native) ---- + "OWN015": "stack-backed buffer cannot escape the current function", + "OWN016": "stack-backed buffer moved to a longer-lived owner", + "OWN017": "movable buffer escape is not supported by code generation (PoC limitation)", + "OWN018": "buffer size must be an integer", + "OWN019": "inline capacity too large for a stack-backed policy", + "OWN021": "stack allocation requires a statically known bound", + "OWN023": "scratch fallback forbidden but the size may exceed the inline limit", # ---- unsupported ---- "OWN020": "unsupported construct (out of scope for the MVP)", # ---- name resolution & structural ---- @@ -60,6 +68,9 @@ class Diagnostic: message: str line: int severity: Severity = Severity.ERROR + # for buffer diagnostics: a stable identity (name#line) of the buffer the + # diagnostic is about, so the report attributes it by symbol, not by name. + subject: str | None = None @property def title(self) -> str: diff --git a/lexer.py b/lexer.py index 5c42a729..a700e63a 100644 --- a/lexer.py +++ b/lexer.py @@ -37,6 +37,7 @@ class Tok(Enum): ELSE = auto() RETURN = auto() MUT = auto() + POLICY = auto() # emit-template keywords (only meaningful inside a resource body) EMIT_TYPE = auto() EMIT_ACQUIRE = auto() @@ -54,6 +55,7 @@ class Tok(Enum): SEMI = auto() AMP = auto() EQ = auto() + DOT = auto() ARROW = auto() EOF = auto() @@ -76,6 +78,7 @@ class Tok(Enum): "else": Tok.ELSE, "return": Tok.RETURN, "mut": Tok.MUT, + "policy": Tok.POLICY, "emit_type": Tok.EMIT_TYPE, "emit_acquire": Tok.EMIT_ACQUIRE, "emit_release": Tok.EMIT_RELEASE, @@ -197,6 +200,7 @@ def advance(k: int = 1) -> None: ";": Tok.SEMI, "&": Tok.AMP, "=": Tok.EQ, + ".": Tok.DOT, } if c in simple: advance() diff --git a/parser.py b/parser.py index 2620d10a..c279bd56 100644 --- a/parser.py +++ b/parser.py @@ -4,10 +4,11 @@ Grammar (informal): module := "module" IDENT item* - item := resource | extern | fn + item := resource | extern | fn | policy resource := "resource" IDENT "{" rmember* "}" rmember := ("acquire" | "release") IDENT | ("emit_type"|"emit_acquire"|"emit_release"|"emit_borrow") STRING + policy := "policy" IDENT "{" (IDENT "=" atom ";")* "}" extern := "extern" "fn" IDENT "(" eparams? ")" ("->" type)? ";" eparams := eparam ("," eparam)* eparam := ("borrow" | "borrow_mut" | "consume")? IDENT // IDENT = type name @@ -18,7 +19,11 @@ block := "{" stmt* "}" stmt := let | release | use | call | borrow | if | return let := "let" IDENT "=" rhs ";" - rhs := "acquire" IDENT "(" args? ")" | "move" IDENT | IDENT | INT + rhs := "acquire" IDENT "(" args? ")" | "move" IDENT + | bufferintent | IDENT | INT + bufferintent:= IDENT "." IDENT "(" bargs? ")" // e.g. Buffer.scratch(...) + bargs := barg ("," barg)* + barg := IDENT "=" atom | atom // named option | positional size release := "release" IDENT ";" use := "use" IDENT ";" call := IDENT "(" args? ")" ";" @@ -111,10 +116,43 @@ def parse_module(self) -> A.Module: mod.externs.append(self.parse_extern()) elif self.at(Tok.FN): mod.functions.append(self.parse_fn()) + elif self.at(Tok.POLICY): + mod.policies.append(self.parse_policy()) else: - raise ParseError("expected 'resource', 'extern' or 'fn'", self.cur) + raise ParseError( + "expected 'resource', 'extern', 'fn' or 'policy'", self.cur) return mod + # -- policies ----------------------------------------------------------- + + def parse_policy(self) -> A.PolicyDecl: + kw = self.eat(Tok.POLICY) + name = self.eat(Tok.IDENT).text + self.eat(Tok.LBRACE) + settings: dict[str, object] = {} + seen: list[str] = [] + while not self.at(Tok.RBRACE): + key = self.eat(Tok.IDENT).text + self.eat(Tok.EQ) + settings[key] = self._policy_value() + self.eat(Tok.SEMI) + seen.append(key) + self.eat(Tok.RBRACE) + dups = tuple(sorted({k for k in seen if seen.count(k) > 1})) + return A.PolicyDecl(name=name, settings=settings, line=kw.line, dups=dups) + + def _policy_value(self) -> object: + """A policy setting value: an int, or an identifier interpreted as a + bool (true/false) or a bare keyword string (pool/forbidden/debug/...).""" + if self.at(Tok.INT): + return int(self.eat(Tok.INT).text) + word = self.eat(Tok.IDENT).text + if word == "true": + return True + if word == "false": + return False + return word + # -- resources ---------------------------------------------------------- def parse_resource(self) -> A.ResourceDecl: @@ -260,8 +298,51 @@ def parse_rhs(self) -> A.Expr: kw = self.eat(Tok.MOVE) v = self.eat(Tok.IDENT) return A.Move(var=v.text, line=kw.line) + # buffer intent: Namespace "." mode "(" bargs? ")" e.g. Buffer.scratch(...) + if self.at(Tok.IDENT) and self.peek().kind == Tok.DOT: + return self.parse_buffer_intent() return self.parse_atom() + def parse_buffer_intent(self) -> A.BufferIntent: + ns = self.eat(Tok.IDENT) # namespace, conventionally "Buffer" + self.eat(Tok.DOT) + mode = self.eat(Tok.IDENT).text + self.eat(Tok.LPAREN) + size: A.Expr | None = None + options: dict[str, A.Expr] = {} + seen: list[str] = [] # option names in order (for dup detect) + first = True + if not self.at(Tok.RPAREN): + size, first = self._buffer_arg(options, seen, first) + while self.accept(Tok.COMMA): + got, first = self._buffer_arg(options, seen, first) + if got is not None: + size = got + self.eat(Tok.RPAREN) + dups = tuple(sorted({k for k in seen if seen.count(k) > 1})) + return A.BufferIntent(mode=mode, size=size, options=options, + line=ns.line, ns=ns.text, col=ns.col, dups=dups) + + def _buffer_arg(self, options: dict, seen: list, first: bool + ) -> tuple[A.Expr | None, bool]: + """Parse one buffer argument: a named option, or the leading positional + size. Returns (size_expr_or_None, still_first).""" + # named option: IDENT "=" atom. `policy` is a keyword token elsewhere but + # is also a valid option name here, so accept it too. + if (self.at(Tok.IDENT) or self.at(Tok.POLICY)) and self.peek().kind == Tok.EQ: + key = self.cur.text + self.pos += 1 + self.eat(Tok.EQ) + options[key] = self.parse_atom() + seen.append(key) + return None, first + atom = self.parse_atom() + if not first: + raise ParseError("only the leading size may be positional in a " + "buffer intent; later arguments must be named", + self.cur) + return atom, False + def parse_args(self) -> list[A.Expr]: args: list[A.Expr] = [] if not self.at(Tok.RPAREN): diff --git a/report.py b/report.py new file mode 100644 index 00000000..60578787 --- /dev/null +++ b/report.py @@ -0,0 +1,113 @@ +""" +Compile-time buffer report — the first of the three logging surfaces. + +The whole point of the scratch/stack line is that it must not be magic. A buffer +that "smartly" chose the pool while the developer thought it lived on the stack +is exactly the kind of abstraction that lies. So at generation time we write down, +per buffer: the mode the user asked for, the inline limit, the fallback, the +escape policy, whether it is cleared on release, the runtime branches codegen +emitted, and which ownership checks held. It goes to stdout for a human and to +`.ownreport.json` for review tooling / CI. + +(The other two surfaces are runtime: the OwnTrace text hook and the OwnCounters, +both emitted into the generated C# by codegen.py under [Conditional] guards.) +""" + +from __future__ import annotations + +from . import ast_nodes as A +from .buffers import resolve as resolve_buffer, Policy, MODE_NAMES +from .diagnostics import Diagnostic + + +# Diagnostics that, if present for a given buffer, mean a specific check failed. +_CHECK_CODES = { + "noEscape": {"OWN015", "OWN016", "OWN017"}, + "releaseOnAllPaths": {"OWN001"}, + "noUseAfterRelease": {"OWN002", "OWN009"}, + "noActiveLoansAtRelease": {"OWN008"}, +} + + +def _walk_buffers(stmts: list[A.Stmt]): + """Yield (let_name, BufferIntent) for every buffer intent in a statement + tree, descending into if-branches and borrow blocks.""" + for st in stmts: + if isinstance(st, A.Let) and isinstance(st.rhs, A.BufferIntent): + yield st.name, st.rhs + elif isinstance(st, A.If): + yield from _walk_buffers(st.then_body) + yield from _walk_buffers(st.else_body) + elif isinstance(st, A.BorrowBlock): + yield from _walk_buffers(st.body) + + +def build_report(mod: A.Module, diags: list[Diagnostic]) -> dict: + policies: dict[str, Policy] = { + p.name: Policy(p.name, dict(p.settings), p.line) for p in mod.policies + } + # diagnostics are attributed to a buffer by its stable identity (name#line), + # carried on Diagnostic.subject — NOT by matching the name in the message, + # which would conflate same-named buffers in sibling scopes. Move-aliases + # already share the original buffer's subject (set in the checker). + by_subject: dict[str, set[str]] = {} + for d in diags: + if d.subject is not None: + by_subject.setdefault(d.subject, set()).add(d.code) + + entries: list[dict] = [] + for fn in mod.functions: + for name, intent in _walk_buffers(fn.body): + # skip a malformed intent (bad namespace or mode, e.g. Foo.stack / + # Buffer.bogus); the checker already reported OWN030, and resolving an + # unknown mode would throw. + if intent.ns != "Buffer" or intent.mode not in MODE_NAMES: + continue + info, _ = resolve_buffer(intent, policies) + mine_codes = by_subject.get(f"{name}#{intent.line}:{intent.col}", set()) + checks = { + check: not (codes & mine_codes) + for check, codes in _CHECK_CODES.items() + } + entries.append({ + "function": fn.name, + "buffer": name, + "mode": info.mode.value, + "inlineBytes": info.inline_bytes, + "fallback": ("ArrayPool" if info.fallback_pool + else ("NativeMemory" if info.mode.value == "native" + else "forbidden")), + "escapePolicy": info.escape_policy, + "clearOnRelease": info.clear_on_release, + "trace": info.trace, + "counters": info.counters, + "policy": info.policy_name, + "branches": info.branches(), + "checks": checks, + }) + return {"module": mod.name, "buffers": entries} + + +def render_report(report: dict) -> str: + lines: list[str] = [f"buffer report for module '{report['module']}'"] + if not report["buffers"]: + lines.append(" (no buffers)") + return "\n".join(lines) + for e in report["buffers"]: + lines.append("") + lines.append(f"Function: {e['function']}") + lines.append(f"Buffer: {e['buffer']}") + lines.append(f" Mode: {e['mode']}") + lines.append(f" InlineLimit: {e['inlineBytes']} bytes") + lines.append(f" Fallback: {e['fallback']}") + lines.append(f" EscapePolicy: {e['escapePolicy']}") + lines.append(f" ClearOnRelease: {str(e['clearOnRelease']).lower()}") + if e["policy"]: + lines.append(f" Policy: {e['policy']}") + lines.append(" Generated branches:") + for b in e["branches"]: + lines.append(f" {b['condition']:<18} -> {b['backend']}") + lines.append(" Checks:") + for k, v in e["checks"].items(): + lines.append(f" {'ok ' if v else 'FAIL'} {k}") + return "\n".join(lines) diff --git a/run_tests.py b/run_tests.py index 58ae2523..40d9c9eb 100644 --- a/run_tests.py +++ b/run_tests.py @@ -16,10 +16,13 @@ from ownlang.parser import parse, ParseError # noqa: E402 from ownlang.lexer import LexError # noqa: E402 -from ownlang.cfg import build_cfg, collect_signatures # noqa: E402 +from ownlang.cfg import (build_cfg, collect_signatures, # noqa: E402 + collect_policies) from ownlang.analysis import analyze # noqa: E402 from ownlang.diagnostics import Severity # noqa: E402 from ownlang.codegen import generate # noqa: E402 +from ownlang.buffers import validate_policies # noqa: E402 +from ownlang.report import build_report # noqa: E402 PRELUDE = ( "module M\n" @@ -38,9 +41,11 @@ def codes(src: str) -> list[str]: return ["OWN020"] rnames = {r.name for r in mod.resources} sigs = collect_signatures(mod) - cs: list[str] = [] + pols = collect_policies(mod) + cs: list[str] = [d.code for d in validate_policies(pols) + if d.severity == Severity.ERROR] for fn in mod.functions: - cfg, d1 = build_cfg(fn, rnames, sigs) + cfg, d1 = build_cfg(fn, rnames, sigs, pols) d2 = analyze(cfg) cs += [d.code for d in (d1 + d2) if d.severity == Severity.ERROR] return cs @@ -154,6 +159,161 @@ def codes(src: str) -> list[str]: ("leak_and_uafr", "fn f(){ let a = acquire Buffer(1); let b = acquire Buffer(2); " "release b; use b; }", ["OWN001", "OWN002"]), + + # ---- buffer storage policies: clean ---- + ("buf_scratch_ok", + "fn f(n: int){ let b = Buffer.scratch(n, inline = 1024); " + "borrow_mut b as m { Fill(m); } release b; }", []), + ("buf_stack_const_ok", + "fn f(){ let b = Buffer.stack(256); borrow_mut b as m { Fill(m); } " + "release b; }", []), + ("buf_stack_dyn_bounded_ok", + "fn f(n: int){ let b = Buffer.stack(n, max = 1024); release b; }", []), + ("buf_inline_ok", + "fn f(){ let b = Buffer.inline(64); release b; }", []), + ("buf_pooled_local_ok", + "fn f(n: int){ let b = Buffer.pooled(n); borrow_mut b as m { Fill(m); } " + "release b; }", []), + ("buf_branchy_release_ok", + "fn f(n: int){ let b = Buffer.pooled(n); if (c) { release b; } " + "else { release b; } }", []), + ("buf_overlapping_fifo_ok", + "fn f(n: int){ let a = Buffer.pooled(n); let b = Buffer.pooled(n); " + "release a; release b; }", []), + ("buf_overlapping_lifo_ok", + "fn f(n: int){ let a = Buffer.pooled(n); let b = Buffer.pooled(n); " + "release b; release a; }", []), + ("buf_scratch_bad_fallback", + "fn f(n: int){ let b = Buffer.scratch(n, fallback = forbiden); " + "release b; }", ["OWN030"]), + ("buf_scratch_nonident_fallback", + "fn f(n: int){ let b = Buffer.scratch(n, fallback = 0); release b; }", + ["OWN030"]), + ("buf_move_release_ok", + "fn f(n: int){ let a = Buffer.pooled(n); let b = move a; release b; }", []), + ("buf_bad_namespace", + "fn f(n: int){ let b = Foo.stack(n, max = 1024); release b; }", ["OWN030"]), + ("buf_move_escapes", + "fn f(n: int) -> Buffer { let a = Buffer.stack(n, max = 100); " + "let b = move a; return b; }", ["OWN015"]), + ("buf_sibling_move_ok", + "fn f(n: int){ let a = Buffer.pooled(n); if (c) { let b = move a; " + "release b; } else { release a; } }", []), + ("buf_disjoint_sequential_ok", + "fn f(n: int){ let a = Buffer.pooled(n); release a; " + "let b = Buffer.pooled(n); release b; }", []), + ("buf_bad_policy_ref", + "fn f(n: int){ let b = Buffer.scratch(n, policy = 0); release b; }", + ["OWN030"]), + ("buf_bad_inline_bound", + "fn f(n: int){ let b = Buffer.scratch(n, inline = bogus); release b; }", + ["OWN030"]), + ("buf_bad_max_bound", + "fn f(n: int){ let b = Buffer.stack(n, max = bogus); release b; }", + ["OWN030"]), + ("buf_unknown_option", + "fn f(n: int){ let b = Buffer.scratch(n, fallbak = forbidden); " + "release b; }", ["OWN030"]), + ("buf_unknown_policy_setting", + "policy P { fallbak = forbidden; } " + "fn f(n: int){ let b = Buffer.scratch(n, policy = P); release b; }", + ["OWN030"]), + ("buf_size_not_int_bool", + "fn f(flag: bool){ let b = Buffer.pooled(flag); release b; }", ["OWN018"]), + ("buf_size_not_int_owned", + "fn f(){ let r = acquire Conn(1); let b = Buffer.pooled(r); " + "release b; release r; }", ["OWN018"]), + ("buf_inline_dynamic_rejected", + "fn f(n: int){ let b = Buffer.inline(n, max = 1024); release b; }", + ["OWN021"]), + ("buf_size_borrow_temp", + "fn f(x: &Buffer){ let n = x; let b = Buffer.pooled(n); release b; }", + ["OWN018"]), + ("buf_local_after_release_ok", + "fn f(n: int){ let b = Buffer.pooled(n); let x = 1; release b; " + "let y = x; }", []), + ("buf_inline_literal_ok", + "fn f(){ let b = Buffer.inline(256); release b; }", []), + ("buf_policy_bad_clear", + "policy Sensitive { clear_on_release = ture; } " + "fn f(n: int){ let b = Buffer.scratch(n, policy = Sensitive); release b; }", + ["OWN030"]), + ("buf_bad_counters", + "fn f(n: int){ let b = Buffer.scratch(n, counters = ture); release b; }", + ["OWN030"]), + ("buf_bad_trace", + "fn f(n: int){ let b = Buffer.scratch(n, trace = ture); release b; }", + ["OWN030"]), + ("buf_policy_bools_ok", + "policy S { clear_on_release = true; counters = false; trace = off; } " + "fn f(n: int){ let b = Buffer.scratch(n, policy = S); release b; }", []), + ("buf_duplicate_option", + "fn f(n: int){ let b = Buffer.scratch(n, fallback = forbidden, " + "fallback = pool); release b; }", ["OWN030"]), + ("buf_duplicate_policy_setting", + "policy P { inline_bytes = 512; inline_bytes = 1024; } " + "fn f(n: int){ let b = Buffer.scratch(n, policy = P); release b; }", + ["OWN030"]), + ("buf_inline_override_ignores_bad_policy", + "policy P { inline_bytes = bogus; } " + "fn f(n: int){ let b = Buffer.scratch(n, policy = P, inline = 128); " + "release b; }", []), + ("buf_bad_policy_inline_no_override", + "policy P { inline_bytes = bogus; } " + "fn f(n: int){ let b = Buffer.scratch(n, policy = P); release b; }", + ["OWN030"]), + ("buf_alias_redecl_ok", + "fn f(n: int){ let a = Buffer.pooled(n); if (c) { let b = move a; " + "release b; } else { release a; } let b = acquire Conn(); release b; }", + []), + ("res_partial_overlap_ok", + "fn f(){ let a = acquire Buffer(1); let c = acquire Conn(1); " + "release a; use c; release c; }", []), + ("buf_nested_release_ok", + "fn f(n: int){ let a = acquire Conn(1); let b = Buffer.pooled(n); " + "borrow a as s { release b; } release a; }", []), + ("buf_local_helper_ok", + "fn helper(x: &mut Buffer){ use x; } " + "fn f(n: int){ let b = Buffer.pooled(n); helper(b); release b; }", []), + ("buf_native_ok", + "fn f(n: int){ let b = Buffer.native(n); release b; }", []), + ("buf_policy_ok", + "policy P { inline_bytes = 512; fallback = pool; } " + "fn f(n: int){ let b = Buffer.scratch(n, policy = P); release b; }", []), + + # ---- buffer storage policies: faults ---- + ("buf_stack_dyn_unbounded", + "fn f(n: int){ let b = Buffer.stack(n); release b; }", ["OWN021"]), + ("buf_stack_too_large", + "fn f(){ let b = Buffer.stack(1000000); release b; }", ["OWN019"]), + ("buf_scratch_escapes_return", + "fn f(n: int) -> Buffer { let b = Buffer.scratch(n); return b; }", + ["OWN015"]), + ("buf_stack_escapes_return", + "fn f() -> Buffer { let b = Buffer.stack(64); return b; }", ["OWN015"]), + ("buf_scratch_escapes_consume", + "fn f(n: int){ let b = Buffer.scratch(n); Store(b); }", ["OWN016"]), + ("buf_pooled_escapes_return", + "fn f(n: int) -> Buffer { let b = Buffer.pooled(n); return b; }", + ["OWN017"]), + ("buf_pooled_escapes_consume", + "fn f(n: int){ let b = Buffer.pooled(n); Store(b); }", ["OWN017"]), + ("buf_native_escapes_return", + "fn f(n: int) -> Buffer { let b = Buffer.native(n); return b; }", + ["OWN017"]), + ("buf_scratch_forbid_dynamic", + "fn f(n: int){ let b = Buffer.scratch(n, fallback = forbidden); " + "release b; }", ["OWN023"]), + ("buf_scratch_leak", + "fn f(n: int){ let b = Buffer.scratch(n); }", ["OWN001"]), + ("buf_unknown_mode", + "fn f(n: int){ let b = Buffer.bogus(n); release b; }", ["OWN030"]), + ("buf_release_while_borrowed", + "fn f(n: int){ let b = Buffer.scratch(n); borrow b as s { release b; } }", + ["OWN008"]), + ("buf_moved_then_used", + "fn f(n: int){ let a = Buffer.pooled(n); let b = move a; use a; " + "release b; }", ["OWN005"]), ] @@ -210,6 +370,504 @@ def golden_smoke() -> list[str]: return fails +BUFFER_GOLDEN = ( + "module ScratchDemo\n" + "policy DefaultScratch { inline_bytes = 1024; fallback = pool; " + "trace = debug; counters = true; clear_on_release = false; }\n" + "extern fn Fill(borrow_mut Buffer);\n" + "extern fn Hash(borrow Buffer);\n" + "fn parse(size: int) {\n" + " let tmp = Buffer.scratch(size, inline = 1024, fallback = pool);\n" + " borrow_mut tmp as bytes { Fill(bytes); }\n" + " borrow tmp as view { Hash(view); }\n" + " release tmp;\n" + "}\n" +) + + +def buffer_smoke() -> list[str]: + """A scratch buffer must check clean, lower to the stack-first/pool-fallback + pattern with the trace + counter hooks, ship the [Conditional] runtime + support, and produce a compile-time report that names the chosen backends.""" + fails: list[str] = [] + cs = [c for c in codes(BUFFER_GOLDEN) if c.startswith("OWN")] + if cs: + fails.append(f"buffer golden should check clean, got {sorted(set(cs))}") + out = generate(parse(BUFFER_GOLDEN)) + must_contain = [ + "Span tmp_backing = stackalloc byte[1024];", + "if (size <= 1024)", + 'OwnTrace.ScratchSelected("parse", "tmp", size, 1024, "stackalloc");', + 'OwnTrace.ScratchSelected("parse", "tmp", size, 1024, "ArrayPool");', + "OwnCounters.StackHit();", + "OwnCounters.PoolFallback(size);", + "ArrayPool.Shared.Rent(size)", + "ArrayPool.Shared.Return(tmp_rented)", + "try", + "finally", + "internal static class OwnTrace", + "internal static class OwnCounters", + 'Conditional("OWNSHARP_TRACE")', + 'Conditional("OWNSHARP_COUNTERS")', + ] + for s in must_contain: + if s not in out: + fails.append(f"buffer C# missing: {s!r}") + # the pool Return must run exactly once, guarded by the rented null-check + if out.count("ArrayPool.Shared.Return(tmp_rented)") != 1: + fails.append("buffer C# should Return the rented array exactly once") + + # compile-time report + mod = parse(BUFFER_GOLDEN) + rep = build_report(mod, []) + if not rep["buffers"]: + fails.append("buffer report produced no entries") + else: + e = rep["buffers"][0] + if e["mode"] != "scratch" or e["inlineBytes"] != 1024: + fails.append(f"buffer report has wrong mode/inline: {e}") + if e["escapePolicy"] != "local-only": + fails.append(f"scratch report should be local-only, got {e['escapePolicy']}") + backends = {b["backend"] for b in e["branches"]} + if backends != {"stackalloc", "ArrayPool"}: + fails.append(f"scratch report branches wrong: {backends}") + + # the committed runnable golden must stay in sync with the emitter and stay + # a real, compilable ArrayPool program (a human can `dotnet run` it). + golden_path = os.path.join(os.path.dirname(__file__), + "buffer_scratch_program.cs.txt") + if os.path.exists(golden_path): + prog = open(golden_path, encoding="utf-8").read() + for s in ("public static void parse(int size)", + "if (size < 0)", + "ArrayPool.Shared.Rent(size)", + "ArrayPool.Shared.Return(tmp_rented)", + "internal static class OwnTrace", + "internal static class OwnCounters", + "public static void Main()"): + if s not in prog: + fails.append(f"runnable golden missing: {s!r}") + # the negative-size guard must precede the trace/counter hooks + if "if (size < 0)" in prog and "ScratchSelected" in prog: + if prog.index("if (size < 0)") > prog.index("ScratchSelected"): + fails.append("runnable golden guard must precede the trace hook") + # its emitted parse body must match what the emitter produces today + if "tmp = tmp_backing[..size];" not in prog: + fails.append("runnable golden parse body drifted from the emitter") + else: + fails.append("runnable golden buffer_scratch_program.cs.txt is missing") + return fails + + +def escape_and_length_smoke() -> list[str]: + """Regression guards for the PR #2 review: + - the checker rejects an escaping movable (pooled/native) buffer rather than + letting codegen emit C# that leaks the rent or fails to compile (OWN017); + - a constant scratch smaller than its inline limit exposes the requested + length, not the reservation; + - a fallback-forbidden scratch reports as stack-only.""" + fails: list[str] = [] + + pooled_ret = ("module M\n" + "fn f(n: int) -> Buffer { let b = Buffer.pooled(n); return b; }\n") + if "OWN017" not in codes(pooled_ret): + fails.append("escaping pooled (return) must be rejected with OWN017") + + pooled_consume = ("module M\nextern fn Store(consume Buffer);\n" + "fn f(n: int){ let b = Buffer.pooled(n); Store(b); }\n") + if "OWN017" not in codes(pooled_consume): + fails.append("escaping pooled (consume) must be rejected with OWN017") + + native_ret = ("module M\n" + "fn f(n: int) -> Buffer { let b = Buffer.native(n); return b; }\n") + if "OWN017" not in codes(native_ret): + fails.append("escaping native (return) must be rejected with OWN017") + + # a locally-released pooled buffer is fine and DOES Return to the pool + pooled_local = ("module M\nextern fn Fill(borrow_mut Buffer);\n" + "fn f(n: int){ let b = Buffer.pooled(n); " + "borrow_mut b as m { Fill(m); } release b; }\n") + if codes(pooled_local): + fails.append(f"local pooled buffer should check clean, got {codes(pooled_local)}") + if "ArrayPool.Shared.Return" not in generate(parse(pooled_local)): + fails.append("local pooled buffer must Return to the pool") + + scratch_const = ("module M\nextern fn Fill(borrow_mut Buffer);\n" + "fn f(){ let b = Buffer.scratch(64, inline = 1024, " + "fallback = forbidden); borrow_mut b as m { Fill(m); } " + "release b; }\n") + out = generate(parse(scratch_const)) + if "b_backing[..64]" not in out: + fails.append("constant forbidden-fallback scratch must expose length 64") + + rep = build_report(parse(scratch_const), []) + e = rep["buffers"][0] + if e["fallback"] != "forbidden": + fails.append(f"forbidden scratch report fallback should be 'forbidden', got {e['fallback']}") + backends = {b["backend"] for b in e["branches"]} + if backends != {"stackalloc"}: + fails.append(f"forbidden scratch report should be stack-only, got {backends}") + return fails + + +def branchy_and_malformed_smoke() -> list[str]: + """Two follow-up review guards: + - a buffer released inside branches is NOT an escape: codegen must emit the + real pool cleanup at each release site, never a generic Dispose on a Span; + - the report must skip a malformed buffer mode (Buffer.bogus) instead of + throwing, leaving the checker's OWN030 to stand.""" + fails: list[str] = [] + + branchy = ("module M\n" + "fn f(n: int){ let b = Buffer.pooled(n); " + "if (c) { release b; } else { release b; } }\n") + if codes(branchy): + fails.append(f"branchy buffer release should check clean, got {codes(branchy)}") + out = generate(parse(branchy)) + if "Dispose()" in out: + fails.append("branchy buffer release must not emit a generic Dispose()") + if out.count("ArrayPool.Shared.Return(b_array)") != 2: + fails.append("branchy pooled release must Return at both release sites") + if "ArrayPool.Shared.Rent(n)" not in out: + fails.append("branchy pooled buffer must still Rent once") + + bogus = parse("module M\nfn f(n: int){ let b = Buffer.bogus(n); release b; }\n") + try: + rep = build_report(bogus, []) + except Exception as e: # noqa: BLE001 + fails.append(f"report crashed on a malformed buffer mode: {type(e).__name__}: {e}") + else: + if rep["buffers"]: + fails.append("report should skip an unresolved buffer mode") + + # overlapping buffer lifetimes released in non-LIFO (FIFO) order must lower + # without a CodegenError, and both arrays must be returned to the pool. + fifo = ("module M\nfn f(n: int){ let a = Buffer.pooled(n); " + "let b = Buffer.pooled(n); release a; release b; }\n") + if codes(fifo): + fails.append(f"FIFO overlapping buffers should check clean, got {codes(fifo)}") + try: + out = generate(parse(fifo)) + except Exception as e: # noqa: BLE001 + fails.append(f"FIFO overlapping buffers crashed codegen: {type(e).__name__}: {e}") + else: + if out.count("ArrayPool.Shared.Return(a_array)") != 1: + fails.append("FIFO: a must be returned to the pool exactly once") + if out.count("ArrayPool.Shared.Return(b_array)") != 1: + fails.append("FIFO: b must be returned to the pool exactly once") + + # a misspelled forbidden-fallback must be diagnosed, never silently pooled + bad_fb = ("module M\nfn f(n: int){ " + "let b = Buffer.scratch(n, fallback = forbiden); release b; }\n") + if "OWN030" not in codes(bad_fb): + fails.append("misspelled scratch fallback must produce OWN030") + + # an inline override must skip a malformed policy default (override wins) and + # the effective inline limit must be the override + ov = ("module M\npolicy P { inline_bytes = bogus; }\n" + "fn f(n: int){ let b = Buffer.scratch(n, policy = P, inline = 128); " + "release b; }\n") + if codes(ov): + fails.append(f"inline override should ignore bad policy default, got {codes(ov)}") + if "stackalloc byte[128]" not in generate(parse(ov)): + fails.append("inline override (128) must be the effective inline limit") + + # a non-identifier fallback (fallback = 0) must likewise fail safe: OWN030 + # AND no ArrayPool fallback enabled (it must not silently heap-allocate). + from ownlang.buffers import resolve as _resolve + from ownlang.ast_nodes import BufferIntent, IntLit + intent = BufferIntent(mode="scratch", size=IntLit(8, 1), + options={"fallback": IntLit(0, 1)}, line=1) + info, idiags = _resolve(intent, {}) + if "OWN030" not in {d.code for d in idiags}: + fails.append("non-identifier scratch fallback must produce OWN030") + if info.fallback_pool: + fails.append("non-identifier scratch fallback must not enable the pool") + + # the report's noEscape check must agree with the OWN017 checker diagnostic + esc = parse("module M\nfn f(n: int) -> Buffer { let b = Buffer.pooled(n); " + "return b; }\n") + if _report_check(esc, "noEscape"): + fails.append("report noEscape must be false for an OWN017-rejected buffer") + + # a buffer released through a moved alias must lower (not raise) and the + # cleanup must attach to the moved-to name's release. + moved = ("module M\nfn f(n: int){ let a = Buffer.pooled(n); " + "let b = move a; release b; }\n") + if codes(moved): + fails.append(f"move-then-release buffer should check clean, got {codes(moved)}") + try: + out = generate(parse(moved)) + except Exception as e: # noqa: BLE001 + fails.append(f"move-then-release buffer crashed codegen: {type(e).__name__}: {e}") + else: + if out.count("ArrayPool.Shared.Return(a_array)") != 1: + fails.append("moved buffer must still Return its original backing once") + + # a wrong namespace (Foo.stack) must be diagnosed, not silently lowered + bad_ns = "module M\nfn f(n: int){ let b = Foo.stack(n, max = 1024); release b; }\n" + if "OWN030" not in codes(bad_ns): + fails.append("non-Buffer namespace must produce OWN030") + + # an escape reported on a moved-to alias must fail the buffer's noEscape check + moved_esc = parse("module M\nfn f(n: int) -> Buffer { " + "let a = Buffer.stack(n, max = 100); let b = move a; " + "return b; }\n") + if _report_check(moved_esc, "noEscape"): + fails.append("report noEscape must be false when a moved alias escapes") + return fails + + +def _report_check(mod, check): + """Run the full checker over a parsed module and return the named report + check (True/False) for its first buffer.""" + from ownlang.cfg import build_cfg, collect_signatures, collect_policies + rn = {r.name for r in mod.resources} + sg = collect_signatures(mod) + pl = collect_policies(mod) + diags = [] + for fn in mod.functions: + cfg, d1 = build_cfg(fn, rn, sg, pl) + diags += d1 + analyze(cfg) + rep = build_report(mod, diags) + return rep["buffers"][0]["checks"][check] if rep["buffers"] else None + + +def nesting_native_trace_smoke() -> list[str]: + """Three follow-up review guards: + - an ordinary resource inside a buffer's lifetime keeps its own finally + (exception from the body must not leak it while the buffer is returned); + - a native buffer with a dynamic size guards against a negative request + before NativeMemory.Alloc; + - a policy `trace = false` is honoured (no OwnTrace, report trace:false).""" + fails: list[str] = [] + + # Issue 1: buffer + ordinary resource each get their own finally + mix = ("module M\nresource Conn { acquire open release close }\n" + "extern fn Work(borrow_mut Buffer);\n" + "fn f(n: int){ let b = Buffer.scratch(n); let c = acquire Conn(); " + "Work(b); release c; release b; }\n") + if codes(mix): + fails.append(f"buffer+resource should check clean, got {codes(mix)}") + out = generate(parse(mix)) + if out.count("finally") != 2: + fails.append(f"buffer+resource must produce two finally blocks, got {out.count('finally')}") + if "c.close();" not in out: + fails.append("Conn must be closed") + elif out.index("c.close();") > out.index("ArrayPool.Shared.Return"): + fails.append("Conn finally must be nested inside the buffer's try (close before Return)") + + # Issue 2: native dynamic size guards against a negative request + nat_dyn = ("module M\nextern fn Fill(borrow_mut Buffer);\n" + "fn g(n: int){ let b = Buffer.native(n); " + "borrow_mut b as m { Fill(m); } release b; }\n") + out = generate(parse(nat_dyn)) + if "if (n < 0)" not in out or "ArgumentOutOfRangeException(nameof(n))" not in out: + fails.append("native dynamic size must guard against a negative request") + nat_const = ("module M\nextern fn Fill(borrow_mut Buffer);\n" + "fn g(){ let b = Buffer.native(256); " + "borrow_mut b as m { Fill(m); } release b; }\n") + if "< 0" in generate(parse(nat_const)): + fails.append("native constant size should not emit a negative guard") + + # Issue 3: a policy trace = false disables tracing + from ownlang.buffers import resolve as _resolve, Policy + from ownlang.ast_nodes import BufferIntent, VarRef + intent = BufferIntent(mode="scratch", size=VarRef("n", 1), + options={"policy": VarRef("Quiet", 1)}, line=1) + pol = {"Quiet": Policy("Quiet", {"trace": False, "counters": True})} + info, _ = _resolve(intent, pol) + if info.trace: + fails.append("policy trace = false must disable tracing") + quiet = ("module M\npolicy Quiet { trace = false; }\n" + "extern fn Fill(borrow_mut Buffer);\n" + "fn h(n: int){ let b = Buffer.scratch(n, policy = Quiet); " + "borrow_mut b as m { Fill(m); } release b; }\n") + hbody = generate(parse(quiet)).split("public static void h")[1].split("internal static")[0] + if "OwnTrace" in hbody: + fails.append("policy trace = false must emit no OwnTrace calls") + return fails + + +def ordering_counters_smoke() -> list[str]: + """Three follow-up review guards: + - a moved buffer's cleanup survives in sibling branches (a `move` in one + branch must not strip the original's cleanup from the other); + - pooled buffers do not emit the Scratch.* counters (they would corrupt the + stack-hit-rate metric); + - simple-mode keeps a buffer prelude after earlier plain statements it can + depend on (no Rent(n) before `var n = 64;`).""" + fails: list[str] = [] + + # Issue 1: move in one branch must not strip cleanup from the sibling branch + sib = ("module M\nfn f(n: int){ let a = Buffer.pooled(n); " + "if (c) { let b = move a; release b; } else { release a; } }\n") + if codes(sib): + fails.append(f"sibling-branch move should check clean, got {codes(sib)}") + out = generate(parse(sib)) + if "Dispose()" in out: + fails.append("sibling-branch release must return to the pool, not Dispose") + if out.count("ArrayPool.Shared.Return(a_array)") != 2: + fails.append("both branches must return a_array to the pool") + + # Issue 2: pooled buffers must not touch the Scratch.* counters (check the + # function body only — the OwnCounters class itself is always emitted) + pooled = ("module M\nextern fn Fill(borrow_mut Buffer);\n" + "fn p(n: int){ let b = Buffer.pooled(n); " + "borrow_mut b as m { Fill(m); } release b; }\n") + pbody = generate(parse(pooled)).split("public static void p")[1].split("internal static")[0] + if "OwnCounters" in pbody: + fails.append("pooled buffer must not emit Scratch.* counters") + # ...but scratch still must (sanity that the gate is mode-specific) + scratch = ("module M\nextern fn Fill(borrow_mut Buffer);\n" + "fn s(n: int){ let b = Buffer.scratch(n); " + "borrow_mut b as m { Fill(m); } release b; }\n") + sbody = generate(parse(scratch)).split("public static void s")[1].split("internal static")[0] + if "OwnCounters.StackHit()" not in sbody: + fails.append("scratch buffer must still emit Scratch.* counters") + + # Issue 3: a plain statement before a buffer stays before its prelude + order = "module M\nfn q(){ let n = 64; let b = Buffer.pooled(n); release b; }\n" + out = generate(parse(order)) + if "var n = 64;" not in out or "Rent(n)" not in out: + fails.append("ordering smoke missing expected lines") + elif out.index("var n = 64;") > out.index("Rent(n)"): + fails.append("buffer prelude must come after the plain statement it depends on") + + # disjoint sequential buffers: a must be returned before b is rented (its + # source release point must not be swallowed into b's lifetime) + disj = ("module M\nextern fn Use1(borrow_mut Buffer);\n" + "fn d(n: int){ let a = Buffer.pooled(n); Use1(a); release a; " + "let b = Buffer.pooled(n); Use1(b); release b; }\n") + out = generate(parse(disj)) + if "ArrayPool.Shared.Return(a_array)" not in out or "b_array = ArrayPool" not in out: + fails.append("disjoint smoke missing expected lines") + elif out.index("Return(a_array)") > out.index("b_array = ArrayPool"): + fails.append("buffer a must be returned before buffer b is rented") + + # a plain local declared in a buffer's body and used after the release must + # not be trapped in a hoisted try (it would go out of C# scope); the buffer + # is lowered inline (no try) so the local stays at function scope. + aftrel = ("module M\nfn f(n: int){ let b = Buffer.pooled(n); let x = 1; " + "release b; let y = x; }\n") + if codes(aftrel): + fails.append(f"local-after-release should check clean, got {codes(aftrel)}") + body = generate(parse(aftrel)).split("public static void f")[1].split("internal static")[0] + if "try" in body: + fails.append("a buffer with a plain local in its body must not be hoisted") + if "var x = 1;" not in body or "var y = x;" not in body: + fails.append("local-after-release smoke missing expected lines") + elif body.index("var x = 1;") > body.index("var y = x;"): + fails.append("x must be declared before y") + + # a negative scratch size is rejected BEFORE any trace/counter runs + negsc = ("module M\nextern fn Fill(borrow_mut Buffer);\n" + "fn g(n: int){ let b = Buffer.scratch(n); " + "borrow_mut b as m { Fill(m); } release b; }\n") + out = generate(parse(negsc)) + if "if (n < 0)" not in out: + fails.append("scratch dynamic size must guard against a negative request") + elif out.index("if (n < 0)") > out.index("ScratchSelected"): + fails.append("scratch negative guard must run before any trace/counter") + + # partially-overlapping lifetimes (a released while b still live) must NOT be + # hoisted into nested try/finally — that would force b's release before its + # source point. They fall back to faithful inline (release where written). + overlap = ("module M\nresource A { acquire oa release ca }\n" + "resource B { acquire ob release cb }\n" + "fn f(){ let a = acquire A(); let b = acquire B(); " + "release a; use b; release b; }\n") + out = generate(parse(overlap)) + if "Use(b)" not in out or "b.cb()" not in out: + fails.append("overlap smoke missing expected lines") + elif out.index("Use(b)") > out.index("b.cb()"): + fails.append("partial overlap must not emit use-after-release for b") + if "a.ca()" in out and out.index("a.ca()") > out.index("Use(b)"): + fails.append("a must be released at its source point, before use b") + + # a buffer whose release is nested in another owner's borrow block must NOT + # be hoisted (that would double-clean: a stray Dispose plus the finally). + nested = ("module M\nresource Conn { acquire open release close }\n" + "fn f(n: int){ let a = acquire Conn(1); let b = Buffer.pooled(n); " + "borrow a as s { release b; } release a; }\n") + if codes(nested): + fails.append(f"nested buffer release should check clean, got {codes(nested)}") + out = generate(parse(nested)) + if "Dispose()" in out: + fails.append("nested buffer release must not emit a generic Dispose()") + if out.count("ArrayPool.Shared.Return(b_array)") != 1: + fails.append("nested buffer release must return b_array exactly once") + + # native buffers expose a Span view (so a borrow/call sees the same + # logical type as pooled/stack), and free the backing pointer on release + native = ("module M\nextern fn Fill(borrow_mut Buffer);\n" + "fn g(n: int){ let b = Buffer.native(n); " + "borrow_mut b as m { Fill(m); } release b; }\n") + out = generate(parse(native)) + if "new Span(b_ptr, n)" not in out: + fails.append("native buffer must expose a Span view for borrows/calls") + if "NativeMemory.Free(b_ptr)" not in out: + fails.append("native release must free the backing pointer") + if "var m = b;" not in out: + fails.append("native borrow must bind the span view, not the raw pointer") + + # a moved buffer's cleanup alias must not leak onto a later same-named, but + # unrelated, resource declared in another scope + redecl = ("module M\nresource Conn { acquire open release close }\n" + "fn f(n: int){ let a = Buffer.pooled(n); " + "if (c) { let b = move a; release b; } else { release a; } " + "let b = acquire Conn(); release b; }\n") + if codes(redecl): + fails.append(f"alias redeclaration should check clean, got {codes(redecl)}") + out = generate(parse(redecl)) + if "b.close();" not in out: + fails.append("redeclared Conn must be closed, not treated as the buffer alias") + if out.count("ArrayPool.Shared.Return(a_array)") != 2: + fails.append("a_array must be returned exactly once per branch (no stale alias)") + return fails + + +def helper_and_report_smoke() -> list[str]: + """Two review guards: + - a local helper with a `&mut Buffer` / `&Buffer` param lowers to a + Span / ReadOnlySpan signature, so passing a buffer value (a + Span) compiles; + - the report attributes diagnostics by buffer identity (name#line), so two + same-named buffers in sibling scopes are not conflated.""" + fails: list[str] = [] + + helper = ("module M\nfn helper(x: &mut Buffer){ use x; }\n" + "fn view(y: &Buffer){ use y; }\n" + "fn f(n: int){ let b = Buffer.pooled(n); helper(b); view(b); " + "release b; }\n") + if codes(helper): + fails.append(f"local Buffer helper should check clean, got {codes(helper)}") + out = generate(parse(helper)) + if "public static void helper(Span x)" not in out: + fails.append("&mut Buffer param must lower to Span") + if "public static void view(ReadOnlySpan y)" not in out: + fails.append("&Buffer param must lower to ReadOnlySpan") + if "ref Buffer" in out or "ref readonly Buffer" in out: + fails.append("Buffer borrow params must not lower to ref Buffer") + + # same name in sibling scopes: only the leaking buffer fails releaseOnAllPaths + sib = ("module M\nfn f(n: int){ if (c) { let b = Buffer.pooled(n); } " + "else { let b = Buffer.pooled(n); release b; } }\n") + mod = parse(sib) + rn = {r.name for r in mod.resources} + sg = collect_signatures(mod) + pl = collect_policies(mod) + diags = list(validate_policies(pl)) + for fn in mod.functions: + cfg, d1 = build_cfg(fn, rn, sg, pl) + diags += d1 + analyze(cfg) + rep = build_report(mod, diags) + flags = sorted(e["checks"]["releaseOnAllPaths"] for e in rep["buffers"]) + if flags != [False, True]: + fails.append(f"sibling same-name buffers must be attributed separately, got {flags}") + return fails + + def run() -> int: passed = 0 failed = 0 @@ -240,11 +898,43 @@ def run() -> int: for f in golden_fails: print(f"GOLDEN FAIL: {f}") + buffer_fails = buffer_smoke() + for f in buffer_fails: + print(f"BUFFER FAIL: {f}") + + escape_fails = escape_and_length_smoke() + for f in escape_fails: + print(f"ESCAPE FAIL: {f}") + + branchy_fails = branchy_and_malformed_smoke() + for f in branchy_fails: + print(f"BRANCHY FAIL: {f}") + + nest_fails = nesting_native_trace_smoke() + for f in nest_fails: + print(f"NESTING FAIL: {f}") + + order_fails = ordering_counters_smoke() + for f in order_fails: + print(f"ORDERING FAIL: {f}") + + helper_fails = helper_and_report_smoke() + for f in helper_fails: + print(f"HELPER FAIL: {f}") + total = passed + failed print(f"\nanalysis: {passed}/{total} passed, {failed} failed") print(f"codegen: {cg_total - cg_fail}/{cg_total} generated cleanly") print(f"golden: {'PASS' if not golden_fails else 'FAIL'}") - return 1 if (failed or cg_fail or golden_fails) else 0 + print(f"buffer: {'PASS' if not buffer_fails else 'FAIL'}") + print(f"escape: {'PASS' if not escape_fails else 'FAIL'}") + print(f"branchy: {'PASS' if not branchy_fails else 'FAIL'}") + print(f"nesting: {'PASS' if not nest_fails else 'FAIL'}") + print(f"ordering: {'PASS' if not order_fails else 'FAIL'}") + print(f"helper: {'PASS' if not helper_fails else 'FAIL'}") + return 1 if (failed or cg_fail or golden_fails or buffer_fails + or escape_fails or branchy_fails or nest_fails + or order_fails or helper_fails) else 0 if __name__ == "__main__":