Compare commits

...
16 changed files with 2504 additions and 4 deletions
@@ -3,9 +3,11 @@
mod budget;
mod cursor;
mod declarations;
mod index;
mod lex;
mod lower;
mod syntax;
mod update;
mod write;
#[cfg(test)]
@@ -143,6 +143,14 @@ impl Budget {
}
Ok(())
}
/// Aggregate owned text for an index/update phase, checked before copying.
pub fn index_bytes(&self, count: usize, span: Span) -> Result<(), Error> {
self.active()?;
if count > self.limits.bytes {
return Err(error(span, "Source8 index/output byte limit exceeded."));
}
Ok(())
}
pub fn lowering_occurrences(&self, count: usize, span: Span) -> Result<(), Error> {
self.active()?;
// Parameter declarations are syntax entries, not reference uses.
@@ -0,0 +1,709 @@
//! Temporary typed source witnesses. No arena index becomes a model identity.
//! The staged entry point parses each supplied typed file once. A future bundle
//! parser must retain the same opaque parse/lower provenance. Occurrence
//! completeness trusts that lowerer; the private builder does not prove the
//! presence of every syntax reference in a caller-fabricated Lowered value.
use super::cursor::Cursor;
use super::lower::{self, Child, Field, Lowered, Role, Target, Unit};
use super::{Budget, Error, Span, declarations, error, lex};
use crate::{Id, Project, analysis as a};
use std::cmp::Ordering;
use std::collections::{BTreeMap, BTreeSet};
use std::fmt;
use std::sync::Arc;
/// Persistent structural path: cloning shares ancestors, and equality/order
/// compare the complete edge sequence, never allocation or arena identities.
/// Depth is checked against the inherited frame limit before allocation.
/// Comparison and Debug walk leaf-to-root without allocating or recursing;
/// this private total order need not be root-first lexicographic order.
#[derive(Clone, Default)]
pub(super) struct Path(Option<Arc<Step>>);
struct Step {
parent: Path,
edge: Child,
}
impl PartialEq for Path {
fn eq(&self, other: &Self) -> bool {
self.cmp(other) == Ordering::Equal
}
}
impl Eq for Path {}
impl PartialOrd for Path {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl Ord for Path {
fn cmp(&self, other: &Self) -> Ordering {
let (mut left, mut right) = (self, other);
loop {
match (left.0.as_deref(), right.0.as_deref()) {
(Some(a), Some(b)) => {
let order = a.edge.cmp(&b.edge);
if order != Ordering::Equal {
return order;
}
(left, right) = (&a.parent, &b.parent);
}
(None, None) => return Ordering::Equal,
(None, Some(_)) => return Ordering::Less,
(Some(_), None) => return Ordering::Greater,
}
}
}
}
impl fmt::Debug for Path {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
// Debug intentionally shows the same leaf-to-root order as comparison.
let mut list = formatter.debug_list();
let mut path = self;
while let Some(step) = &path.0 {
list.entry(&step.edge);
path = &step.parent;
}
list.finish()
}
}
impl Drop for Path {
fn drop(&mut self) {
let mut next = self.0.take();
while let Some(link) = next {
// Shared ancestors remain owned elsewhere. The final owner peels
// unique links here, taking the parent before Step is dropped.
let Some(mut step) = Arc::into_inner(link) else {
break;
};
next = step.parent.0.take();
}
}
}
impl Path {
pub fn edges(&self) -> Vec<Child> {
let mut result = Vec::new();
let mut path = self;
while let Some(step) = &path.0 {
result.push(step.edge);
path = &step.parent;
}
result.reverse();
result
}
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub(super) struct ExpressionKey {
pub owner: Id,
pub field: Field,
pub path: Path,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub(super) enum TokenRole {
Declaration,
Use(Role),
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub(super) struct OccurrenceKey {
pub owner: Id,
/// None is a global declaration header, not a binding field.
pub field: Option<Field>,
/// None is outside an expression; Some(empty) is its root call site.
pub path: Option<Path>,
pub role: TokenRole,
pub target: Target,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) struct Location {
pub file: usize,
pub span: Span,
}
#[derive(Debug)]
pub(super) struct Index<'s> {
// Private provenance: callers can inspect locations, not supply replacements
// for the source witness, keys, catalog or occurrence spans.
sources: &'s BTreeMap<usize, String>,
names: BTreeMap<Target, String>,
occurrences: BTreeMap<OccurrenceKey, Location>,
expressions: BTreeMap<ExpressionKey, Location>,
}
impl<'s> Index<'s> {
pub fn sources(&self) -> &BTreeMap<usize, String> {
self.sources
}
pub fn names(&self) -> &BTreeMap<Target, String> {
&self.names
}
pub fn occurrences(&self) -> &BTreeMap<OccurrenceKey, Location> {
&self.occurrences
}
pub fn expressions(&self) -> &BTreeMap<ExpressionKey, Location> {
&self.expressions
}
/// Typed-only staged adapter. `ordinary` has already been parsed by its
/// owner. Missing typed IDs fail in lowering; this never invents UUIDs.
/// The caller merges the returned additions and admits the complete model.
pub fn parse(
ordinary: &Project,
sources: &'s BTreeMap<usize, String>,
budget: &mut Budget,
) -> Result<(Self, Lowered), lower::Failure> {
let mut units = Vec::new();
let mut witnesses = BTreeMap::new();
for (&file, source) in sources {
let failure = |error| lower::Failure {
file: Some(file),
error,
};
budget
.lowering_entries(units.len() + 1, Span::default())
.map_err(failure)?;
let tokens = lex::tokens(source, budget).map_err(failure)?;
let mut cursor = Cursor::new(source, &tokens, budget);
let mut parsed = Vec::new();
while cursor.cursor < tokens.len() {
parsed.push(declarations::parse(&mut cursor, 8, None).map_err(failure)?);
}
units.push(Unit {
file,
declarations: parsed,
});
witnesses.insert(file, tokens);
}
let lowered = lower::lower(ordinary, &units, budget)?;
// A build failure after our own parse/lower is a phase/invariant error,
// not an invented file assignment for an ambiguous numeric span.
let index = Self::build(ordinary, sources, &units, &witnesses, &lowered, budget)
.map_err(|error| lower::Failure { file: None, error })?;
Ok((index, lowered))
}
// Same-parse/lower internal seam only. Checks below reject damaged supplied
// sites but occurrence completeness is an invariant of our own lowerer.
// Do not expose a constructor accepting independently fabricated Lowered.
fn build(
ordinary: &Project,
sources: &'s BTreeMap<usize, String>,
units: &[Unit],
witnesses: &BTreeMap<usize, Vec<lex::Token>>,
lowered: &Lowered,
budget: &mut Budget,
) -> Result<Self, Error> {
budget.active()?;
budget.lowering_entries(lowered.expressions.len(), Span::default())?;
budget.lowering_occurrences(lowered.occurrences.len(), Span::default())?;
let mut names = BTreeMap::new();
let mut name_bytes = 0usize;
let mut insert_name = |target, name: &str| -> Result<(), Error> {
budget.lowering_entries(names.len() + 1, Span::default())?;
name_bytes = name_bytes
.checked_add(name.len())
.ok_or_else(|| error(Span::default(), "Index name size overflow."))?;
budget.index_bytes(name_bytes, Span::default())?;
if names.insert(target, name.to_owned()).is_some() {
return Err(error(
Span::default(),
"Duplicate qualified catalog identity.",
));
}
Ok(())
};
// Lowering has already bounded and validated the complete ordinary
// identity catalog before this existing convenience visitor allocates.
for (id, alias) in ordinary.identities() {
insert_name(Target::Global(id), alias)?;
}
for definition in ordinary
.definitions
.values()
.chain(lowered.definitions.values())
{
if lowered.definitions.contains_key(&definition.id) {
insert_name(Target::Global(definition.id), &definition.alias)?;
}
if let a::DefinitionBody::Calculation(calculation) = &definition.body {
for parameter in &calculation.parameters {
insert_name(
Target::Parameter {
calculation: definition.id,
parameter: parameter.id,
},
&parameter.name,
)?;
}
}
}
for element in lowered.elements.values() {
insert_name(Target::Global(element.id), &element.alias)?;
}
let mut index = Self {
sources,
names,
occurrences: BTreeMap::new(),
expressions: BTreeMap::new(),
};
let mut owners = BTreeMap::new();
let mut physical = BTreeSet::new();
for unit in units {
for declaration in &unit.declarations {
let owner = declaration.header.identity.value.ok_or_else(|| {
error(
declaration.span,
"Missing ID requires source stabilization.",
)
})?;
let location = Location {
file: unit.file,
span: declaration.span,
};
checked_source(sources, location)?;
budget.lowering_entries(owners.len() + 1, declaration.span)?;
if owners.insert(owner, location).is_some() {
return Err(error(declaration.span, "Duplicate source owner."));
}
let key = OccurrenceKey {
owner,
field: None,
path: None,
role: TokenRole::Declaration,
target: Target::Global(owner),
};
index.insert(
key,
Location {
file: unit.file,
span: declaration.header.alias.span,
},
witnesses,
&mut physical,
budget,
)?;
}
}
// Parent-before-child is part of the lowering contract. One forward
// pass rejects forward, cyclic, cross-file/owner/field and duplicate
// paths. Each node allocates one shared edge, not a copied path vector.
let mut paths: Vec<(ExpressionKey, usize)> = Vec::new();
for (position, site) in lowered.expressions.iter().enumerate() {
if !matches!(
site.field,
Field::Body | Field::Base | Field::Predicate | Field::Override(_)
) {
return Err(error(
site.span,
"Expression occurs outside a binding field.",
));
}
let location = Location {
file: site.file,
span: site.span,
};
checked_source(sources, location)?;
let owner = owners
.get(&site.owner)
.ok_or_else(|| error(site.span, "Unknown expression owner."))?;
contained(*owner, location)?;
let (path, depth) = if let Some((parent, edge)) = site.parent {
if parent >= position {
return Err(error(
site.span,
"Expression parent must precede its child.",
));
}
let ancestor = &lowered.expressions[parent];
if ancestor.owner != site.owner
|| ancestor.field != site.field
|| ancestor.file != site.file
{
return Err(error(
site.span,
"Expression parent crosses its owner, field or file.",
));
}
contained(
Location {
file: ancestor.file,
span: ancestor.span,
},
location,
)?;
if ancestor.span == site.span {
return Err(error(
site.span,
"Child expression must be a proper source subtree.",
));
}
let depth = paths[parent].1 + 1;
budget.frame(depth, site.span)?;
(
Path(Some(Arc::new(Step {
parent: paths[parent].0.path.clone(),
edge,
}))),
depth,
)
} else {
budget.frame(1, site.span)?;
(Path::default(), 1)
};
let key = ExpressionKey {
owner: site.owner,
field: site.field,
path,
};
if index.expressions.insert(key.clone(), location).is_some() {
return Err(error(site.span, "Duplicate qualified expression path."));
}
paths.push((key, depth));
}
for occurrence in &lowered.occurrences {
let location = Location {
file: occurrence.file,
span: occurrence.span,
};
let owner = owners
.get(&occurrence.owner)
.ok_or_else(|| error(occurrence.span, "Unknown occurrence owner."))?;
contained(*owner, location)?;
let path = match occurrence.expression {
Some(site) => {
let expression = lowered
.expressions
.get(site)
.ok_or_else(|| error(occurrence.span, "Unknown occurrence expression."))?;
if expression.owner != occurrence.owner
|| expression.field != occurrence.field
|| expression.file != occurrence.file
{
return Err(error(
occurrence.span,
"Occurrence crosses its expression owner, field or file.",
));
}
contained(
Location {
file: expression.file,
span: expression.span,
},
location,
)?;
Some(paths[site].0.path.clone())
}
None => None,
};
match (occurrence.role, occurrence.target) {
(
Role::ParameterDeclaration,
Target::Parameter {
calculation,
parameter,
},
) if calculation == occurrence.owner
&& occurrence.field == Field::Input(parameter)
&& path.is_none() => {}
(Role::ParameterUse, Target::Parameter { calculation, .. })
if calculation == occurrence.owner
&& occurrence.field == Field::Body
&& path.is_some() => {}
(Role::ArgumentLabel, Target::Parameter { .. }) if path.is_some() => {}
(Role::OverrideTarget, Target::Global(id))
if occurrence.field == Field::Override(id) && path.is_none() => {}
(Role::CallTarget, Target::Global(_)) if path.is_some() => {}
(Role::Reference, Target::Global(_)) => {}
_ => return Err(error(occurrence.span, "Invalid qualified occurrence role.")),
}
let key = OccurrenceKey {
owner: occurrence.owner,
field: Some(occurrence.field),
path,
role: TokenRole::Use(occurrence.role),
target: occurrence.target,
};
index.insert(key, location, witnesses, &mut physical, budget)?;
}
// Argument labels belong to this precise call site and its callee, not
// merely to a parameter UUID/name present somewhere in the catalog.
let mut calls = BTreeMap::new();
for (key, at) in &index.occurrences {
if key.role == TokenRole::Use(Role::CallTarget) {
let site = (key.owner, key.field, key.path.clone());
if calls.insert(site, key.target).is_some() {
return Err(error(at.span, "Multiple callees at one expression site."));
}
}
}
for (key, at) in &index.occurrences {
if let (TokenRole::Use(Role::ArgumentLabel), Target::Parameter { calculation, .. }) =
(key.role, key.target)
&& calls.get(&(key.owner, key.field, key.path.clone()))
!= Some(&Target::Global(calculation))
{
return Err(error(
at.span,
"Argument label crosses its called calculation.",
));
}
}
for (position, site) in lowered.expressions.iter().enumerate() {
if let Some((parent, Child::Argument(parameter))) = site.parent {
let key = &paths[parent].0;
let Some(Target::Global(calculation)) =
calls.get(&(key.owner, Some(key.field), Some(key.path.clone())))
else {
return Err(error(site.span, "Argument child has no owning call."));
};
let label = OccurrenceKey {
owner: site.owner,
field: Some(site.field),
path: Some(key.path.clone()),
role: TokenRole::Use(Role::ArgumentLabel),
target: Target::Parameter {
calculation: *calculation,
parameter,
},
};
if !index.occurrences.contains_key(&label) || paths[position].0.path == key.path {
return Err(error(site.span, "Argument child has no qualified label."));
}
}
}
index.check_syntax_edges(units, lowered, budget)?;
Ok(index)
}
/// Check structural edges against the already parsed syntax, not just the
/// arena's internally consistent claims. This is one iterative tree walk,
/// with indexed span lookup; it does not lex or copy source/path strings.
fn check_syntax_edges<'a>(
&self,
units: &'a [Unit],
lowered: &Lowered,
budget: &Budget,
) -> Result<(), Error> {
let mut globals = BTreeMap::new();
let mut parameters = BTreeMap::new();
for (target, name) in &self.names {
match target {
Target::Global(id) => {
globals.insert(name.as_str(), *id);
}
Target::Parameter {
calculation,
parameter,
} => {
parameters.insert((*calculation, name.as_str()), *parameter);
}
}
}
let mut sites = BTreeMap::new();
for (position, site) in lowered.expressions.iter().enumerate() {
if sites
.insert(
(
site.file,
site.owner,
site.field,
site.span.start,
site.span.end,
),
position,
)
.is_some()
{
return Err(error(site.span, "Duplicate expression source ownership."));
}
}
let mut work = Vec::new();
let mut root = |file, owner, field, binding: &'a super::Binding| -> Result<(), Error> {
if let super::Binding::Expression(expression) = binding {
budget.lowering_entries(work.len() + 1, expression.span)?;
work.push((file, owner, field, expression, None));
}
Ok(())
};
for unit in units {
for declaration in &unit.declarations {
let owner = declaration.header.identity.value.expect("validated owner");
match &declaration.body {
declarations::Body::Calculation { body, .. } => {
root(unit.file, owner, Field::Body, &body.value)?
}
declarations::Body::Property { base, .. } => {
root(unit.file, owner, Field::Base, &base.value)?
}
declarations::Body::Check { predicate, .. } => {
root(unit.file, owner, Field::Predicate, &predicate.value)?
}
declarations::Body::Case { overrides, .. } => {
for entry in &overrides.value {
let target =
*globals.get(entry.target.name.as_str()).ok_or_else(|| {
error(entry.target.span, "Unknown syntax override target.")
})?;
root(unit.file, owner, Field::Override(target), &entry.binding)?;
}
}
_ => {}
}
}
}
let mut visited = 0usize;
while let Some((file, owner, field, expression, parent)) = work.pop() {
budget.lowering_entries(visited + 1, expression.span)?;
visited += 1;
let site = *sites
.get(&(
file,
owner,
field,
expression.span.start,
expression.span.end,
))
.ok_or_else(|| {
error(
expression.span,
"Parsed expression is absent from the source arena.",
)
})?;
if lowered.expressions[site].parent != parent {
return Err(error(
expression.span,
"Arena parent/child edge differs from original syntax.",
));
}
let mut child = |value: &'a super::Expression, edge| -> Result<(), Error> {
budget.lowering_entries(work.len() + 1, value.span)?;
work.push((file, owner, field, value, Some((site, edge))));
Ok(())
};
use super::ExpressionKind as E;
match &expression.kind {
E::List(items) => {
for (index, item) in items.iter().enumerate() {
child(item, Child::Item(index))?;
}
}
E::Unary { operand, .. } => child(operand, Child::Operand)?,
E::Binary { left, right, .. } => {
child(left, Child::Left)?;
child(right, Child::Right)?;
}
E::If {
condition,
when_true,
when_false,
} => {
child(condition, Child::Condition)?;
child(when_true, Child::WhenTrue)?;
child(when_false, Child::WhenFalse)?;
}
E::Call {
definition,
arguments,
} => {
let calculation = *globals
.get(definition.name.as_str())
.ok_or_else(|| error(definition.span, "Unknown syntax call target."))?;
for argument in arguments {
let parameter = *parameters
.get(&(calculation, argument.parameter.name.as_str()))
.ok_or_else(|| {
error(argument.parameter.span, "Unknown syntax call parameter.")
})?;
child(&argument.value, Child::Argument(parameter))?;
}
}
E::Sum { values, initial } => {
child(values, Child::Values)?;
child(initial, Child::Initial)?;
}
E::Power { base, .. } => child(base, Child::Base)?,
E::AsKind { value, .. } => child(value, Child::Value)?,
_ => {}
}
}
if visited != lowered.expressions.len() {
return Err(error(
Span::default(),
"Arena contains expressions absent from parsed syntax.",
));
}
Ok(())
}
fn insert(
&mut self,
key: OccurrenceKey,
location: Location,
witnesses: &BTreeMap<usize, Vec<lex::Token>>,
physical: &mut BTreeSet<(usize, usize, usize)>,
budget: &Budget,
) -> Result<(), Error> {
budget.lowering_occurrences(self.occurrences.len() + 1, location.span)?;
let text = checked_source(self.sources, location)?;
if self.names.get(&key.target).is_none_or(|name| name != text) {
return Err(error(
location.span,
"Token differs from its qualified target name.",
));
}
let tokens = witnesses
.get(&location.file)
.ok_or_else(|| error(location.span, "Missing lexical witness."))?;
let token = tokens
.binary_search_by_key(&location.span.start, |token| token.span.start)
.ok()
.map(|at| tokens[at]);
if token.is_none_or(|token| token.span != location.span || token.class != lex::Class::Word)
{
return Err(error(
location.span,
"Occurrence is not an original whole name token.",
));
}
if self.occurrences.contains_key(&key)
|| !physical.insert((location.file, location.span.start, location.span.end))
{
return Err(error(
location.span,
"Duplicate qualified occurrence or token ownership.",
));
}
self.occurrences.insert(key, location);
Ok(())
}
}
pub(super) fn checked_source(
sources: &BTreeMap<usize, String>,
location: Location,
) -> Result<&str, Error> {
if location.span.start >= location.span.end {
return Err(error(location.span, "Empty or reversed source span."));
}
sources
.get(&location.file)
.and_then(|source| source.get(location.span.start..location.span.end))
.ok_or_else(|| {
error(
location.span,
"Source span is outside its original UTF-8 file.",
)
})
}
fn contained(parent: Location, child: Location) -> Result<(), Error> {
if parent.file != child.file
|| child.span.start < parent.span.start
|| child.span.end > parent.span.end
{
return Err(error(
child.span,
"Source span crosses its containing syntax.",
));
}
Ok(())
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,630 @@
use super::*;
use crate::language::ParseLimits;
const OA: &str = include_str!("../lower/fixtures/typed-oa.arcadia");
const ORDINARY: &str = include_str!("../lower/fixtures/ordinary-oa.arcadia");
fn budget() -> Budget {
Budget::new(super::super::budget::Limits::source8(
ParseLimits::UNLIMITED,
))
.unwrap()
}
fn ordinary() -> Project {
crate::language::parse(ORDINARY).unwrap()
}
fn sources() -> BTreeMap<usize, String> {
let split = OA.find("calculation combine").unwrap();
BTreeMap::from([
(7, OA[..split].replace('\n', "\r\n")),
(19, OA[split..].to_owned()),
])
}
fn raw(
sources: &BTreeMap<usize, String>,
budget: &mut Budget,
) -> (Vec<Unit>, BTreeMap<usize, Vec<lex::Token>>, Lowered) {
let mut units = Vec::new();
let mut witnesses = BTreeMap::new();
for (&file, source) in sources {
let tokens = lex::tokens(source, budget).unwrap();
let mut cursor = Cursor::new(source, &tokens, budget);
let mut declarations = Vec::new();
while cursor.cursor < tokens.len() {
declarations.push(declarations::parse(&mut cursor, 8, None).unwrap());
}
units.push(Unit { file, declarations });
witnesses.insert(file, tokens);
}
let lowered = lower::lower(&ordinary(), &units, budget).unwrap();
(units, witnesses, lowered)
}
#[test]
fn staged_index_preserves_complete_oa_sa_models_and_original_file_spans() {
for (layer, ordinary_source, typed) in [
(crate::Layer::Oa, ORDINARY, OA),
(
crate::Layer::Sa,
include_str!("../lower/fixtures/ordinary-sa.arcadia"),
include_str!("../lower/fixtures/typed-sa.arcadia"),
),
] {
let sources = BTreeMap::from([(37, typed.replace('\n', "\r\n"))]);
let skeleton = crate::language::parse(ordinary_source).unwrap();
let mut work = budget();
let (index, lowered) = Index::parse(&skeleton, &sources, &mut work).unwrap();
let mut actual = skeleton;
actual.definitions.extend(lowered.definitions);
actual.elements.extend(lowered.elements);
assert_eq!(actual, super::super::model_oracle::project(layer));
a::admit(&actual, a::AdmissionLimits::REFERENCE).unwrap();
assert_eq!(work.usage.bytes, sources[&37].len(), "one lexical pass");
for (key, at) in index.occurrences() {
assert_eq!(at.file, 37);
assert_eq!(
&sources[&37][at.span.start..at.span.end],
index.names[&key.target]
);
}
assert!(!index.expressions().is_empty());
}
}
#[test]
fn repeated_reordered_calls_have_stable_full_paths_and_four_labels() {
let expression = "@combine(b: @dispatch_time, a: @receive_time) // first 雪\n + @combine(a: @receive_time, b: @dispatch_time)";
let original = OA.replace("@combine(a: @receive_time, b: @dispatch_time)", expression);
let reordered = original.replace(
"@combine(b: @dispatch_time, a: @receive_time)",
"@combine(a: @receive_time, b: @dispatch_time)",
);
let first = BTreeMap::from([(7, original)]);
let second = BTreeMap::from([(7, reordered)]);
let (left, _) = Index::parse(&ordinary(), &first, &mut budget()).unwrap();
let (right, _) = Index::parse(&ordinary(), &second, &mut budget()).unwrap();
assert_eq!(
left.occurrences.keys().collect::<Vec<_>>(),
right.occurrences.keys().collect::<Vec<_>>()
);
assert_eq!(
left.expressions.keys().collect::<Vec<_>>(),
right.expressions.keys().collect::<Vec<_>>()
);
let labels: Vec<_> = left
.occurrences
.iter()
.filter(|(key, _)| key.role == TokenRole::Use(Role::ArgumentLabel))
.collect();
assert_eq!(labels.len(), 4);
let paths: BTreeSet<_> = labels
.iter()
.map(|(key, _)| key.path.as_ref().unwrap().edges())
.collect();
assert_eq!(
paths,
BTreeSet::from([vec![Child::Left], vec![Child::Right]])
);
let spans: BTreeSet<_> = labels
.iter()
.map(|(_, at)| (at.span.start, at.span.end))
.collect();
assert_eq!(spans.len(), 4);
}
#[test]
fn index_checks_every_expression_edge_and_keeps_override_fields_distinct() {
let source = OA
.replace("body $a + $b;", "body if(if(true, 1 < 2, false), sum([$a, abs(-$b)], 0 [@second]), as_kind(@duration, power(1 [@second], 2) / 1 [@second]));")
.replace("@receive_time = 1250 [@millisecond];", "@receive_time = @combine(a: 1 [@second], b: 2 [@second]); @dispatch_time = @combine(b: 2 [@second], a: 1 [@second]);");
let sources = BTreeMap::from([(17, source)]);
let (index, lowered) = Index::parse(&ordinary(), &sources, &mut budget()).unwrap();
let mut project = ordinary();
project.definitions.extend(lowered.definitions);
project.elements.extend(lowered.elements);
a::admit(&project, a::AdmissionLimits::REFERENCE).unwrap();
let edges: BTreeSet<_> = index
.expressions
.keys()
.flat_map(|key| key.path.edges())
.collect();
for edge in [
Child::Item(0),
Child::Item(1),
Child::Operand,
Child::Left,
Child::Right,
Child::Condition,
Child::WhenTrue,
Child::WhenFalse,
Child::Values,
Child::Initial,
Child::Base,
Child::Value,
] {
assert!(edges.contains(&edge), "missing {edge:?}");
}
assert!(edges.iter().any(|edge| matches!(edge, Child::Argument(_))));
let labels: Vec<_> = index
.occurrences
.keys()
.filter(|key| {
key.role == TokenRole::Use(Role::ArgumentLabel)
&& matches!(key.field, Some(Field::Override(_)))
})
.collect();
assert_eq!(labels.len(), 4);
assert_eq!(
labels
.iter()
.map(|key| key.field)
.collect::<BTreeSet<_>>()
.len(),
2
);
assert!(
labels
.iter()
.all(|key| key.path.as_ref().unwrap().edges().is_empty())
);
}
#[test]
fn arena_rejects_forward_cycles_and_cross_owner_field_file_parents() {
for damage in 0..6 {
let sources = sources();
let mut work = budget();
let (units, witnesses, mut lowered) = raw(&sources, &mut work);
let child = lowered
.expressions
.iter()
.position(|site| site.parent.is_some())
.unwrap();
let parent = lowered.expressions[child].parent.unwrap().0;
match damage {
0 => lowered.expressions[child].parent = Some((child, Child::Left)),
1 => lowered.expressions[parent].parent = Some((child, Child::Right)),
2 => lowered.expressions[child].owner = ordinary().id,
3 => lowered.expressions[child].field = Field::Predicate,
4 => lowered.expressions[child].file = 12345,
_ => lowered.expressions[child].parent = Some((parent, Child::Operand)),
}
assert!(
Index::build(
&ordinary(),
&sources,
&units,
&witnesses,
&lowered,
&mut work
)
.is_err(),
"damage {damage}"
);
}
}
#[test]
fn arena_rejects_duplicate_sites_occurrences_and_wrong_callee_labels() {
for damage in 0..4 {
let mut sources = sources();
// The other callee is fully valid and reuses both parameter identities
// and spellings. Catalog/token validation must not explain rejection.
sources.insert(
31,
r#"calculation other "Other" id "00000000-0000-0000-0000-00000000cafe" {
input a id "28ff40b0-1b8f-4735-8536-5a6ab21b56cb" : Quantity(@duration);
input b id "7a47cc1e-8ae5-4551-b2d8-8a68108e0976" : Quantity(@duration);
result Quantity(@duration); body $a + $b;
}"#
.into(),
);
let mut work = budget();
let (units, witnesses, mut lowered) = raw(&sources, &mut work);
let valid = Index::build(
&ordinary(),
&sources,
&units,
&witnesses,
&lowered,
&mut work,
)
.unwrap();
let mut expected_mismatch_span = None;
match damage {
0 => {
let site = &lowered.expressions[0];
lowered.expressions.push(lower::ExpressionSite {
file: site.file,
owner: site.owner,
field: site.field,
parent: site.parent,
span: site.span,
});
}
1 => {
let row = &lowered.occurrences[0];
lowered.occurrences.push(lower::Occurrence {
file: row.file,
owner: row.owner,
field: row.field,
expression: row.expression,
role: row.role,
target: row.target,
span: row.span,
});
}
2 => {
let label = lowered
.occurrences
.iter_mut()
.find(|row| row.role == Role::ArgumentLabel)
.unwrap();
let original = label.target;
let Target::Parameter {
calculation,
parameter,
} = &mut label.target
else {
panic!("argument label must target a parameter");
};
let other = Target::Parameter {
calculation: Id::from_u128(0xcafe),
parameter: *parameter,
};
assert_eq!(valid.names[&original], valid.names[&other]);
*calculation = Id::from_u128(0xcafe);
expected_mismatch_span = Some(label.span);
}
_ => {
let row = lowered
.occurrences
.iter_mut()
.find(|row| row.role == Role::OverrideTarget)
.unwrap();
row.field = Field::Override(ordinary().id);
}
}
let failure = Index::build(
&ordinary(),
&sources,
&units,
&witnesses,
&lowered,
&mut work,
)
.unwrap_err();
if let Some(span) = expected_mismatch_span {
assert_eq!(
failure.message,
"Argument label crosses its called calculation."
);
assert_eq!(failure.span, span);
}
}
}
#[test]
fn index_rejects_comment_partial_and_non_utf8_occurrence_spans() {
for damage in 0..3 {
let mut sources = sources();
let with_comment =
sources[&7].replace("subject @receive;", "subject @receive; // duration 雪\r\n");
sources.insert(7, with_comment);
let mut work = budget();
let (units, witnesses, mut lowered) = raw(&sources, &mut work);
let occurrence = lowered
.occurrences
.iter_mut()
.find(|row| row.file == 7 && &sources[&7][row.span.start..row.span.end] == "duration")
.unwrap();
match damage {
0 => {
let start = sources[&7].find("// duration").unwrap() + 3;
occurrence.span = Span {
start,
end: start + 8,
};
}
1 => occurrence.span.end -= 1,
_ => {
let start = sources[&7].find('雪').unwrap() + 1;
occurrence.span = Span {
start,
end: start + 1,
};
}
}
assert!(
Index::build(
&ordinary(),
&sources,
&units,
&witnesses,
&lowered,
&mut work
)
.is_err()
);
}
}
#[test]
fn index_has_shared_budget_and_never_stabilizes_missing_ids() {
let source = "calculation f \"F\" { result Number; body unbound; }";
let files = BTreeMap::from([(91, source.into())]);
let missing = Index::parse(&ordinary(), &files, &mut budget()).unwrap_err();
assert_eq!(missing.file, Some(91));
assert!(missing.error.message.contains("stabilization"));
let mut limits = super::super::budget::Limits::source8(ParseLimits::UNLIMITED);
limits.bytes = 1;
let mut work = Budget::new(limits).unwrap();
assert!(Index::parse(&ordinary(), &sources(), &mut work).is_err());
assert!(work.active().is_err());
}
fn on_small_stack(work: impl FnOnce() + Send + 'static) {
// Match the established core language_nesting_tests worker-stack request.
// This is a requested platform stack size, not a measured peak-byte claim.
std::thread::Builder::new()
.name("analysis-index-128k".into())
.stack_size(128 * 1024)
.spawn(work)
.unwrap()
.join()
.unwrap();
}
fn chain(first: Child, edges: usize) -> Path {
let mut path = Path::default();
for position in 0..edges {
path = Path(Some(Arc::new(Step {
parent: path,
edge: if position == 0 { first } else { Child::Operand },
})));
}
path
}
#[test]
fn maximum_index_paths_compare_lookup_debug_and_drop_on_128k_stack() {
on_small_stack(|| {
let frames = super::super::budget::Limits::source8(ParseLimits::UNLIMITED).frames;
assert_eq!(frames, 256);
let edges = frames - 1; // root has frame depth one
let mut work = budget();
work.frame(frames, Span::default()).unwrap();
let left = chain(Child::Left, edges);
let equal = chain(Child::Left, edges); // independent allocations
let right = chain(Child::Right, edges);
assert!(!Arc::ptr_eq(
left.0.as_ref().unwrap(),
equal.0.as_ref().unwrap()
));
assert_eq!(left, equal);
assert_eq!(left.partial_cmp(&equal), Some(Ordering::Equal));
// The difference is at the root-most edge: leaf-first walks every link.
assert_eq!(left.cmp(&right), Child::Left.cmp(&Child::Right));
assert_eq!(right.cmp(&left), left.cmp(&right).reverse());
assert!(Path::default() < left);
assert!(chain(Child::Operand, edges - 1) < chain(Child::Operand, edges));
let shared = left.clone();
assert!(Arc::ptr_eq(
left.0.as_ref().unwrap(),
shared.0.as_ref().unwrap()
));
let leaf = Arc::downgrade(left.0.as_ref().unwrap());
let key = |path| ExpressionKey {
owner: Id::from_u128(1),
field: Field::Body,
path,
};
let mut expressions = BTreeMap::new();
assert_eq!(expressions.insert(key(left.clone()), 1), None);
assert_eq!(expressions.insert(key(equal.clone()), 2), Some(1));
assert_eq!(expressions.insert(key(right.clone()), 3), None);
assert_eq!(expressions.get(&key(equal.clone())), Some(&2));
assert_eq!(expressions.get(&key(chain(Child::Right, edges))), Some(&3));
assert_eq!(expressions.len(), 2);
let occurrence = |path| OccurrenceKey {
owner: Id::from_u128(1),
field: Some(Field::Body),
path: Some(path),
role: TokenRole::Use(Role::ParameterUse),
target: Target::Parameter {
calculation: Id::from_u128(1),
parameter: a::ParameterId(Id::from_u128(2)),
},
};
let mut occurrences = BTreeSet::new();
assert!(occurrences.insert(occurrence(left.clone())));
assert!(!occurrences.insert(occurrence(equal.clone())));
assert!(occurrences.contains(&occurrence(chain(Child::Left, edges))));
let debug = format!("{left:?}");
assert_eq!(debug.matches("Operand").count(), edges - 1);
assert!(debug.ends_with("Left]"));
assert!(!format!("{expressions:?} {occurrences:?}").is_empty());
drop(expressions);
drop(occurrences);
drop(shared);
drop(left);
assert!(leaf.upgrade().is_none(), "maps/clones release their chain");
drop(equal);
drop(right);
// Dropping a child stops at a shared ancestor, which remains usable.
let parent = chain(Child::Left, edges - 1);
let ancestor = Arc::downgrade(parent.0.as_ref().unwrap());
let child = Path(Some(Arc::new(Step {
parent: parent.clone(),
edge: Child::Right,
})));
drop(child);
assert_eq!(parent.edges().len(), edges - 1);
drop(parent);
assert!(ancestor.upgrade().is_none());
// No other strong owners: final destruction must peel the full chain.
let unique = chain(Child::Left, edges);
let final_leaf = Arc::downgrade(unique.0.as_ref().unwrap());
drop(unique);
assert!(final_leaf.upgrade().is_none());
});
}
#[test]
fn deep_parse_index_plan_and_failed_build_cleanup_on_128k_stack() {
on_small_stack(|| {
use super::super::update::Plan;
let deep = format!("{}$a{}", "abs(".repeat(127), ")".repeat(127));
let source = format!(
r#"calculation f "Deep 雪" id "00000000-0000-0000-0000-000000000081" {{
input a id "00000000-0000-0000-0000-000000000083" : Number;
result Number; body {deep};
}}
calculation caller "Caller" id "00000000-0000-0000-0000-000000000082" {{
result Number; body @f(a: 1);
}}
"#
);
let sources = BTreeMap::from([(47, source.clone())]);
let skeleton = ordinary();
let mut work = budget();
let (index, lowered) = Index::parse(&skeleton, &sources, &mut work).unwrap();
assert_eq!(work.usage.bytes, source.len());
assert_eq!(index.expressions.len(), 130); // 128 unary/leaf + call/argument
assert_eq!(
index.expressions.keys().map(|k| k.path.edges().len()).max(),
Some(127)
);
let renames = BTreeMap::from([
(Target::Global(Id::from_u128(0x81)), "renamed_deep".into()),
(
Target::Parameter {
calculation: Id::from_u128(0x81),
parameter: a::ParameterId(Id::from_u128(0x83)),
},
"renamed_arg".into(),
),
]);
let usage = work.usage;
let plan = Plan::renames(&index, &renames, &work).unwrap();
assert_eq!(plan.edit_count(), 5); // declaration/call + input/use/label
let output = plan.apply(&sources, &work).unwrap();
assert_eq!(work.usage, usage);
// Independent complete original-token oracle; no index-derived offsets.
let expected_source = source
.replace("calculation f ", "calculation renamed_deep ")
.replace("input a id", "input renamed_arg id")
.replace("$a", "$renamed_arg")
.replace("@f(a: 1)", "@renamed_deep(renamed_arg: 1)");
assert_eq!(output, BTreeMap::from([(47, expected_source)]));
let (renamed_index, renamed) = Index::parse(&skeleton, &output, &mut budget()).unwrap();
assert_eq!(
index.occurrences.keys().collect::<Vec<_>>(),
renamed_index.occurrences.keys().collect::<Vec<_>>()
);
// Compare complete independently constructed models, including deepest
// parameter identity and the call's parameter-keyed argument map.
for (lowered, renamed) in [(lowered, false), (renamed, true)] {
let mut expected = skeleton.clone();
let parameter = a::ParameterId(Id::from_u128(0x83));
let mut expression = a::Expr::Parameter(parameter);
for _ in 0..127 {
expression = a::Expr::Unary {
op: a::UnaryOp::Absolute,
operand: Box::new(expression),
};
}
for (id, alias, name, parameters, expression) in [
(
0x81,
if renamed { "renamed_deep" } else { "f" },
"Deep 雪",
vec![a::Parameter {
id: parameter,
name: if renamed { "renamed_arg" } else { "a" }.into(),
value_type: a::AnalysisType::Scalar(a::ScalarType::Number),
}],
expression,
),
(
0x82,
"caller",
"Caller",
Vec::new(),
a::Expr::Call {
definition: a::DefinitionRef::Local(Id::from_u128(0x81)),
arguments: BTreeMap::from([(
parameter,
a::Expr::Number(crate::analysis_numeric::Exact::from_integer(1)),
)]),
},
),
] {
let id = Id::from_u128(id);
expected.definitions.insert(
id,
a::Definition {
id,
alias: alias.into(),
name: name.into(),
description: String::new(),
body: a::DefinitionBody::Calculation(a::CalculationDefinition {
parameters,
result: a::AnalysisType::Scalar(a::ScalarType::Number),
body: a::Binding::Expression(expression),
}),
},
);
}
let mut actual = skeleton.clone();
actual.definitions.extend(lowered.definitions);
actual.elements.extend(lowered.elements);
assert_eq!(actual, expected);
a::admit(&actual, a::AdmissionLimits::REFERENCE).unwrap();
drop(actual);
drop(expected);
}
drop(plan);
drop(renamed_index);
drop(index);
// Failure after the deepest keys have been allocated must clean up on
// the same stack; no modified Lowered escapes this private test seam.
let mut work = budget();
let (units, witnesses, mut lowered) = raw(&sources, &mut work);
let deepest = lowered
.expressions
.iter()
.find(|site| &sources[&site.file][site.span.start..site.span.end] == "$a")
.unwrap();
lowered.expressions.push(lower::ExpressionSite {
file: deepest.file,
owner: deepest.owner,
field: deepest.field,
parent: deepest.parent,
span: deepest.span,
});
let failure =
Index::build(&skeleton, &sources, &units, &witnesses, &lowered, &mut work).unwrap_err();
assert_eq!(failure.message, "Duplicate qualified expression path.");
lowered.expressions.pop();
let mut limits = super::super::budget::Limits::source8(ParseLimits::UNLIMITED);
limits.frames = 127;
let mut bounded = Budget::new(limits).unwrap();
let failure = Index::build(
&skeleton,
&sources,
&units,
&witnesses,
&lowered,
&mut bounded,
)
.unwrap_err();
assert!(failure.message.contains("syntax frames"));
assert!(bounded.active().is_err());
drop(lowered);
drop(witnesses);
drop(units);
let rejected = BTreeMap::from([(47, source.replace(&deep, &format!("abs({deep})")))]);
let failure = Index::parse(&skeleton, &rejected, &mut budget()).unwrap_err();
assert!(failure.error.message.contains("logical depth"));
});
}
@@ -34,7 +34,7 @@ pub(super) enum Category {
Calculation,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub(super) enum Target {
Global(Id),
Parameter {
@@ -43,7 +43,7 @@ pub(super) enum Target {
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub(super) enum Field {
Algebra,
Kind,
@@ -59,7 +59,7 @@ pub(super) enum Field {
Override(Id),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub(super) enum Role {
Reference,
ParameterDeclaration,
@@ -70,7 +70,7 @@ pub(super) enum Role {
}
/// Stable structural edges, independent of alias spelling and argument order.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub(super) enum Child {
Item(usize),
Operand,
@@ -349,6 +349,11 @@ impl Cursor<'_, '_> {
_ => 0,
};
if child_depth >= crate::analysis::MAX_EXPRESSION_DEPTH {
// The rejected parent still owns accepted, potentially depth-128
// children. Recursive derived cleanup here runs beneath the parser
// frames and can exhaust the bounded native stack before returning
// the depth error. Consume those children with an explicit worklist.
drop_rejected_kind(kind);
return Err(error(span, "Expression logical depth exceeds 128."));
}
Ok(Expression {
@@ -399,6 +404,42 @@ impl Cursor<'_, '_> {
self.node(kind, span)
}
}
fn drop_rejected_kind(kind: ExpressionKind) {
let mut pending = vec![kind];
while let Some(kind) = pending.pop() {
match kind {
ExpressionKind::Unary { operand, .. } => pending.push(operand.kind),
ExpressionKind::Binary { left, right, .. } => {
pending.push(left.kind);
pending.push(right.kind);
}
ExpressionKind::If {
condition,
when_true,
when_false,
} => {
pending.push(condition.kind);
pending.push(when_true.kind);
pending.push(when_false.kind);
}
ExpressionKind::List(values) => pending.extend(values.into_iter().map(|v| v.kind)),
ExpressionKind::Call { arguments, .. } => {
pending.extend(arguments.into_iter().map(|a| a.value.kind));
}
ExpressionKind::Sum { values, initial } => {
pending.push(values.kind);
pending.push(initial.kind);
}
ExpressionKind::Power { base, .. } => pending.push(base.kind),
ExpressionKind::AsKind { value, .. } => pending.push(value.kind),
ExpressionKind::Boolean(_)
| ExpressionKind::Number(_)
| ExpressionKind::Quantity { .. }
| ExpressionKind::Property(_)
| ExpressionKind::Parameter(_) => {}
}
}
}
fn comparison(op: BinaryOp) -> bool {
matches!(
op,
@@ -108,6 +108,49 @@ fn flat_binary_depth_and_malformed_constructor_retain_bounded_completed_children
assert_eq!(number(&expression("1")), "1");
}
#[test]
fn every_parent_shape_rejects_depth129_and_cleans_children_on_128k_stack() {
std::thread::Builder::new()
.name("analysis-syntax-rejection-128k".into())
.stack_size(128 * 1024)
.spawn(|| {
// Rotate through every recursive syntax shape, so the rejected
// parent's complete child tree also exercises every cleanup arm.
let wrap = |shape, child: &str| match shape {
0 => format!("abs({child})"),
1 => format!("({child}) + 1"),
2 => format!("if(true, {child}, 0)"),
3 => format!("[{child}]"),
4 => format!("@f(a: {child})"),
5 => format!("sum({child}, 0)"),
6 => format!("power({child}, 1)"),
7 => format!("as_kind(@k, {child})"),
_ => unreachable!(),
};
let mut deep = "1".to_owned();
for layer in 0..127 {
deep = wrap(layer % 8, &deep);
}
assert_eq!(expression(&deep).depth, 128);
for shape in 0..8 {
let source = wrap(shape, &deep);
let failure = parse(&source, &mut budget()).unwrap_err();
assert_eq!(failure.message, "Expression logical depth exceeds 128.");
assert_eq!(
failure.span,
Span {
start: 0,
end: source.len()
}
);
assert_eq!(number(&expression("1")), "1");
}
})
.unwrap()
.join()
.unwrap();
}
#[test]
fn significant_ratio_and_quantity_trivia_preserve_original_spans() {
let source = "ratio // explanation café\n ( - // sign\n 3, // denominator\n 4 ) // measurement\n [@millisecond]";
@@ -0,0 +1,198 @@
//! Simultaneous typed token edits. This is a staged plan, not source acceptance:
//! the bundle owner must merge ordinary edits, reparse/recompile the exact bytes,
//! admit once and compare the complete intended Project before installation.
use super::index::{Index, Location, OccurrenceKey, checked_source};
use super::lower::Target;
use super::{Budget, Error, Span, error};
use std::collections::{BTreeMap, BTreeSet};
#[derive(Debug, Clone, PartialEq, Eq)]
struct Edit {
key: OccurrenceKey,
location: Location,
}
#[derive(Debug)]
pub(super) struct Plan<'i, 's> {
index: &'i Index<'s>,
names: BTreeMap<Target, String>,
edits: Vec<Edit>,
}
impl<'i, 's> Plan<'i, 's> {
/// All names are the desired final names. Validate the final namespaces,
/// rather than applying one rename at a time: alias/parameter swaps are legal.
pub fn renames(
index: &'i Index<'s>,
renames: &BTreeMap<Target, String>,
budget: &Budget,
) -> Result<Self, Error> {
budget.active()?;
budget.lowering_entries(renames.len(), Span::default())?;
let mut bytes = 0usize;
for (target, name) in renames {
if !index.names().contains_key(target) {
return Err(error(
Span::default(),
"Rename has no qualified source target.",
));
}
if !crate::valid_alias(name) {
return Err(error(
Span::default(),
"Invalid alias or scoped parameter name.",
));
}
bytes = bytes
.checked_add(name.len())
.ok_or_else(|| error(Span::default(), "Rename size overflow."))?;
budget.index_bytes(bytes, Span::default())?;
}
let mut final_names = BTreeSet::new();
for (target, original) in index.names() {
let name = renames.get(target).unwrap_or(original);
let scope = match target {
Target::Global(_) => None,
Target::Parameter { calculation, .. } => Some(*calculation),
};
budget.lowering_entries(final_names.len() + 1, Span::default())?;
if !final_names.insert((scope, name.as_str())) {
return Err(error(
Span::default(),
"Rename creates a duplicate name in its namespace.",
));
}
}
let mut edits = Vec::new();
for (key, location) in index.occurrences() {
if renames
.get(&key.target)
.is_some_and(|name| index.names().get(&key.target) != Some(name))
{
budget.lowering_occurrences(edits.len() + 1, location.span)?;
edits.push(Edit {
key: key.clone(),
location: *location,
});
}
}
edits.sort_by_key(|edit| {
(
edit.location.file,
edit.location.span.start,
edit.location.span.end,
)
});
let plan = Self {
index,
names: renames.clone(),
edits,
};
plan.validate(index.sources(), budget)?;
Ok(plan)
}
pub fn edit_count(&self) -> usize {
self.edits.len()
}
/// Recheck all files, all edit ownership and all final sizes before any
/// output String is allocated. A changed unrelated file is stale too.
fn validate(
&self,
current: &BTreeMap<usize, String>,
budget: &Budget,
) -> Result<BTreeMap<usize, usize>, Error> {
budget.active()?;
if current != self.index.sources() {
return Err(error(
Span::default(),
"Source index is stale: original file set or bytes changed.",
));
}
budget.lowering_entries(current.len(), Span::default())?;
budget.lowering_occurrences(self.edits.len(), Span::default())?;
let mut lengths: BTreeMap<_, _> = current
.iter()
.map(|(&file, source)| (file, source.len()))
.collect();
let mut previous: Option<Location> = None;
let mut seen = BTreeSet::new();
for edit in &self.edits {
let at = edit.location;
if self.index.occurrences().get(&edit.key) != Some(&at) || !seen.insert(&edit.key) {
return Err(error(at.span, "Forged or duplicate edit occurrence."));
}
let old = checked_source(current, at)?;
if self
.index
.names()
.get(&edit.key.target)
.is_none_or(|name| name != old)
{
return Err(error(at.span, "Stale original token spelling."));
}
let replacement = self
.names
.get(&edit.key.target)
.ok_or_else(|| error(at.span, "Edit has no final target name."))?;
if previous.is_some_and(|prior| {
prior.file > at.file || (prior.file == at.file && prior.span.end > at.span.start)
}) {
return Err(error(at.span, "Overlapping or unordered source edits."));
}
previous = Some(at);
let length = lengths
.get_mut(&at.file)
.ok_or_else(|| error(at.span, "Unknown edit file."))?;
*length = length
.checked_sub(old.len())
.and_then(|size| size.checked_add(replacement.len()))
.ok_or_else(|| error(at.span, "Updated source size overflow."))?;
}
for (key, at) in self.index.occurrences() {
let required = self
.names
.get(&key.target)
.is_some_and(|name| self.index.names().get(&key.target) != Some(name));
if required != seen.contains(key) {
return Err(error(
at.span,
"Edit set differs from the complete qualified rename plan.",
));
}
}
let mut total = 0usize;
for length in lengths.values() {
total = total
.checked_add(*length)
.ok_or_else(|| error(Span::default(), "Updated bundle size overflow."))?;
budget.index_bytes(total, Span::default())?;
}
Ok(lengths)
}
pub fn apply(
&self,
current: &BTreeMap<usize, String>,
budget: &Budget,
) -> Result<BTreeMap<usize, String>, Error> {
let lengths = self.validate(current, budget)?;
let mut result = BTreeMap::new();
let mut edits = self.edits.iter().peekable();
for (&file, source) in current {
let mut output = String::with_capacity(lengths[&file]);
let mut start = 0;
while edits.peek().is_some_and(|edit| edit.location.file == file) {
let edit = edits.next().expect("peeked edit");
output.push_str(&source[start..edit.location.span.start]);
output.push_str(&self.names[&edit.key.target]);
start = edit.location.span.end;
}
output.push_str(&source[start..]);
result.insert(file, output);
}
Ok(result)
}
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,324 @@
use super::super::budget::Limits;
use super::*;
use crate::analysis as a;
use crate::analysis_numeric::Exact;
use crate::language::ParseLimits;
use crate::{Id, Layer, Project};
const DEFINITIONS: &str = "// 雪 @f(a:) $a: comment spellings stay exact\r\n\
calculation f \"f a b\" id \"00000000-0000-0000-0000-000000000100\" {\r\n\
input a id \"00000000-0000-0000-0000-000000000200\" : Number;\r\n\
input b id \"00000000-0000-0000-0000-000000000201\" : Number;\r\n\
result Number; body $a + $b;\r\n}\r\n\
calculation g \"g a b\" id \"00000000-0000-0000-0000-000000000200\" {\r\n\
input a id \"00000000-0000-0000-0000-000000000200\" : Number;\r\n\
input b id \"00000000-0000-0000-0000-000000000201\" : Number;\r\n\
result Number; body $a + $b;\r\n}\r\n";
const CALLER: &str = "calculation caller \"Caller\" id \"00000000-0000-0000-0000-000000000300\" {\n\
result Number; body @f(b: 2, a: 1) // first 雪 @f(a:)\n\
+ @f(a: 3, b: 4) // second\n\
+ @g(a: 5, b: 6);\n}\n";
fn budget() -> Budget {
Budget::new(Limits::source8(ParseLimits::UNLIMITED)).unwrap()
}
fn global(id: u128) -> Target {
Target::Global(Id::from_u128(id))
}
fn parameter(calculation: u128, parameter: u128) -> Target {
Target::Parameter {
calculation: Id::from_u128(calculation),
parameter: a::ParameterId(Id::from_u128(parameter)),
}
}
fn sources() -> BTreeMap<usize, String> {
BTreeMap::from([(11, DEFINITIONS.into()), (23, CALLER.into())])
}
fn skeleton() -> Project {
let mut project = super::super::model_oracle::project(Layer::Oa);
project.definitions.clear();
project
.elements
.retain(|_, element| !element.kind().is_analysis());
project
}
fn binary(left: a::Expr, right: a::Expr) -> a::Expr {
a::Expr::Binary {
op: a::BinaryOp::Add,
left: Box::new(left),
right: Box::new(right),
}
}
fn expected(renamed: bool) -> Project {
// Independent model construction: no parsed AST, source index, writer or
// generated edit participates in this expected complete Project.
let mut project = skeleton();
let p = |id| a::ParameterId(Id::from_u128(id));
for (id, alias, name) in [
(0x100, if renamed { "g" } else { "f" }, "f a b"),
(0x200, if renamed { "f" } else { "g" }, "g a b"),
] {
let names = if renamed && id == 0x100 {
["b", "a"]
} else {
["a", "b"]
};
let body = a::CalculationDefinition {
parameters: [(0x200, names[0]), (0x201, names[1])]
.into_iter()
.map(|(id, name)| a::Parameter {
id: p(id),
name: name.into(),
value_type: a::AnalysisType::Scalar(a::ScalarType::Number),
})
.collect(),
result: a::AnalysisType::Scalar(a::ScalarType::Number),
body: a::Binding::Expression(binary(
a::Expr::Parameter(p(0x200)),
a::Expr::Parameter(p(0x201)),
)),
};
let id = Id::from_u128(id);
project.definitions.insert(
id,
a::Definition {
id,
alias: alias.into(),
name: name.into(),
description: String::new(),
body: a::DefinitionBody::Calculation(body),
},
);
}
let call = |id, x, y| a::Expr::Call {
definition: a::DefinitionRef::Local(Id::from_u128(id)),
arguments: BTreeMap::from([
(p(0x200), a::Expr::Number(Exact::from_integer(x))),
(p(0x201), a::Expr::Number(Exact::from_integer(y))),
]),
};
let id = Id::from_u128(0x300);
project.definitions.insert(
id,
a::Definition {
id,
alias: "caller".into(),
name: "Caller".into(),
description: String::new(),
body: a::DefinitionBody::Calculation(a::CalculationDefinition {
parameters: Vec::new(),
result: a::AnalysisType::Scalar(a::ScalarType::Number),
body: a::Binding::Expression(binary(
binary(call(0x100, 1, 2), call(0x100, 3, 4)),
call(0x200, 5, 6),
)),
}),
},
);
project
}
fn merge(mut project: Project, lowered: super::super::lower::Lowered) -> Project {
project.definitions.extend(lowered.definitions);
project.elements.extend(lowered.elements);
project
}
fn rename_map() -> BTreeMap<Target, String> {
BTreeMap::from([
(global(0x100), "g".into()),
(global(0x200), "f".into()),
(parameter(0x100, 0x200), "b".into()),
(parameter(0x100, 0x201), "a".into()),
])
}
#[test]
fn simultaneous_global_and_scoped_swaps_match_original_token_and_whole_model_oracles() {
let sources = sources();
let mut work = budget();
let (index, lowered) = Index::parse(&skeleton(), &sources, &mut work).unwrap();
assert_eq!(merge(skeleton(), lowered), expected(false));
let plan = Plan::renames(&index, &rename_map(), &work).unwrap();
// List expected edits against the original authored bytes independently of
// the index. Full contexts deliberately exclude equal strings/comments.
let mut edits = Vec::new();
let mut add = |file, context: &str, token: &str, replacement: &str| {
let source = &sources[&file];
let context_start = source.find(context).unwrap();
let start = context_start + context.find(token).unwrap();
edits.push((file, start, start + token.len(), replacement.to_owned()));
};
add(11, "calculation f ", "f", "g");
add(11, "calculation g ", "g", "f");
add(11, "input a id", "a", "b");
add(11, "input b id", "b", "a");
add(11, "body $a + $b;", "a", "b");
add(11, "body $a + $b;", "b;", "a;");
// The last oracle edit above includes ';' to disambiguate 'b' in 'body';
// restrict it back to the exact name token for the span oracle.
edits.last_mut().unwrap().2 -= 1;
edits.last_mut().unwrap().3.pop();
for call in ["@f(b: 2, a: 1)", "@f(a: 3, b: 4)"] {
add_call_edits(&sources, &mut edits, call);
}
let start = CALLER.find("@g(a: 5, b: 6)").unwrap() + 1;
edits.push((23, start, start + 1, "f".into()));
edits.sort_by_key(|edit| (edit.0, edit.1));
assert_eq!(plan.edit_count(), 13);
assert_eq!(
plan.edits
.iter()
.map(|edit| (
edit.location.file,
edit.location.span.start,
edit.location.span.end,
plan.names[&edit.key.target].clone()
))
.collect::<Vec<_>>(),
edits
);
let mut exact = BTreeMap::new();
for (&file, source) in &sources {
let mut output = source.clone();
for (_, start, end, value) in edits.iter().filter(|edit| edit.0 == file).rev() {
output.replace_range(*start..*end, value);
}
exact.insert(file, output);
}
let output = plan.apply(&sources, &work).unwrap();
assert_eq!(output, exact);
assert!(output[&11].contains("// 雪 @f(a:) $a: comment spellings stay exact\r\n"));
assert!(output[&23].contains("// first 雪 @f(a:)\n"));
assert!(output[&11].contains("\"f a b\""));
let (_, lowered) = Index::parse(&skeleton(), &output, &mut budget()).unwrap();
let actual = merge(skeleton(), lowered);
assert_eq!(actual, expected(true));
a::admit(&actual, a::AdmissionLimits::REFERENCE).unwrap();
}
fn add_call_edits(
sources: &BTreeMap<usize, String>,
edits: &mut Vec<(usize, usize, usize, String)>,
call: &str,
) {
let start = sources[&23].find(call).unwrap();
for (offset, replacement) in [
(1, "g"),
(call.find("a:").unwrap(), "b"),
(call.find("b:").unwrap(), "a"),
] {
edits.push((23, start + offset, start + offset + 1, replacement.into()));
}
}
#[test]
fn alias_edits_preserve_unit_type_override_tokens_and_unrelated_authored_bytes() {
let source = include_str!("../lower/fixtures/typed-oa.arcadia")
.replace("750 [@millisecond]", "750 // exact 雪\r\n [@millisecond]");
let files = BTreeMap::from([(59, source.clone())]);
let mut work = budget();
let (index, _) = Index::parse(&skeleton(), &files, &mut work).unwrap();
let mut expected_project = super::super::model_oracle::project(Layer::Oa);
let id = |name: &str| {
index
.names()
.iter()
.find(|(target, alias)| matches!(target, Target::Global(_)) && alias.as_str() == name)
.map(|(target, _)| *target)
.unwrap()
};
let renames = BTreeMap::from([
(id("duration"), "time_kind".into()),
(id("millisecond"), "ms_unit".into()),
(id("receive_time"), "arrival".into()),
]);
let plan = Plan::renames(&index, &renames, &work).unwrap();
let output = plan.apply(&files, &work).unwrap();
let expected_source = source
.replace("quantity_kind duration ", "quantity_kind time_kind ")
.replace("@duration", "@time_kind")
.replace("unit millisecond ", "unit ms_unit ")
.replace("@millisecond", "@ms_unit")
.replace("oa_property receive_time ", "oa_property arrival ")
.replace("@receive_time", "@arrival");
assert_eq!(output[&59], expected_source);
for definition in expected_project.definitions.values_mut() {
if let Some(name) = renames.get(&Target::Global(definition.id)) {
definition.alias = name.clone();
}
}
for element in expected_project.elements.values_mut() {
if let Some(name) = renames.get(&Target::Global(element.id)) {
element.alias = name.clone();
}
}
let (_, lowered) = Index::parse(&skeleton(), &output, &mut budget()).unwrap();
assert_eq!(merge(skeleton(), lowered), expected_project);
}
#[test]
fn rename_rejects_unknown_invalid_and_colliding_final_names_but_allows_noop() {
let sources = sources();
let work = &mut budget();
let (index, _) = Index::parse(&skeleton(), &sources, work).unwrap();
for renames in [
BTreeMap::from([(global(999), "new".into())]),
BTreeMap::from([(global(0x100), "bad name".into())]),
BTreeMap::from([(global(0x100), "g".into())]),
BTreeMap::from([(parameter(0x100, 0x200), "b".into())]),
] {
assert!(Plan::renames(&index, &renames, work).is_err());
}
let plan = Plan::renames(&index, &BTreeMap::from([(global(0x100), "f".into())]), work).unwrap();
assert_eq!(plan.edit_count(), 0);
assert_eq!(plan.apply(&sources, work).unwrap(), sources);
}
#[test]
fn stale_file_sets_bytes_forged_spans_and_duplicate_edits_fail_before_output() {
let sources = sources();
let mut work = budget();
let (index, _) = Index::parse(&skeleton(), &sources, &mut work).unwrap();
for damage in 0..6 {
let mut current = sources.clone();
let mut plan = Plan::renames(&index, &rename_map(), &work).unwrap();
match damage {
0 => current
.get_mut(&23)
.unwrap()
.push_str("// unrelated change"),
1 => {
current.remove(&23);
}
2 => {
current.insert(99, String::new());
}
3 => plan.edits[0].location.span.start += 1,
4 => plan.edits.insert(1, plan.edits[0].clone()),
_ => {
plan.edits.pop();
}
}
assert!(plan.apply(&current, &work).is_err(), "damage {damage}");
}
}
#[test]
fn output_and_plan_limits_are_aggregate_and_do_not_reset_parse_work() {
let sources = sources();
let mut work = budget();
let (index, _) = Index::parse(&skeleton(), &sources, &mut work).unwrap();
let usage = work.usage;
let numeric = work.numeric_units();
let plan = Plan::renames(&index, &rename_map(), &work).unwrap();
let output = plan.apply(&sources, &work).unwrap();
assert_eq!(work.usage, usage);
assert_eq!(work.numeric_units(), numeric);
let bytes = output.values().map(String::len).sum();
let mut limits = Limits::source8(ParseLimits::UNLIMITED);
limits.bytes = bytes;
assert!(plan.apply(&sources, &Budget::new(limits).unwrap()).is_ok());
limits.bytes -= 1;
assert!(plan.apply(&sources, &Budget::new(limits).unwrap()).is_err());
limits = Limits::source8(ParseLimits::UNLIMITED);
limits.references = 0;
limits.entries = 0;
assert!(Plan::renames(&index, &rename_map(), &Budget::new(limits).unwrap()).is_err());
}
@@ -0,0 +1 @@
* -text
@@ -0,0 +1,46 @@
# Private S3 Windows evidence
Source `1b1c3e3abb6fc3d0934c9dbfbd21fddc8ae4f594`, following `90fae14818f684bf778640be2e613f34e3c7e999`. This checkpoint verifies the private typed index/rename slice and the rejected-depth syntax cleanup. **It does not establish current full-core runtime, public source8, integration, native UI or release acceptance.** Parent owns review/publication and integration.
| Recorded scope | Result |
| --- | --- |
| Full library listing | 167 test names; original 166 retained plus one cleanup test |
| Typed source runtime | 59 passed, including all 9 index and 5 update tests |
| Existing raw-token control | 1 passed |
| Three focused 128 KiB controls | Original whole-flow, eight-parent cleanup and synthetic 256-frame Path tests passed |
| Strict core, all targets | Clippy with `-D warnings`, exit0 |
| Original90fae typed gate | Stack overflow retained; aborted run is not a passing full gate |
| Temporary diagnostics/probes | Two dirty overflow reproductions; four dirty direct-parser malformed-input probes passed |
Actual platform was Windows x86_64-pc-windows-gnullvm, Rust/Cargo 1.98.1, one compiler job. Every clean gate binds 538 committed inputs before/after, exact commands and source. Raw source manifests, logs, receipts, helper scripts and diagnostic diffs remain byte-for-byte unchanged in [evidence.zip](evidence.zip). Its SHA256 is `4486f4c42b140ae4b9b3caf0aacdf151a2acb1fd85eda09a57764fd69e980574`; 658251 bytes, 61 members. No executables, caches or profiles are included.
The failure occurred when `Cursor::node` recursively dropped a rejected depth129 syntax tree below parser frames. Complete depth128 parsing/indexing/renaming/output/model equality/admission and earlier cleanup had finished. The source fix uses iterative rejected-kind disposal and preserves the existing stack request, depth limit and assertions. The 14 original S3 tests and the added mixed-shape cleanup test now have actual runtime evidence. Ordinary AST destruction and all possible parser-error paths are not claimed iterative or universally stack-safe.
## Reproduce evidence verification
With Python 3 standard library and Git, from any working directory:
```text
python -B PATH/verify.py --repo PATH/TO/GIT/REPOSITORY
python -B PATH/negative_controls.py --repo PATH/TO/GIT/REPOSITORY
```
The repository needs immutable commits `1b1c3e3`, `90fae148` and `1f828323` (the publication commit contains that ancestry). Verification reads commit blobs; it neither depends on nor changes the current checkout. `--evidence-dir PATH` optionally selects another complete extracted payload directory. No archived helper, compiler or native artifact is executed. Optimized Python is refused.
The verifier checks both manifests/ZIP byte integrity, actual listing and exact selected runtime membership, 538 committed inputs, recorded commands/times/environment, strict completion, historical source-artifact manifests and the original aborted gate. It reconstructs the dirty source patches in memory and verifies the corresponding 538 test-input hashes. Broader tracked-file hashes also remain in the original captures; the verifier does not claim to audit unrelated tracked files outside that test-input scope. Binary identities are checked against raw executable-path receipts, not rehashed executable bytes, because binaries are deliberately absent. Initial focused tests and final restored typed/token/listing tests were separate builds of identical source and have their own binary receipts.
[checks.json](checks.json) records the positive working evidence audit and 13 discriminating negative controls. Those controls corrupt evidence only in memory; raw archives stay intact. Semantic cases reach their specific checks instead of failing only at the outer ZIP checksum. Package verification also runs against the saved commit and a fresh extraction; exact publication-commit readback is reported in the handoff to the parent, since embedding a commit's own hash in its contents is not possible.
## Review and historical evidence navigation
Extract the ZIP only if desired; the verifier itself inspects it in memory. Current authoritative raw closure is `closure.json` plus `raw/`. Relevant retained notes inside the ZIP:
- `notes/analysis-source8-s3-index-independent-review-20260913.md`: original S3 source review and required corrections.
- `notes/analysis-source8-s3-correction-independent-review-20260922.md`: corrected Path/index source review before runtime.
- `notes/analysis-source3-stack-cleanup-review-20260922.md`: independent review of iterative rejected-kind cleanup.
- `notes/analysis-source3-stack-cleanup-handoff-20260922.md`: exact failure isolation, fix and completed Windows gates.
- Original index/correction source manifests bind their respective immutable Git versions; their historical `unrun` labels are preserved and superseded only by the later actual receipts.
The ZIP's initial `README.md` contains the earlier portable-verifier proposal. It is retained as historical package text; this outer README and actual `verify.py` describe the completed verifier. No peer should execute the archived local orchestration helpers to verify evidence.
Source-bound original-span export, mixed ordinary/typed composition, complete output recompile/admission/intended-Project equality and public dispatch/migration/save integration remain separate work. This evidence does not close those requirements.
@@ -0,0 +1,107 @@
{
"source": "1b1c3e3abb6fc3d0934c9dbfbd21fddc8ae4f594",
"positive_audit": {
"source": "1b1c3e3abb6fc3d0934c9dbfbd21fddc8ae4f594",
"verified_git_inputs": 538,
"listing": 167,
"typed_pass": 59,
"index_pass": 9,
"update_pass": 5,
"token_pass": 1,
"strict_exit": 0,
"historical_overflow_retained": true,
"dirty_diagnostics": 2,
"dirty_direct_parser_probes_pass": 4,
"recorded_test_artifacts": {
"stack-fixed-original-1b1c3e3": {
"path": "C:\\Users\\User\\Documents\\Git\\ARCADIA-Studio\\.local\\analysis-checked-save-target-20260913\\debug\\deps\\oa_core-73987e0d7089bf1e.exe",
"bytes": 39014400,
"sha256": "56c0e5cd44ac5cb96b814d25ea403356195271a1aa9b6c6d1165fed4ffd09ddf"
},
"stack-fixed-shapes-1b1c3e3": {
"path": "C:\\Users\\User\\Documents\\Git\\ARCADIA-Studio\\.local\\analysis-checked-save-target-20260913\\debug\\deps\\oa_core-73987e0d7089bf1e.exe",
"bytes": 39014400,
"sha256": "56c0e5cd44ac5cb96b814d25ea403356195271a1aa9b6c6d1165fed4ffd09ddf"
},
"stack-fixed-paths-1b1c3e3": {
"path": "C:\\Users\\User\\Documents\\Git\\ARCADIA-Studio\\.local\\analysis-checked-save-target-20260913\\debug\\deps\\oa_core-73987e0d7089bf1e.exe",
"bytes": 39014400,
"sha256": "56c0e5cd44ac5cb96b814d25ea403356195271a1aa9b6c6d1165fed4ffd09ddf"
},
"typed-fixed-1b1c3e3": {
"path": "C:\\Users\\User\\Documents\\Git\\ARCADIA-Studio\\.local\\analysis-checked-save-target-20260913\\debug\\deps\\oa_core-73987e0d7089bf1e.exe",
"bytes": 39014400,
"sha256": "7b38b54ad94678171bd8b5964b1e00dd7da8346d96276c1ad790d1a6eb335208"
},
"token-fixed-1b1c3e3": {
"path": "C:\\Users\\User\\Documents\\Git\\ARCADIA-Studio\\.local\\analysis-checked-save-target-20260913\\debug\\deps\\oa_core-73987e0d7089bf1e.exe",
"bytes": 39014400,
"sha256": "7b38b54ad94678171bd8b5964b1e00dd7da8346d96276c1ad790d1a6eb335208"
},
"list-fixed-1b1c3e3": {
"path": "C:\\Users\\User\\Documents\\Git\\ARCADIA-Studio\\.local\\analysis-checked-save-target-20260913\\debug\\deps\\oa_core-73987e0d7089bf1e.exe",
"bytes": 39014400,
"sha256": "7b38b54ad94678171bd8b5964b1e00dd7da8346d96276c1ad790d1a6eb335208"
}
},
"scope": "Private S3 only; no full-core runtime/public8/integration/native/release acceptance. Recorded binary identities verified against receipts, not rehashed executable bytes; binaries intentionally absent."
},
"negative_control_count": 13,
"negative_controls": [
{
"name": "changed_zip_bytes",
"refusal": "Evidence ZIP SHA256 differs"
},
{
"name": "duplicate_json_key",
"refusal": "Duplicate JSON key: source"
},
{
"name": "duplicate_zip_member_with_test_only_pin",
"refusal": "Exact unique ZIP membership"
},
{
"name": "unsafe_zip_member_with_test_only_pin",
"refusal": "Unsafe ZIP path"
},
{
"name": "truncated_strict_completion",
"refusal": "Closed clean strict log"
},
{
"name": "missing_typed_runtime_member",
"refusal": "Exact ordered runtime results: typed-fixed-1b1c3e3"
},
{
"name": "altered_gate_command",
"refusal": "Exact command: token-fixed-1b1c3e3"
},
{
"name": "altered_committed_input_hash",
"refusal": "Exact538 source bindings: typed-fixed-1b1c3e3"
},
{
"name": "dirty_run_mislabeled_as_fixed_source",
"refusal": "Dirty capture identity/stability"
},
{
"name": "historical_abort_mislabeled_success",
"refusal": "Historical aborted gate"
},
{
"name": "diagnostic_patch_disagrees_with_captured_hash",
"refusal": "Reconstructed diagnostic input: crates/oa-core/src/language/analysis_source/index/tests.rs"
},
{
"name": "probe_case_mislabeled",
"refusal": "Dirty probe result/case"
},
{
"name": "optimized_python_refused",
"refusal": "Optimized Python refused"
}
],
"verifier_sha256": "4da79ba0c6fe9dc0ae9b81da6589756d3a215620f2e2fd6cbefaa621f646ebc9",
"controls_sha256": "a969264c98ab2fb95a0585b81c9d27a277ebe629488f20f45216d4483c26b396",
"method": "Raw archive untouched. Semantic corruptions pass directly to audit() after authentic archive validation; ZIP structural negatives temporarily substitute only the test module pin. No compiler, artifact execution or worktree write."
}
@@ -0,0 +1,36 @@
{
"source": "1b1c3e3abb6fc3d0934c9dbfbd21fddc8ae4f594",
"archive_sha256": "4486f4c42b140ae4b9b3caf0aacdf151a2acb1fd85eda09a57764fd69e980574",
"files": [
{
"path": ".gitattributes",
"bytes": 8,
"sha256": "705fd4d6451a31d36b3df7de96f83f30ac976c9b4a6d1e51671d8e2f33e2d0da"
},
{
"path": "README.md",
"bytes": 5456,
"sha256": "8059aa1794544272e3cdaa622ff48a9d0fdf4c257a4a1ca0daf9cc1adcc8b061"
},
{
"path": "verify.py",
"bytes": 17589,
"sha256": "4da79ba0c6fe9dc0ae9b81da6589756d3a215620f2e2fd6cbefaa621f646ebc9"
},
{
"path": "negative_controls.py",
"bytes": 5767,
"sha256": "a969264c98ab2fb95a0585b81c9d27a277ebe629488f20f45216d4483c26b396"
},
{
"path": "checks.json",
"bytes": 4506,
"sha256": "295acaadd2b1fe0f55c08c71c66c193635f70c2f7c63311ea42b039035065139"
},
{
"path": "evidence.zip",
"bytes": 658251,
"sha256": "4486f4c42b140ae4b9b3caf0aacdf151a2acb1fd85eda09a57764fd69e980574"
}
]
}
@@ -0,0 +1,98 @@
"""Read-only in-memory falsifiers for verify.py; does not modify archived evidence."""
import argparse
import importlib.util
import io
import json
from pathlib import Path
import subprocess
import sys
import warnings
import zipfile
def run(repo, directory):
spec = importlib.util.spec_from_file_location('s3_evidence_verifier', directory / 'verify.py')
verifier = importlib.util.module_from_spec(spec)
spec.loader.exec_module(verifier)
verifier.require(__debug__, 'Optimized Python refused')
raw = (directory / 'evidence.zip').read_bytes()
payload = verifier.archive_payload(raw)
positive = verifier.audit(repo, payload)
cases = []
def refused(name, expected, action):
try:
action()
except RuntimeError as error:
verifier.require(expected in str(error), f'{name}: wrong refusal: {error}')
cases.append({'name': name, 'refusal': str(error)})
else:
raise RuntimeError(name + ': invalid evidence was accepted')
def changed_json(name, change):
candidate = dict(payload)
data = verifier.document(candidate[name])
change(data)
candidate[name] = json.dumps(data).encode()
return candidate
refused('changed_zip_bytes', 'Evidence ZIP SHA256 differs', lambda: verifier.archive_payload(raw + b'tampered'))
refused('duplicate_json_key', 'Duplicate JSON key', lambda: verifier.document(b'{"source":1,"source":2}'))
def changed_zip(mode):
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, 'w') as archive:
for name, content in payload.items():
archive.writestr('../README.md' if mode == 'unsafe' and name == 'README.md' else name, content)
if mode == 'duplicate':
with warnings.catch_warnings():
warnings.simplefilter('ignore', UserWarning)
archive.writestr('README.md', payload['README.md'])
malformed = buffer.getvalue()
# Test-only pin substitution reaches ZIP structure checks. Production
# archive_payload always requires its checked-in immutable ZIP digest.
original_pin = verifier.ZIP_SHA
try:
verifier.ZIP_SHA = verifier.sha(malformed)
verifier.archive_payload(malformed)
finally:
verifier.ZIP_SHA = original_pin
refused('duplicate_zip_member_with_test_only_pin', 'Exact unique ZIP membership', lambda: changed_zip('duplicate'))
refused('unsafe_zip_member_with_test_only_pin', 'Unsafe ZIP path', lambda: changed_zip('unsafe'))
changed = dict(payload)
changed['raw/strict-fixed-1b1c3e3.log'] = b' Compiling oa-core\n'
refused('truncated_strict_completion', 'Closed clean strict log', lambda: verifier.audit(repo, changed))
changed = dict(payload)
name = 'raw/typed-fixed-1b1c3e3.log'
changed[name] = b'\n'.join(line for line in changed[name].split(b'\n') if verifier.NEW.encode() not in line)
refused('missing_typed_runtime_member', 'Exact ordered runtime results', lambda: verifier.audit(repo, changed))
changed = changed_json('raw/token-fixed-1b1c3e3.result.json', lambda data: data['command'].remove('--offline'))
refused('altered_gate_command', 'Exact command', lambda: verifier.audit(repo, changed))
changed = changed_json('raw/typed-fixed-1b1c3e3.inputs.json', lambda data: data['inputs'][0].update(sha256='0' * 64))
refused('altered_committed_input_hash', 'Exact538 source bindings', lambda: verifier.audit(repo, changed))
changed = changed_json('raw/stack-diag-02-dirty.inputs.json', lambda data: data.update(head=verifier.SOURCE))
refused('dirty_run_mislabeled_as_fixed_source', 'Dirty capture identity/stability', lambda: verifier.audit(repo, changed))
changed = changed_json('raw/typed-90fae148.result.json', lambda data: data.update(exit=0))
refused('historical_abort_mislabeled_success', 'Historical aborted gate', lambda: verifier.audit(repo, changed))
changed = changed_json('raw/stack-diag-01-dirty.inputs.json', lambda data: data.update(dirty=data['dirty'].replace('STAGE entered', 'STAGE changed')))
refused('diagnostic_patch_disagrees_with_captured_hash', 'Reconstructed diagnostic input', lambda: verifier.audit(repo, changed))
changed = dict(payload)
changed['raw/stack-probe-if-01-dirty.log'] = changed['raw/stack-probe-if-01-dirty.log'].replace(b'PROBE after malformed case 1:', b'PROBE after malformed case 0:')
refused('probe_case_mislabeled', 'Dirty probe result/case', lambda: verifier.audit(repo, changed))
result = subprocess.run([sys.executable, '-B', '-O', str(directory / 'verify.py'), '--repo', str(repo)], capture_output=True, text=True)
verifier.require(result.returncode != 0 and 'Optimized Python refused' in result.stderr, 'Optimized verifier must refuse before audit')
cases.append({'name': 'optimized_python_refused', 'refusal': 'Optimized Python refused'})
return {'source': verifier.SOURCE, 'positive_audit': positive, 'negative_control_count': len(cases), 'negative_controls': cases, 'verifier_sha256': verifier.sha((directory / 'verify.py').read_bytes()), 'controls_sha256': verifier.sha(Path(__file__).read_bytes()), 'method': 'Raw archive untouched. Semantic corruptions pass directly to audit() after authentic archive validation; ZIP structural negatives temporarily substitute only the test module pin. No compiler, artifact execution or worktree write.'}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--repo', type=Path, required=True)
args = parser.parse_args()
print(json.dumps(run(args.repo.resolve(), Path(__file__).resolve().parent), indent=2))
if __name__ == '__main__':
main()
@@ -0,0 +1,257 @@
"""Read-only portable S3 evidence verification; never compiles or runs artifacts."""
import argparse
from datetime import datetime
import hashlib
import io
import json
from pathlib import Path, PurePosixPath, PureWindowsPath
import re
import subprocess
import zipfile
SOURCE = '1b1c3e3abb6fc3d0934c9dbfbd21fddc8ae4f594'
PRIOR = '90fae14818f684bf778640be2e613f34e3c7e999'
ZIP_SHA = '4486f4c42b140ae4b9b3caf0aacdf151a2acb1fd85eda09a57764fd69e980574'
RECORDER = '328970b226d9f4c3b8d3feb7d64478fb9756310ae5dfd3caa43b23c8b9d6f245'
PREFIX = ['cargo', 'test', '--locked', '--offline', '-p', 'oa-core', '--lib']
SCOPES = ['Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', '.cargo', 'build', 'crates', 'assets', 'docs', 'examples', 'harness/fixtures', 'vendor']
STACK = 'deep_parse_index_plan_and_failed_build_cleanup_on_128k_stack'
NEW = 'every_parent_shape_rejects_depth129_and_cleans_children_on_128k_stack'
PATHS = 'maximum_index_paths_compare_lookup_debug_and_drop_on_128k_stack'
OVERFLOW = 3221225725
def require(ok, message):
if not ok:
raise RuntimeError(message)
def sha(raw):
return hashlib.sha256(raw).hexdigest()
def unique(pairs):
value = {}
for key, item in pairs:
require(key not in value, 'Duplicate JSON key: ' + key)
value[key] = item
return value
def document(raw):
return json.loads(raw, object_pairs_hook=unique)
def git(repo, *args, data=None):
return subprocess.check_output(['git', *args], cwd=repo, input=data)
def source_inputs(repo, commit):
paths = git(repo, 'ls-tree', '-r', '--name-only', commit, '--', *SCOPES).decode().splitlines()
require(len(paths) == len(set(paths)) == 538, 'Exactly538 committed test inputs')
batch = git(repo, 'cat-file', '--batch', data=''.join(f'{commit}:{p}\n' for p in paths).encode())
offset, inputs, blobs = 0, [], {}
for path in paths:
end = batch.index(b'\n', offset)
_, kind, size = batch[offset:end].split()
size = int(size)
raw = batch[end + 1:end + 1 + size]
require(kind == b'blob' and len(raw) == size and batch[end + 1 + size:end + 2 + size] == b'\n', 'Git blob framing: ' + path)
inputs.append({'path': path, 'bytes': size, 'sha256': sha(raw)})
blobs[path] = raw
offset = end + size + 2
require(offset == len(batch), 'Complete Git blob stream')
return inputs, blobs
def archive_payload(raw):
require(sha(raw) == ZIP_SHA, 'Evidence ZIP SHA256 differs')
with zipfile.ZipFile(io.BytesIO(raw)) as archive:
names = archive.namelist()
require(len(names) == len(set(names)) == 61, 'Exact unique ZIP membership')
for name in names:
path = PurePosixPath(name)
require(not path.is_absolute() and '..' not in path.parts and str(path) == name and '\\' not in name and ':' not in name, 'Unsafe ZIP path')
payload = {name: archive.read(name) for name in names}
manifest = document(payload['manifest.json'])
require(manifest['source'] == SOURCE, 'Archive manifest source')
listed = [item['path'] for item in manifest['files']]
require(len(listed) == len(set(listed)) and set(listed) | {'manifest.json'} == set(payload), 'Archive manifest membership')
for item in manifest['files']:
require(len(payload[item['path']]) == item['bytes'] and sha(payload[item['path']]) == item['sha256'], 'Archive member binding: ' + item['path'])
return payload
def apply_recorded_diff(blobs, diff):
"""Reconstruct diagnostic source without touching an index or worktree."""
changed = {}
sections = re.split(r'(?m)^diff --git ', diff)[1:]
require(sections, 'Missing dirty diff')
for section in sections:
lines = section.splitlines(keepends=True)
names = re.fullmatch(r'a/(\S+) b/(\S+)\n', lines[0])
require(names is not None and names[1] == names[2] and names[1] in blobs, 'Unexpected diagnostic diff path')
path = names[1]
old = blobs[path].decode().splitlines(keepends=True)
output, cursor, index = [], 0, 1
while index < len(lines) and not lines[index].startswith('@@ '):
index += 1
require(index < len(lines), 'Missing diagnostic hunk')
while index < len(lines):
match = re.match(r'@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@', lines[index])
require(match is not None, 'Invalid diagnostic hunk header')
old_start, old_count = int(match[1]), int(match[2] or 1)
new_start, new_count = int(match[3]), int(match[4] or 1)
start = old_start - 1 if old_count else old_start
require(cursor <= start <= len(old), 'Diagnostic hunk position')
output.extend(old[cursor:start])
cursor = start
require(len(output) == (new_start - 1 if new_count else new_start), 'Diagnostic new hunk position')
consumed = emitted = 0
index += 1
while index < len(lines) and not lines[index].startswith('@@ '):
line = lines[index]
require(line[:1] in [' ', '+', '-'], 'Unsupported diagnostic patch line')
if line[0] in ' -':
require(cursor < len(old) and old[cursor] == line[1:], 'Diagnostic old source mismatch')
cursor += 1
consumed += 1
if line[0] in ' +':
output.append(line[1:])
emitted += 1
index += 1
require((consumed, emitted) == (old_count, new_count), 'Diagnostic hunk counts')
output.extend(old[cursor:])
require(path not in changed, 'Repeated diagnostic path')
changed[path] = ''.join(output).encode()
return changed
def audit(repo, payload):
read = lambda name: payload['raw/' + name]
doc = lambda name: document(read(name))
closure = document(payload['closure.json'])
expected, blobs = source_inputs(repo, SOURCE)
prior_inputs, prior_blobs = source_inputs(repo, PRIOR)
require(git(repo, 'rev-parse', SOURCE + '^').decode().strip() == PRIOR, 'Correction parent')
require(sha(payload['helpers/run-analysis-source3-gate-20260922.py']) == RECORDER, 'Recorder bytes')
for manifest_name, commit, count in [
('analysis-source8-s3-index-source-manifest-20260913.json', '1f8283236f8ccdefca13814003e7f046860b4f22', 7),
('analysis-source8-s3-index-correction-source-manifest-20260913.json', PRIOR, 2),
]:
manifest = document(payload['notes/' + manifest_name])
require(manifest['source'] == commit and len(manifest['files']) == count, 'Historical source artifact manifest')
for item in manifest['files']:
raw = git(repo, 'show', commit + ':' + item['path'])
require(len(raw) == item['bytes'] and sha(raw) == item['sha256'] and git(repo, 'rev-parse', commit + ':' + item['path']).decode().strip() == item['git_blob'], 'Historical source artifact: ' + item['path'])
names = re.findall(r'^([^\r\n]+): test\r?$', read('list-fixed-1b1c3e3.log').decode(), re.M)
old_names = re.findall(r'^([^\r\n]+): test\r?$', read('list-90fae148.log').decode(), re.M)
require(len(names) == len(set(names)) == 167 and len(old_names) == len(set(old_names)) == 166, 'Actual listing sizes')
qualified_new = 'language::analysis_source::tests::' + NEW
require(set(names) == set(old_names) | {qualified_new} and qualified_new not in old_names, 'Retained166 plus one new case')
require(len([n for n in names if 'language::analysis_source' in n]) == 59 and len([n for n in names if 'language::raw_tokens' in n]) == 1, 'Typed/token listing counts')
for module, count in [('index', 9), ('update', 5)]:
path = f'crates/oa-core/src/language/analysis_source/{module}/tests.rs'
authored = [f'language::analysis_source::{module}::tests::{name}' for name in re.findall(r'#\[test\]\s*fn (\w+)\(', blobs[path].decode())]
require(len(authored) == count and set(authored) == {n for n in names if n.startswith(f'language::analysis_source::{module}::tests::')}, 'Source-derived membership: ' + module)
selectors = {'stack-fixed-original-1b1c3e3': STACK, 'stack-fixed-shapes-1b1c3e3': NEW, 'stack-fixed-paths-1b1c3e3': PATHS, 'typed-fixed-1b1c3e3': 'language::analysis_source', 'token-fixed-1b1c3e3': 'language::raw_tokens', 'list-fixed-1b1c3e3': None, 'strict-fixed-1b1c3e3': 'clippy'}
environment, artifacts, last_end = None, {}, None
for stem, selector in selectors.items():
record, log = doc(stem + '.result.json'), read(stem + '.log').decode()
require(record['source'] == record['head_after'] == SOURCE and type(record['exit']) is int and record['exit'] == 0 and record['changed_inputs'] == [], 'Clean source/result: ' + stem)
require(record['inputs'] == stem + '.inputs.json' and doc(record['inputs']) == {'source': SOURCE, 'inputs': expected}, 'Exact538 source bindings: ' + stem)
require(record['launcher_sha256'] == RECORDER, 'Recorded launcher')
start, end = (datetime.fromisoformat(record[k]) for k in ['start', 'end'])
require(start.tzinfo is not None and end.tzinfo is not None and start <= end and (last_end is None or last_end <= start), 'Closed ordered gate times')
last_end = end
if environment is None:
environment = record['environment']
require(record['environment'] == environment and environment['CARGO_BUILD_JOBS'] == '1' and PureWindowsPath(environment['CARGO_TARGET_DIR']).name == 'analysis-checked-save-target-20260913', 'Recorded environment')
if selector == 'clippy':
command = ['cargo', 'clippy', '--locked', '--offline', '-p', 'oa-core', '--all-targets', '--', '-D', 'warnings']
require(record['artifacts'] == [] and 'Finished `dev` profile [unoptimized]' in log and not re.search(r'^\s*(warning|error)(\[|:)', log, re.M), 'Closed clean strict log')
selected = []
else:
require(len(record['artifacts']) == 1, 'One recorded test artifact')
artifact = record['artifacts'][0]
emitted = re.findall(r'Running unittests [^\r\n]+ \(([^\r\n]+\.exe)\)', log)
require(len(emitted) == 1 and PureWindowsPath(emitted[0]) == PureWindowsPath(artifact['path']) and artifact['bytes'] > 0 and re.fullmatch('[0-9a-f]{64}', artifact['sha256']), 'Recorded executable/log binding')
artifacts[stem] = artifact
if selector is None:
command = PREFIX + ['--', '--list']
selected = names
require(record['listing_members'] == names and record['listing_count'] == 167 and re.findall(r'^(\d+) tests, (\d+) benchmarks\s*$', log, re.M) == [('167', '0')], 'Listing receipt and summary')
else:
command = PREFIX + [selector, '--', '--test-threads=1']
selected = [name for name in names if selector in name]
rows = re.findall(r'^test ([^\r\n]+) \.\.\. (ok|FAILED|ignored)\r?$', log, re.M)
require(rows == [(name, 'ok') for name in selected], 'Exact ordered runtime results: ' + stem)
require(re.findall(r'^test result: ok\. (\d+) passed; (\d+) failed; (\d+) ignored; (\d+) measured; (\d+) filtered out;', log, re.M) == [(str(len(selected)), '0', '0', '0', str(167-len(selected)))], 'Runtime summary')
require(record['command'] == command, 'Exact command: ' + stem)
if selector is not None:
require(record['listing_count'] == 0 and record['listing_members'] == [], 'Non-list metadata')
saved = closure['gates'][stem]
require([saved[k] for k in ['command', 'exit', 'start', 'end', 'names']] == [command, 0, record['start'], record['end'], selected], 'Closure gate facts')
require(saved['pins'] == {stem+s: sha(read(stem+s)) for s in ['.inputs.json', '.log', '.result.json']}, 'Closure raw pins')
require(artifacts['stack-fixed-original-1b1c3e3'] == artifacts['stack-fixed-shapes-1b1c3e3'] == artifacts['stack-fixed-paths-1b1c3e3'], 'First focused build receipt')
require(artifacts['typed-fixed-1b1c3e3'] == artifacts['token-fixed-1b1c3e3'] == artifacts['list-fixed-1b1c3e3'], 'Final restored clean build receipt')
require([closure[k] for k in ['source', 'parent', 'verified_inputs', 'listing', 'typed_pass', 'index_pass', 'update_pass', 'token_pass', 'strict_exit']] == [SOURCE, PRIOR, 538, 167, 59, 9, 5, 1, 0], 'Closure counts')
for stem, mode in [('list-90fae148', 'list'), ('typed-90fae148', 'failed')]:
record, log = doc(stem + '.result.json'), read(stem + '.log').decode()
require(record['source'] == record['head_after'] == PRIOR and record['changed_inputs'] == [] and record['launcher_sha256'] == RECORDER, 'Historical frozen source')
require(doc(stem + '.inputs.json') == {'source': PRIOR, 'inputs': prior_inputs}, 'Historical538 inputs')
if mode == 'list':
require(record['exit'] == 0 and record['listing_members'] == old_names and record['listing_count'] == 166 and record['command'] == PREFIX + ['--', '--list'], 'Historical listing')
else:
require(record['exit'] == OVERFLOW and record['command'] == PREFIX + ['language::analysis_source', '--', '--test-threads=1'] and 'STATUS_STACK_OVERFLOW' in log and STACK + ' ...' in log and 'test result:' not in log, 'Historical aborted gate')
selected = [n for n in old_names if 'language::analysis_source' in n]
failing = 'language::analysis_source::index::tests::' + STACK
rows = re.findall(r'^test ([^\r\n]+) \.\.\. (ok|FAILED|ignored)\r?$', log, re.M)
require(rows == [(n, 'ok') for n in selected[:selected.index(failing)]] and 'running 58 tests' in log, 'Historical completed prefix before abort')
dirty = ['stack-diag-01-dirty', 'stack-diag-02-dirty', 'stack-probe-sum-01-dirty', 'stack-probe-if-01-dirty', 'stack-probe-power-01-dirty', 'stack-probe-trailing-01-dirty']
for index, stem in enumerate(dirty):
capture, result, log = doc(stem + '.inputs.json'), doc(stem + '.result.json'), read(stem + '.log').decode()
diagnostic = index < 2
commit, original = (PRIOR, prior_blobs) if diagnostic else (SOURCE, blobs)
require(capture['head'] == commit and result['changed'] == [], 'Dirty capture identity/stability')
patched = apply_recorded_diff(original, capture['dirty'])
expected_paths = {'crates/oa-core/src/language/analysis_source/index/tests.rs'} if diagnostic else {'crates/oa-core/src/language/analysis_source/tests.rs'}
if index == 1:
expected_paths.add('crates/oa-core/src/language/analysis_source/syntax.rs')
require(set(patched) == expected_paths, 'Bounded diagnostic paths')
for path, raw in original.items():
require(capture['inputs'][path] == sha(patched.get(path, raw)), 'Reconstructed diagnostic input: ' + path)
expected_filter = STACK if diagnostic else 'diagnostic_malformed_cleanup_on_128k_stack'
require(capture['command'] == PREFIX + [expected_filter, '--', '--test-threads=1', '--nocapture'], 'Diagnostic command')
if diagnostic:
require(result['exit'] == OVERFLOW and 'STATUS_STACK_OVERFLOW' in log and 'STAGE failed builds dropped' in log and 'STAGE finished' not in log, 'Diagnostic abort')
if index == 1:
require('STAGE node rejecting depth 129 before kind drop' in log and 'STAGE node rejected kind dropped' not in log, 'Isolated rejected-kind cleanup')
else:
case = index - 2
require(result['exit'] == 0 and f'PROBE before malformed case {case}' in log and f'PROBE after malformed case {case}:' in log and '1 passed; 0 failed; 0 ignored' in log, 'Dirty probe result/case')
return {'source': SOURCE, 'verified_git_inputs': 538, 'listing': 167, 'typed_pass': 59, 'index_pass': 9, 'update_pass': 5, 'token_pass': 1, 'strict_exit': 0, 'historical_overflow_retained': True, 'dirty_diagnostics': 2, 'dirty_direct_parser_probes_pass': 4, 'recorded_test_artifacts': artifacts, 'scope': 'Private S3 only; no full-core runtime/public8/integration/native/release acceptance. Recorded binary identities verified against receipts, not rehashed executable bytes; binaries intentionally absent.'}
def verify_package(directory):
manifest = document((directory / 'manifest.json').read_bytes())
require(manifest['source'] == SOURCE and manifest['archive_sha256'] == ZIP_SHA, 'Outer manifest identity')
names = [item['path'] for item in manifest['files']]
require(set(names) == {'.gitattributes', 'README.md', 'verify.py', 'negative_controls.py', 'checks.json', 'evidence.zip'} and len(names) == len(set(names)), 'Outer manifest membership')
for item in manifest['files']:
raw = (directory / item['path']).read_bytes()
require(len(raw) == item['bytes'] and sha(raw) == item['sha256'], 'Outer manifest bytes: ' + item['path'])
return archive_payload((directory / 'evidence.zip').read_bytes())
def main():
require(__debug__, 'Optimized Python refused')
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--repo', type=Path, required=True, help='Git repo containing immutable source commits; current checkout is not used')
parser.add_argument('--evidence-dir', type=Path, default=Path(__file__).resolve().parent)
args = parser.parse_args()
print(json.dumps(audit(args.repo, verify_package(args.evidence_dir)), indent=2))
if __name__ == '__main__':
main()