355 lines
22 KiB
Python
355 lines
22 KiB
Python
#!/usr/bin/env python3
|
|
"""Root-only native collection. Does not run regression or claim acceptance."""
|
|
import argparse
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
import re
|
|
|
|
sys.dont_write_bytecode = True
|
|
HERE = Path(__file__).resolve().parent
|
|
CONTRACT = 'docs/plans/original-completion-2026-10-02/execution/completion_marker_contracts.json'
|
|
CONDITIONS = ('room_occupancy_and_operation', 'queues_and_overflow',
|
|
'elevator_state_and_passengers', 'staff_state_and_jobs',
|
|
'supplies_and_carried_goods', 'same_demand_route_improvement',
|
|
'normal_os_and_distinct_pid_resume')
|
|
OS_CHILD_TIMEOUT_SECONDS = 3600
|
|
RECORDER = 'tests/fixtures/p23-03/passive_os_recorder.gd'
|
|
|
|
def collect_screenshots(out, pair, role):
|
|
"""Bind every external CUA screenshot to its request and decoded PNG.
|
|
|
|
The external receipt supplies screenshots[{path,raw_sha256,tool_call_id}].
|
|
Paths are relative to the evidence root; raw receipt bytes remain unchanged.
|
|
"""
|
|
checker = module('p23_screenshot_binding', HERE/'check_p23_life_observation_evidence.py')
|
|
bindings = []
|
|
requests = json.loads((pair/(role+'-requests.json')).read_text())['requests']
|
|
for ack_name in requests:
|
|
ack_path = out/ack_name
|
|
checker.require(ack_path.resolve().is_relative_to(out.resolve()), 'ACK path escape')
|
|
ack = json.loads(ack_path.read_text())
|
|
request_path = ack_path.with_name('request.json')
|
|
request = json.loads(request_path.read_text())
|
|
receipt_path = out/ack['input_receipt']
|
|
checker.require(receipt_path.resolve().is_relative_to(out.resolve()), 'CUA receipt path escape')
|
|
cua = json.loads(receipt_path.read_text())
|
|
checker.require(ack['request_sha256'] == checker.sha(request_path), 'CUA request hash')
|
|
checker.require(cua['screenshots'], 'CUA screenshots missing')
|
|
for number, shot in enumerate(cua['screenshots']):
|
|
raw = out/shot['path']
|
|
checker.require(raw.resolve().is_relative_to(out.resolve()) and not raw.is_symlink(), 'screenshot path escape')
|
|
checker.require(checker.sha(raw) == shot['raw_sha256'] and shot['tool_call_id'], 'raw screenshot CUA binding')
|
|
kind, _, _ = checker.image_kind(raw.read_bytes())
|
|
conversion = None
|
|
if kind == 'jpeg':
|
|
png = ack_path.with_name('decoded-%02d.png' % number)
|
|
receipt = ack_path.with_name('conversion-%02d.json' % number)
|
|
convert_screenshot(raw.resolve(), png.resolve(), shot, receipt.resolve())
|
|
conversion = str(receipt.relative_to(out))
|
|
else:
|
|
png = raw
|
|
width, height, _ = checker.png_pixels(png.read_bytes())
|
|
bindings.append(dict(request=str(request_path.relative_to(out)), request_sha256=checker.sha(request_path),
|
|
sequence=request['sequence'], target_pid=request['target_pid'],
|
|
input_receipt=ack['input_receipt'], input_receipt_sha256=checker.sha(receipt_path),
|
|
screenshot_index=number, tool_call_id=shot['tool_call_id'],
|
|
raw_path=shot['path'], raw_sha256=shot['raw_sha256'], conversion=conversion,
|
|
png_path=str(png.relative_to(out)), png_sha256=checker.sha(png), width=width, height=height))
|
|
checker.require(bindings, 'empty screenshot collection')
|
|
destination = pair/(role+'-screenshots.json')
|
|
with destination.open('x') as output:
|
|
json.dump(bindings, output, indent=2)
|
|
output.write('\n')
|
|
return str(destination.relative_to(out))
|
|
|
|
def convert_screenshot(raw_path, png_path, cua_receipt, receipt_path):
|
|
"""Root-only conversion; raw CUA bytes are never renamed or rewritten."""
|
|
checker = module('p23_conversion_checker', HERE/'check_p23_life_observation_evidence.py')
|
|
raw, png, receipt = map(Path, (raw_path, png_path, receipt_path))
|
|
checker.require(raw.is_absolute() and png.is_absolute() and receipt.is_absolute(),
|
|
'absolute conversion paths required')
|
|
checker.require(raw.is_file() and not raw.is_symlink() and not png.exists()
|
|
and not receipt.exists() and raw.resolve() != png.resolve(), 'fresh separate conversion output')
|
|
raw_hash = checker.sha(raw)
|
|
checker.require(raw_hash == cua_receipt['raw_sha256'] and cua_receipt['tool_call_id'],
|
|
'CUA conversion input hash/call mismatch')
|
|
kind, width, height = checker.image_kind(raw.read_bytes())
|
|
checker.require(kind == 'jpeg', 'conversion requires JPEG input')
|
|
converter = '/usr/bin/sips'
|
|
converter_hash = checker.sha(converter)
|
|
argv = [converter, '-s', 'format', 'png', str(raw), '--out', str(png)]
|
|
process = subprocess.run(argv, capture_output=True, timeout=900, check=False)
|
|
rec = dict(raw_path=str(raw), raw_sha256=raw_hash, raw_kind=kind,
|
|
tool_call_id=cua_receipt['tool_call_id'], converter=converter,
|
|
converter_sha256=converter_hash, argv=argv, exit_code=process.returncode,
|
|
stdout_sha256=hashlib.sha256(process.stdout).hexdigest(),
|
|
stderr_sha256=hashlib.sha256(process.stderr).hexdigest(),
|
|
png_path=str(png), png_sha256=checker.sha(png) if png.is_file() else None,
|
|
width=width, height=height)
|
|
# Preserve failure receipts too. These logs allow independent hash checking.
|
|
with receipt.open('x') as output:
|
|
json.dump(rec, output, indent=2)
|
|
output.write('\n')
|
|
with Path(str(receipt) + '.stdout').open('xb') as output:
|
|
output.write(process.stdout)
|
|
with Path(str(receipt) + '.stderr').open('xb') as output:
|
|
output.write(process.stderr)
|
|
checker.require(checker.sha(raw) == raw_hash, 'raw CUA bytes changed during conversion')
|
|
checker.require(checker.sha(converter) == converter_hash, 'converter changed during conversion')
|
|
checker.inspect_conversion(rec, cua_receipt, converter_hash)
|
|
return rec
|
|
|
|
def collect_os(r, root, out, stage, engine, commit, refs, protected, outer):
|
|
"""Adapted P08-03 collect_os protocol, retaining owned-process guards.
|
|
|
|
Root supplies CUA ack/receipt files in the requested directories. Neither
|
|
controller nor passive endpoints generate input or change permissions.
|
|
"""
|
|
r.require(not outer.native.helper['live_engines'](), 'external engine present')
|
|
checker = module('p23_os_pins', HERE/'check_p23_life_observation_evidence.py')
|
|
before = r.inventory(stage, stage=True)
|
|
public = (stage/'project.godot').read_bytes()
|
|
(out/'os-pair').mkdir()
|
|
phases = []
|
|
for label in ('A-writer', 'A-reader', 'B-writer', 'B-reader'):
|
|
layout, role = label.split('-')
|
|
pair = out/'os-pair'/layout
|
|
pair.mkdir(exist_ok=role == 'reader')
|
|
r.require(not outer.native.helper['live_engines'](), 'external engine present')
|
|
r.require(r.source_refs(root) == refs and r.default_player(root) == protected, 'OS before guards')
|
|
r.require(r.inventory(stage, stage=True) == before, 'OS stage before guard')
|
|
r.require(shutil.disk_usage(out).free >= 500*1024*1024, 'OS disk admission')
|
|
folder = out/'os-runs'/label
|
|
folder.mkdir(parents=True)
|
|
name = 'TheTree-PCK-tests-'+uuid.uuid4().hex+'-p23-observation-'+label.lower()
|
|
profile = r.profile_path(name)
|
|
r.require(not profile.exists(), 'fresh OS private profile')
|
|
formal_source = pair/'formal' if role == 'reader' else None
|
|
formal_before = r.inventory(formal_source) if formal_source else {}
|
|
if role == 'reader':
|
|
r.require(set(formal_before) == {'world.bin','manifest.json','thumbnail.png','journal.log'}, 'all formal inputs')
|
|
formal = profile/'saves/manual_ui'
|
|
formal.mkdir(parents=True)
|
|
for member, pin in formal_before.items():
|
|
r.copy_source(formal_source/member, formal/member)
|
|
r.require(r.sha(formal/member) == pin['sha256'], 'formal fixture copy SHA')
|
|
recorder = folder/'passive_os.gd'
|
|
r.copy_source(root/RECORDER, recorder)
|
|
proof = folder/'teardown'
|
|
(proof/'owned-audio').mkdir(parents=True)
|
|
env = r.fresh_environment(dict(os.environ))
|
|
env.update(P23_OBSERVATION_ROOT=str(pair), P23_OBSERVATION_ROLE=role, P23_LAYOUT=layout,
|
|
THE_TREE_AUDIO_TEARDOWN_PROOF_DIR=str(proof/'owned-audio'),
|
|
THE_TREE_AUDIO_FIXTURE_CONTROL_PROOF=str(proof/'control.json'))
|
|
command = [engine, '--path', str(stage), '--script', str(recorder)]
|
|
base = {'protocol':'p23-observation-native-os-v1', 'role':role, 'layout':layout, 'commit':commit,
|
|
'source_map':{n:v['sha256'] for n,v in refs.items()}, 'stage':str(stage),
|
|
'stage_before':before, 'profile':str(profile), 'profile_name':name,
|
|
'profile_before':{'existed':profile.exists(), 'files':r.inventory(profile) if profile.exists() else {}}, 'formal_input':formal_before,
|
|
'formal_source':str(formal_source) if formal_source else None, 'owner_pid':os.getpid(),
|
|
'default_before':protected, 'command':command, 'engine_sha256':r.sha(engine),
|
|
'controller_sha256':r.sha(root/'tools/run_p23_life_observation.py'),
|
|
'recorder_sha256':r.sha(recorder), 'formal_acceptance':False,
|
|
'timeout_seconds':OS_CHILD_TIMEOUT_SECONDS,
|
|
'inputs':checker.input_map(root), 'spec_sha256':checker.SPEC_SHA,
|
|
'settings_sha256':hashlib.sha256(public).hexdigest(),
|
|
'environment':{k:v for k,v in env.items() if k.startswith(('P23_', 'THE_TREE_'))},
|
|
'environment_sha256':hashlib.sha256(json.dumps(env,sort_keys=True).encode()).hexdigest()}
|
|
r.write(folder/'started.json', base)
|
|
error = None; children = []; guards = []; exit_code = None; owner_proof = None
|
|
try:
|
|
(stage/'project.godot').write_bytes(r.private_project(public, name))
|
|
with (folder/'engine-native.log').open('x') as log:
|
|
process = subprocess.Popen(command, cwd=stage, env=env, stdout=log, stderr=subprocess.STDOUT)
|
|
children.append(process)
|
|
r.write(folder/'process-started.json', dict(base, actual_pid=process.pid,
|
|
monotonic_ns=time.monotonic_ns(), state='waiting_for_root_cua_requests'))
|
|
print(json.dumps({'actual_pid':process.pid, 'request_directory':str(pair),
|
|
'state':'waiting_for_root_cua', 'role':role}), flush=True)
|
|
exit_code = process.wait(timeout=base['timeout_seconds'])
|
|
r.require(exit_code == 0, 'ordinary native OS exit0')
|
|
raw = (folder/'engine-native.log').read_text()
|
|
r.require('ERROR:' not in raw and 'SCRIPT ERROR:' not in raw, 'OS native diagnostics')
|
|
matches = re.findall(r'(?m)^P23 passive OS observation: ([0-9]+) checks, 0 failures$', raw)
|
|
r.require(len(matches) == 1 and int(matches[0]) > 0, 'passive observer completion')
|
|
observed = r.read(pair/(role+'-profile.json'))
|
|
r.require(observed['pid'] == process.pid and Path(observed['user_data_dir']).resolve() == profile.resolve()
|
|
and observed['display'] != 'headless', 'actual OS private profile/display')
|
|
collect_screenshots(out, pair, role)
|
|
owners = r.module('p23_os_owner_gate', r.SUPPORT/'ordinary_proof_gate.py')
|
|
owner_proof = owners.ordinary_gate(proof, process.pid, folder, ['single'])
|
|
except BaseException as caught:
|
|
error = type(caught).__name__+': '+str(caught)
|
|
finally:
|
|
try:
|
|
cleanup, cleanup_errors = outer.finish_captured(children)
|
|
finally:
|
|
(stage/'project.godot').write_bytes(public)
|
|
try:
|
|
r.require(r.source_refs(root) == refs, 'OS source unchanged')
|
|
r.require(r.inventory(stage, stage=True) == before, 'OS stage unchanged')
|
|
r.require(r.default_player(root) == protected, 'OS default player unchanged')
|
|
if formal_source:
|
|
r.require(r.inventory(formal_source) == formal_before, 'formal input unchanged')
|
|
r.require(r.sha(recorder) == base['recorder_sha256'], 'recorder unchanged')
|
|
except BaseException as caught:
|
|
guards.append(type(caught).__name__+': '+str(caught))
|
|
passed = not (error or guards or cleanup_errors) and len(children) == 1 and exit_code == 0
|
|
r.write(folder/'ended.json', dict(base, actual_pid=children[0].pid if children else None,
|
|
exit_code=exit_code, engine_ended=bool(cleanup) and all(v['engine_ended'] for v in cleanup),
|
|
error=error, guard_errors=guards, cleanup_errors=cleanup_errors, owned_cleanup=cleanup,
|
|
ordinary_owner_proof=owner_proof, stage_after=r.inventory(stage,stage=True),
|
|
engine_sha256_after=r.sha(engine), inputs_after=checker.input_map(root),
|
|
settings_sha256_after=hashlib.sha256((stage/'project.godot').read_bytes()).hexdigest(),
|
|
environment_after={k:v for k,v in env.items() if k.startswith(('P23_', 'THE_TREE_'))},
|
|
environment_sha256_after=hashlib.sha256(json.dumps(env,sort_keys=True).encode()).hexdigest(),
|
|
default_after=r.default_player(root), source_after=r.source_refs(root),
|
|
profile_after=r.inventory(profile), passed=passed))
|
|
phases.append('os-runs/'+label+'/ended.json')
|
|
r.require(passed, 'native OS collection failed: '+str(error or guards or cleanup_errors))
|
|
return phases
|
|
|
|
def module(name, path):
|
|
spec = importlib.util.spec_from_file_location(name, path)
|
|
obj = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(obj)
|
|
return obj
|
|
|
|
def absolute(value):
|
|
path = Path(value)
|
|
if not path.is_absolute():
|
|
raise argparse.ArgumentTypeError('absolute path required')
|
|
return path.resolve()
|
|
|
|
def collect(args):
|
|
root, out = args.project_root, args.output
|
|
if out.is_relative_to(root):
|
|
raise ValueError('evidence output must be outside the frozen source checkout')
|
|
r = module('p23_standard_runner', root/'tools/run_all_tests.py')
|
|
commit = subprocess.check_output(['git', '-C', str(root), 'rev-parse', 'HEAD'], text=True).strip()
|
|
r.require(commit == args.commit, 'actual final submitted commit required')
|
|
r.require(not subprocess.check_output(['git','-C',str(root),'status','--porcelain'], text=True), 'frozen clean worktree required')
|
|
r.require(not out.exists(), 'fresh output; never overwrite earlier proof')
|
|
refs = r.source_refs(root)
|
|
checker = module('p23_initial_inputs', root/'tools/check_p23_life_observation_evidence.py')
|
|
inputs_before = checker.input_map(root)
|
|
outer = r.module('p23_owned_outer', r.SUPPORT/'owned_outer.py')
|
|
r.require(not outer.native.helper['live_engines'](), 'one native engine')
|
|
engine = shutil.which(args.godot)
|
|
r.require(engine is not None, 'native engine exists')
|
|
engine = str(Path(engine).resolve())
|
|
out.mkdir(parents=True)
|
|
stage = out/'stage'; stage.mkdir()
|
|
(out/'starters').mkdir(); (out/'bootstrap').mkdir()
|
|
for name, pin in r.read(r.SUPPORT/'bootstrap-scripts.json').items():
|
|
data = pin['text'].encode()
|
|
r.require(hashlib.sha256(data).hexdigest() == pin['sha256'], 'bootstrap pin')
|
|
(out/'bootstrap'/name).write_bytes(data)
|
|
for name, pin in refs.items():
|
|
target = stage/name; target.parent.mkdir(parents=True, exist_ok=True)
|
|
r.copy_source(Path(pin['path']), target)
|
|
r.require(r.sha(target) == pin['sha256'], 'whole copied source '+name)
|
|
expected = {n: {'bytes':v['bytes'], 'sha256':v['sha256']} for n,v in refs.items()}
|
|
selected = {'root_before':refs, 'map':{n:v['sha256'] for n,v in refs.items()}, 'timeout':180}
|
|
r.require(r.inventory(stage, stage=True) == expected, 'complete private source')
|
|
protected = r.default_player(root); public = (stage/'project.godot').read_bytes()
|
|
paths = ['res://'+n for n in sorted(refs) if n.endswith('.gd')]
|
|
r.write(out/'parse-paths.json', paths)
|
|
markers = r.module('p23_strict_markers', root/'tools/strict_test_markers.py')
|
|
contracts = markers.load_contracts(root/CONTRACT)
|
|
policy = r.read(r.SUPPORT/'suite-policy.json')
|
|
diagnostics = r.module('p23_diagnostics', r.SUPPORT/'diagnostic_gate.py')
|
|
owners = r.module('p23_owners', r.SUPPORT/'ordinary_proof_gate.py')
|
|
audio = r.module('p23_audio', r.SUPPORT/'audio_evidence_gate.py')
|
|
r.write(out/'runner-input.json', {'protocol':'p23-normal-observation-v1', 'commit':commit,
|
|
'selected_source_map':selected['map'], 'engine':engine, 'engine_sha256':r.sha(engine),
|
|
'owner_pid':os.getpid(), 'default_player':protected, 'diagnostic_seed':806042,
|
|
'inputs':inputs_before, 'provenance':checker.input_provenance(root),
|
|
'pixel_decoder':{'interpreter':sys.executable,
|
|
'PIL_available':importlib.util.find_spec('PIL') is not None,
|
|
'raw_screenshots':'preserve bytes; Root lossless conversion required for JPEG without PIL'}})
|
|
phases = []; previous = {}; cases = []; error = None; os_proof = None
|
|
deadline = time.monotonic()+900
|
|
saved_env = {k:os.environ.get(k) for k in ('P23_LAYOUT','P23_OUTPUT')}
|
|
def phase(label, test):
|
|
remaining = int(deadline-time.monotonic())
|
|
r.require(remaining >= (180 if label == 'import' else 1), '900s collection deadline')
|
|
selected['timeout'] = min(180, remaining)
|
|
phase_out = out/'runs'/label
|
|
phase_out.mkdir(parents=True)
|
|
(phase_out/'starters').mkdir(); (phase_out/'bootstrap').mkdir()
|
|
for original in (out/'bootstrap').iterdir():
|
|
r.copy_source(original,phase_out/'bootstrap'/original.name)
|
|
r.require(r.sha(original)==r.sha(phase_out/'bootstrap'/original.name),'bootstrap full copy')
|
|
r.write(phase_out/'parse-paths.json',paths)
|
|
row = r.execute_phase(engine, stage, phase_out, label, test, selected, expected, previous,
|
|
policy, contracts, paths, public, protected, root, outer, markers, diagnostics, owners, audio)
|
|
# Supplement the existing runner receipt; preserve it verbatim.
|
|
r.write(phase_out/'p23-inputs.json', dict(source_map=selected['map'], source_after=r.source_refs(root),
|
|
engine_sha256=r.sha(engine), inputs=inputs_before, inputs_after=checker.input_map(root),
|
|
spec_sha256=checker.SPEC_SHA, settings_sha256=hashlib.sha256(public).hexdigest(),
|
|
stage_before=expected, stage_after=r.inventory(stage,stage=True),
|
|
default_before=protected, default_after=r.default_player(root),
|
|
receipt_sha256=r.sha(phase_out/(label+'-receipt.json'))))
|
|
phases.append('runs/'+label+'/'+label+'-receipt.json')
|
|
r.require(not row['failed'], 'strict phase failed: '+label)
|
|
try:
|
|
phase('import', None); phase('parse', None)
|
|
for layout in ('A', 'B'):
|
|
directory = out/('layout-'+layout)
|
|
os.environ.update(P23_LAYOUT=layout, P23_OUTPUT=str(directory))
|
|
phase('layout-'+layout.lower(), 'tests/ui/test_p23_life_observation.gd')
|
|
cases.append({'directory':directory.name, 'phase':phases[-1], 'files':r.inventory(directory)})
|
|
if args.input_mode == 'native_os':
|
|
os_phases = collect_os(r, root, out, stage, engine, commit, refs, protected, outer)
|
|
checker = module('p23_os_input_map', root/'tools/check_p23_life_observation_evidence.py')
|
|
os_proof = 'os-proof.json'
|
|
r.write(out/os_proof, {'source_map':selected['map'], 'inputs':checker.input_map(root),
|
|
'input_method':'native_os', 'observer_mutations':[], 'phases':os_phases, 'directory':'os-pair'})
|
|
except BaseException as caught:
|
|
error = type(caught).__name__+': '+str(caught)
|
|
finally:
|
|
cleanup, cleanup_errors = outer.finish_captured(outer.OWNED)
|
|
for key,value in saved_env.items():
|
|
if value is None: os.environ.pop(key, None)
|
|
else: os.environ[key] = value
|
|
if r.source_refs(root) != refs or r.default_player(root) != protected:
|
|
error = (error+'; ' if error else '')+'protected root/profile changed'
|
|
if checker.input_map(root) != inputs_before:
|
|
error = (error+'; ' if error else '')+'spec/fixture/controller inputs changed'
|
|
r.write(out/'runner-ended.json', {'error':error, 'cleanup':cleanup, 'cleanup_errors':cleanup_errors,
|
|
'phases':phases, 'formal_acceptance':False, 'native_Windows':'pending'})
|
|
checker = module('p23_observation_checker', root/'tools/check_p23_life_observation_evidence.py')
|
|
r.write(out/'index.json', {'schema':'p23-life-observation-v1', 'commit':commit,
|
|
'source_map':selected['map'], 'input_map':inputs_before,
|
|
'provenance':checker.input_provenance(root),
|
|
'marker_contract_sha256':r.sha(root/CONTRACT),
|
|
'spec_sha256':hashlib.sha256((root/'docs/spec/acceptance/P23-03.json').read_text().strip().encode()).hexdigest(),
|
|
'cases':cases, 'phases':phases, 'conditions':list(CONDITIONS),
|
|
'artifacts':{str(p.relative_to(out)):r.sha(p)
|
|
for p in out.rglob('*') if p.is_file() and 'stage' not in p.relative_to(out).parts},
|
|
'os_proof':os_proof, 'regression':None,
|
|
'acceptance':'pending', 'collection_error':error})
|
|
return 1 if error or cleanup_errors else 0
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--project-root', type=absolute, required=True)
|
|
parser.add_argument('--output', type=absolute, required=True)
|
|
parser.add_argument('--commit', required=True)
|
|
parser.add_argument('--godot', default='godot')
|
|
parser.add_argument('--input-mode', choices=('handler', 'native_os'), default='handler')
|
|
parser.add_argument('--collect', action='store_true', required=True)
|
|
return collect(parser.parse_args(argv))
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|