203 lines
14 KiB
Python
203 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Read-only P24 evidence gate. Success is machine validation, never acceptance."""
|
|
import argparse
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
sys.dont_write_bytecode = True
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
CONDITIONS = ('facility_lifecycle', 'guest_lifecycle', 'staff_lifecycle',
|
|
'layer_and_values', 'invalid_and_failed_load', 'normal_os_input',
|
|
'final_validation_review')
|
|
|
|
|
|
def require(ok, message):
|
|
if not ok:
|
|
raise ValueError(message)
|
|
|
|
|
|
def sha(path):
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def read(path):
|
|
return json.loads(path.read_bytes())
|
|
|
|
|
|
def load_module(name, path):
|
|
spec = importlib.util.spec_from_file_location(name, path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def verify(base):
|
|
specification = ROOT / 'docs/spec/acceptance/P24-03-code-r02.json'
|
|
require(hashlib.sha256(specification.read_text().strip().encode('utf-8')).hexdigest() == 'dbb885b47a3ee4af3d4180bb97fe746e26a0e08d75af2cd3b5b727ed337bd37c', 'fixed P24 specification mismatch')
|
|
index = read(base / 'index.json')
|
|
require(index.get('schema_version') == 1, 'index schema_version must be 1')
|
|
runner = load_module('p24_runner_source', ROOT / 'tools/run_all_tests.py')
|
|
current = {name: pin['sha256'] for name, pin in runner.source_refs(ROOT).items()}
|
|
require(index.get('source_map') == current, 'index source_map differs from current whole candidate')
|
|
artifacts = index.get('artifacts', {})
|
|
require(isinstance(artifacts, dict) and artifacts, 'missing artifacts')
|
|
files = {}
|
|
for name, pin in artifacts.items():
|
|
relative = Path(pin['path'])
|
|
require(not relative.is_absolute() and '..' not in relative.parts, 'artifact path must stay inside evidence')
|
|
path = base / relative
|
|
require(not any(p.is_symlink() for p in [path, *path.parents]), 'artifact symlink refused')
|
|
require(path.is_file() and path.stat().st_size > 0 and sha(path) == pin['sha256'], 'missing/changed artifact: ' + name)
|
|
files[name] = path
|
|
|
|
def artifact(name):
|
|
require(name in files, 'unregistered artifact: ' + str(name))
|
|
return files[name]
|
|
|
|
matrix = index.get('conditions', {})
|
|
require(set(matrix) == set(CONDITIONS), 'condition IDs missing=%s unknown=%s' % (sorted(set(CONDITIONS) - set(matrix)), sorted(set(matrix) - set(CONDITIONS))))
|
|
for name in CONDITIONS:
|
|
row = matrix[name]
|
|
require(row.get('source_map') == current, 'condition candidate mismatch: ' + name)
|
|
# Formal independent review follows machine validation in Runner.
|
|
# Keep its condition explicit without requiring a prior duplicate review.
|
|
if name == 'final_validation_review':
|
|
require(row.get('status') in ('pending_review', 'executed') and row.get('artifacts'), 'final regression evidence missing')
|
|
else:
|
|
require(row.get('status') == 'executed' and row.get('artifacts'), 'pending condition: ' + name)
|
|
for reference in row['artifacts']:
|
|
artifact(reference)
|
|
|
|
# Corroborate the full roster against existing strict runner receipts and raw logs.
|
|
regression = index['full_regression']
|
|
native = index['native']
|
|
expected_local = ['tests/ui/test_actor_selection.gd', 'tests/ui/test_selection_lifecycle.gd',
|
|
'tests/ui/test_selection_autosave.gd', 'tests/ui/test_selection_departure.gd']
|
|
require(native.get('suites') == expected_local, 'native selection suite coverage missing')
|
|
start = read(artifact(regression['input']))
|
|
end = read(artifact(regression['ended']))
|
|
rows_path = artifact(regression['rows'])
|
|
rows = read(rows_path)
|
|
roster = runner.roster(ROOT)
|
|
require(start.get('selected_source_map') == current and start.get('roster') == roster, 'regression source/whole roster mismatch')
|
|
require(start.get('protocol') == 'guarded-standard-v1' and end.get('protocol') == start['protocol'], 'regression protocol')
|
|
require(start.get('run_id') and end.get('run_id') == start['run_id'] and end.get('owner_runner_pid') == start.get('owner_runner_pid'), 'regression ownership mismatch')
|
|
require(end.get('passed') is True and end.get('engine_ended') is True and not end.get('error') and not end.get('cleanup_errors') and not end.get('guard_errors'), 'regression incomplete/failed')
|
|
require(end.get('all_tests_sha256') == sha(rows_path) and [r['test'] for r in rows] == roster, 'regression rows checksum/roster mismatch')
|
|
markers = load_module('p24_strict_markers', ROOT / 'tools/strict_test_markers.py')
|
|
contracts = markers.load_contracts(ROOT / 'docs/plans/original-completion-2026-10-02/execution/completion_marker_contracts.json')
|
|
require(start.get('marker_contract_sha256') == sha(ROOT / 'docs/plans/original-completion-2026-10-02/execution/completion_marker_contracts.json'), 'marker contract mismatch')
|
|
for row in rows:
|
|
require(type(row.get('actual_pid')) is int and row['actual_pid'] > 0 and row['actual_pid'] in end['actual_captured_PIDs'], 'native PID mismatch')
|
|
require(row.get('passed') is True and row.get('exit_code') == 0 and row.get('engine_ended') is True, 'native suite failed/unexecuted')
|
|
require(row.get('source_sha256') == current[row['test']] and row.get('source_sha256_end') == current[row['test']], 'native suite source mismatch')
|
|
log = artifact(regression['logs'][row['test']])
|
|
require(sha(log) == row['log_sha256'], 'native raw log checksum mismatch')
|
|
parsed = markers.parse_raw(row['test'], log.read_text(), contracts, current[row['test']])
|
|
require(parsed['passed'] and parsed['checks'] == row['reported_checks'], 'native strict marker failed')
|
|
required_logs = {
|
|
'facility_lifecycle': ['tests/ui/test_selection_lifecycle.gd', 'tests/ui/test_selection_departure.gd'],
|
|
'guest_lifecycle': ['tests/ui/test_actor_selection.gd', 'tests/ui/test_selection_departure.gd'],
|
|
'staff_lifecycle': ['tests/ui/test_actor_selection.gd', 'tests/ui/test_selection_departure.gd'],
|
|
'layer_and_values': ['tests/ui/test_selection_autosave.gd', 'tests/ui/test_actor_selection.gd'],
|
|
'invalid_and_failed_load': ['tests/ui/test_selection_lifecycle.gd', 'tests/ui/test_actor_selection.gd'],
|
|
}
|
|
for condition, suites in required_logs.items():
|
|
require(set(matrix[condition]['artifacts']) >= {regression['logs'][s] for s in suites}, 'condition lacks actual native logs: ' + condition)
|
|
phases = end.get('phases', [])
|
|
require(len(phases) == len(roster) + 2 and all(p.get('passed') is True for p in phases), 'import/whole parse/whole regression incomplete')
|
|
|
|
# A producer and consumer must observe one unmodified formal save package.
|
|
cross = index['cross_process']
|
|
for condition in ('facility_lifecycle', 'guest_lifecycle', 'staff_lifecycle', 'layer_and_values'):
|
|
require(set(matrix[condition]['artifacts']) >= {cross[k] for k in ('producer', 'consumer', 'manifest', 'world')}, 'condition lacks cross-process/save artifacts: ' + condition)
|
|
observations = [read(artifact(cross[k])) for k in ('producer', 'consumer')]
|
|
manifest_path = artifact(cross['manifest'])
|
|
world = artifact(cross['world'])
|
|
manifest = read(manifest_path)
|
|
data = world.read_bytes()
|
|
require(data[:4] == b'TTWB' and len(data) > 16 and manifest['checksum_world'] == hashlib.sha256(data[16:]).hexdigest(), 'formal world header/checksum mismatch')
|
|
require(int.from_bytes(data[4:8], 'little') == manifest['format_version'] and int.from_bytes(data[8:16], 'little') == manifest['uncompressed_size'], 'formal world version/size mismatch')
|
|
for report in observations:
|
|
require(report.get('source_map') == current and report.get('scene') == 'res://scenes/main/main.tscn', 'cross-process normal Main/source mismatch')
|
|
require(type(report.get('pid')) is int and report['pid'] > 0, 'cross-process PID missing')
|
|
require(report.get('manifest_sha256') == sha(manifest_path) and report.get('world_sha256') == sha(world), 'cross-process save mismatch')
|
|
log = artifact(report['log'])
|
|
require(('P24_PROCESS pid=%d scene=res://scenes/main/main.tscn' % report['pid']) in log.read_text(), 'cross-process PID/scene absent from raw log')
|
|
require(('P24_OBSERVATION ' + json.dumps(report['observation'], sort_keys=True, ensure_ascii=False)) in log.read_text(), 'cross-process observation absent from raw log')
|
|
require(report['observation']['interface_selection'] == manifest['interface_selection'], 'formal saved selection mismatch')
|
|
require(observations[0]['pid'] != observations[1]['pid'], 'restart must use distinct processes')
|
|
require(observations[0]['observation'] == observations[1]['observation'], 'restored selections/values/layer differ')
|
|
observed = observations[0]['observation']
|
|
require(set(observed) >= {'interface_selection', 'layer', 'facility', 'guest', 'staff'}, 'cross-process value coverage missing')
|
|
require(type(observed['layer']) is int and observed['layer'] in (0, 1), 'cross-process layer invalid')
|
|
for kind in ('facility', 'guest', 'staff'):
|
|
value = observed[kind]
|
|
require(isinstance(value, dict) and value.get('id') and value.get('model_values') and value.get('screen_values') == value.get('model_values'), 'model/display values missing or inconsistent: ' + kind)
|
|
key = 'id' if kind == 'facility' else kind + '_id'
|
|
require(observed['interface_selection'].get(key) == value['id'], 'observed/saved ID mismatch: ' + kind)
|
|
require(observed['facility'].get('highlight_id') == observed['facility']['id'], 'restored map highlight mismatch')
|
|
|
|
# OS collector events must identify actual input, target process and captured frames.
|
|
os_report = read(artifact(index['os_input']))
|
|
require(index['os_input'] in matrix['normal_os_input']['artifacts'], 'OS condition lacks collector artifact')
|
|
require(os_report.get('source_map') == current and os_report.get('collector') in ('macOS-CGEventTap', 'Windows-RawInput', 'CUA-native-app'), 'OS collector/source mismatch')
|
|
cua = os_report.get('collector') == 'CUA-native-app'
|
|
if cua:
|
|
require(os_report.get('trace_provenance') in ('original_tool_receipts', 'faithful_transcription') and isinstance(os_report.get('limitations'), str) and os_report['limitations'].strip(), 'CUA trace provenance/limitations required')
|
|
events = os_report.get('events', [])
|
|
required_actions = {'select_facility', 'select_guest', 'select_staff', 'select_layer', 'save', 'resume'}
|
|
require({e.get('action') for e in events} >= required_actions, 'OS action coverage missing')
|
|
require(any(e.get('action') == 'save' and e.get('pid') == observations[0]['pid'] for e in events) and any(e.get('action') == 'resume' and e.get('pid') == observations[1]['pid'] for e in events), 'OS save/resume must target producer/consumer respectively')
|
|
for event in events:
|
|
require(type(event.get('pid')) is int and event['pid'] in [r['pid'] for r in observations] and type(event.get('monotonic_ns')) is int and event['monotonic_ns'] > 0, 'OS target/time mismatch')
|
|
require(event.get('native_type') in ('mouseDown', 'keyDown'), 'engine function/emit is not OS input')
|
|
if cua:
|
|
require(event.get('delivery') == 'native_os' and event.get('injected') is True, 'CUA must disclose tool-generated native OS delivery')
|
|
step = read(artifact(event['tool_step']))
|
|
require(step.get('pid') == event['pid'] and step.get('action') == event['action'] and step.get('monotonic_ns') == event['monotonic_ns'], 'CUA step target/action/time mismatch')
|
|
require(step.get('api') == 'native_app' and step.get('operation') in ('click', 'key') and isinstance(step.get('arguments'), dict) and step['arguments'], 'actual native app call arguments missing')
|
|
require(step.get('screenshot') == event['screenshot'] and step.get('source_map') == current, 'CUA returned screen/source mismatch')
|
|
else:
|
|
require(type(event.get('injected')) is bool and event.get('delivery') == 'native_os', 'OS delivery/origin disclosure required')
|
|
frame = artifact(event['screenshot'])
|
|
require({event['screenshot'], event['raw_trace']} <= set(matrix['normal_os_input']['artifacts']), 'OS condition lacks actual frame/trace artifacts')
|
|
if cua:
|
|
require(event['tool_step'] in matrix['normal_os_input']['artifacts'], 'OS condition lacks native app step artifact')
|
|
require(frame.read_bytes().startswith(b'\x89PNG\r\n\x1a\n'), 'OS screenshot missing/invalid')
|
|
trace = artifact(event['raw_trace']).read_text()
|
|
require(json.dumps(event, sort_keys=True, ensure_ascii=False) in trace, 'OS event missing in collector trace')
|
|
|
|
require(set(matrix['final_validation_review']['artifacts']) >= {regression['input'], regression['ended'], regression['rows']}, 'final condition lacks full regression artifacts')
|
|
review_status = 'pending'
|
|
if matrix['final_validation_review']['status'] == 'executed':
|
|
review = read(artifact(index['independent_review']))
|
|
require(index['independent_review'] in matrix['final_validation_review']['artifacts'], 'final condition lacks review artifact')
|
|
require(review.get('source_map') == current and review.get('reviewer') == 'Claude' and review.get('independent_session') and review.get('verdict') == 'pass', 'independent final Claude review missing')
|
|
require(set(review.get('condition_ids', [])) == set(CONDITIONS), 'independent review coverage missing')
|
|
review_status = 'executed'
|
|
return {'machine_validation': True, 'formal_acceptance': False, 'conditions': list(CONDITIONS), 'independent_review': review_status}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--verify-only', action='store_true', required=True)
|
|
args = parser.parse_args()
|
|
build = os.environ.get('BUILD_OUTPUT_DIR')
|
|
try:
|
|
require(bool(build), 'BUILD_OUTPUT_DIR is required')
|
|
result = verify(Path(build).resolve() / 'p24-evidence')
|
|
print(json.dumps(result, ensure_ascii=False))
|
|
return 0
|
|
except (OSError, ValueError, KeyError, TypeError, AttributeError, IndexError, AssertionError) as error:
|
|
print(json.dumps({'machine_validation': False, 'formal_acceptance': False, 'error': str(error)}, ensure_ascii=False))
|
|
return 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|