From 7f1920b329c71f1a86738beedd02caac686f2be5 Mon Sep 17 00:00:00 2001 From: Agent Control Runner Date: Fri, 16 Oct 2026 14:41:54 +0900 Subject: [PATCH] =?UTF-8?q?P23-03:=20=E7=94=9F=E6=B4=BB=E3=81=AE=E8=A6=B3?= =?UTF-8?q?=E5=AF=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/check_p23_life_observation_evidence.py | 112 ++++++++++-- tools/test_p23_life_observation_evidence.py | 169 ++++++++++++++++++- 2 files changed, 262 insertions(+), 19 deletions(-) diff --git a/tools/check_p23_life_observation_evidence.py b/tools/check_p23_life_observation_evidence.py index 3ac3c6f..b9b7210 100644 --- a/tools/check_p23_life_observation_evidence.py +++ b/tools/check_p23_life_observation_evidence.py @@ -1839,7 +1839,58 @@ def git_bytes(root, *arguments): return process.stdout -def verify_composition(root, roles, source_map, inputs, read_git=None): +_VERIFIED_VIEWS = {} + + +def blob_oid(content): + return hashlib.sha1(b'blob %d\0' % len(content) + content).hexdigest() + + +def git_blob_stream(root, oids, read_batch=None): + """Pace requests to one owned child; adapters return (binary stream, status).""" + requested = list(oids) + require(all(isinstance(oid, str) and re.fullmatch('[0-9a-f]{40}', oid) + for oid in requested), 'invalid blob oid') + process = stream = None + try: + if read_batch is None: + process = subprocess.Popen(['git', '-C', str(root), 'cat-file', '--batch'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + stream = process.stdout + else: + stream, status = read_batch(requested) + count = 0 + for oid in requested: + if process is not None: + process.stdin.write((oid + '\n').encode()) + process.stdin.flush() + header = stream.readline(128) + match = re.fullmatch(rb'([0-9a-f]{40}) blob (0|[1-9][0-9]*)\n', header) + require(match is not None and match[1].decode() == oid, 'invalid/out-of-order blob header') + size = int(match[2]) + content = stream.read(size) + require(len(content) == size, 'short blob read') + require(stream.read(1) == b'\n', 'missing blob LF') + require(blob_oid(content) == oid, 'blob content SHA-1 mismatch') + count += 1 + yield oid, content + if process is not None: + process.stdin.close() + require(stream.read(1) == b'', 'extra trailing blob output') + require((process.wait() if process is not None else status) == 0, 'cat-file failed') + require(count == len(requested), 'blob output count mismatch') + finally: + if process is not None: + if process.poll() is None: + process.kill() + process.wait() + process.stdin.close() + process.stdout.close() + elif stream is not None: + stream.close() + + +def verify_composition(root, roles, source_map, inputs, read_git=None, view=None, read_batch=None): require(isinstance(roles, dict), 'missing externally bound S/M/T/Q') names = ('submitted_sha', 'main_context_sha', 'tested_tree', 'tested_commit') require(all(isinstance(roles.get(k), str) and re.fullmatch('[0-9a-f]{40}', roles[k]) for k in names), @@ -1865,6 +1916,7 @@ def verify_composition(root, roles, source_map, inputs, read_git=None): continue metadata, path = record.split(b'\t', 1) mode, kind, oid = metadata.decode().split() + require(path.decode() not in result, 'duplicate T path') result[path.decode()] = (mode, kind, oid) return result base, submitted, main, tested = map(tree, (BASE_COMMIT, s, m, t)) @@ -1882,30 +1934,43 @@ def verify_composition(root, roles, source_map, inputs, read_git=None): return strict_json(git('show', revision + ':' + CONTRACT)) verify_marker_join(document(BASE_COMMIT), document(s), document(m), document(t)) runner = module('p23_composition_source_filter', ROOT/'tools/run_all_tests.py') - with tested_checkout(root, roles, git) as view: - require(source_map == {p: row['sha256'] for p, row in runner.source_refs(view).items()}, + if view is not None: + registered = _VERIFIED_VIEWS.get(Path(view).resolve()) + require(registered is not None, 'unregistered verified view') + require(registered[0] == t, 'verified view tree mismatch') + require(registered[1] == {p: e for p, e in tested.items() if e[1] == 'blob'}, + 'verified view manifest mismatch') + with (contextlib.nullcontext(Path(view)) if view is not None else + tested_checkout(root, roles, git, read_batch)) as source_view: + require(source_map == {p: row['sha256'] for p, row in runner.source_refs(source_view).items()}, 'incomplete tested-tree source/settings/QA map') # Source refs include all QA suites and settings, not just S-changed files. required = set(source_map) | {p.split('#', 1)[0] for p in inputs} | {PLAN, CONTRACT, 'project.godot', 'tests/business/test_required_kitchen.gd', 'tools/regression/suite-policy.json'} bindings = roles.get('blob_bindings') require(isinstance(bindings, dict) and set(bindings) == required, 'incomplete/extraneous T blob bindings') - for path in sorted(required): + paths = sorted(required) + for path in paths: entry = tested.get(path) require(entry is not None and entry[1] == 'blob' and bindings[path] == entry[2], 'T blob binding: ' + path) - content = git('show', t + ':' + path) - if head != s or path in changed_s-changed_m: - require((root/path).read_bytes() == content, 'checkout bytes differ from T: ' + path) - digest = hashlib.sha256(content).hexdigest() - if path in source_map: - require(source_map[path] == digest, 'evidence source is not T: ' + path) - if path in inputs: - require(inputs[path] == digest, 'evidence input is not T: ' + path) + contents = (((tested[p][2], (Path(view)/p).read_bytes()) for p in paths) if view is not None + else git_blob_stream(root, [tested[p][2] for p in paths], read_batch)) + with contextlib.closing(contents): + for number, (oid, content) in enumerate(contents): + path = paths[number] + require(blob_oid(content) == tested[path][2] == oid, 'binding blob SHA-1 mismatch: ' + path) + if head != s or path in changed_s-changed_m: + require((root/path).read_bytes() == content, 'checkout bytes differ from T: ' + path) + digest = hashlib.sha256(content).hexdigest() + if path in source_map: + require(source_map[path] == digest, 'evidence source is not T: ' + path) + if path in inputs: + require(inputs[path] == digest, 'evidence input is not T: ' + path) return {k: roles[k] for k in names} @contextlib.contextmanager -def tested_checkout(root, roles, read_git=None): +def tested_checkout(root, roles, read_git=None, read_batch=None): """Disposable Git-blob view shared by the existing filters and readers.""" require(isinstance(roles, dict), 'missing externally bound S/M/T/Q') tree = roles.get('tested_tree', '') @@ -1913,6 +1978,7 @@ def tested_checkout(root, roles, read_git=None): git = read_git or (lambda *args: git_bytes(root, *args)) with tempfile.TemporaryDirectory(prefix='p23-tested-tree-') as temporary: view = Path(temporary) + manifest = {} for record in git('ls-tree', '-r', '-z', tree).split(b'\0'): if not record: continue @@ -1921,10 +1987,20 @@ def tested_checkout(root, roles, read_git=None): path = Path(raw_path.decode()) require(not path.is_absolute() and '..' not in path.parts, 'unsafe T path') require(kind == 'blob' and mode in ('100644', '100755'), 'T source aliases refused') - target = view/path - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(git('show', tree + ':' + path.as_posix())) - yield view + require(path.as_posix() not in manifest, 'duplicate T path') + manifest[path.as_posix()] = (mode, kind, oid) + paths = list(manifest) + contents = git_blob_stream(root, [manifest[p][2] for p in paths], read_batch) + with contextlib.closing(contents): + for number, (_, content) in enumerate(contents): + target = view/paths[number] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + _VERIFIED_VIEWS[view.resolve()] = (tree, manifest) + try: + yield view + finally: + del _VERIFIED_VIEWS[view.resolve()] def root_context(directory, index, explicit=None): @@ -2164,7 +2240,7 @@ def check_tested(checkout, directory, root_roles, root): 'historical v1 evidence has diagnostic applicability only') runner = module('p23_source_applicability', root/'tools/run_all_tests.py') require(index.get('source_map') == {n: v['sha256'] for n, v in runner.source_refs(root).items()}, 'exact frozen source map') - roles = verify_composition(checkout, root_roles, index['source_map'], input_map(root)) + roles = verify_composition(checkout, root_roles, index['source_map'], input_map(root), view=root) if git_bytes(checkout, 'rev-parse', 'HEAD').decode().strip() != roles['submitted_sha']: require(index['source_map'] == {n: v['sha256'] for n, v in runner.source_refs(checkout).items()}, 'exact checkout source map') diff --git a/tools/test_p23_life_observation_evidence.py b/tools/test_p23_life_observation_evidence.py index c8c5c4d..1f147f9 100644 --- a/tools/test_p23_life_observation_evidence.py +++ b/tools/test_p23_life_observation_evidence.py @@ -183,6 +183,12 @@ class Repair28CompositionTests(unittest.TestCase): return self.files[revision][path] raise AssertionError(args) + def batch(self, oids): + self.calls.append(('cat-file', '--batch')) + blobs = {self.blob(v): v for files in self.files.values() for v in files.values()} + return io.BytesIO(b''.join(oid.encode()+b' blob '+str(len(blobs[oid])).encode()+b'\n' + +blobs[oid]+b'\n' for oid in oids)), 0 + def verify(self): original = Path.read_bytes def read(path): @@ -190,7 +196,8 @@ class Repair28CompositionTests(unittest.TestCase): return self.files[self.t][str(path.relative_to(check.ROOT))] return original(path) with mock.patch.object(Path, 'read_bytes', read): - return check.verify_composition(check.ROOT,self.roles,self.source,self.inputs,self.git) + return check.verify_composition(check.ROOT,self.roles,self.source,self.inputs,self.git, + read_batch=self.batch) def test_valid_route_b_keeps_unchanged_allowed_path_from_main(self): self.assertEqual(self.verify()['tested_tree'], self.t) @@ -252,6 +259,145 @@ class Repair28CompositionTests(unittest.TestCase): self.roles, self.source = original_roles, original_source +class Repair31StreamTests(unittest.TestCase): + def test_owned_child_cleanup_on_parse_failure_and_early_close(self): + oid = check.blob_oid(b'x') + for malformed in (True, False): + child = mock.Mock() + child.stdin = io.BytesIO() + child.stdout = io.BytesIO(b'bad\n' if malformed else oid.encode()+b' blob 1\nx\n') + child.poll.return_value = None + child.wait.return_value = -9 + with mock.patch.object(subprocess, 'Popen', return_value=child): + stream = check.git_blob_stream(check.ROOT, [oid]) + if malformed: + with self.assertRaises(ValueError): next(stream) + else: + self.assertEqual(next(stream), (oid, b'x')) + stream.close() + child.kill.assert_called_once_with() + child.wait.assert_called_once_with() + self.assertTrue(child.stdin.closed) + self.assertTrue(child.stdout.closed) + + def test_strict_stream_negatives_and_repeated_oids(self): + content = b'abc' + oid = check.blob_oid(content) + other = check.blob_oid(b'def') + record = oid.encode()+b' blob 3\nabc\n' + def read(data, oids=(oid,), status=0): + return list(check.git_blob_stream(check.ROOT, oids, + lambda requested: (io.BytesIO(data), status))) + self.assertEqual(read(record*2, (oid, oid)), [(oid, content)]*2) + invalid = [record[:-2], record[:-1], record.replace(b'blob 3', b'blob 2'), + record.replace(b'blob 3', b'blob 4'), record.replace(b'blob', b'tree'), + oid.encode()+b' missing\n', oid.encode()+b' ambiguous\n', + record+record, record.replace(b'abc', b'def'), b'', + record.replace(oid.encode(), other.encode())] + for data in invalid: + with self.subTest(data=data), self.assertRaises(ValueError): + read(data) + with self.assertRaises(ValueError): + read(record, status=1) + with self.assertRaises(ValueError): + read(record*2, (oid, other)) + + def test_checkout_paths_and_modes(self): + oid = check.blob_oid(b'x') + roles = dict(tested_tree='1'*40) + for path, mode, kind, duplicate in [('a', '100644', 'blob', True), + ('a','120000','blob',False), ('a','160000','commit',False), + ('/a','100644','blob',False), ('a/../b','100644','blob',False)]: + record = f'{mode} {kind} {oid}\t{path}\0'.encode() + with self.subTest(path=path, mode=mode), self.assertRaises(ValueError): + with check.tested_checkout(check.ROOT, roles, + lambda *args: record*(2 if duplicate else 1), + lambda oids: (io.BytesIO(oid.encode()+b' blob 1\nx\n'),0)): + self.fail('unsafe checkout yielded') + + def test_view_authority_and_binding_recheck(self): + fixture = Repair28CompositionTests() + fixture.setUp() + def verify(view): + return check.verify_composition(check.ROOT, fixture.roles, fixture.source, + fixture.inputs, fixture.git, view=view, read_batch=fixture.batch) + with tempfile.TemporaryDirectory() as temporary: + with self.assertRaisesRegex(ValueError, 'unregistered'): + verify(Path(temporary)) + with check.tested_checkout(check.ROOT, fixture.roles, fixture.git, fixture.batch) as view: + registered = check._VERIFIED_VIEWS[view.resolve()] + check._VERIFIED_VIEWS[view.resolve()] = ('0'*40, registered[1]) + with self.assertRaisesRegex(ValueError, 'tree mismatch'): verify(view) + check._VERIFIED_VIEWS[view.resolve()] = (registered[0], {}) + with self.assertRaisesRegex(ValueError, 'manifest mismatch'): verify(view) + check._VERIFIED_VIEWS[view.resolve()] = registered + # A normative document is outside source_refs; its bytes must still be checked. + (view/check.PLAN).write_bytes(b'tampered') + with mock.patch.object(Path, 'read_bytes', autospec=True) as reader: + # Preserve real view reads and synthesize only the source checkout comparison. + original = io.open + def read(path): + if path.is_relative_to(view): + with original(path, 'rb') as stream: return stream.read() + return fixture.files[fixture.t][str(path.relative_to(check.ROOT))] + reader.side_effect = read + with self.assertRaisesRegex(ValueError, 'SHA-1 mismatch'): verify(view) + self.assertNotIn(view.resolve(), check._VERIFIED_VIEWS) + + def test_constant_calls_full_composition_with_5000_paths(self): + fixture = Repair28CompositionTests() + fixture.setUp() + additions = {f'assets/bulk/{i:05}.txt': b'repeated small blob' for i in range(5000)} + for files in fixture.files.values(): files.update(additions) + fixture.source.update({p: hashlib.sha256(v).hexdigest() for p,v in additions.items()}) + fixture.roles['blob_bindings'].update({p: check.blob_oid(v) for p,v in additions.items()}) + original = Path.read_bytes + verified = set() + def read(path): + if path.is_relative_to(check.ROOT): + return fixture.files[fixture.t][str(path.relative_to(check.ROOT))] + data = original(path) + if 'bulk' in path.parts: + self.assertEqual(data, b'repeated small blob') + verified.add(path.name) + return data + with mock.patch.object(Path, 'read_bytes', read): + with check.tested_checkout(check.ROOT, fixture.roles, fixture.git, fixture.batch) as view: + check.verify_composition(check.ROOT, fixture.roles, fixture.source, fixture.inputs, + fixture.git, view=view, read_batch=fixture.batch) + self.assertEqual(len(verified), 5000) + self.assertEqual(fixture.calls.count(('cat-file','--batch')), 1) + self.assertEqual(len([c for c in fixture.calls if c[0] == 'show']), 4) + + def test_real_pipe_overflow_requests_and_repeated_paths(self): + runtime = os.environ.get('TMPDIR') + self.assertTrue(runtime, 'assigned TMPDIR required') + with tempfile.TemporaryDirectory(dir=runtime, prefix='p23-repair31-') as temporary: + repo = Path(temporary) + def git(*args, data=None): + return subprocess.run(['git','-C',str(repo),*args], input=data, + capture_output=True, check=True, timeout=30).stdout + git('init','-q') + content = b'pipe pacing verified\n' + oid = git('hash-object','-w','--stdin', data=content).decode().strip() + count = 5000 + tree = git('mktree', data=b''.join( + f'100644 blob {oid}\tpath-{i:05}\n'.encode() for i in range(count))).decode().strip() + # A watchdog bounds a deadlock regression; elapsed time is not a pass metric. + program = '''import sys +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +import check_p23_life_observation_evidence as c +repo, tree = Path(sys.argv[2]), sys.argv[3] +with c.tested_checkout(repo, {'tested_tree': tree}) as view: + paths = list(view.iterdir()) + assert len(paths) == 5000 + assert all(p.read_bytes() == b'pipe pacing verified\\n' for p in paths) +''' + subprocess.run([sys.executable, '-B', '-c', program, str(check.ROOT/'tools'), + str(repo), tree], check=True, timeout=60) + + class Repair30EntryTests(unittest.TestCase): """Synthetic labelled Git history; zero native or OS credit.""" def setUp(self): @@ -274,6 +420,8 @@ class Repair30EntryTests(unittest.TestCase): 'scenes/main/world_preview.gd', 'tests/business/test_required_kitchen.gd', 'tools/regression/suite-policy.json'): self.write(path, 'synthetic baseline\n') + for i in range(220): + self.write(f'assets/repair31/{i:04}.txt', f'verified path {i}\n') self.marker = dict(schema_version=1, rows=[dict(test='history-'+str(i), required=True) for i in range(6)]) self.write_marker(self.marker) self.base = self.commit('synthetic base') @@ -357,6 +505,25 @@ class Repair30EntryTests(unittest.TestCase): with self.subTest(head=head, verify=verify): self.assert_entry(verify) + def test_whole_entry_has_constant_git_calls_and_verifies_every_path(self): + spawn = subprocess.Popen + calls, read_paths = [], set() + original = Path.read_bytes + def counted(*args, **kwargs): + calls.append(args[0]) + return spawn(*args, **kwargs) + def read(path): + data = original(path) + if 'repair31' in path.parts: + self.assertEqual(data, f'verified path {int(path.stem)}\n'.encode()) + read_paths.add(path.name) + return data + with mock.patch.object(subprocess, 'Popen', counted), mock.patch.object(Path, 'read_bytes', read): + self.assert_entry() + self.assertEqual(len(read_paths), 220) + self.assertEqual(sum(command[-2:] == ['cat-file','--batch'] for command in calls), 1) + self.assertEqual(sum('show' in command for command in calls), 4) + def test_wrong_roles_and_missing_Q(self): for key, value in (('submitted_sha',self.base), ('main_context_sha',self.base), ('tested_tree',self.git('rev-parse', self.base+'^{tree}')),