140 lines
8.4 KiB
Python
140 lines
8.4 KiB
Python
"""Archive superseded proof, verify every member, remove only cloud-readback-verified copies."""
|
|
import argparse
|
|
import datetime
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import tarfile
|
|
|
|
R = Path('/Users/korizatou114/Projects/とりにく工房ゲーム制作部')
|
|
A = R/'artifacts/original-completion-2026-10-02'
|
|
D = R/'docs/plans/original-completion-2026-10-02/execution/drive-offload-20261015'
|
|
S = Path('/private/tmp/the-tree-drive-offload-20261015')
|
|
FOLDER = '1Xa2b6rQmwBFq4jWIDFcJgiCbcNYfWXsK'
|
|
|
|
def need(ok, message):
|
|
if not ok: raise ValueError(message)
|
|
|
|
def sha(p):
|
|
h=hashlib.sha256()
|
|
with Path(p).open('rb') as f:
|
|
for b in iter(lambda:f.read(1048576),b''):h.update(b)
|
|
return h.hexdigest()
|
|
|
|
def write(p, data):
|
|
Path(p).write_text(json.dumps(data,ensure_ascii=False,indent=2)+'\n')
|
|
|
|
def target(name):
|
|
selected=json.loads((D/'OFFLOAD_SELECTION.json').read_text())['entries'] if (D/'OFFLOAD_SELECTION.json').exists() else []
|
|
need('/' not in name and (name=='windows-current1237-release-candidate-r03' or name in {r['name'] for r in selected}),'explicit selected historical proof only')
|
|
need('1299' not in name,'retain latest historical Windows baseline')
|
|
root=A/name
|
|
need(root.is_dir() and not root.is_symlink(),'ordinary selected evidence directory')
|
|
return root
|
|
|
|
def inventory(root):
|
|
rows=[]
|
|
for p in sorted(root.rglob('*')):
|
|
s=p.lstat(); row={'path':p.relative_to(root).as_posix(),'mode':s.st_mode & 0o7777,'mtime_ns':s.st_mtime_ns}
|
|
if p.is_symlink():row.update(type='symlink',target=os.readlink(p))
|
|
elif p.is_file():row.update(type='file',bytes=s.st_size,sha256=sha(p))
|
|
elif p.is_dir():row.update(type='directory')
|
|
else:raise ValueError('unsupported special file '+str(p))
|
|
rows.append(row)
|
|
return rows
|
|
|
|
def build(name):
|
|
root=target(name);S.mkdir(exist_ok=True);out=S/(name+'.tar.gz')
|
|
need(not out.exists(),'fresh archive output')
|
|
tracked=subprocess.check_output(['git','-C',str(R),'ls-files','--',str(root)],text=True)
|
|
need(not tracked.strip(),'retain tracked repository files')
|
|
need(shutil.disk_usage(R).free>700*1024**2,'archive disk admission')
|
|
members=inventory(root);need(any(x['type']=='file' for x in members),'nonempty proof')
|
|
m={'schema':'tree-proof-cloud-archive-v1','original_root':str(root),'name':name,
|
|
'reason':'Superseded proof selected in OFFLOAD_SELECTION.json; no current acceptance or active work removed',
|
|
'members':members,'member_count':len(members),'file_count':sum(x['type']=='file' for x in members),
|
|
'logical_bytes':sum(x.get('bytes',0) for x in members),'created_at_jst':datetime.datetime.now(datetime.timezone(datetime.timedelta(hours=9))).isoformat()}
|
|
raw=(json.dumps(m,ensure_ascii=False,indent=2)+'\n').encode()
|
|
with tarfile.open(out,'w:gz',compresslevel=6,format=tarfile.PAX_FORMAT) as t:
|
|
t.add(root,arcname=name,recursive=False)
|
|
for row in members:t.add(root/row['path'],arcname=name+'/'+row['path'],recursive=False)
|
|
info=tarfile.TarInfo('__OFFLOAD_MANIFEST__.json');info.size=len(raw);t.addfile(info,io.BytesIO(raw))
|
|
with tarfile.open(out,'r:gz') as t:
|
|
names=t.getnames();need(len(names)==len(set(names))==len(members)+2,'archive exact member roster')
|
|
need(t.extractfile('__OFFLOAD_MANIFEST__.json').read()==raw,'embedded manifest')
|
|
for row in members:
|
|
p=t.getmember(name+'/'+row['path'])
|
|
if row['type']=='file':
|
|
need(p.isfile() and p.size==row['bytes'],'archive member size/type')
|
|
h=hashlib.sha256()
|
|
with t.extractfile(p) as f:
|
|
for b in iter(lambda:f.read(1048576),b''):h.update(b)
|
|
need(h.hexdigest()==row['sha256'],'archive member SHA')
|
|
elif row['type']=='symlink':need(p.issym() and p.linkname==row['target'],'archive symlink')
|
|
else:need(p.isdir(),'archive directory')
|
|
need(inventory(root)==members,'source changed while archiving')
|
|
m.update(archive=str(out),archive_bytes=out.stat().st_size,archive_sha256=sha(out),archive_members_verified=True)
|
|
write(D/(name+'.manifest.json'),m)
|
|
print(json.dumps({k:m[k] for k in ['name','file_count','logical_bytes','archive','archive_bytes','archive_sha256']}),flush=True)
|
|
|
|
def split(name):
|
|
p=D/(name+'.manifest.json');m=json.loads(p.read_text());parts=[]
|
|
with Path(m['archive']).open('rb') as f:
|
|
n=0
|
|
while True:
|
|
b=f.read(64*1024*1024)
|
|
if not b:break
|
|
q=Path(m['archive']+'.part-%04d'%n);need(not q.exists(),'fresh split output')
|
|
q.write_bytes(b);parts.append({'order':n,'name':q.name,'local_path':str(q),'bytes':len(b),'sha256':hashlib.sha256(b).hexdigest()});n+=1
|
|
m['cloud_parts']=parts;m['transfer_limit_bytes']=104857600;write(p,m)
|
|
print(json.dumps({k:m[k] for k in ['name','archive','archive_bytes','archive_sha256','cloud_parts']}),flush=True)
|
|
|
|
def enrich(name):
|
|
p=D/(name+'.manifest.json');m=json.loads(p.read_text());v=json.loads((D/(name+'.cloud.json')).read_text())
|
|
# Python preserves nanosecond integer metadata; never round-trip this manifest in JS.
|
|
m['cloud_parts']=v['cloud_parts'];write(p,m)
|
|
|
|
def finalize(name):
|
|
root=target(name);m=json.loads((D/(name+'.manifest.json')).read_text());v=json.loads((D/(name+'.cloud.json')).read_text())
|
|
need(v['cloud_folder_id']==FOLDER and v['cloud_id'] and v['cloud_url'],'actual private cloud destination')
|
|
need(v['remote_metadata_bytes']==m['archive_bytes']==v['readback_bytes'],'remote complete size')
|
|
need(v['readback_sha256']==m['archive_sha256'] and v['readback_verified'],'cloud whole-file readback SHA')
|
|
need(m['archive_members_verified'] and sha(m['archive'])==m['archive_sha256'],'local verified archive unchanged')
|
|
need(inventory(root)==m['members'],'source changed before retirement')
|
|
opened=subprocess.run(['lsof','-n','-F','n'],capture_output=True,text=True).stdout
|
|
need(not any(s.startswith('n'+str(root)+'/') for s in opened.splitlines()),'active evidence files must stay local')
|
|
before=shutil.disk_usage(R).free
|
|
# Exact manifest-bound unlink, never recursive deletion of an unverified tree.
|
|
for row in m['members']:
|
|
p=root/row['path']
|
|
if row['type']!='directory':p.unlink()
|
|
for row in sorted(m['members'],key=lambda x:len(Path(x['path']).parts),reverse=True):
|
|
if row['type']=='directory':(root/row['path']).rmdir()
|
|
pointer={'state':'cloud_archived','original_root':str(root),'cloud_url':v['cloud_url'],'cloud_id':v['cloud_id'],
|
|
'archive_name':Path(m['archive']).name,'archive_sha256':m['archive_sha256'],
|
|
'cloud_parts':v.get('cloud_parts',[]),
|
|
'local_manifest':str(D/(name+'.manifest.json')),'cloud_manifest_id':v.get('cloud_manifest_id'),
|
|
'restore':'Download every ordered cloud_part and concatenate in numeric order to archive_name (or download the single archive if parts are absent). Verify archive_sha256, then extract its named root into artifacts/original-completion-2026-10-02. Embedded manifest preserves all bytes and exact original mtimes.',
|
|
'accepted_current_proof_affected':False}
|
|
write(root/'CLOUD_ARCHIVE.json',pointer)
|
|
Path(m['archive']).unlink()
|
|
download=Path(v['readback_path'])
|
|
need(download.parent==S and download.name==name+'.readback.tar.gz','owned readback temporary file')
|
|
need(sha(download)==m['archive_sha256'],'readback unchanged before temporary cleanup');download.unlink()
|
|
for part in m.get('cloud_parts',[]):
|
|
p=Path(part['local_path']);need(p.parent==S and p.name.startswith(name+'.tar.gz.part-'),'owned split temporary file')
|
|
need(sha(p)==part['sha256'],'split temporary SHA');p.unlink()
|
|
result={**pointer,'file_count':m['file_count'],'logical_bytes_offloaded':m['logical_bytes'],
|
|
'free_before_retirement_bytes':before,'free_after_retirement_bytes':shutil.disk_usage(R).free,
|
|
'local_originals_removed_after_cloud_readback':True,'member_hashes_verified':True}
|
|
write(D/(name+'.retired.json'),result)
|
|
print(json.dumps({k:result[k] for k in ['file_count','logical_bytes_offloaded','free_after_retirement_bytes']}),flush=True)
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('operation',choices=['build','split','enrich','finalize']);p.add_argument('name');a=p.parse_args()
|
|
{'build':build,'split':split,'enrich':enrich,'finalize':finalize}[a.operation](a.name)
|