extends SceneTree ## P06-03 outage recovery across a real process boundary. "-- writer" saves ## mid-outage and records its own recovery; "-- reader" restores that save in a ## new Godot process, recovers the same way and must match exactly. Recovery ## must never clear DESTROYED or RAIDED suspension. const ManaOutage = preload("res://tests/helpers/mana_outage.gd") const OUTAGE_SAVE := "user://saves/p06_03_recovery_new_pid" const EXPECTED_PATH := "user://saves/p06_03_recovery_new_pid_expected.json" const RECOVERY_MINUTES := 30 var _checks := 0 var _failures := 0 func _initialize() -> void: call_deferred("_run") func _run() -> void: var mode := "reader" if "reader" in OS.get_cmdline_user_args() else "writer" print("S2: P06-03 Mana recovery new-PID %s (pid %d)" % [mode, OS.get_process_id()]) if mode == "writer": _write() else: _read() print("%s: %d checks" % ["PASS" if _failures == 0 else "FAIL", _checks]) quit(_failures) func _new_runtime() -> Array: var clock: Variant = load("res://autoload/game_clock.gd").new() var state: Variant = load("res://autoload/game_state.gd").new() var runtime: Variant = load("res://autoload/simulation_runtime.gd").new() clock.reset(1, 6, 0) state.reset() runtime.initialize(clock, state, 6606) clock.set_paused(true) return [clock, state, runtime] func _facility(runtime: Variant, definition_id: StringName) -> FacilityInstance: for facility: FacilityInstance in runtime.facility_service.get_instances(): if facility.definition_id == definition_id: return facility return null func _summary(runtime: Variant, state: Variant) -> Dictionary: var rows := {} for facility: FacilityInstance in runtime.facility_service.get_instances(): rows[String(facility.definition_id)] = { "power": facility.power_available, "suspension": String(facility.suspension_reason), "operation": facility.operation_state, } return {"mana": state.get_resource(&"mana"), "facilities": rows} func _recover(clock: Variant, state: Variant, runtime: Variant) -> void: ManaOutage.release(runtime, state, 0) clock.advance_minutes(RECOVERY_MINUTES) func _write() -> void: var nodes := _new_runtime() var clock: Variant = nodes[0] var state: Variant = nodes[1] var runtime: Variant = nodes[2] _expect(runtime.submit_command(&"build_facility", {&"definition_id": &"reception", &"origin": Vector3i(-6, 0, 0)}).is_success(), "build reception") clock.advance_minutes(120) _expect(runtime.submit_command(&"build_facility", {&"definition_id": &"standard_room", &"origin": Vector3i(-6, 2, 0)}).is_success(), "build room on supported floor") var cafeteria_built := false for x: int in [-2, 2, -8, -10]: if runtime.submit_command(&"build_facility", {&"definition_id": &"cafeteria", &"origin": Vector3i(x, 0, 0)}).is_success(): cafeteria_built = true break _expect(cafeteria_built, "build cafeteria on ground floor") clock.advance_minutes(180) var reception := _facility(runtime, &"reception") var cafeteria := _facility(runtime, &"cafeteria") var room := _facility(runtime, &"standard_room") _expect(reception != null and cafeteria != null and room != null, "facilities complete normally") if reception == null or cafeteria == null or room == null: return runtime.facility_service.set_operational_status(cafeteria.instance_id, {&"suspension_reason": &"DESTROYED"}, clock.absolute_minute) runtime.facility_service.set_operational_status(room.instance_id, {&"suspension_reason": &"RAIDED"}, clock.absolute_minute) ManaOutage.hold(runtime, state) clock.advance_minutes(15) _expect(not _facility(runtime, &"reception").power_available, "outage stops the tower") var outage: Dictionary = runtime.create_snapshot() _expect(SaveService.save_snapshot(OUTAGE_SAVE, outage)[&"ok"], "writer saves mid-outage") _recover(clock, state, runtime) var summary := _summary(runtime, state) _check_recovered(summary) var file := FileAccess.open(EXPECTED_PATH, FileAccess.WRITE) _expect(file != null, "writer records recovery") if file != null: file.store_string(JSON.stringify({ "writer_pid": OS.get_process_id(), "summary": summary, "digest": SimulationDigest.compute(runtime.create_snapshot()), })) file.close() for node: Variant in nodes: node.free() func _read() -> void: var expected: Variant = JSON.parse_string(FileAccess.get_file_as_string(EXPECTED_PATH)) _expect(expected is Dictionary, "reader finds writer record") if not expected is Dictionary: return _expect(int(expected["writer_pid"]) != OS.get_process_id(), "reader runs in a different process") var nodes := _new_runtime() var clock: Variant = nodes[0] var state: Variant = nodes[1] var runtime: Variant = nodes[2] var loaded := SaveService.load_snapshot(OUTAGE_SAVE) _expect(loaded[&"ok"] and runtime.restore_snapshot(loaded[&"snapshot"]), "reader restores mid-outage save") _expect(state.get_resource(&"mana") == 0 and not _facility(runtime, &"reception").power_available, "restored tower is still stopped") _recover(clock, state, runtime) var summary := _summary(runtime, state) _check_recovered(summary) _expect(JSON.stringify(JSON.parse_string(JSON.stringify(summary))) == JSON.stringify(expected["summary"]), "recovered facility states match the writer") _expect(SimulationDigest.compute(runtime.create_snapshot()) == String(expected["digest"]), "whole recovered state digest matches the writer") SaveService.delete_save(OUTAGE_SAVE) DirAccess.remove_absolute(ProjectSettings.globalize_path(EXPECTED_PATH)) for node: Variant in nodes: node.free() func _check_recovered(summary: Dictionary) -> void: var rows: Dictionary = summary["facilities"] _expect(int(summary["mana"]) > 0, "world tree refills the reserve after the outage") _expect(bool(rows["reception"]["power"]) and String(rows["reception"]["suspension"]) == "", "ordinary facility power returns") _expect(bool(rows["cafeteria"]["power"]) and String(rows["cafeteria"]["suspension"]) == "DESTROYED" and int(rows["cafeteria"]["operation"]) == FacilityInstance.OperationState.SUSPENDED, "recovery does not clear destruction") _expect(bool(rows["standard_room"]["power"]) and String(rows["standard_room"]["suspension"]) == "RAIDED" and int(rows["standard_room"]["operation"]) == FacilityInstance.OperationState.SUSPENDED, "recovery does not clear raid suspension") func _expect(condition: bool, label: String) -> void: _checks += 1 if condition: print(" ✓ %s" % label) else: _failures += 1 push_error(" ✗ %s" % label)