diff --git a/docs/plans/original-completion-2026-10-02/execution/p28-01/collector/check_p28_01_report.py b/docs/plans/original-completion-2026-10-02/execution/p28-01/collector/check_p28_01_report.py index e5fbab1..36a0663 100644 --- a/docs/plans/original-completion-2026-10-02/execution/p28-01/collector/check_p28_01_report.py +++ b/docs/plans/original-completion-2026-10-02/execution/p28-01/collector/check_p28_01_report.py @@ -132,7 +132,8 @@ def normalize_report(value, path=()): key = path[-1] if path else "" declared = key in INTEGER_FIELDS or key.endswith("_us") declared = declared or (key == "frames" and not isinstance(value, list)) - declared = declared or (len(path) >= 2 and path[-2] in ("samples_us", "kernel_stall_samples_us", "sample_counts")) + declared = declared or (len(path) >= 2 and (path[-2] == "sample_counts" or + (key == "[]" and path[-2] in ("samples_us", "kernel_stall_samples_us")))) declared = declared or ("budget_verdict" in path and key == "value") declared = declared or ("count_ranges" in path and key in ("min", "max")) declared = declared or (len(path) >= 2 and path[-2] in @@ -146,6 +147,40 @@ def normalize_report(value, path=()): return value +def check_budget_snapshot(snapshot): + need(isinstance(snapshot, dict), "budget snapshot object") + for entry in snapshot.values(): + keys(entry, ("samples_us", "lifetime_count", "reason", "provenance")) + provenance = entry["provenance"] + columns = ("samples_us", "lifetime_count") + keys(provenance, columns) + for column in columns: + keys(provenance[column], ("dictionary_readable", "key_form")) + need(type(provenance[column]["dictionary_readable"]) is bool, + "budget dictionary availability") + both_readable = all(provenance[c]["dictionary_readable"] for c in columns) + for column, zero in (("samples_us", []), ("lifetime_count", 0)): + observed = provenance[column] + form, value = observed["key_form"], entry[column] + if not observed["dictionary_readable"]: + need(form is None and value is None, "budget unreadable dictionary must remain null") + elif form in ("absent", "absent_means_zero_per_product"): + if both_readable: + need(form == "absent_means_zero_per_product" and type(value) is type(zero) + and value == zero, "budget readable absent key must use product zero") + else: + need(form == "absent" and value is None, + "budget absent key without readable dictionaries") + else: + need(form in ("String", "StringName"), "budget present key form") + if value is not None: + need((isinstance(value, list) and all(type(x) is int and x >= 0 for x in value)) + if column == "samples_us" else type(value) is int and value >= 0, + "budget sample value") + expected = "counts_unavailable" if entry["samples_us"] is None or entry["lifetime_count"] is None else None + need(entry["reason"] == expected, "budget availability reason") + + def budget_verdict(before, after, speed): result = {} for metric, aggregation, limit in ((f"simulation_x{speed}", "average" if speed == 1 else "p95", @@ -284,6 +319,8 @@ def interval(report, name): def check_report(report): check_budget_scope(report, ((),)) + for name in ("budget_at_warmup_end", "budget_at_phase_end"): + check_budget_snapshot(report[name]["data"]) for name in RAW_FIELDS: keys(report[name], ("source", "data")) equal(report["probe_deadline"], {"limit_seconds": 1740, "source": DEADLINE_SOURCE}, "probe deadline") diff --git a/docs/plans/original-completion-2026-10-02/execution/p28-01/collector/measure_p28_01.gd b/docs/plans/original-completion-2026-10-02/execution/p28-01/collector/measure_p28_01.gd index 05d2f4b..c33538a 100644 --- a/docs/plans/original-completion-2026-10-02/execution/p28-01/collector/measure_p28_01.gd +++ b/docs/plans/original-completion-2026-10-02/execution/p28-01/collector/measure_p28_01.gd @@ -76,11 +76,32 @@ func field(d: Variant, name: String) -> Variant: func budget_snapshot() -> Dictionary: var result := {} var profiler: Variant = clock.performance_budget + var sample_dictionary: Variant = profiler.get("_samples") + var count_dictionary: Variant = profiler.get("_sample_counts") + var both_readable := sample_dictionary is Dictionary and count_dictionary is Dictionary for metric: String in ["simulation_x1", "simulation_x3", "main_thread_stall"]: - var samples: Variant = field(profiler.get("_samples"), metric) - var count: Variant = field(profiler.get("_sample_counts"), metric) - result[metric] = {"samples_us": samples.duplicate() if samples is Array else null, - "lifetime_count": count, "reason": "counts_unavailable" if count == null or samples == null else null} + var entry := {} + var provenance := {} + for column: String in ["samples_us", "lifetime_count"]: + var dictionary: Variant = sample_dictionary if column == "samples_us" else count_dictionary + var value: Variant = null + var key_form: Variant = null + if dictionary is Dictionary: + key_form = "absent" + for key: Variant in dictionary.keys(): + if (key is StringName or key is String) and String(key) == metric: + key_form = "StringName" if key is StringName else "String" + value = dictionary[key] + break + if key_form == "absent" and both_readable: + # performance_budget.gd uses get(metric, []) / get(metric, 0). + value = [] if column == "samples_us" else 0 + key_form = "absent_means_zero_per_product" + provenance[column] = {"dictionary_readable": dictionary is Dictionary, "key_form": key_form} + entry[column] = (value.duplicate() if value is Array else null) if column == "samples_us" else value + entry["provenance"] = provenance + entry["reason"] = "counts_unavailable" if entry.samples_us == null or entry.lifetime_count == null else null + result[metric] = entry return result func kernel_verdict(before: Dictionary, after: Dictionary) -> Dictionary: diff --git a/docs/plans/original-completion-2026-10-02/execution/p28-01/collector/test_collect_p28_01.py b/docs/plans/original-completion-2026-10-02/execution/p28-01/collector/test_collect_p28_01.py index 1a3ea0e..09611e4 100644 --- a/docs/plans/original-completion-2026-10-02/execution/p28-01/collector/test_collect_p28_01.py +++ b/docs/plans/original-completion-2026-10-02/execution/p28-01/collector/test_collect_p28_01.py @@ -144,6 +144,11 @@ def synthetic_report(): "statistics": {"count": 10, "average_us": 3000, "p95_us": 3000, "maximum_us": 3000}} for k, aggregate, limit in (("simulation_x1", "average", 4000), ("main_thread_stall", "maximum", 50000))} + for name in ("budget_at_warmup_end", "budget_at_phase_end"): + for entry in report[name].values(): + entry.update(reason=None, provenance={ + c: {"dictionary_readable": True, "key_form": "StringName"} + for c in ("samples_us", "lifetime_count")}) for name in checker.RAW_FIELDS: report[name] = {"source": checker.RAW_SOURCE, "data": report[name]} report["probe_deadline"] = {"limit_seconds": 1740, "source": checker.DEADLINE_SOURCE} @@ -616,6 +621,10 @@ class ReportTests(unittest.TestCase): ("no_interval_samples", 30), ("counts_unavailable", None)): r = synthetic_report() r["budget_at_phase_end"]["data"]["simulation_x1"]["lifetime_count"] = count + if count is None: + entry = r["budget_at_phase_end"]["data"]["simulation_x1"] + entry["reason"] = "counts_unavailable" + entry["provenance"]["lifetime_count"] = {"dictionary_readable": False, "key_form": None} r["budget_verdict"] = checker.budget_verdict(r["budget_at_warmup_end"]["data"], r["budget_at_phase_end"]["data"], 1) self.assertEqual(r["budget_verdict"]["simulation_x1"]["reason"], reason) @@ -625,6 +634,66 @@ class ReportTests(unittest.TestCase): with self.assertRaises(ValueError): checker.check_report(r) + def test_c8_readable_absent_baseline_uses_actual_interval(self): + r = synthetic_report() + samples = [401870, 368451, 374607, 363011, 371589, + 465303, 359596, 361040, 362532, 354702] + for metric in ("simulation_x1", "main_thread_stall"): + entry = r["budget_at_warmup_end"]["data"][metric] + entry.update(lifetime_count=0, samples_us=[]) + for observed in entry["provenance"].values(): + observed["key_form"] = "absent_means_zero_per_product" + r["budget_at_phase_end"]["data"][metric].update( + lifetime_count=10, samples_us=samples) + r["budget_verdict"] = checker.budget_verdict( + r["budget_at_warmup_end"]["data"], r["budget_at_phase_end"]["data"], 1) + checker.check_report(r) + for metric, value in (("simulation_x1", sum(samples) // 10), + ("main_thread_stall", max(samples))): + verdict = r["budget_verdict"][metric] + self.assertEqual(verdict["value"], value) + self.assertEqual(verdict["statistics"]["count"], 10) + self.assertIs(verdict["passed"], False) + self.assertIsNone(verdict["reason"]) + for column in ("samples_us", "lifetime_count"): + broken = copy.deepcopy(r) + entry = broken["budget_at_warmup_end"]["data"]["simulation_x1"] + entry[column] = None + entry["reason"] = "counts_unavailable" + with self.subTest(column=column), self.assertRaisesRegex( + ValueError, "budget readable absent key must use product zero"): + checker.check_report(broken) + broken = copy.deepcopy(r) + broken["budget_at_warmup_end"]["data"]["simulation_x1"]["reason"] = "counts_unavailable" + with self.assertRaisesRegex(ValueError, "budget availability reason"): + checker.check_report(broken) + + def test_c8_unreadable_dictionaries_remain_unavailable(self): + for unavailable in (("samples_us",), ("lifetime_count",), + ("samples_us", "lifetime_count")): + r = synthetic_report() + entry = r["budget_at_warmup_end"]["data"]["simulation_x1"] + for column in unavailable: + entry[column] = None + entry["provenance"][column] = {"dictionary_readable": False, "key_form": None} + entry["reason"] = "counts_unavailable" + r["budget_verdict"] = checker.budget_verdict( + r["budget_at_warmup_end"]["data"], r["budget_at_phase_end"]["data"], 1) + # An unreadable samples dictionary alone leaves the actual count usable; + # retain the existing verdict extraction rule (it uses the end samples). + checker.check_report(r) + if "lifetime_count" in unavailable: + self.assertEqual(r["budget_verdict"]["simulation_x1"]["reason"], "counts_unavailable") + self.assertIsNone(r["budget_verdict"]["simulation_x1"]["passed"]) + for column in unavailable: + broken = copy.deepcopy(r) + bad = broken["budget_at_warmup_end"]["data"]["simulation_x1"] + bad[column] = [] if column == "samples_us" else 0 + bad["provenance"][column]["key_form"] = "absent_means_zero_per_product" + with self.subTest(unavailable=unavailable, column=column), self.assertRaisesRegex( + ValueError, "budget unreadable dictionary must remain null"): + checker.check_report(broken) + def test_h3_save_stall_correlation(self): r = synthetic_report() r["save_events"] = [{"frame": 2, "game_minute": 363, "ok": True,