summarize_priority_reconfiguration.py
按训练 seed 汇总配对效应
原文件 · 216 行 · 7653 字节
scripts/analysis/summarize_priority_reconfiguration.py
SHA-256: db6cfe025628a6f833c6bf040d6e9bbbd9b39f15439faa3220b071b6e81113e0
1"""Create training-seed-aware summaries for priority-reveal diagnostics."""23from __future__ import annotations45import argparse6import csv7import json8import math9from collections import defaultdict10from pathlib import Path1112import numpy as np131415METRICS = (16 "mean_event_task_probability",17 "immediate_target_assignments",18 "immediate_policy_switches",19 "departures_from_fragile_sources",20 "executed_target_assignments",21 "executed_switches",22 "executed_non_target_switches",23 "executed_fragile_departures",24 "success",25 "task_completion_ratio",26 "priority_weighted_completion",27 "event_task_completed",28 "event_task_response_delay",29 "remaining_demand",30 "episode_length",31 "team_reward_after_event",32)333435def _read(path):36 with Path(path).open(encoding="utf-8", newline="") as stream:37 return list(csv.DictReader(stream))383940def _write(path, rows):41 path = Path(path)42 if not rows:43 path.write_text("", encoding="utf-8")44 return45 with path.open("w", encoding="utf-8", newline="") as stream:46 writer = csv.DictWriter(stream, fieldnames=list(rows[0]))47 writer.writeheader()48 writer.writerows(rows)495051def _number(row, key):52 value = row.get(key, "")53 return float(value) if value not in (None, "") else math.nan545556def _paired_rows(rows):57 index = {58 (r["model"], int(r["scenario_id"]), r["condition"]): r for r in rows59 }60 result = []61 contrasts = (62 ("priority_high_minus_low", "high", "low"),63 (64 "react_minus_keep_current",65 "high",66 "high_keep_current_at_event",67 ),68 )69 for (model, scenario, condition), high in sorted(index.items()):70 if condition != "high":71 continue72 for label, left_name, right_name in contrasts:73 left = index.get((model, scenario, left_name))74 right = index.get((model, scenario, right_name))75 if right is None:76 continue77 item = {78 "model": model,79 "model_role": left["model_role"],80 "training_condition": left["training_condition"],81 "ablation": left["ablation"],82 "train_seed": int(left["train_seed"]),83 "scenario_id": scenario,84 "contrast": label,85 "pre_event_state_hash_equal": int(86 left["pre_event_state_hash"] == right["pre_event_state_hash"]87 ),88 }89 for metric in METRICS:90 if metric.startswith("executed_"):91 source = {92 "executed_target_assignments": "immediate_target_assignments",93 "executed_switches": "immediate_policy_switches",94 "executed_non_target_switches": None,95 "executed_fragile_departures": "departures_from_fragile_sources",96 }[metric]97 if metric == "executed_non_target_switches":98 left_value = _number(99 left, "immediate_policy_switches"100 ) - _number(left, "immediate_target_assignments")101 right_value = (102 0.0103 if label == "react_minus_keep_current"104 else _number(right, "immediate_policy_switches")105 - _number(right, "immediate_target_assignments")106 )107 else:108 left_value = _number(left, source)109 right_value = (110 0.0111 if label == "react_minus_keep_current"112 else _number(right, source)113 )114 item[metric] = left_value - right_value115 else:116 item[metric] = _number(left, metric) - _number(right, metric)117 result.append(item)118 return result119120121def _aggregate_training_seeds(paired):122 by_seed = defaultdict(list)123 for row in paired:124 key = (125 row["training_condition"],126 row["model_role"],127 row["ablation"],128 row["train_seed"],129 row["contrast"],130 )131 by_seed[key].append(row)132133 seed_rows = []134 for key, items in sorted(by_seed.items()):135 condition, role, ablation, seed, contrast = key136 record = {137 "training_condition": condition,138 "model_role": role,139 "ablation": ablation,140 "train_seed": seed,141 "contrast": contrast,142 "eval_scenarios": len(items),143 "paired_state_check": int(144 all(int(x["pre_event_state_hash_equal"]) == 1 for x in items)145 ),146 }147 for metric in METRICS:148 values = np.asarray([float(x[metric]) for x in items], dtype=float)149 record[metric] = float(np.nanmean(values))150 seed_rows.append(record)151152 grouped = defaultdict(list)153 for row in seed_rows:154 key = (155 row["training_condition"],156 row["model_role"],157 row["ablation"],158 row["contrast"],159 )160 grouped[key].append(row)161162 aggregate = []163 for key, items in sorted(grouped.items()):164 condition, role, ablation, contrast = key165 record = {166 "training_condition": condition,167 "model_role": role,168 "ablation": ablation,169 "contrast": contrast,170 "training_seeds": len(items),171 "eval_scenarios_per_seed": min(int(x["eval_scenarios"]) for x in items),172 "paired_state_check": int(all(x["paired_state_check"] for x in items)),173 }174 for metric in METRICS:175 values = np.asarray([float(x[metric]) for x in items], dtype=float)176 mean = float(np.mean(values))177 sd = float(np.std(values, ddof=1)) if len(values) > 1 else math.nan178 # With three training seeds this is descriptive uncertainty. Keep179 # the individual seed rows for every paper-facing interpretation.180 half = 4.303 * sd / math.sqrt(len(values)) if len(values) == 3 else math.nan181 record[f"{metric}_mean"] = mean182 record[f"{metric}_training_seed_sd"] = sd183 record[f"{metric}_t95_low"] = mean - half184 record[f"{metric}_t95_high"] = mean + half185 aggregate.append(record)186 return seed_rows, aggregate187188189def main():190 parser = argparse.ArgumentParser()191 parser.add_argument("output_dir")192 args = parser.parse_args()193 output = Path(args.output_dir).resolve()194 rows = _read(output / "all_results.csv")195 paired = _paired_rows(rows)196 seed_rows, aggregate = _aggregate_training_seeds(paired)197 _write(output / "paired_effects_checked.csv", paired)198 _write(output / "per_training_seed_summary.csv", seed_rows)199 _write(output / "across_training_seed_summary.csv", aggregate)200 audit = {201 "raw_rows": len(rows),202 "paired_rows": len(paired),203 "models": len({r["model"] for r in rows}),204 "all_within_model_pairs_share_pre_event_state": all(205 int(r["pre_event_state_hash_equal"]) == 1 for r in paired206 ),207 "uncertainty_unit": "independent training seed",208 "eval_scenarios_are_paired_repeated_measurements": True,209 }210 (output / "analysis_audit.json").write_text(211 json.dumps(audit, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"212 )213214215if __name__ == "__main__":216 main()