run_staged_validation_queue.py
分阶段冻结 checkpoint 评估队列
原文件 · 742 行 · 30292 字节
scripts/experiments/run_staged_validation_queue.py
SHA-256: 6f1e797915e3c0532b70ab3bf12a3645fee99bc4eb9bef905beef7530ddab874
1"""Backfill and follow periodic validation for an active formal campaign.23This process is independent from training. It watches the atomic sweep4status/checkpoint files, evaluates every 50-episode actor on the selected GPU, locks a5validation-best checkpoint after a job finishes, then evaluates6validation-best and last on a disjoint 128-seed test suite.7"""89import argparse10import json11import os12import re13import subprocess14import sys15import threading16import time17from concurrent.futures import ThreadPoolExecutor18from datetime import datetime19from pathlib import Path2021import yaml222324REPO_ROOT = Path(__file__).resolve().parents[2]25STAGES = ("delivcoal", "capcoal-legacy", "capcoal-event")26# Above this temperature a GPU is treated as a fallback for new evaluators, so27# a shared-cooling box sheds load from the warmer card instead of piling more28# heat onto it. Well below the thermal guard's 97C SIGKILL threshold, leaving29# margin for the guard to still be the final safety net.30HOT_GPU_C = 88.0313233def _now():34 return datetime.now().astimezone().isoformat(timespec="seconds")353637def _load_json(path):38 try:39 return json.loads(Path(path).read_text(encoding="utf-8"))40 except (OSError, ValueError, json.JSONDecodeError):41 return None424344def _atomic_json(path, payload):45 path = Path(path)46 path.parent.mkdir(parents=True, exist_ok=True)47 temporary = path.with_name(f"{path.name}.{os.getpid()}.tmp")48 temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")49 temporary.replace(path)505152def _atomic_yaml(path, payload):53 path = Path(path)54 path.parent.mkdir(parents=True, exist_ok=True)55 temporary = path.with_name(f"{path.name}.{os.getpid()}.tmp")56 temporary.write_text(57 yaml.safe_dump(payload, sort_keys=False, allow_unicode=True),58 encoding="utf-8",59 )60 temporary.replace(path)616263class ValidationQueue:64 def __init__(65 self,66 campaign_dir,67 max_workers,68 cuda=0,69 output_name="staged_evaluation",70 stages=None,71 recovery_stages=None,72 job_pattern=None,73 eval_memory_mb=1800,74 reserve_memory_mb=4096,75 periodic_interval=50,76 periodic_eval_episodes=32,77 periodic_seed_start=10000,78 final_eval_episodes=128,79 final_seed_start=50000,80 ):81 self.campaign_dir = Path(campaign_dir).resolve()82 self.root = self.campaign_dir / output_name83 self.state_path = self.root / "status.json"84 self.lock = threading.Lock()85 self.gpu_condition = threading.Condition()86 self.max_workers = int(max_workers)87 self.cudas = [88 int(g) for g in (cuda if isinstance(cuda, (list, tuple)) else [cuda])89 ]90 self.cuda = self.cudas[0] # kept for single-GPU callers / state default91 self.eval_memory_mb = int(eval_memory_mb)92 self.reserve_memory_mb = int(reserve_memory_mb)93 self.periodic_interval = int(periodic_interval)94 self.periodic_eval_episodes = int(periodic_eval_episodes)95 self.periodic_seed_start = int(periodic_seed_start)96 self.final_eval_episodes = int(final_eval_episodes)97 self.final_seed_start = int(final_seed_start)98 self.active_tasks_per_gpu = {gpu: 0 for gpu in self.cudas}99 self.job_pattern = re.compile(job_pattern) if job_pattern else None100 self.stage_dirs = {101 name: Path(path).resolve()102 for name, path in (stages or {103 name: self.campaign_dir / name for name in STAGES104 }).items()105 }106 self.recovery_stage_dirs = {107 name: [Path(path).resolve() for path in paths]108 for name, paths in (recovery_stages or {}).items()109 }110 self.executor = ThreadPoolExecutor(max_workers=self.max_workers)111 self.futures = {}112 self.state = _load_json(self.state_path) or {113 "name": f"{self.campaign_dir.name}-evaluation",114 "created_at": _now(),115 "state": "running",116 "cuda": self.cudas,117 "max_workers": self.max_workers,118 "periodic_interval": self.periodic_interval,119 "periodic_eval_episodes": self.periodic_eval_episodes,120 "periodic_seed_start": self.periodic_seed_start,121 "final_eval_episodes": self.final_eval_episodes,122 "final_seed_start": self.final_seed_start,123 "tasks": {},124 "jobs": {},125 }126 self.state["cuda"] = self.cudas127 if self.state.get("name") == "validation-best-heldout-v1":128 self.state["name"] = f"{self.campaign_dir.name}-evaluation"129 self.state["max_workers"] = self.max_workers130 self.state["eval_memory_mb"] = self.eval_memory_mb131 self.state["reserve_memory_mb"] = self.reserve_memory_mb132 self.state["periodic_interval"] = self.periodic_interval133 self.state["periodic_eval_episodes"] = self.periodic_eval_episodes134 self.state["periodic_seed_start"] = self.periodic_seed_start135 self.state["final_eval_episodes"] = self.final_eval_episodes136 self.state["final_seed_start"] = self.final_seed_start137 # A stopped/crashed queue leaves submitted tasks in pending/running.138 # Requeue them on restart; completed summaries remain immutable.139 self.recovery_tasks = [140 dict(task)141 for task in self.state["tasks"].values()142 if task.get("state") in ("pending", "running", "failed")143 ]144 for task in self.recovery_tasks:145 self.state["tasks"].pop(task["task_id"], None)146 self._write_state()147148 def _gpu_memory(self, gpu=None):149 if gpu is None:150 gpu = self.cuda151 try:152 result = subprocess.run(153 [154 "nvidia-smi",155 f"--id={gpu}",156 "--query-gpu=memory.used,memory.total",157 "--format=csv,noheader,nounits",158 ],159 capture_output=True,160 text=True,161 timeout=10,162 )163 if result.returncode == 0:164 used, total = result.stdout.strip().splitlines()[0].split(",")165 return int(used.strip()), int(total.strip())166 except (OSError, ValueError, subprocess.SubprocessError):167 pass168 return None169170 def _gpu_temperature(self, gpu):171 try:172 result = subprocess.run(173 [174 "nvidia-smi",175 f"--id={gpu}",176 "--query-gpu=temperature.gpu",177 "--format=csv,noheader,nounits",178 ],179 capture_output=True,180 text=True,181 timeout=10,182 )183 if result.returncode == 0:184 return int(result.stdout.strip().splitlines()[0])185 except (OSError, ValueError, subprocess.SubprocessError):186 pass187 return None188189 def _pick_gpu(self):190 """Choose the best GPU for one more evaluator.191192 Prefers the coolest GPU with room for one more worker, so a shared-193 cooling box sheds load from its warmer card. Hot GPUs (``>= HOT_GPU_C``)194 are a last resort, used only when every cool GPU is memory-full. Within195 the cool tier the actual temperature -- not just load -- orders the196 cards, which matters at restart: a card that briefly cooled below the197 threshold during the idle window still loses to a genuinely cooler card198 instead of taking a whole wave of workers. ``memory.used`` already199 contains running evaluators, so a GPU that has just admitted a worker200 still shows near-zero ``used`` until the worker allocates; scoring with201 ``external_used + active * estimate`` makes the freshly reserved worker202 count forward, and ``external_used`` keeps room for unrelated training203 processes that may share a GPU.204 """205206 if len(self.cudas) == 1:207 return self.cudas[0]208 candidates = []209 for gpu in self.cudas:210 memory = self._gpu_memory(gpu)211 if memory is None:212 continue213 used, total = memory214 active = self.active_tasks_per_gpu.get(gpu, 0)215 external_used = max(0, used - active * self.eval_memory_mb)216 projected = external_used + (active + 1) * self.eval_memory_mb217 if projected > total - self.reserve_memory_mb:218 continue219 score = external_used + active * self.eval_memory_mb220 temp = self._gpu_temperature(gpu)221 cool = 0 if (temp is not None and temp >= HOT_GPU_C) else 1222 # An unreadable temperature sorts as hottest so we never pile onto a223 # card whose thermal state we cannot see.224 sort_temp = temp if temp is not None else 10**9225 candidates.append((cool, sort_temp, score, gpu))226 if not candidates:227 return None228 # Cool GPUs first, then coolest, then least projected load.229 candidates.sort(key=lambda item: (-item[0], item[1], item[2]))230 return candidates[0][3]231232 def _acquire_gpu_memory(self, task_id):233 """Admit one evaluator onto a GPU, returning the chosen GPU index.234235 Blocks until some GPU has room; dynamically re-picks on each attempt so236 a full or temporarily unavailable GPU does not stall the queue.237 """238239 while True:240 with self.gpu_condition:241 gpu = self._pick_gpu()242 if gpu is not None:243 self.active_tasks_per_gpu[gpu] = (244 self.active_tasks_per_gpu.get(gpu, 0) + 1245 )246 return gpu247 with self.lock:248 task = self.state["tasks"].get(task_id)249 if task is not None:250 task["state"] = "waiting_for_gpu_memory"251 task["heartbeat_at"] = _now()252 self._write_state()253 self.gpu_condition.wait(timeout=5.0)254255 def _release_gpu_memory(self, gpu):256 with self.gpu_condition:257 self.active_tasks_per_gpu[gpu] = max(258 0, self.active_tasks_per_gpu.get(gpu, 0) - 1259 )260 self.gpu_condition.notify_all()261262 def _write_state(self):263 self.state["updated_at"] = _now()264 tasks = list(self.state["tasks"].values())265 self.state["counts"] = {266 key: sum(task.get("state") == key for task in tasks)267 for key in ("pending", "running", "completed", "failed")268 }269 phases = {}270 active_tasks = []271 progress_heartbeats = []272 for phase_name, episodes_per_task in (273 ("periodic_validation", self.periodic_eval_episodes),274 ("final_test", self.final_eval_episodes),275 ):276 phase_tasks = [277 task for task in tasks if task.get("phase") == phase_name278 ]279 submitted_units = sum(280 int(task.get("eval_episodes", episodes_per_task))281 for task in phase_tasks282 )283 completed_units = 0284 failed_units = 0285 for task in phase_tasks:286 task_units = int(task.get("eval_episodes", episodes_per_task))287 if task.get("state") == "completed":288 completed_units += task_units289 continue290 if task.get("state") == "failed":291 failed_units += task_units292 continue293 if task.get("state") != "running":294 continue295 progress = _load_json(self._task_dir(task) / "progress.json") or {}296 progress_units = min(297 task_units,298 max(0, int(progress.get("completed_episodes", 0) or 0)),299 )300 completed_units += progress_units301 heartbeat = progress.get("updated_at")302 if heartbeat:303 progress_heartbeats.append(heartbeat)304 active_tasks.append(305 {306 "job_name": task["job_name"],307 "checkpoint": task["selector"],308 "completed_episodes": progress_units,309 "total_episodes": task_units,310 }311 )312313 if phase_name == "periodic_validation":314 planned_units = sum(315 int(job.get("planned_periodic_checkpoints", 0))316 * episodes_per_task317 for job in self.state["jobs"].values()318 )319 else:320 planned_units = len(self.state["jobs"]) * 2 * episodes_per_task321 phase_state = "pending"322 if failed_units:323 phase_state = "partial_failure"324 elif planned_units and completed_units >= planned_units:325 phase_state = "completed"326 elif submitted_units:327 phase_state = "running"328 phases[phase_name] = {329 "state": phase_state,330 "planned_units": planned_units,331 "submitted_units": submitted_units,332 "completed_units": completed_units,333 "failed_units": failed_units,334 }335336 planned_units = sum(phase["planned_units"] for phase in phases.values())337 submitted_units = sum(phase["submitted_units"] for phase in phases.values())338 completed_units = sum(phase["completed_units"] for phase in phases.values())339 failed_units = sum(phase["failed_units"] for phase in phases.values())340 active_phase = next(341 (342 name343 for name in ("periodic_validation", "final_test")344 if phases[name]["state"] in ("running", "partial_failure")345 ),346 None,347 )348 eval_state = (349 "partial_failure"350 if failed_units351 else "completed"352 if planned_units and completed_units >= planned_units353 else "running"354 )355 self.state["eval"] = {356 "state": eval_state,357 "active_phase": active_phase,358 "heartbeat_at": max(progress_heartbeats, default=None),359 "planned_tasks": planned_units,360 "total_tasks": planned_units,361 "submitted_tasks": submitted_units,362 "completed_tasks": completed_units,363 "failed_tasks": failed_units,364 "active_tasks": active_tasks,365 "phases": phases,366 }367 _atomic_json(self.state_path, self.state)368369 def _task_dir(self, task):370 suffix = (371 f"ep{task['episode']:06d}"372 if task["phase"] == "periodic_validation"373 else task["selector"]374 )375 return self.root / task["phase"] / task["stage"] / task["job_name"] / suffix376377 def _prepare_task(self, task, gpu):378 task_dir = self._task_dir(task)379 config = yaml.safe_load(Path(task["config_path"]).read_text(encoding="utf-8")) or {}380 config.update(381 cuda=gpu,382 use_eval=True,383 model_dir=None,384 n_eval_rollout_threads=1,385 eval_episodes=int(task["eval_episodes"]),386 eval_seed=int(task["eval_seed"]),387 eval_reset_mode="full_random",388 eval_deterministic=True,389 eval_progress_file=None,390 )391 config_path = task_dir / "config.yaml"392 manifest_path = task_dir / "manifest.yaml"393 _atomic_yaml(config_path, config)394 _atomic_yaml(395 manifest_path,396 {397 "name": task["task_id"].replace(":", "-"),398 "output_root": str(task_dir / "results"),399 "models": [400 {401 "name": task["selector"],402 "checkpoint": task["checkpoint"],403 "overrides": {},404 }405 ],406 },407 )408 return task_dir, config_path, manifest_path409410 def _run_task(self, task_id):411 gpu = self._acquire_gpu_memory(task_id)412 with self.lock:413 task = dict(self.state["tasks"][task_id])414 self.state["tasks"][task_id].update(415 state="running", started_at=_now(), cuda=gpu416 )417 self._write_state()418 task_dir, config_path, manifest_path = self._prepare_task(task, gpu)419 progress_path = task_dir / "progress.json"420 command = [421 sys.executable,422 str(REPO_ROOT / "scripts" / "gpu_thermal_guard.py"),423 "--gpu", str(gpu),424 # 97C: above the ~93C point where the GPU throttles (safe, just slow),425 # but below the ~100C emergency shutdown. The guard SIGKILLs (not426 # SIGSTOPs) on violation, so a trigger is a clean terminate+retry,427 # never the stuck-kernel freeze that deadlocked the queue.428 "--pause-at", "97",429 "--resume-at", "90",430 "--poll-seconds", "2",431 "--safe-readings", "3",432 "--query-failures", "2",433 "--status-file", str(task_dir / "thermal_status.json"),434 "--",435 sys.executable,436 str(REPO_ROOT / "train" / "eval.py"),437 "--config_file", str(config_path),438 "--models_file", str(manifest_path),439 "--output_root", str(task_dir / "results"),440 "--progress_file", str(progress_path),441 ]442 try:443 result = subprocess.run(444 command,445 cwd=REPO_ROOT,446 stdin=subprocess.DEVNULL,447 capture_output=True,448 text=True,449 env={**os.environ, "PYTHONUNBUFFERED": "1"},450 )451 finally:452 self._release_gpu_memory(gpu)453 (task_dir / "eval.log").write_text(454 result.stdout + ("\n[stderr]\n" + result.stderr if result.stderr else ""),455 encoding="utf-8",456 )457 summaries = sorted((task_dir / "results").rglob("summary.json"))458 record = None459 if result.returncode == 0 and summaries:460 payload = _load_json(summaries[-1]) or {}461 records = payload.get("models") or []462 if len(records) == 1:463 record = records[0]464 _atomic_json(task_dir / "summary.json", record)465 with self.lock:466 target = self.state["tasks"][task_id]467 target["return_code"] = result.returncode468 target["finished_at"] = _now()469 target["state"] = "completed" if record is not None else "failed"470 if record is not None:471 target["summary"] = {472 key: record.get(key)473 for key in (474 "success", "task_completion_ratio", "sent_percentage",475 "episode_length", "successful_episode_length", "episode_reward",476 )477 if key in record478 }479 self._write_state()480 return record is not None481482 def _submit(self, task):483 task_id = task["task_id"]484 if task_id in self.state["tasks"]:485 return486 task["state"] = "pending"487 task["submitted_at"] = _now()488 with self.lock:489 self.state["tasks"][task_id] = task490 self._write_state()491 future = self.executor.submit(self._run_task, task_id)492 self.futures[task_id] = future493494 @staticmethod495 def _best(periodic):496 def rank(item):497 summary = item.get("summary", {})498 length = summary.get("episode_length")499 return (500 float(summary.get("success", 0.0) or 0.0),501 float(summary.get("task_completion_ratio", 0.0) or 0.0),502 -float(length if length is not None else 1e30),503 -int(item["episode"]),504 )505 return max(periodic, key=rank)506507 def _stage_jobs(self, stage):508 group_dir = self.stage_dirs[stage]509 group_status = _load_json(group_dir / "status.json")510 if not group_status:511 return None, {}512 jobs = dict(group_status.get("jobs", {}))513 for recovery_dir in self.recovery_stage_dirs.get(stage, []):514 recovery = _load_json(recovery_dir / "status.json") or {}515 for job_name, job in recovery.get("jobs", {}).items():516 if job.get("state") == "completed":517 jobs[job_name] = job518 return group_status, jobs519520 def _scan_stage(self, stage):521 group_status, jobs = self._stage_jobs(stage)522 if not group_status:523 return524 for job_name, job in jobs.items():525 if self.job_pattern and not self.job_pattern.search(job_name):526 continue527 config_path = Path(job["config"])528 config = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}529 max_episode = int(config["max_train_episodes"])530 models = Path(job["run_dir"]) / "models"531 episode_checkpoints = {}532 for checkpoint in models.glob("actor_ep*.pt"):533 match = re.fullmatch(r"actor_ep(\d+)\.pt", checkpoint.name)534 if match:535 episode_checkpoints[int(match.group(1))] = checkpoint536 job_key = f"{stage}:{job_name}"537 self.state["jobs"].setdefault(538 job_key,539 {540 "stage": stage,541 "job_name": job_name,542 "planned_periodic_checkpoints": max_episode // self.periodic_interval,543 "state": "training",544 },545 )546 for scheduled_episode in range(547 self.periodic_interval,548 max_episode + 1,549 self.periodic_interval,550 ):551 # A PPO rollout can contain more than one completed environment552 # episode. The runner freezes the updated actor after the553 # rollout, so the first checkpoint at/after a 50-episode554 # threshold is the checkpoint generated by that trigger.555 if (556 scheduled_episode == max_episode557 and job.get("state") == "completed"558 and (models / "actor.pt").is_file()559 ):560 episode = max_episode561 checkpoint = models / "actor.pt"562 else:563 candidates = [564 episode for episode in episode_checkpoints565 if (566 scheduled_episode567 <= episode568 < scheduled_episode + self.periodic_interval569 )570 ]571 if not candidates:572 continue573 episode = min(candidates)574 checkpoint = episode_checkpoints[episode]575 if not checkpoint.is_file():576 continue577 task_id = (578 f"{job_key}:periodic:slot{scheduled_episode:06d}:"579 f"ep{episode:06d}"580 )581 self._submit(582 {583 "task_id": task_id,584 "phase": "periodic_validation",585 "stage": stage,586 "job_name": job_name,587 "episode": episode,588 "scheduled_episode": scheduled_episode,589 "selector": f"ep{episode:06d}",590 "checkpoint": str(checkpoint),591 "config_path": str(config_path),592 "eval_episodes": self.periodic_eval_episodes,593 "eval_seed": self.periodic_seed_start,594 }595 )596597 if job.get("state") != "completed":598 continue599 periodic = [600 task for task in self.state["tasks"].values()601 if task["stage"] == stage602 and task["job_name"] == job_name603 and task["phase"] == "periodic_validation"604 and task["state"] == "completed"605 ]606 if len(periodic) != max_episode // self.periodic_interval:607 continue608 best = self._best(periodic)609 job_state = self.state["jobs"][job_key]610 job_state.update(611 state="validation_locked",612 validation_best_episode=best["episode"],613 validation_best_checkpoint=best["checkpoint"],614 validation_best_summary=best.get("summary", {}),615 )616 selectors = (617 ("validation-best", best["episode"], Path(best["checkpoint"])),618 ("last", max_episode, models / "actor.pt"),619 )620 for selector, episode, checkpoint in selectors:621 if not checkpoint.is_file():622 continue623 task_id = f"{job_key}:final:{selector}"624 self._submit(625 {626 "task_id": task_id,627 "phase": "final_test",628 "stage": stage,629 "job_name": job_name,630 "episode": episode,631 "selector": selector,632 "checkpoint": str(checkpoint),633 "config_path": str(config_path),634 "eval_episodes": self.final_eval_episodes,635 "eval_seed": self.final_seed_start,636 }637 )638639 def run(self):640 try:641 for task in self.recovery_tasks:642 task.pop("return_code", None)643 task.pop("finished_at", None)644 task.pop("summary", None)645 self._submit(task)646 while True:647 for stage in self.stage_dirs:648 self._scan_stage(stage)649 with self.lock:650 self._write_state()651 active = any(not future.done() for future in self.futures.values())652 selected_jobs = []653 for stage in self.stage_dirs:654 _, jobs = self._stage_jobs(stage)655 selected_jobs.extend(656 (stage, name, job)657 for name, job in jobs.items()658 if not self.job_pattern or self.job_pattern.search(name)659 )660 expected_final_tasks = len(selected_jobs) * 2661 final_tasks = [task for task in self.state["tasks"].values()662 if task["phase"] == "final_test"]663 if (664 selected_jobs665 and all(job.get("state") == "completed"666 for _, _, job in selected_jobs)667 and not active668 and len(final_tasks) == expected_final_tasks669 and all(task["state"] == "completed" for task in final_tasks)670 ):671 break672 time.sleep(10)673 with self.lock:674 self.state["state"] = "completed"675 self.state["finished_at"] = _now()676 self._write_state()677 return 0678 finally:679 self.executor.shutdown(wait=True)680681682def main(argv=None):683 parser = argparse.ArgumentParser()684 parser.add_argument("--campaign-dir", required=True)685 parser.add_argument("--max-workers", type=int, default=8)686 parser.add_argument(687 "--cuda", type=int, nargs="+", default=[0],688 help="one or more GPU ids; tasks are placed on the least-loaded GPU",689 )690 parser.add_argument("--output-name", default="staged_evaluation")691 parser.add_argument(692 "--stage", action="append", default=[], metavar="NAME=PATH",693 help="evaluate a named sweep directory; repeat for multiple stages",694 )695 parser.add_argument(696 "--recovery-stage", action="append", default=[], metavar="NAME=PATH",697 help="overlay completed jobs from a recovery sweep onto NAME",698 )699 parser.add_argument("--job-pattern")700 parser.add_argument("--eval-memory-mb", type=int, default=1800)701 parser.add_argument("--reserve-memory-mb", type=int, default=4096)702 parser.add_argument("--periodic-interval", type=int, default=50)703 parser.add_argument("--periodic-eval-episodes", type=int, default=32)704 parser.add_argument("--periodic-seed-start", type=int, default=10000)705 parser.add_argument("--final-eval-episodes", type=int, default=128)706 parser.add_argument("--final-seed-start", type=int, default=50000)707 args = parser.parse_args(argv)708709 def parse_stage(values, multiple=False):710 parsed = {} if not multiple else {}711 for value in values:712 if "=" not in value:713 parser.error(f"expected NAME=PATH, got {value!r}")714 name, path = value.split("=", 1)715 if multiple:716 parsed.setdefault(name, []).append(path)717 else:718 parsed[name] = path719 return parsed720721 stages = parse_stage(args.stage) if args.stage else None722 recovery_stages = parse_stage(args.recovery_stage, multiple=True)723 return ValidationQueue(724 args.campaign_dir,725 args.max_workers,726 cuda=args.cuda,727 output_name=args.output_name,728 stages=stages,729 recovery_stages=recovery_stages,730 job_pattern=args.job_pattern,731 eval_memory_mb=args.eval_memory_mb,732 reserve_memory_mb=args.reserve_memory_mb,733 periodic_interval=args.periodic_interval,734 periodic_eval_episodes=args.periodic_eval_episodes,735 periodic_seed_start=args.periodic_seed_start,736 final_eval_episodes=args.final_eval_episodes,737 final_seed_start=args.final_seed_start,738 ).run()739740741if __name__ == "__main__":742 raise SystemExit(main())