config.py
参数定义与默认值；冻结有效配置字段见 evidence
原文件 · 809 行 · 28374 字节
config.py
SHA-256: bb66031ca941968f0bacc76c319ccbe945b193b6b7141e8537fe9222e8912f75
1import argparse2import yaml34def get_config(args):5 path = getattr(args, 'config_file', None)6 if path:7 with open(path, 'r', encoding='utf-8') as f:8 cfg = yaml.safe_load(f) or {}910 for k, v in cfg.items():11 setattr(args, k, v)1213def get_args():14 """15 The configuration parser for common hyperparameters of all environment.16 Please reach each `scripts/train/<env>_runner.py` file to find private hyperparameters17 only used in <env>.1819 Prepare parameters:20 --algorithm_name <algorithm_name>21 specifiy the algorithm, including `["rmappo", "mappo", "rmappg", "mappg", "trpo"]`22 --experiment_name <str>23 an identifier to distinguish different experiment.24 --seed <int>25 set seed for numpy and torch26 --cuda27 by default True, will use GPU to train; or else will use CPU;28 --cuda_deterministic29 by default, make sure random seed effective. if set, bypass such function.30 --n_training_threads <int>31 number of training threads working in parallel. by default 132 --n_rollout_threads <int>33 number of parallel envs for training rollout. by default 3234 --n_eval_rollout_threads <int>35 number of parallel envs for evaluating rollout. by default 136 --n_render_rollout_threads <int>37 number of parallel envs for rendering, could only be set as 1 for some environments.38 --num_env_steps <int>39 number of env steps to train (default: 10e6)40 --user_name <str>41 [for wandb usage], to specify user's name for simply collecting training data.42 --use_wandb43 [for wandb usage], by default True, will log date to wandb server. or else will use tensorboard to log data.4445 Env parameters:46 --env_name <str>47 specify the name of environment48 --use_obs_instead_of_state49 [only for some env] by default False, will use global state; or else will use concatenated local obs.5051 Replay Buffer parameters:52 --episode_length <int>53 the max length of episode in the buffer.5455 Network parameters:56 --share_policy57 by default True, all agents will share the same network; set to make training agents use different policies.58 --use_centralized_V59 by default True, use centralized training mode; or else will decentralized training mode.60 --stacked_frames <int>61 Number of input frames which should be stack together.62 --hidden_size <int>63 Dimension of hidden layers for actor/critic networks64 --layer_N <int>65 Number of layers for actor/critic networks66 --use_ReLU67 by default True, will use ReLU. or else will use Tanh.68 --use_popart69 by default True, use PopArt to normalize rewards.70 --use_valuenorm71 by default True, use running mean and std to normalize rewards.72 --use_feature_normalization73 by default True, apply layernorm to normalize inputs.74 --use_orthogonal75 by default True, use Orthogonal initialization for weights and 0 initialization for biases. or else, will use xavier uniform inilialization.76 --gain77 by default 0.01, use the gain # of last action layer78 --use_naive_recurrent_policy79 by default False, use the whole trajectory to calculate hidden states.80 --use_recurrent_policy81 by default, use Recurrent Policy. If set, do not use.82 --recurrent_N <int>83 The number of recurrent layers ( default 1).84 --data_chunk_length <int>85 Time length of chunks used to train a recurrent_policy, default 10.8687 Optimizer parameters:88 --lr <float>89 learning rate parameter, (default: 5e-4, fixed).90 --critic_lr <float>91 learning rate of critic (default: 5e-4, fixed)92 --opti_eps <float>93 RMSprop optimizer epsilon (default: 1e-5)94 --weight_decay <float>95 coefficience of weight decay (default: 0)9697 PPO parameters:98 --ppo_epoch <int>99 number of ppo epochs (default: 15)100 --use_clipped_value_loss101 by default, clip loss value. If set, do not clip loss value.102 --clip_param <float>103 ppo clip parameter (default: 0.2)104 --num_mini_batch <int>105 number of batches for ppo (default: 1)106 --entropy_coef <float>107 entropy term coefficient (default: 0.01)108 --use_max_grad_norm109 by default, use max norm of gradients. If set, do not use.110 --max_grad_norm <float>111 max norm of gradients (default: 0.5)112 --use_gae113 by default, use generalized advantage estimation. If set, do not use gae.114 --gamma <float>115 discount factor for rewards (default: 0.99)116 --gae_lambda <float>117 gae lambda parameter (default: 0.95)118 --use_proper_time_limits119 by default, the return value does consider limits of time. If set, compute returns with considering time limits factor.120 --use_huber_loss121 by default, use huber loss. If set, do not use huber loss.122 --use_value_active_masks123 by default True, whether to mask useless data in value loss.124 --huber_delta <float>125 coefficient of huber loss.126127 PPG parameters:128 --aux_epoch <int>129 number of auxiliary epochs. (default: 4)130 --clone_coef <float>131 clone term coefficient (default: 0.01)132133 Run parameters：134 --use_linear_lr_decay135 by default, do not apply linear decay to learning rate. If set, use a linear schedule on the learning rate136137 Save & Log parameters:138 --save_interval <int>139 time duration between contiunous twice models saving.140 --log_interval <int>141 time duration between contiunous twice log printing.142143 Eval parameters:144 --use_eval145 by default, do not start evaluation. If set`, start evaluation alongside with training.146 --eval_interval <int>147 time duration between contiunous twice evaluation progress.148 --eval_episodes <int>149 number of episodes of a single evaluation.150151 Render parameters:152 --save_gifs153 by default, do not save render video. If set, save video.154 --use_render155 by default, do not render the env during training. If set, start render. Note: something, the environment has internal render process which is not controlled by this hyperparam.156 --render_episodes <int>157 the number of episodes to render a given env158 --ifi <float>159 the play interval of each rendered image in saved video.160161 Pretrained parameters:162 --model_dir <str>163 by default None. set the path to pretrained model.164 """165 parser = argparse.ArgumentParser(166 description="onpolicy", formatter_class=argparse.RawDescriptionHelpFormatter167 )168169 # prepare parameters170 parser.add_argument("--algorithm_name", type=str, default="mappo")171 parser.add_argument(172 "--trainer_algorithm_name",173 type=str,174 default=None,175 choices=["rmappo", "mappo", "qmix", "vdn", "hatt_qmix"],176 help="internal trainer implementation; algorithm_name is used as experiment label",177 )178179 parser.add_argument(180 "--experiment_name",181 type=str,182 default="check",183 help="an identifier to distinguish different experiment.",184 )185 parser.add_argument("--seed", type=int, default=1, help="Random seed for numpy/torch")186 parser.add_argument(187 "--cuda",188 type=int,189 default=0,190 help="by default True, will use GPU to train; or else will use CPU;",191 )192 parser.add_argument(193 "--cuda_deterministic",194 action="store_false",195 default=True,196 help="by default, make sure random seed effective. if set, bypass such function.",197 )198 parser.add_argument(199 "--n_training_threads",200 type=int,201 default=2,202 help="Number of torch threads for training",203 )204 parser.add_argument(205 "--n_rollout_threads",206 type=int,207 default=5,208 help="Number of parallel envs for training rollouts",209 )210 parser.add_argument(211 "--n_eval_rollout_threads",212 type=int,213 default=2,214 help="Number of parallel envs for evaluating rollouts",215 )216 parser.add_argument(217 "--n_render_rollout_threads",218 type=int,219 default=1,220 help="Number of parallel envs for rendering rollouts",221 )222 parser.add_argument(223 "--num_env_steps",224 type=int,225 default=10e6,226 help="Number of environment steps to train (default: 10e6)",227 )228 parser.add_argument(229 "--user_name",230 type=str,231 default="marl",232 help="[for wandb usage], to specify user's name for simply collecting training data.",233 )234235 # env parameters236 parser.add_argument("--env_name", type=str, default="MyEnv", help="specify the name of environment")237 parser.add_argument(238 "--use_obs_instead_of_state",239 action="store_true",240 default=False,241 help="Whether to use global state or concatenated obs",242 )243244 # replay buffer parameters245 parser.add_argument("--episode_length", type=int, default=200, help="Max length for any episode")246247 parser.add_argument(248 "--explore_action_mode",249 type=str,250 default="nearest",251 choices=["nearest", "sector", "survey"],252 help="exploration action semantics: nearest target oracle, sector directions, or explicit survey tasks",253 )254 parser.add_argument(255 "--num_explore_actions",256 type=int,257 default=1,258 help="number of directional exploration actions when explore_action_mode=sector",259 )260 parser.add_argument(261 "--explore_step_size",262 type=float,263 default=None,264 help="sector exploration waypoint distance; defaults to 25%% of map_size",265 )266 parser.add_argument(267 "--explore_random_jitter",268 type=float,269 default=0.0,270 help="fraction of sector width used as random angular jitter for sector exploration",271 )272 parser.add_argument(273 "--sector_redecision_on_waypoint",274 action=argparse.BooleanOptionalAction,275 default=True,276 help="reactivate a sector explorer after reaching its current waypoint",277 )278 parser.add_argument(279 "--explore_event_redecision",280 action=argparse.BooleanOptionalAction,281 default=True,282 help="reactivate all living explorers when a new delivery task is discovered",283 )284 parser.add_argument(285 "--sector_min_leg_mobility_steps",286 type=float,287 default=1.0,288 help="mask sector actions whose nominal in-map leg is shorter than this many mobility steps",289 )290 parser.add_argument(291 "--delivery_tasks_initially_discovered",292 action=argparse.BooleanOptionalAction,293 default=False,294 help="make every delivery task visible at reset without changing task/action dimensions",295 )296 parser.add_argument(297 "--coalition_task_feature_mode",298 choices=("compact", "full"),299 default="compact",300 help=(301 "task feature schema for coalition delivery: compact preserves the "302 "historical 6-D observation; full exposes all capability requirements"303 ),304 )305 parser.add_argument(306 "--mask_explore_actions",307 action=argparse.BooleanOptionalAction,308 default=False,309 help="mask all Explore action slots while preserving the checkpoint action dimension",310 )311 parser.add_argument(312 "--idle_time_cost_mode",313 choices=("legacy", "uniform"),314 default="legacy",315 help=(316 "legacy keeps the shaped reward's action-type cost; uniform "317 "charges the same per-step time cost to Idle and non-Idle agents"318 ),319 )320 parser.add_argument(321 "--explore_observer_relative_geometry",322 action=argparse.BooleanOptionalAction,323 default=False,324 help=(325 "describe sector candidates with each observer's executable goal "326 "and leg length, without enabling searched-area coverage memory"327 ),328 )329 parser.add_argument(330 "--explore_coverage_features",331 action=argparse.BooleanOptionalAction,332 default=False,333 help=(334 "add searched-area coverage to sector candidates; this also "335 "enables their observer-relative executable geometry"336 ),337 )338 parser.add_argument(339 "--explore_coverage_grid_resolution",340 type=int,341 default=20,342 help="cells per map axis used by optional searched-area coverage features",343 )344 # network parameters345 parser.add_argument(346 "--share_policy",347 action="store_false",348 default=False,349 help="Whether agent share the same policy",350 )351 parser.add_argument(352 "--use_centralized_V",353 action="store_false",354 default=True,355 help="Whether to use centralized V function",356 )357 parser.add_argument(358 "--stacked_frames",359 type=int,360 default=1,361 help="Dimension of hidden layers for actor/critic networks",362 )363 parser.add_argument(364 "--use_stacked_frames",365 action="store_true",366 default=False,367 help="Whether to use stacked_frames",368 )369 parser.add_argument(370 "--hidden_size",371 type=int,372 default=64,373 help="Dimension of hidden layers for actor/critic networks",374 )375 parser.add_argument(376 "--layer_N",377 type=int,378 default=1,379 help="Number of layers for actor/critic networks",380 )381 parser.add_argument("--use_ReLU", action="store_false", default=True, help="Whether to use ReLU")382 parser.add_argument(383 "--use_popart",384 action="store_true",385 default=False,386 help="by default False, use PopArt to normalize rewards.",387 )388 parser.add_argument(389 "--use_valuenorm",390 action="store_false",391 default=True,392 help="by default True, use running mean and std to normalize rewards.",393 )394 parser.add_argument(395 "--use_feature_normalization",396 action="store_false",397 default=True,398 help="Whether to apply layernorm to the inputs",399 )400 parser.add_argument(401 "--use_orthogonal",402 action="store_false",403 default=True,404 help="Whether to use Orthogonal initialization for weights and 0 initialization for biases",405 )406 parser.add_argument("--gain", type=float, default=0.01, help="The gain # of last action layer")407408 # recurrent parameters409 parser.add_argument(410 "--use_naive_recurrent_policy",411 action="store_true",412 default=False,413 help="Whether to use a naive recurrent policy",414 )415 parser.add_argument(416 "--use_recurrent_policy",417 action="store_false",418 default=False,419 help="use a recurrent policy",420 )421 parser.add_argument("--recurrent_N", type=int, default=1, help="The number of recurrent layers.")422 parser.add_argument(423 "--data_chunk_length",424 type=int,425 default=10,426 help="Time length of chunks used to train a recurrent_policy",427 )428429 # optimizer parameters430 parser.add_argument("--lr", type=float, default=5e-4, help="learning rate (default: 5e-4)")431 parser.add_argument(432 "--critic_lr",433 type=float,434 default=5e-4,435 help="critic learning rate (default: 5e-4)",436 )437 parser.add_argument(438 "--opti_eps",439 type=float,440 default=1e-5,441 help="RMSprop optimizer epsilon (default: 1e-5)",442 )443 parser.add_argument("--weight_decay", type=float, default=0)444445 # ppo parameters446 parser.add_argument("--ppo_epoch", type=int, default=15, help="number of ppo epochs (default: 15)")447 parser.add_argument(448 "--use_clipped_value_loss",449 action="store_false",450 default=True,451 help="by default, clip loss value. If set, do not clip loss value.",452 )453 parser.add_argument(454 "--clip_param",455 type=float,456 default=0.2,457 help="ppo clip parameter (default: 0.2)",458 )459 parser.add_argument(460 "--ppo_target_kl",461 type=float,462 default=0.0,463 help="stop remaining PPO minibatch updates when approx KL exceeds this value; <=0 disables",464 )465 parser.add_argument(466 "--ppo_max_clip_fraction",467 type=float,468 default=0.0,469 help="stop remaining PPO minibatch updates when clipped sample fraction exceeds this value; <=0 disables",470 )471 parser.add_argument(472 "--num_mini_batch",473 type=int,474 default=1,475 help="number of batches for ppo (default: 1)",476 )477 parser.add_argument(478 "--entropy_coef",479 type=float,480 default=0.01,481 help="entropy term coefficient (default: 0.01)",482 )483 parser.add_argument(484 "--value_loss_coef",485 type=float,486 default=1,487 help="value loss coefficient (default: 0.5)",488 )489 parser.add_argument(490 "--use_max_grad_norm",491 action="store_false",492 default=True,493 help="by default, use max norm of gradients. If set, do not use.",494 )495 parser.add_argument(496 "--max_grad_norm",497 type=float,498 default=10.0,499 help="max norm of gradients (default: 0.5)",500 )501 parser.add_argument(502 "--use_gae",503 action="store_false",504 default=True,505 help="use generalized advantage estimation",506 )507 parser.add_argument(508 "--gamma",509 type=float,510 default=0.99,511 help="discount factor for rewards (default: 0.99)",512 )513 parser.add_argument(514 "--gae_lambda",515 type=float,516 default=0.95,517 help="gae lambda parameter (default: 0.95)",518 )519 parser.add_argument(520 "--use_proper_time_limits",521 action="store_true",522 default=False,523 help="compute returns taking into account time limits",524 )525 parser.add_argument(526 "--use_huber_loss",527 action="store_false",528 default=True,529 help="by default, use huber loss. If set, do not use huber loss.",530 )531 parser.add_argument(532 "--use_value_active_masks",533 action="store_false",534 default=True,535 help="by default True, whether to mask useless data in value loss.",536 )537 parser.add_argument(538 "--use_policy_active_masks",539 action="store_false",540 default=True,541 help="by default True, whether to mask useless data in policy loss.",542 )543 parser.add_argument("--huber_delta", type=float, default=10.0, help=" coefficience of huber loss.")544545 # Recurrent value decomposition (QMIX / VDN / HATT-QMIX)546 parser.add_argument("--qmix_agent_hidden_dim", type=int, default=128)547 parser.add_argument("--qmix_mixer_hidden_dim", type=int, default=64)548 parser.add_argument("--qmix_hypernet_hidden_dim", type=int, default=128)549 parser.add_argument("--qmix_buffer_size", type=int, default=128, help="episode replay capacity")550 parser.add_argument("--qmix_batch_size", type=int, default=8)551 parser.add_argument("--qmix_sequence_length", type=int, default=32)552 parser.add_argument("--qmix_burn_in", type=int, default=8)553 parser.add_argument(554 "--qmix_use_recurrent_agent",555 type=lambda value: str(value).lower() in ("1", "true", "yes", "y"),556 default=True,557 help="use a GRU in the per-agent utility network",558 )559 parser.add_argument(560 "--qmix_transition_mode",561 choices=["physical_legacy", "physical_constrained", "event_smdp"],562 default="physical_legacy",563 help="replay/target semantics for event-driven coalition actions",564 )565 parser.add_argument(566 "--qmix_hatt_sequence_chunk_size",567 type=int,568 default=8,569 help=(570 "number of recurrent time steps whose memoryless HATT backbone and "571 "candidate decoder are evaluated together"572 ),573 )574 parser.add_argument("--qmix_learning_starts", type=int, default=16, help="episodes before updates")575 parser.add_argument(576 "--qmix_train_interval",577 type=int,578 default=1,579 help="completed episodes between update groups",580 )581 parser.add_argument("--qmix_updates_per_train", type=int, default=1)582 parser.add_argument(583 "--qmix_target_update_interval",584 type=int,585 default=200,586 help="completed episodes between hard target-network updates",587 )588 parser.add_argument(589 "--qmix_optimizer", choices=["rmsprop", "adam"], default="rmsprop"590 )591 parser.add_argument("--qmix_rmsprop_alpha", type=float, default=0.99)592 parser.add_argument(593 "--qmix_max_abs_q",594 type=float,595 default=10000.0,596 help="fail fast when joint online/target Q exceeds this magnitude; <=0 disables",597 )598 parser.add_argument(599 "--qmix_double_q", type=lambda value: str(value).lower() in ("1", "true", "yes", "y"),600 default=True,601 )602 parser.add_argument("--qmix_epsilon_start", type=float, default=1.0)603 parser.add_argument("--qmix_epsilon_finish", type=float, default=0.05)604 parser.add_argument("--qmix_epsilon_anneal_steps", type=int, default=500000)605 parser.add_argument("--qmix_reward_scale", type=float, default=1.0)606 parser.add_argument(607 "--mappo_team_mean_reward",608 type=lambda value: str(value).lower() in ("1", "true", "yes", "y"),609 default=False,610 help="train MAPPO/HATT-MAPPO with the team-mean reward broadcast to all agents",611 )612613 # HATT utility head auxiliary loss614 parser.add_argument(615 "--utility_loss_coef",616 type=float,617 default=0.01,618 help="coefficient for the auxiliary utility-head MSE loss (v3.2d+)",619 )620621 # run parameters622 parser.add_argument(623 "--use_linear_lr_decay",624 action="store_true",625 default=False,626 help="use a linear schedule on the learning rate",627 )628 # save parameters629 parser.add_argument(630 "--save_interval",631 type=int,632 default=1,633 help="time duration between contiunous twice models saving.",634 )635 parser.add_argument(636 "--save_episode_checkpoints",637 action="store_true",638 default=False,639 help="also save tagged actor/critic checkpoints at each save interval",640 )641642 # log parameters643 parser.add_argument(644 "--log_interval",645 type=int,646 default=5,647 help="time duration between contiunous twice log printing.",648 )649650 # eval parameters651 parser.add_argument(652 "--use_eval",653 action="store_true",654 default=False,655 help="by default, do not start evaluation. If set`, start evaluation alongside with training.",656 )657 parser.add_argument(658 "--eval_interval",659 type=int,660 default=25,661 help="time duration between contiunous twice evaluation progress.",662 )663 parser.add_argument(664 "--eval_episodes",665 type=int,666 default=32,667 help="number of episodes of a single evaluation.",668 )669 parser.add_argument(670 "--max_train_episodes",671 type=int,672 default=None,673 help="optional upper bound on runner training episodes",674 )675 parser.add_argument(676 "--eval_seed",677 type=int,678 default=10000,679 help="first seed in the fixed evaluation scenario suite",680 )681 parser.add_argument(682 "--eval_deterministic",683 action="store_true",684 default=True,685 help="use greedy actions during evaluation; set false in YAML to sample actions",686 )687 parser.add_argument(688 "--training_eval_cuda",689 type=int,690 default=None,691 help="optional CUDA device for periodic training evaluation subprocesses",692 )693 parser.add_argument(694 "--eval_progress_file",695 type=str,696 default=None,697 help="internal atomic per-episode evaluation progress file",698 )699700 # render parameters701 parser.add_argument(702 "--save_gifs",703 action="store_true",704 default=False,705 help="by default, do not save render video. If set, save video.",706 )707 parser.add_argument(708 "--use_render",709 action="store_true",710 default=False,711 help="by default, do not render the env during training. If set, start render. Note: something, the environment has internal render process which is not controlled by this hyperparam.",712 )713 parser.add_argument(714 "--render_episodes",715 type=int,716 default=5,717 help="the number of episodes to render a given env",718 )719 parser.add_argument(720 "--ifi",721 type=float,722 default=0.1,723 help="the play interval of each rendered image in saved video.",724 )725726 # PCFA baseline parameters727 parser.add_argument(728 "--use_pcfa",729 action="store_true",730 default=False,731 help="use the Project Manager-oriented Coalition Formation Algorithm baseline instead of learning",732 )733 parser.add_argument(734 "--pcfa_comm_range",735 type=float,736 default=-1.0,737 help="communication range for PCFA consensus; -1 means global communication",738 )739 parser.add_argument(740 "--pcfa_lambda",741 type=float,742 default=0.01,743 help="distance decay coefficient in PCFA fitness",744 )745 parser.add_argument(746 "--pcfa_consensus_loops",747 type=int,748 default=-1,749 help="number of PCFA consensus loops; -1 means num_agents",750 )751 parser.add_argument(752 "--pcfa_max_coalition_size",753 type=int,754 default=3,755 help="maximum coalition size selected by PCFA",756 )757 parser.add_argument(758 "--pcfa_min_coalition_size",759 type=int,760 default=1,761 help="minimum coalition size selected by PCFA",762 )763 parser.add_argument(764 "--pcfa_match_metric",765 type=str,766 default="cosine",767 choices=["cosine", "dot"],768 help="metric used to match agent capabilities with task requirements",769 )770 parser.add_argument(771 "--pcfa_resupply_battery_threshold",772 type=float,773 default=0.2,774 help="DelivCoal PCFA battery ratio below which an active agent resupplies",775 )776 parser.add_argument(777 "--pcfa_resupply_load_threshold",778 type=float,779 default=0.05,780 help="DelivCoal PCFA payload ratio below which an active agent resupplies",781 )782 parser.add_argument(783 "--pcfa_priority_weight",784 type=float,785 default=1.0,786 help="DelivCoal PCFA task-priority weight in project-manager ordering",787 )788 parser.add_argument(789 "--pcfa_urgency_weight",790 type=float,791 default=1.0,792 help="DelivCoal PCFA deadline-pressure weight in project-manager ordering",793 )794 parser.add_argument(795 "--pcfa_explore_distance_weight",796 type=float,797 default=0.15,798 help="DelivCoal PCFA penalty for long observer-relative Explore legs",799 )800801 # pretrained parameters802 parser.add_argument(803 "--model_dir",804 type=str,805 default=None,806 help="by default None. set the path to pretrained model.",807 )808809 return parser