pcfa_policy.py
规则策略接口
原文件 · 208 行 · 6821 字节
algorithms/algorithm/pcfa_policy.py
SHA-256: 27c0b6552f27dde39de3b24e40a2af1c500d147f2d5d713728cf5169d34b1416
1"""PCFA Policy wrapper.23Provides a policy-style interface around the non-learning PCFA market algorithm4so that it can be plugged into the existing runner architecture.5"""67import numpy as np8from algorithms.algorithm.pcfa import run_pcfa_policy91011class PCFAPolicy:12 """Non-learning policy wrapper for PCFA.1314 The interface mirrors GAMAPPOPolicy/RMAPPOPolicy where convenient, but the15 underlying algorithm is deterministic and does not require gradients or16 training updates.17 """1819 def __init__(self, args, obs_space, cent_obs_space, act_space, device=None):20 self.args = args21 self.obs_space = obs_space22 self.cent_obs_space = cent_obs_space23 self.act_space = act_space24 self.device = device2526 self.num_agents = getattr(args, "num_agents", 1)27 self.num_tasks = act_space.n28 self.hidden_size = getattr(args, "hidden_size", 64)29 self.recurrent_N = getattr(args, "recurrent_N", 1)3031 # Will be set by runner after env is available.32 self.idle_task_id = 033 self.explore_task_id = 034 self.resupply_task_id = 035 self.delivery_task_num = None36 self.explore_task_start = 037 self.explore_task_end = 138 self.use_coalition_delivery = False3940 def set_task_ids(self, idle_task_id, explore_task_id, resupply_task_id):41 """Set special task IDs used by the environment."""42 self.idle_task_id = idle_task_id43 self.explore_task_id = explore_task_id44 self.resupply_task_id = resupply_task_id45 self.explore_task_start = explore_task_id46 self.explore_task_end = explore_task_id + 14748 def set_task_layout(49 self,50 *,51 delivery_task_num,52 explore_task_start,53 explore_task_end,54 idle_task_id,55 resupply_task_id,56 use_coalition_delivery,57 ):58 """Record the environment task-slot layout.5960 DelivCoal exposes several observer-relative Explore-sector slots. The61 historical PCFA wrapper only received one Explore ID, which caused the62 remaining sector slots to be interpreted as delivery tasks.63 """64 self.delivery_task_num = int(delivery_task_num)65 self.explore_task_start = int(explore_task_start)66 self.explore_task_end = int(explore_task_end)67 self.explore_task_id = self.explore_task_start68 self.idle_task_id = int(idle_task_id)69 self.resupply_task_id = int(resupply_task_id)70 self.use_coalition_delivery = bool(use_coalition_delivery)7172 def _parse_batch_args(self, obs, available_actions, active_masks):73 """Return per-env args and detect batch size.7475 Returns:76 batch_size: int77 obs_list: list of obs dicts with no batch dimension78 avail_list: list of available_actions arrays79 active_list: list of active_masks arrays80 """81 batch_size = obs["ego"].shape[0]8283 obs_list = []84 for b in range(batch_size):85 obs_list.append({k: v[b] for k, v in obs.items()})8687 if available_actions is None:88 avail_list = [None] * batch_size89 else:90 avail_list = [available_actions[b] for b in range(batch_size)]9192 if active_masks is None:93 active_list = [np.ones(self.num_agents, dtype=np.float32)] * batch_size94 else:95 active_list = [active_masks[b] for b in range(batch_size)]9697 return batch_size, obs_list, avail_list, active_list9899 def get_actions(100 self,101 cent_obs,102 obs,103 rnn_states_actor,104 rnn_states_critic,105 masks,106 available_actions=None,107 active_masks=None,108 deterministic=False,109 ):110 """Compute actions for the given observations.111112 Args follow the MAPPO policy signature for compatibility, but rnn states,113 masks, and cent_obs are ignored.114115 Returns:116 values: zeros (batch, n_agents, 1)117 actions: (batch, n_agents) int118 action_log_probs: zeros (batch, n_agents, 1)119 rnn_states_actor: same shape as input, unchanged120 rnn_states_critic: same shape as input, unchanged121 """122 batch_size, obs_list, avail_list, active_list = self._parse_batch_args(123 obs, available_actions, active_masks124 )125126 actions_list = []127 for b in range(batch_size):128 acts = run_pcfa_policy(129 obs_list[b],130 active_list[b],131 avail_list[b],132 self.args,133 self.idle_task_id,134 self.explore_task_id,135 self.resupply_task_id,136 delivery_task_num=self.delivery_task_num,137 explore_task_start=self.explore_task_start,138 explore_task_end=self.explore_task_end,139 use_coalition_delivery=self.use_coalition_delivery,140 )141 actions_list.append(acts)142143 actions = np.stack(actions_list, axis=0) # (batch, n_agents)144145 values = np.zeros((batch_size, self.num_agents, 1), dtype=np.float32)146 action_log_probs = np.zeros((batch_size, self.num_agents, 1), dtype=np.float32)147148 return values, actions, action_log_probs, rnn_states_actor, rnn_states_critic149150 def get_values(self, cent_obs, rnn_states_critic, masks):151 """Return dummy values."""152 batch_size = cent_obs.shape[0] if hasattr(cent_obs, "shape") else 1153 return np.zeros((batch_size, self.num_agents, 1), dtype=np.float32)154155 def eval_actions(156 self, cent_obs, obs, rnn_states_actor, rnn_states_critic, action, masks,157 available_actions=None, active_masks=None,158 ):159 """Return dummy evaluation quantities."""160 batch_size = action.shape[0]161 action_log_probs = np.zeros((batch_size, self.num_agents, 1), dtype=np.float32)162 dist_entropy = np.zeros((batch_size, self.num_agents, 1), dtype=np.float32)163 values = np.zeros((batch_size, self.num_agents, 1), dtype=np.float32)164 return values, action_log_probs, dist_entropy165166 def prep_rollout(self):167 pass168169 def prep_training(self):170 pass171172 def train(self, *args, **kwargs):173 pass174175 def save(self, save_dir, episode):176 pass177178 def restore(self, model_dir):179 pass180181 def get_max_std(self):182 return 0.0183184 def update_actor(self, *args, **kwargs):185 pass186187 def update_critic(self, *args, **kwargs):188 pass189190 @property191 def actor(self):192 return self193194 @property195 def critic(self):196 return self197198 def parameters(self):199 return []200201 def named_parameters(self):202 return {}203204 def state_dict(self):205 return {}206207 def load_state_dict(self, state_dict):208 pass