env_core.py
CapCoal 任务生成与执行语义
原文件 · 1422 行 · 60369 字节
envs/capcoal/env_core.py
SHA-256: b124a2d1d4923295f7c5a73b56eeacd0fa9faaa11102345ea1626fcac82c6a72
1import copy2import numpy as np3import pickle4import os5from collections import deque6import yaml78eps = 1e-6910from enum import IntEnum1112class AgentType(IntEnum):13 Drone = 014 Vehicle = 11516class TaskType(IntEnum):17 Explore = 018 Deliver = 119 Idle = 220 Resupply = 32122def dist_point_segment(p, a, b):23 # distance from point p to segment ab24 p = np.array(p, dtype=float)25 a = np.array(a, dtype=float)26 b = np.array(b, dtype=float)27 ab = b - a28 if np.allclose(ab, 0):29 return np.linalg.norm(p - a)30 t = np.dot(p - a, ab) / np.dot(ab, ab)31 t = np.clip(t, 0.0, 1.0)32 proj = a + t * ab33 return np.linalg.norm(p - proj)3435def fast_path(start, goal, danger_points=None, danger_radius=1.0, clearance=0.5, max_iter=8):36 '''37 Continuous-space approximate path planner.3839 Strategy:40 - Try direct straight-line from start to goal.41 - If the line intersects any danger circle (centered at danger_points with42 radius `danger_radius`), add a simple perpendicular-offset waypoint to43 go around the obstacle, then retry. Repeat up to `max_iter` times.4445 Returns a list of 2-tuples (x,y) representing waypoints including start46 and goal. May return [] only if start==goal.47 '''48 if danger_points is None:49 danger_points = []5051 def segment_intersects_circle(a, b, c, r):52 return dist_point_segment(c, a, b) <= r5354 s = tuple(map(float, start))55 g = tuple(map(float, goal))56 if np.allclose(s, g):57 return [s]5859 waypoints = [s, g]60 iter_count = 061 # iterative fixing: check each segment, if intersects an obstacle, insert detour62 while iter_count < max_iter:63 changed = False64 new_waypoints = [waypoints[0]]65 for i in range(len(waypoints)-1):66 a = waypoints[i]67 b = waypoints[i+1]68 # find first obstacle intersecting this segment69 hit = None70 for c in danger_points:71 if segment_intersects_circle(a, b, c, danger_radius + clearance):72 hit = c73 break74 if hit is None:75 new_waypoints.append(b)76 continue77 # compute detour point: closest point on segment to obstacle, then shift perpendicular78 a_np = np.array(a); b_np = np.array(b); c_np = np.array(hit)79 ab = b_np - a_np80 if np.allclose(ab, 0):81 # degenerate, just sidestep82 dir_perp = np.array([1.0, 0.0])83 else:84 t = np.dot(c_np - a_np, ab) / np.dot(ab, ab)85 t = np.clip(t, 0.0, 1.0)86 closest = a_np + t * ab87 # perpendicular direction88 dir_perp = np.array([-ab[1], ab[0]])89 if np.allclose(dir_perp, 0):90 dir_perp = np.array([1.0, 0.0])91 dir_perp = dir_perp / np.linalg.norm(dir_perp)92 # choose sign by testing which side is farther from other obstacles93 # create two candidate detours (left/right)94 det_dist = danger_radius + clearance95 cand1 = tuple((closest + dir_perp * det_dist).tolist()) if 'closest' in locals() else tuple((a_np + dir_perp * det_dist).tolist())96 cand2 = tuple((closest - dir_perp * det_dist).tolist()) if 'closest' in locals() else tuple((a_np - dir_perp * det_dist).tolist())97 # pick candidate that doesn't immediately collide with any obstacle (or the one with larger margin)98 def collision_score(pt):99 ssum = 0.0100 for c2 in danger_points:101 d = np.linalg.norm(np.array(pt) - np.array(c2)) - danger_radius102 ssum += max(0.0, (clearance - d))103 return ssum104105 s1 = collision_score(cand1)106 s2 = collision_score(cand2)107 detour = cand1 if s1 <= s2 else cand2108 # insert detour before b109 new_waypoints.append(detour)110 new_waypoints.append(b)111 changed = True112 waypoints = new_waypoints113 iter_count += 1114 if not changed:115 break116117 # finally compress waypoints (remove near-duplicates)118 out = [tuple(map(float, waypoints[0]))]119 for p in waypoints[1:]:120 if np.linalg.norm(np.array(p) - np.array(out[-1])) > eps:121 out.append(tuple(map(float, p)))122 return out123124class AgentLastState:125 __slots__ = ['coalition', 'new_task_found', 'current_task_completed', 'any_task_completed', 'coalition_completed', 'dist', 'load', 'idle']126 127 def __init__(self):128 self.coalition = EnvCore.idle_task_id129 self.new_task_found = False130 self.current_task_completed = False131 self.any_task_completed = False132 self.coalition_completed = False133 self.dist = 0.0134 self.load = 0.0135 self.idle = True136137class Agent:138 __slots__ = ['aid', 'position', 'capbilities', 'perception', 'mobility', 'load',139 'battery', 'type', 'living', 'idle', 'coalition', 'path', 'last_state',140 'dead', 'new_task_found', 'coalition_changed', 'discovery', 'delivery', 'coalition_delivery', 'resupply_ratio', 'init_load', 'resupplied',141 'current_task_completed', 'any_task_completed', 'coalition_completed', 'dist', 'unchanged_time',142 'idle_wait_steps']143144 def __init__(self, aid: int, pos: np.ndarray, capbilities: np.ndarray, rng=None):145 self.aid = aid146 self.position = np.array(pos, dtype=float)147 self.capbilities = capbilities148 self.perception = float(capbilities[0] * 50.0)149 self.mobility = float(capbilities[1] * 10) + 10150 self.init_load = int(capbilities[2] * 70) + 30151 self.load = self.init_load152 self.battery = int(capbilities[3] * 500) + 500153 self.type = rng.choice([AgentType.Drone, AgentType.Vehicle]) if rng is not None else np.random.choice([AgentType.Drone, AgentType.Vehicle])154 self.living = True155 self.idle = True156 self.coalition = EnvCore.idle_task_id157 self.path = []158159 self.dead = False160 self.new_task_found = False161 self.current_task_completed = False162 self.any_task_completed = False163 self.coalition_completed = False164 self.coalition_changed = False165 self.discovery = 0.0166 self.delivery = 0.0167 self.coalition_delivery = 0.0168 self.dist = 0.0169 self.unchanged_time = 0.0170 self.idle_wait_steps = 0171172 self.resupply_ratio = 0.0173 self.resupplied = False174175 self.last_state = AgentLastState()176177 def act(self, cid: int):178 if cid != self.coalition:179 self.coalition = cid180 self.coalition_changed = True181 self.path = []182 self.unchanged_time = 0.0183 else:184 self.unchanged_time += 1.0185186 if cid == EnvCore.idle_task_id:187 self.idle = True188 else:189 self.idle_wait_steps = 0190191 def consume(self):192 if self.living:193 self.battery -= 1194 if self.battery == 0:195 self.living = False196 self.dead = True197198 def deliver(self, demand):199 delivery = np.min([self.load, demand])200 self.load -= delivery201 self.delivery = delivery if delivery > 0 else -10202 self.idle = True203204 if self.delivery < 0:205 pass206 207 return delivery208209 def complete(self):210 self.current_task_completed = True211 self.idle = True212213 def resupply(self):214 self.living = True215 self.dead = False216 self.idle = True217 self.battery = int(self.capbilities[3] * 500) + 500218 self.resupply_ratio = (self.init_load - self.load) / self.init_load219 self.resupplied = True220 self.load = self.init_load221222 def step(self):223 for key in self.last_state.__slots__:224 attr = getattr(self, key)225 setattr(self.last_state, key, attr)226227 self.dead = False228 self.new_task_found = False229 self.current_task_completed = False230 self.any_task_completed = False231 self.coalition_completed = False232 self.coalition_changed = False233 self.discovery = 0.0234 self.delivery = 0.0235 self.coalition_delivery = 0.0236 self.resupply_ratio = 0.0237 self.resupplied = False238 self.idle = False239240241class Task:242 __slots__ = [243 'id', 'cap_dim', 'position', 'demand', 'initial_demand', 'urgency',244 'req_vec', 'req_ratio', 'type', 'discovered', 'completed',245 'discovered_step', 'first_assigned_step', 'response_time'246 ]247248 def __init__(self, cid: int, pos: np.ndarray, demand: int, urgency: float, cap_dim: int, rng=None):249 self.id = cid250 self.cap_dim = cap_dim251 self.position = np.array(pos, dtype=float)252 self.demand = demand253 self.initial_demand = demand254 self.urgency = urgency255 self.req_vec = ((rng.rand(cap_dim) if rng is not None else np.random.rand(cap_dim)) * (demand / 100.0)).astype(float)256 self.req_ratio = np.zeros(cap_dim)257 self.type = TaskType.Deliver258 self.discovered = False259 self.completed = False260261 self.discovered_step = None262 self.first_assigned_step = None263 self.response_time = None264265 def calc_req_ratio(self, cap_now):266 if self.discovered and not self.completed:267 req_vec = self.req_vec268 for dim in range(self.cap_dim):269 if np.isnan(req_vec[dim]) or req_vec[dim] <= 0.0:270 self.req_ratio[dim] = 0.0271 else:272 self.req_ratio[dim] = (cap_now[dim] - req_vec[dim]) / req_vec[dim]273 else:274 self.req_ratio = np.zeros(self.cap_dim)275276277278class EnvCore:279280 explore_task_id = -1281 idle_task_id = -1282 resupply_task_id = -1283284 def __init__(self, config):285 deliver_target_num = config.target_num286 self.explore_action_mode = getattr(config, "explore_action_mode", "nearest")287 if self.explore_action_mode not in ("nearest", "sector"):288 raise ValueError(f"Unsupported explore_action_mode: {self.explore_action_mode}")289 self.num_explore_actions = (290 int(getattr(config, "num_explore_actions", 1))291 if self.explore_action_mode == "sector"292 else 1293 )294 if self.num_explore_actions <= 0:295 raise ValueError("num_explore_actions must be positive.")296 self._configured_explore_step_size = getattr(config, "explore_step_size", None)297 self.explore_random_jitter = float(getattr(config, "explore_random_jitter", 0.0))298 self.explore_task_start = deliver_target_num299 self.explore_task_end = deliver_target_num + self.num_explore_actions300 self.__class__.explore_task_id = self.explore_task_start301 self.__class__.idle_task_id = self.explore_task_end302 self.__class__.resupply_task_id = self.explore_task_end + 1303 # initialize using constructor parameters (use defaults if not provided)304 self.map_size = float(config.map_size) # 地图尺寸 / m (continuous)305 self.explore_step_size = (306 float(self._configured_explore_step_size)307 if self._configured_explore_step_size is not None308 else self.map_size * 0.25309 )310 # obstacle/danger geometry params311 self.danger_radius = float(getattr(config, 'danger_radius', 1.0))312 self.clearance = float(getattr(config, 'clearance', 0.5))313 # keep UAV/UGV split for legacy logic (agent_num param is available but code uses uav/ugv)314 self.uav_num = config.num_uav # 无人机数量315 self.ugv_num = config.num_ugv # 无人车数量316 self.uav_perception_rg = tuple(config.uav_perception_rg[0:2])317 self.ugv_perception_rg = tuple(config.ugv_perception_rg[0:2])318 self.delivery_task_num = config.target_num319 self.task_num = config.target_num + self.num_explore_actions + 2320 self.gat_dim = config.gat_dim # GAT 输出维度321 self.max_steps = config.max_steps # 最大步数322 self.cap_dim = config.cap_dim # 能力维度323 self.reconfiguration_mode = str(getattr(config, "reconfiguration_mode", "legacy"))324 valid_reconfiguration_modes = {"legacy", "always", "periodic", "event", "urgency", "learned"}325 if self.reconfiguration_mode not in valid_reconfiguration_modes:326 raise ValueError(f"Unsupported reconfiguration_mode: {self.reconfiguration_mode!r}")327 self.reconfiguration_interval = int(getattr(config, "reconfiguration_interval", 10))328 if self.reconfiguration_interval <= 0:329 raise ValueError("reconfiguration_interval must be positive.")330 self.reconfiguration_urgency_threshold = float(331 getattr(config, "reconfiguration_urgency_threshold", 0.7)332 )333 self.reconfiguration_urgency_aggregation = str(334 getattr(config, "reconfiguration_urgency_aggregation", "mean")335 )336 if self.reconfiguration_urgency_aggregation not in ("mean", "sum"):337 raise ValueError("reconfiguration_urgency_aggregation must be 'mean' or 'sum'.")338 self.reconfiguration_switch_cost = float(339 getattr(config, "reconfiguration_switch_cost", 0.0)340 )341 if self.reconfiguration_switch_cost < 0.0:342 raise ValueError("reconfiguration_switch_cost must be non-negative.")343 self.capcoal_req_mode = getattr(config, "capcoal_req_mode", "random_demand")344 req_mode_aliases = {345 "v0": "random_demand",346 "original": "random_demand",347 "rand_demand": "random_demand",348 "v2a": "demand4",349 "demand": "demand4",350 "demand_vector": "demand4",351 "v2b": "demand1",352 "demand_scalar": "demand1",353 "v2c": "random_identity",354 "identity": "random_identity",355 }356 self.capcoal_req_mode = req_mode_aliases.get(self.capcoal_req_mode, self.capcoal_req_mode)357 if self.capcoal_req_mode not in ("random_demand", "demand4", "demand1", "random_identity"):358 raise ValueError(359 "capcoal_req_mode must be 'random_demand', 'demand4', 'demand1', "360 f"or 'random_identity', got {self.capcoal_req_mode!r}"361 )362363 self.reward_keys = ['switch_coalition', 'dead', 'discover', 'deliver', 'resupply', 'coalition_deliver', 'coalition_complete', 'dist', 'end']364365 # total agents (legacy uses uav+ugv)366 self.agent_num = self.uav_num + self.ugv_num367368 self.step_count = 0 # 当前步数369 # ego obs = position(2), capabilities(cap_dim), coalition_id(1), coalition_type(1),370 # load(1), discovered_ratio(1), battery(1).371 self.ego_feat_dim = self.cap_dim + 7372 # task obs = normalized position(2), req_vec(cap_dim), demand(1), type(1).373 self.per_task_feat = 2 + self.cap_dim + 1 + 1374 # other-agent obs = position(2), capabilities(cap_dim), load(1),375 # battery(1), coalition_id(1), agent_type(1), living(1).376 self.other_agent_feat_dim = 2 + self.cap_dim + 1 + 1 + 1 + 1 + 1377 self.global_agent_feat_dim = self.other_agent_feat_dim378 # task state additionally exposes discovery/completion flags to the critic.379 self.global_task_feat_dim = self.per_task_feat + 2380 self.global_feat_dim = 1381382 # 观测维度：GAT383 # self.obs_dim = self.cap_dim + 1 + self.task_num * self.per_task_feat + self.gat_dim * self.task_num384385 # 观测维度：无GAT386 self.obs_dim = (387 self.ego_feat_dim388 + self.task_num * self.per_task_feat389 + self.agent_num * self.other_agent_feat_dim390 )391392 # 观测维度：最简393 # self.obs_dim = 1 + 3 + 3 # 设置智能体的观测维度394 # self.obs_dim = self.cap_dim + 1 + self.task_num + self.task_num395 # 动作维度：任务个数 + 1（额外的 Idle 动作，索引为 task_num）396 # 为了兼容性，也接受 -1 表示 Idle397 # RL 输出范围建议为 [0, task_num]（或使用 -1 作为 Idle），其中 task_num 表示 Idle（空闲，不选择任何任务）398 self.action_dim = self.task_num399400 self.event_new_task_found = 0401 self.event_task_completed = [0 for i in range(self.agent_num)]402 self.event_any_task_completed = 0403404 self.data_dir = 'data'405406 if config.rewards_config_file:407 with open(config.rewards_config_file, 'r', encoding='utf-8') as f:408 rewards_config = yaml.safe_load(f) or {}409 410 for key, value in rewards_config.items():411 setattr(self, key, value)412413 else:414415 self.pun_change_coalition_event = 0.001416 self.pun_change_coalition = 1417 self.w_short = 2.0 # 能力不足权重（重罚）418 self.w_surplus = 0.3 # 能力过多权重（轻罚）419 self.b_short = 8.0 # 能力不足权重（重罚）420 self.b_surplus = 20.0 # 能力过多权重（轻罚）421 self.rwd_ratio_discover_task = 5422 self.rwd_ratio_discover_dp = 1.0423 self.pun_ag_dead = 10424 self.rwd_ratio_send = 1425 self.rwd_coalition_ratio_send = 0.2426 self.rwd_task_completed = 200427 self.rwd_coalition_task_completed = 50428 self.rwd_dist_ratio = 3.0429 self.pun_time_ratio = 0.1430431 # 初始化全局状态432 self.global_state = np.zeros((self.agent_num, self.task_num, self.gat_dim))433434 self._seed = None435 self.rng = None436 self._base_initialized = False437 self._base_danger_points = None438 self._base_tasks = None439 self._base_agents = None440 self.task_position_jitter = float(getattr(config, 'task_position_jitter', 0.05))441 self.env_reset_mode = getattr(config, 'env_reset_mode', 'jitter')442443 def seed(self, seed):444 self._seed = int(seed)445 self.rng = np.random.RandomState(self._seed)446 self._base_initialized = False447 self._base_danger_points = None448 self._base_tasks = None449 self._base_agents = None450 return [self._seed]451452 def _rand(self, *args):453 if self.rng is not None:454 return self.rng.rand(*args)455 return np.random.rand(*args)456457 def _randint(self, *args, **kwargs):458 if self.rng is not None:459 return self.rng.randint(*args, **kwargs)460 return np.random.randint(*args, **kwargs)461462 def _choice(self, a):463 if self.rng is not None:464 return self.rng.choice(a)465 return np.random.choice(a)466467 def _explore_sector_goal(self, ag):468 sector = int(ag.coalition - self.explore_task_start)469 sector = max(0, min(self.num_explore_actions - 1, sector))470 sector_width = 2.0 * np.pi / float(self.num_explore_actions)471 angle = sector * sector_width472 if self.explore_random_jitter > 0.0:473 jitter = (float(self._rand()) * 2.0 - 1.0) * self.explore_random_jitter * sector_width474 angle += jitter475 direction = np.array([np.cos(angle), np.sin(angle)], dtype=float)476 goal = ag.position + direction * self.explore_step_size477 return np.clip(goal, 0.0, self.map_size)478479 def _save_base_env(self):480 self._base_danger_points = copy.deepcopy(self.danger_points)481 self._base_tasks = copy.deepcopy(self.tasks)482 self._base_agents = copy.deepcopy(self.agents)483484 def _restore_base_env(self):485 self.danger_points = copy.deepcopy(self._base_danger_points)486 self.tasks = copy.deepcopy(self._base_tasks)487 self.agents = copy.deepcopy(self._base_agents)488489 def _jitter_task_positions(self):490 if self.task_position_jitter <= 0.0:491 return492 scale = self.task_position_jitter * self.map_size493 for task in self.tasks:494 if task.type != TaskType.Deliver:495 continue496 base_pos = task.position.copy()497 for _ in range(20):498 offset = self._rand(2) * 2.0 * scale - scale499 new_pos = np.clip(base_pos + offset, 0.0, self.map_size)500 if not any(501 np.linalg.norm(new_pos - np.array(dp['position'])) <= (self.danger_radius + self.clearance)502 for dp in self.danger_points503 ):504 task.position = new_pos505 break506 else:507 task.position = np.clip(base_pos, 0.0, self.map_size)508509 def _init_map(self):510 # Continuous map: represent only danger points as circle centers.511 # 连续地图：只记录危险点圆心512 self.danger_points = []513 # number of danger points scaled by area514 # 危险点数量按面积比例决定515 num_danger = 10#max(1, int(self.map_size * self.map_size * 0.002))516 for _ in range(num_danger):517 p = (self._rand(2) * self.map_size).astype(float)518 self.danger_points.append({519 'position': p,520 'discovered': False521 })522523 def _init_agents(self):524 self.agents: list[Agent] = []525 for i in range(self.agent_num):526 capbilities = self._rand(self.cap_dim)527 # 采样连续位置，避开危险点中心528 pos = self._rand(2) * self.map_size529 # 保证不在危险半径内530 tries = 0531 while any(np.linalg.norm(pos - np.array(dp['position'])) <= (self.danger_radius + self.clearance) for dp in self.danger_points) and tries < 50:532 pos = self._rand(2) * self.map_size533 tries += 1534535 agent = Agent(i, pos, capbilities, rng=self.rng)536537 self.agents.append(agent)538539 def _make_task_req_vec(self, demand_val):540 """生成配送任务 req，用于区分 demand 尺度和随机 task 指纹的作用。"""541542 if self.capcoal_req_mode == "random_demand":543 return None544 if self.capcoal_req_mode == "demand4":545 return np.ones(self.cap_dim, dtype=float) * (float(demand_val) / 200.0)546 if self.capcoal_req_mode == "demand1":547 req = np.zeros(self.cap_dim, dtype=float)548 req[0] = float(demand_val) / 200.0549 return req550 if self.capcoal_req_mode == "random_identity":551 return (self._rand(self.cap_dim) * 1.5).astype(float)552 raise RuntimeError(f"Unsupported capcoal_req_mode: {self.capcoal_req_mode}")553554 def _init_tasks(self):555 '''在空地 (map == 0) 上随机放置任务，不会卡死。'''556 self.tasks: list[Task] = []557558 # 采样任务为连续点，避开危险点559 for i in range(self.task_num):560 demand_val = int(self._randint(100, 500)) # 50–100561 pos = self._rand(2) * self.map_size562 tries = 0563 while any(np.linalg.norm(pos - np.array(dp['position'])) <= (self.danger_radius + self.clearance) for dp in self.danger_points) and tries < 200:564 pos = self._rand(2) * self.map_size565 tries += 1566 task = Task(i, np.array(pos, dtype=float), demand_val, self._rand(), self.cap_dim, rng=self.rng)567 req_vec = self._make_task_req_vec(demand_val)568 if req_vec is not None:569 task.req_vec = req_vec570 self.tasks.append(task)571572 # 指定探索联盟的初始任务。sector 模式下每个探索动作代表一个方向。573 for explore_id in range(self.explore_task_start, self.explore_task_end):574 exploring_task = Task(explore_id, np.array((np.nan, np.nan)), 0, 0.5, self.cap_dim)575 exploring_task.req_vec = np.array([3, 3, 0, 1])576 exploring_task.type = TaskType.Explore577 exploring_task.discovered = True578 self.tasks[explore_id] = exploring_task579580 idle_task = Task(self.__class__.idle_task_id, np.array((np.nan, np.nan)), 0, 0, self.cap_dim)581 idle_task.req_vec = np.array([0, 0, 0, 0])582 idle_task.type = TaskType.Idle583 idle_task.discovered = True584 self.tasks[self.__class__.idle_task_id] = idle_task585586 resupply_task = Task(self.__class__.resupply_task_id, np.array((0, 0)), 0, 0, self.cap_dim)587 resupply_task.req_vec = np.array([0, 0, 0, 0])588 resupply_task.type = TaskType.Resupply589 resupply_task.discovered = True590 self.tasks[self.__class__.resupply_task_id] = resupply_task591592593 def set_reset_mode(self, mode):594 """Switch reset mode dynamically (used for curriculum learning)."""595 assert mode in ("jitter", "full_random")596 self.env_reset_mode = mode597598 def reset(self, episode=-1):599 self.step_count = 0600 if self.env_reset_mode == "full_random":601 self._init_map()602 self._init_tasks()603 self._init_agents()604 elif not self._base_initialized:605 self._init_map()606 self._init_tasks()607 self._init_agents()608 self._save_base_env()609 self._base_initialized = True610 else:611 self._restore_base_env()612 self._jitter_task_positions()613 self.sent_goal = np.sum([task.initial_demand for task in self.tasks])614 self.event_new_task_found = 0615 self.event_task_completed = [0 for i in range(self.agent_num)]616 self.event_any_task_completed = 0617 self.global_state = np.zeros((self.agent_num, self.task_num, self.gat_dim))618619 self.total_coalition_changes = 0620 self.total_reconfiguration_decisions = 0621 self._current_decision_mask = np.ones((self.agent_num,), dtype=np.float32)622 self.total_alive_agent_steps = 0623 self.response_times = []624625 if episode != -1:626 self.episode = episode627 observations = self._get_observations()628629 active_mask = np.ones((self.agent_num,), dtype=np.float32)630 active_mask = active_mask[..., None]631632 available_actions = self._get_available_actions()633634 return observations, active_mask, available_actions#, info635636 def _get_available_actions(self):637 task_mask = np.fromiter(638 (1.0 if task.discovered and not task.completed else 0.0 for task in self.tasks),639 dtype=np.float32,640 count=self.task_num,641 )642 return np.repeat(task_mask[None, :], self.agent_num, axis=0)643644 def _reconfiguration_urgency_pressure(self):645 time_ratio = self.step_count / max(1.0, float(self.max_steps))646 pressures = [647 1.0 - min(1.0, max(0.0, float(task.urgency) - time_ratio))648 for task in self.tasks649 if task.type == TaskType.Deliver and task.discovered and not task.completed650 ]651 if not pressures:652 return 0.0653 aggregate = np.sum if self.reconfiguration_urgency_aggregation == "sum" else np.mean654 return float(aggregate(pressures))655656 def _get_reconfiguration_active_mask(self, key_event, all_still):657 living = np.array([ag.living for ag in self.agents], dtype=bool)658 completed = np.array(659 [ag.living and ag.current_task_completed for ag in self.agents], dtype=bool660 )661 if all_still:662 return living.astype(np.float32)663 if self.reconfiguration_mode == "legacy":664 return np.array(665 [666 1.0 if ag.living and (667 ag.current_task_completed or (ag.idle and key_event)668 ) else 0.0669 for ag in self.agents670 ],671 dtype=np.float32,672 )673 if self.reconfiguration_mode == "always":674 return living.astype(np.float32)675 interval_due = self.step_count % self.reconfiguration_interval == 0676 if self.reconfiguration_mode == "periodic":677 reconsider_all = interval_due678 elif self.reconfiguration_mode == "event":679 reconsider_all = key_event680 elif self.reconfiguration_mode == "urgency":681 reconsider_all = self._reconfiguration_urgency_pressure() >= self.reconfiguration_urgency_threshold682 else:683 reconsider_all = interval_due or key_event684 return np.where(reconsider_all, living, completed).astype(np.float32)685686 def _allocation_diagnostics(self):687 """Return load-based allocation diagnostics without changing dynamics."""688 committed_load = np.zeros(self.task_num, dtype=np.float64)689 for ag in self.agents:690 if not ag.living or ag.coalition is None:691 continue692 cid = int(ag.coalition)693 if not (0 <= cid < self.task_num):694 continue695 task = self.tasks[cid]696 if task.type == TaskType.Deliver and task.discovered and not task.completed:697 committed_load[cid] += max(0.0, float(ag.load))698699 task_rows = []700 total_shortage = 0.0701 total_surplus = 0.0702 has_payload_deficit = False703 for task in self.tasks:704 if task.type != TaskType.Deliver or not task.discovered or task.completed:705 continue706 demand = max(0.0, float(task.demand))707 committed = float(committed_load[task.id])708 shortage = max(0.0, demand - committed)709 surplus = max(0.0, committed - demand)710 total_shortage += shortage711 total_surplus += surplus712 has_payload_deficit = has_payload_deficit or shortage > 1e-6713 task_rows.append({714 "task_id": int(task.id),715 "remaining_demand": demand,716 "committed_load": committed,717 "payload_shortage": shortage,718 "payload_surplus": surplus,719 })720721 idle_agents = [722 ag for ag in self.agents723 if ag.living and self.tasks[int(ag.coalition)].type == TaskType.Idle724 ]725 harmful_idle = sum(726 1 for ag in idle_agents727 if float(ag.load) > 1e-6 and has_payload_deficit728 )729 normalizer = max(1.0, float(self.sent_goal))730 return {731 "idle_agents": int(len(idle_agents)),732 "harmful_idle_agents": int(harmful_idle),733 "useful_idle_agents": int(len(idle_agents) - harmful_idle),734 "committed_payload_shortage_ratio": float(total_shortage / normalizer),735 "committed_payload_surplus_ratio": float(total_surplus / normalizer),736 "tasks": task_rows,737 }738739 def discover_along_segment(self, agent_idx, a, b):740 # 标记任务和危险点为已发现（如果在感知距离内）741 ag = self.agents[agent_idx]742 per = float(ag.perception)743 discovered_tasks = []744 discovered_dp = 0745746 a = np.asarray(a, dtype=float)747 b = np.asarray(b, dtype=float)748 ab = b - a749 denom = float(np.dot(ab, ab))750751 def point_segment_distances(points):752 if len(points) == 0:753 return np.empty((0,), dtype=float)754 if denom <= eps:755 return np.linalg.norm(points - a[None, :], axis=1)756 t = ((points - a[None, :]) @ ab) / denom757 t = np.clip(t, 0.0, 1.0)758 proj = a[None, :] + t[:, None] * ab[None, :]759 return np.linalg.norm(points - proj, axis=1)760761 task_ids = [762 idx763 for idx, task in enumerate(self.tasks)764 if (not task.discovered)765 and task.type == TaskType.Deliver766 and not np.any(np.isnan(task.position))767 ]768 if task_ids:769 task_positions = np.stack([self.tasks[idx].position for idx in task_ids])770 discovered_mask = point_segment_distances(task_positions) <= per771 for idx in np.asarray(task_ids, dtype=np.int64)[discovered_mask]:772 task = self.tasks[int(idx)]773 task.discovered = True774 task.discovered_step = self.step_count775 discovered_tasks.append(int(idx))776777 danger_ids = [778 idx for idx, dp in enumerate(self.danger_points) if not dp["discovered"]779 ]780 if danger_ids:781 danger_positions = np.stack(782 [self.danger_points[idx]["position"] for idx in danger_ids]783 )784 discovered_mask = point_segment_distances(danger_positions) <= per785 for idx in np.asarray(danger_ids, dtype=np.int64)[discovered_mask]:786 self.danger_points[int(idx)]["discovered"] = True787 discovered_dp += 1788 return discovered_tasks, discovered_dp789790 def move_along_path(self, path_pts, start_pos, max_dist):791 # path_pts: 从 start 到 goal 的经过点列表792 # start_pos: 当前的位置793 # 返回 new_pos (ndarray) 和 remaining_path (位置列表)794 pos = np.array(start_pos, dtype=float)795 rem = []796 # 保证路径从当前位置开始797 pts = [tuple(pos)] + [tuple(p) for p in path_pts[1:]] if len(path_pts) > 0 else [tuple(pos)]798 dist_left = float(max_dist)799 i = 0800 while dist_left > 1e-8 and i < len(pts)-1:801 a = np.array(pts[i], dtype=float)802 b = np.array(pts[i+1], dtype=float)803 seg_len = np.linalg.norm(b - a)804 if seg_len <= 1e-8:805 i += 1806 continue807 need = min(seg_len - np.linalg.norm(pos - a), dist_left)808 # 如果位置不在线段开头，计算剩下在线段上还有多长809 to_end = np.linalg.norm(b - pos)810 move = min(to_end, dist_left)811 if move >= to_end - 1e-8:812 # 到达 b813 # reach b814 pos = b815 dist_left -= to_end816 i += 1817 else:818 # 按比例往 b 移动819 direction = (b - pos) / np.linalg.norm(b - pos)820 pos = pos + direction * move821 dist_left = 0.0822 break823 # 剩下的路径：从当前位置开始，然后是原路径中 i 之后的点824 remaining = [tuple(pos)]825 for j in range(i+1, len(pts)):826 remaining.append(tuple(pts[j]))827 return pos, remaining828829830 def step(self, actions):831 '''832 参数833 -------834 actions : List[int] 长度 = self.agent_num 835 每个元素为智能体选中的任务 ID 836 动作语义：0..(task_num-1) 对应任务 ID，task_num 表示 Idle（空闲，不选择任何任务）。837 为兼容旧版，也接受 -1 表示 Idle。838 返回839 -------840 observations : np.ndarray [N_agents, obs_dim]841 rewards : np.ndarray [N_agents]842 dones : np.ndarray [N_agents] (全部任务完成或到 max_steps)843 info : dict {'agent_cap':..., 'task_reqs':...}844 '''845 from datetime import datetime846 d = d0 = datetime.now().timestamp()847 self.step_count += 1848 step_reconfiguration_decisions = int(np.count_nonzero(self._current_decision_mask > 0.5))849 self.total_reconfiguration_decisions += step_reconfiguration_decisions850 self.total_alive_agent_steps += int(sum(ag.living for ag in self.agents))851852 new_discovered = set()853854 self.t = np.array([0, 0, 0, 0, 0], float)855856 # -------------------------------------------------857 # 1) 联盟决策：只能选 active_task_ids858 # -------------------------------------------------859 for i, raw_a in enumerate(actions):860 # 兼容 raw_a 是 tensor 或普通 int 的情况861 try:862 a = int(raw_a.item())863 except Exception:864 a = int(raw_a)865866 # 约定：动作索引在 [0, task_num]，其中 task_num 表示 Idle；也接受 -1 表示 Idle867 if a < 0 or a >= self.task_num:868 # 非法动作，忽略869 continue870871 self.agents[i].act(a)872 # 若选择非法任务 ID，忽略，保持原状态873874 step_coalition_changes = int(sum(ag.coalition_changed for ag in self.agents))875 self.total_coalition_changes += step_coalition_changes876877 # ---- 新增：记录任务首次响应时间 ----878 for task in self.tasks:879 if task.type != TaskType.Deliver:880 continue881 if not task.discovered or task.completed:882 continue883 if task.discovered_step is None:884 continue885 if task.first_assigned_step is not None:886 continue887888 assigned = any(889 ag.living and ag.coalition == task.id890 for ag in self.agents891 )892 if assigned:893 task.first_assigned_step = self.step_count894 task.response_time = task.first_assigned_step - task.discovered_step895 self.response_times.append(task.response_time)896897 # 重置事件898 self.event_new_task_found = 0899 self.event_task_completed = [0 for i in range(self.agent_num)]900 self.event_any_task_completed = 0901902 task_caps = np.zeros((self.task_num, self.cap_dim))903 coalition_sizes = np.zeros(self.task_num, dtype=int)904 coalitions = [[] for i in range(self.task_num)]905906 # 聚合各任务当前联盟的能力 & 统计联盟规模907 for i, ag in enumerate(self.agents):908 c = ag.coalition909 if c is None or c == -1 or c == self.task_num:910 continue911 task_caps[c] += ag.capbilities912 coalition_sizes[c] += 1913 coalitions[c].append(ag.aid)914915 # -------------------------------916 # 联盟-任务 匹配度惩罚（按任务算，再分摊到联盟内agent）917 # -------------------------------918919 for j, task in enumerate(self.tasks):920 # 不考虑 Idle 任务和补给任务921 if j == self.__class__.idle_task_id or j == self.__class__.resupply_task_id:922 continue923 # 只对 active 的、未完成的任务算匹配度924 if coalition_sizes[j] == 0:925 continue926 if not task.completed:927 continue928929 req = task.req_vec930 if req is None:931 continue932933 cap = task_caps[j]934 min_val = 0.2935 # 避免除零936 denom = np.where((cap > -min_val) & (cap < min_val), min_val, cap)937938 diff = cap - req939 shortage = np.clip(-diff, 0.0, None) # cap < req 的部分940 surplus = np.clip( diff, 0.0, None) # cap > req 的部分941942 # 按平方惩罚，让严重不匹配的惩罚更大943 penalty_j = (944 self.w_short * (float(np.sum(shortage ** 2)) - self.b_short) +945 self.w_surplus * (float(np.sum(surplus ** 2)) - self.b_surplus)946 )947948 if penalty_j <= 0:949 continue950951 # 把惩罚均摊给联盟内所有 agent952 share = penalty_j / float(coalition_sizes[j])953954 share = np.clip(share, -100, 2)955956 # for i in coalitions[j]:957 # rewards[i] -= share958959 d = datetime.now().timestamp()960 self.t[0] = d - d0961 d0 = d962 963 # rewards_by_step[1] = rewards - rewards_by_step[0]964965 # -------------------------------------------------966 # 2) 为无目标非空闲智能体分配目标967 # -------------------------------------------------968969 danger_positions = [dp['position'] for dp in self.danger_points]970971 for i, ag in enumerate(self.agents):972973 # 尊重死者974 if not ag.living:975 continue976977 # 没走完的不分配978 if len(ag.path) != 0:979 continue980981 task = self.tasks[ag.coalition]982983 # 空闲智能体不分配984 if task.type == TaskType.Idle:985 continue986987 # 计算目标点和开始点988 if task.type == TaskType.Explore:989 if self.explore_action_mode == "sector":990 goal = self._explore_sector_goal(ag)991 else:992 # 找到未发现的非探索任务993 cand_tasks = [task for task in self.tasks if (not task.discovered) and (task.type == TaskType.Deliver)]994 if cand_tasks:995 # 欧几里得距离排序996 cand_tasks.sort(key=lambda task: np.linalg.norm(task.position - ag.position))997 goal = tuple(cand_tasks[0].position)998 else:999 # 随机探索路径点1000 goal = tuple((self._rand(2) * self.map_size))10011002 goal = np.array(goal, dtype=float)10031004 else: # elif coalition.type == TaskType.Deliver or coalition.type == TaskType.Resupply:10051006 goal = task.position1007 1008 # 更新路径1009 ag.path = fast_path(ag.position, goal, danger_positions, danger_radius=self.danger_radius, clearance=self.clearance)10101011 # 找不到路径，跳过移动1012 if len(ag.path) == 0:1013 print(f'step {self.step_count}, agent {i}: cannot find path for exploration agent')10141015 # -------------------------------------------------1016 # 3) 沿路径探索、送货、补给1017 # -------------------------------------------------10181019 sent = 01020 completed = 010211022 for i, ag in enumerate(self.agents):10231024 # 尊重死者1025 if not ag.living:1026 continue10271028 # 路径分配失败的不走1029 if len(ag.path) == 0:1030 continue10311032 task = self.tasks[ag.coalition]10331034 # 空闲智能体不动1035 if task.type == TaskType.Idle:1036 continue10371038 # 电量消耗1039 ag.consume()10401041 # 根据 mobility 按当前路径移动1042 start = ag.position1043 new_pos, remaining = self.move_along_path(ag.path, start, ag.mobility)1044 # 沿着线段发现任务和危险点1045 discovered_tasks, discovered_dp = self.discover_along_segment(i, start, new_pos)10461047 # 探索更新1048 if len(discovered_tasks) > 0:1049 ag.discovery = len(discovered_tasks) + 0.1 * discovered_dp1050 self.event_new_task_found = 11051 for task_id in discovered_tasks:1052 new_discovered.add(task_id)10531054 ag.position = np.array(new_pos, dtype=float)10551056 # 更新剩下的路径1057 ag.path = remaining if len(remaining) > 1 else []10581059 # 如果是运输或补给智能体，需要记录距离1060 if task.type in [TaskType.Deliver, TaskType.Resupply]:1061 dist = np.linalg.norm(ag.position - task.position)1062 ag.dist = float(dist)10631064 if task.type == TaskType.Explore:1065 if len(discovered_tasks) > 0:#ag.unchanged_time >= 10:1066 ag.current_task_completed = True10671068 # 如果运输智能体到达任务点，执行交付1069 elif task.type == TaskType.Deliver:1070 if not np.any(np.isnan(task.position)) and ag.dist < 1e-3:1071 ag.complete()1072 send_amount = ag.deliver(task.demand)1073 sent += send_amount1074 task.demand -= send_amount1075 for i in coalitions[task.id]:1076 self.agents[i].coalition_delivery += ag.delivery10771078 if task.demand <= 0:1079 task.completed = True1080 completed += 11081 1082 for i in coalitions[task.id]:1083 self.event_task_completed[i] = 11084 self.agents[i].coalition_completed = True1085 self.agents[i].complete()10861087 self.event_any_task_completed = 110881089 # 如果补给智能体到达补给点，执行补给1090 elif task.type == TaskType.Resupply:1091 if not np.any(np.isnan(task.position)) and ag.dist < 1e-3:1092 ag.complete()1093 ag.resupply()10941095 d = datetime.now().timestamp()1096 self.t[3] = d - d01097 d0 = d10981099 if len(new_discovered) > 0:1100 for ag in self.agents:1101 if self.tasks[ag.coalition].type == TaskType.Explore:1102 ag.path = []11031104 # -------------------------------------------------1105 # 4) step 结束奖惩1106 # -------------------------------------------------1107 # for i, ag in enumerate(self.agents):1108 # if ag['living']:1109 # # 时间惩罚1110 # rewards[i] -= self.pun_time_ratio * (self.step_count / self.max_steps) # 时间惩罚1111 1112 # -------------------------------------------------1113 # 3) 生成观测1114 # -------------------------------------------------1115 observations = self._get_observations()11161117 # -------------------------------------------------1118 # 4) 终止条件1119 # -------------------------------------------------1120 1121 key_event = (self.event_new_task_found == 1 or1122 self.event_any_task_completed == 1)1123 1124 all_task_completed = all(task.completed for task in self.tasks if task.type == TaskType.Deliver)1125 all_still = all([not ag.living or ag.idle for ag in self.agents])11261127 all_agents_dead = all(not ag.living for ag in self.agents)1128 natural_done = all_task_completed or all_agents_dead1129 time_limit = self.step_count >= self.max_steps11301131 episode_done = natural_done or time_limit1132 dones = np.array([episode_done or not ag.living for ag in self.agents])1133 bad_mask = np.array([0.0 if time_limit and not natural_done else 1.0 for ag in self.agents])1134 bad_mask = bad_mask[..., None]11351136 if episode_done:1137 active_mask = np.ones((len(self.agents),), dtype=np.float32)1138 else:1139 active_mask = self._get_reconfiguration_active_mask(key_event, all_still)1140 active_mask = active_mask[..., None]1141 self._current_decision_mask = active_mask[:, 0].copy()11421143 available_actions = self._get_available_actions()11441145 d = datetime.now().timestamp()1146 self.t[4] = d - d01147 d0 = d11481149 # -------------------------------------------------1150 # rewards1151 # -------------------------------------------------11521153 if self.event_any_task_completed == 1:1154 for ag in self.agents:1155 ag.any_task_completed = True11561157 for i, task in enumerate(self.tasks):1158 task.calc_req_ratio(task_caps[i,:])11591160 rewards_ret, rewards_info = self._get_rewards(sent, completed, new_discovered, all_task_completed, all_still)1161 if self.reconfiguration_switch_cost > 0.0:1162 switched = np.array(1163 [1164 ag.coalition_changed1165 and not ag.last_state.idle1166 and not self.tasks[int(ag.last_state.coalition)].completed1167 for ag in self.agents1168 ],1169 dtype=np.float32,1170 )1171 penalty = self.reconfiguration_switch_cost * switched1172 rewards_ret[:, 0] -= penalty1173 rewards_info[self.reward_keys.index("switch_coalition")] -= penalty11741175 if key_event:1176 pass11771178 # -------------------------------------------------1179 # 5) info1180 # -------------------------------------------------1181 allocation_diagnostics = self._allocation_diagnostics()1182 info = {1183 'agents': [{1184 'position': ag.position,1185 'coalition': ag.coalition,1186 'living': ag.living,1187 'capabilities': ag.capbilities,1188 'load': ag.load,1189 'unchanged': ag.unchanged_time,1190 'idle_wait_steps': ag.idle_wait_steps,1191 } for ag in self.agents],1192 'tasks': [{1193 'position': task.position,1194 'req_vec': task.req_vec,1195 'demand': task.demand,1196 'urgency': task.urgency,1197 'initial_demand': task.initial_demand,1198 'type': int(task.type),1199 'discovered': task.discovered,1200 'completed': task.completed1201 } for task in self.tasks],1202 'danger_points': self.danger_points,1203 't': self.t,1204 'rewards': rewards_info,1205 'reward_keys': self.reward_keys,1206 'sent': sent,1207 'sent_goal': self.sent_goal,1208 'agent_caps': np.stack([ag.capbilities for ag in self.agents]),1209 'task_reqs': np.stack([task.req_vec for task in self.tasks]),1210 'allocation_diagnostics': allocation_diagnostics,1211 'reconfiguration': {1212 'mode': self.reconfiguration_mode,1213 'decisions': step_reconfiguration_decisions,1214 'coalition_changes': step_coalition_changes,1215 'total_decisions': self.total_reconfiguration_decisions,1216 'total_coalition_changes': self.total_coalition_changes,1217 'total_alive_agent_steps': self.total_alive_agent_steps,1218 'urgency_pressure': self._reconfiguration_urgency_pressure(),1219 'mean_response_time': float(np.mean(self.response_times))1220 if self.response_times else 0.0,1221 },1222 'end': self.step_count + 1 if episode_done else 01223 }12241225 for ag in self.agents:1226 if ag.living and self.tasks[int(ag.coalition)].type == TaskType.Idle:1227 ag.idle_wait_steps += 11228 else:1229 ag.idle_wait_steps = 01230 ag.step()123112321233 return observations, rewards_ret, dones, info, active_mask, bad_mask, available_actions123412351236 def _task_feature_matrix(self, include_flags=False, local_visible=False):1237 extra_dim = 2 if include_flags else 01238 features = np.zeros((self.task_num, self.per_task_feat + extra_dim), dtype=np.float32)1239 for idx, task in enumerate(self.tasks):1240 if local_visible and (not task.discovered or task.completed):1241 continue1242 pos = task.position1243 if np.isnan(pos[0]) or np.isnan(pos[1]):1244 pos = np.array([0.0, 0.0])1245 features[idx, 0:2] = pos / self.map_size1246 features[idx, 2:2 + self.cap_dim] = task.req_vec / 5.01247 features[idx, 2 + self.cap_dim] = task.demand / 500.01248 features[idx, 3 + self.cap_dim] = float(task.type) / float(max(TaskType))1249 if include_flags:1250 features[idx, self.per_task_feat] = 1.0 if task.discovered else 0.01251 features[idx, self.per_task_feat + 1] = 1.0 if task.completed else 0.01252 return features12531254 def _agent_feature_matrix(self):1255 features = np.zeros((self.agent_num, self.other_agent_feat_dim), dtype=np.float32)1256 max_coalition = float(max(1, self.task_num - 1))1257 max_agent_type = float(max(AgentType))1258 for idx, ag in enumerate(self.agents):1259 features[idx, 0:2] = ag.position / self.map_size1260 features[idx, 2:2 + self.cap_dim] = ag.capbilities1261 features[idx, 2 + self.cap_dim] = float(ag.load) / float(ag.init_load)1262 features[idx, 3 + self.cap_dim] = ag.battery / 1000.01263 features[idx, 4 + self.cap_dim] = float(ag.coalition) / max_coalition1264 features[idx, 5 + self.cap_dim] = float(ag.type) / max_agent_type1265 features[idx, 6 + self.cap_dim] = 1.0 if ag.living else 0.01266 return features12671268 def _get_observations(self):1269 """Return per-agent local observations grouped as ego/tasks/others."""1270 task_features = self._task_feature_matrix(local_visible=True)1271 task_obs = np.repeat(task_features[None, :, :], self.agent_num, axis=0)12721273 agent_features = self._agent_feature_matrix()1274 other_obs = np.repeat(agent_features[None, :, :], self.agent_num, axis=0)1275 self_indices = np.arange(self.agent_num)1276 other_obs[self_indices, self_indices, :] = 0.012771278 discovered_deliver = sum(1279 1 for task in self.tasks1280 if task.discovered and task.type == TaskType.Deliver1281 )1282 discovered_ratio = float(discovered_deliver) / float(self.delivery_task_num)1283 max_coalition = float(max(1, self.task_num - 1))1284 max_task_type = float(max(TaskType))1285 ego_obs = np.zeros((self.agent_num, self.ego_feat_dim), dtype=np.float32)1286 for idx, ag in enumerate(self.agents):1287 coalition = self.tasks[ag.coalition]1288 ego_obs[idx, 0:2] = ag.position / self.map_size1289 ego_obs[idx, 2:2 + self.cap_dim] = ag.capbilities1290 ego_obs[idx, 2 + self.cap_dim] = float(ag.coalition) / max_coalition1291 ego_obs[idx, 3 + self.cap_dim] = float(coalition.type) / max_task_type1292 ego_obs[idx, 4 + self.cap_dim] = float(ag.load) / float(ag.init_load)1293 ego_obs[idx, 5 + self.cap_dim] = discovered_ratio1294 ego_obs[idx, 6 + self.cap_dim] = ag.battery / 1000.012951296 return {1297 "ego": ego_obs,1298 "tasks": task_obs.astype(np.float32, copy=False),1299 "others": other_obs.astype(np.float32, copy=False),1300 }13011302 def get_global_state(self):1303 """Return the privileged centralized state used only by the critic."""1304 return {1305 "agents": self._agent_feature_matrix(),1306 "tasks": self._task_feature_matrix(include_flags=True),1307 "global": np.array([self.step_count / self.max_steps], dtype=np.float32),1308 }13091310 def _get_rewards(self, sent, completed, new_discovered, all_task_completed, all_still):1311 any_explore = any(ag.coalition == EnvCore.explore_task_id and ag.living for ag in self.agents)1312 # explore_decay = max(0.1, 1 - self.step_count / (0.5 * self.max_steps))1313 discovered_ratio = float([task.discovered and task.type == TaskType.Deliver for task in self.tasks].count(True)) / self.delivery_task_num1314 ep_check_1 = 2001315 ep_check_2 = 6001316 # if self.episode < ep_check_1:1317 # ep_decay = 0.01318 # elif self.episode < ep_check_2:1319 # ep_decay = float(self.episode - ep_check_1) / (ep_check_2 - ep_check_1)1320 # else:1321 # ep_decay = 1.01322 ep_decay = 1.013231324 step_decay = self.step_count / self.max_steps1325 rewards = [{key: 0.0 for key in self.reward_keys} for _ in range(self.agent_num)]1326 rewards_1 = [{key: 0.0 for key in self.reward_keys} for _ in range(self.agent_num)]1327 rewards_2 = [{key: 0.0 for key in self.reward_keys} for _ in range(self.agent_num)]132813291330 for i, ag in enumerate(self.agents):13311332 if not ag.dead and not ag.living:1333 ag.coalition = self.idle_task_id1334 continue13351336 coalition_type = self.tasks[ag.coalition].type1337 last_coalition_type = self.tasks[ag.last_state.coalition].type1338 # 切换联盟，且上回合不空闲，惩罚1339 if ag.coalition_changed and not ag.last_state.idle:1340 # 切换到探索（第一回合不罚）1341 if self.step_count > 0 and coalition_type == TaskType.Explore:1342 # 从运输、补给切换到探索，罚1343 if last_coalition_type in [TaskType.Deliver, TaskType.Resupply]:1344 rewards_2[i]['switch_coalition'] -= 1.013451346 # 从运输或补给切换1347 if last_coalition_type in [TaskType.Deliver, TaskType.Resupply]:1348 rewards_2[i]['switch_coalition'] -= 1.513491350 if ag.coalition_changed and ag.last_state.idle:1351 load_ratio = ag.load / int(ag.init_load)1352 rewards_2[i]['switch_coalition'] += (-1.0 if coalition_type == TaskType.Resupply else 1.0) \1353 * np.sinh((load_ratio - 0.5) * 2) / np.sinh(1.0) * 2.01354 # if coalition_type == TaskType.Resupply:1355 # if load_ratio < 0.4:1356 # rewards_2[i]['switch_coalition'] += ((0.4 - load_ratio) ** 2) * 513571358 # else:1359 # rewards_2[i]['switch_coalition'] += load_ratio - 0.513601361 # 死亡惩罚1362 if ag.dead:1363 rewards_2[i]['dead'] -= 5.01364 1365 # 探索奖励1366 rewards_2[i]['discover'] = ag.discovery * (0.1/(discovered_ratio + 0.1) if discovered_ratio<0.25 else 0.1/0.35)1367 rewards_2[i]['discover'] -= 0.2 if not any_explore and discovered_ratio <= 0.9 else 0.013681369 # 运输奖励1370 rewards_2[i]['deliver'] = ag.delivery * 0.11371 1372 # 补给奖励1373 rewards_2[i]['resupply'] += ag.resupply_ratio * 2.0 - 1.0 if ag.resupplied else 0.013741375 # 联盟运输奖励1376 # rewards_2[i]['coalition_deliver'] = ag.coalition_delivery * 0.0713771378 # 联盟完成任务奖励1379 # rewards_2[i]['coalition_complete'] += ag.coalition_completed * 4.013801381 base = 20.01382 time_coef = 20.01383 completion_reward = (base + time_coef * (1.0 - self.step_count / self.max_steps)) if ag.coalition_completed else 013841385 if all_task_completed:1386 rewards_2[i]['coalition_complete'] += (1 - self.step_count / self.max_steps) * 20 + 2013871388 # 距离奖励1389 if not ag.coalition_changed \1390 and coalition_type != TaskType.Idle \1391 and coalition_type != TaskType.Explore:1392 unchanged_decay = (ag.unchanged_time if ag.unchanged_time < 5.0 else 5.0) / 5.01393 rewards_2[i]['dist'] = 0.002 * (ag.last_state.dist - ag.dist)13941395 # 时间惩罚1396 rewards_2[i]['dist'] -= 0.213971398 if all_still:1399 rewards_2[i]['dist'] -= 1.014001401 # 回合结束平均奖励1402 rewards_2[i]['end'] += sent / self.sent_goal * 101403 rewards_2[i]['end'] += completed / self.delivery_task_num * 501404 rewards_2[i]['end'] /= 2.014051406 for i in range(self.agent_num):1407 for key in self.reward_keys:1408 rewards[i][key] = (1 - ep_decay) * rewards_1[i][key] + ep_decay * rewards_2[i][key]14091410 rewards_ret = np.zeros(self.agent_num, dtype=float)1411 rewards_info = np.zeros((len(self.reward_keys), self.agent_num), dtype=float)14121413 for i, rewards_ag in enumerate(rewards):1414 for key, rw in rewards_ag.items():1415 rewards_ret[i] += rw1416 rewards_info[self.reward_keys.index(key)][i] = rw14171418 rewards_ret = rewards_ret[..., None]14191420 # rewards_info = np.concatenate([rewards_info, np.array([ag.coalition for ag in self.agents]).reshape(1, -1)])14211422 return rewards_ret, rewards_info