env_core.py

当前主实验物理过程、观测、event、覆盖与奖励

原文件 · 2389 行 · 110706 字节

envs/delivcoal/env_core.py
SHA-256: 0012ff2bfea81808b50384d8f915c9c251ad63f81c5f60b290d0e2daca0d9543

1import copy2import numpy as np3import pickle4import os5from collections import deque6import yaml78eps = 1e-6910from enum import IntEnum1112class AgentType(IntEnum):13    Drone = 014    Vehicle = 11516class TaskType(IntEnum):17    Explore = 018    Deliver = 119    Idle = 220    Resupply = 32122def dist_point_segment(p, a, b):23    # 点到线段的最短距离；用于判断路径线段是否穿过感知范围或危险圆。24    p = np.array(p, dtype=float)25    a = np.array(a, dtype=float)26    b = np.array(b, dtype=float)27    ab = b - a28    if np.allclose(ab, 0):29        return np.linalg.norm(p - a)30    t = np.dot(p - a, ab) / np.dot(ab, ab)31    t = np.clip(t, 0.0, 1.0)32    proj = a + t * ab33    return np.linalg.norm(p - proj)3435def fast_path(start, goal, danger_points=None, danger_radius=1.0, clearance=0.5, max_iter=8):36    '''连续空间近似路径规划器。3738    先尝试直线到达目标；如果路径线段穿过危险圆，则在障碍物附近插入垂直偏移路点绕行。39    返回值包含起点和终点的路点列表。40    '''41    if danger_points is None:42        danger_points = []4344    def segment_intersects_circle(a, b, c, r):45        return dist_point_segment(c, a, b) <= r4647    s = tuple(map(float, start))48    g = tuple(map(float, goal))49    if np.allclose(s, g):50        return [s]5152    waypoints = [s, g]53    iter_count = 054    # 逐段检查路径；某段碰到危险圆时，就在该段中插入一个绕行点。55    while iter_count < max_iter:56        changed = False57        new_waypoints = [waypoints[0]]58        for i in range(len(waypoints)-1):59            a = waypoints[i]60            b = waypoints[i+1]61            # 只处理第一个相交障碍，插入路点后下一轮再继续检查。62            hit = None63            for c in danger_points:64                if segment_intersects_circle(a, b, c, danger_radius + clearance):65                    hit = c66                    break67            if hit is None:68                new_waypoints.append(b)69                continue70            # 绕行点取“障碍物到线段最近点”的垂直方向偏移。71            a_np = np.array(a); b_np = np.array(b); c_np = np.array(hit)72            ab = b_np - a_np73            if np.allclose(ab, 0):74                # 线段退化时没有稳定垂线方向，直接使用一个固定侧向偏移。75                dir_perp = np.array([1.0, 0.0])76            else:77                t = np.dot(c_np - a_np, ab) / np.dot(ab, ab)78                t = np.clip(t, 0.0, 1.0)79                closest = a_np + t * ab80                # 正常线段使用垂直方向生成绕行候选点。81                dir_perp = np.array([-ab[1], ab[0]])82                if np.allclose(dir_perp, 0):83                    dir_perp = np.array([1.0, 0.0])84                dir_perp = dir_perp / np.linalg.norm(dir_perp)85            # 通过比较两侧候选点离其他危险圆的距离，选择更安全的一侧。86            # 左右两侧各给一个候选绕行点，选择碰撞风险较小的一侧。87            det_dist = danger_radius + clearance88            cand1 = tuple((closest + dir_perp * det_dist).tolist()) if 'closest' in locals() else tuple((a_np + dir_perp * det_dist).tolist())89            cand2 = tuple((closest - dir_perp * det_dist).tolist()) if 'closest' in locals() else tuple((a_np - dir_perp * det_dist).tolist())90            # 分数越小表示越不容易贴近其他危险圆。91            def collision_score(pt):92                ssum = 0.093                for c2 in danger_points:94                    d = np.linalg.norm(np.array(pt) - np.array(c2)) - danger_radius95                    ssum += max(0.0, (clearance - d))96                return ssum9798            s1 = collision_score(cand1)99            s2 = collision_score(cand2)100            detour = cand1 if s1 <= s2 else cand2101            # 在当前线段终点前插入绕行点。102            new_waypoints.append(detour)103            new_waypoints.append(b)104            changed = True105        waypoints = new_waypoints106        iter_count += 1107        if not changed:108            break109110    # 去掉几乎重合的路点，避免移动时产生零长度线段。111    out = [tuple(map(float, waypoints[0]))]112    for p in waypoints[1:]:113        if np.linalg.norm(np.array(p) - np.array(out[-1])) > eps:114            out.append(tuple(map(float, p)))115    return out116117class AgentLastState:118    """保存上一步智能体状态，奖励函数用它计算切换、距离变化和资源变化。"""119120    __slots__ = ['coalition', 'new_task_found', 'current_task_completed', 'any_task_completed', 'coalition_completed', 'dist', 'load', 'idle']121    122    def __init__(self):123        self.coalition = EnvCore.idle_task_id124        self.new_task_found = False125        self.current_task_completed = False126        self.any_task_completed = False127        self.coalition_completed = False128        self.dist = 0.0129        self.load = 0.0130        self.idle = True131132class Agent:133    """环境内部的智能体实体。134135    capbilities 四维分别驱动感知、移动、载荷和电量上限；每步事件字段在 step 末尾清零。136    """137138    __slots__ = ['aid', 'position', 'capbilities', 'perception', 'mobility', 'load',139                 'battery', 'type', 'living', 'idle', 'coalition', 'path', 'last_state',140                 'dead', 'new_task_found', 'coalition_changed', 'discovery', 'delivery', 'coalition_delivery', 'resupply_ratio', 'init_load', 'resupplied',141                 'current_task_completed', 'any_task_completed', 'coalition_completed', 'dist', 'unchanged_time',142                 'idle_wait_steps']143144    def __init__(self, aid: int, pos: np.ndarray, capbilities: np.ndarray, rng=None):145        self.aid = aid146        self.position = np.array(pos, dtype=float)147        self.capbilities = capbilities148        self.perception = float(capbilities[0] * 50.0)149        self.mobility = float(capbilities[1] * 10) + 10150        self.init_load = int(capbilities[2] * 70) + 30151        self.load = self.init_load152        self.battery = int(capbilities[3] * 500) + 500153        self.type = rng.choice([AgentType.Drone, AgentType.Vehicle]) if rng is not None else np.random.choice([AgentType.Drone, AgentType.Vehicle])154        self.living = True155        self.idle = True156        self.coalition = EnvCore.idle_task_id157        self.path = []158159        self.dead = False160        self.new_task_found = False161        self.current_task_completed = False162        self.any_task_completed = False163        self.coalition_completed = False164        self.coalition_changed = False165        self.discovery = 0.0166        self.delivery = 0.0167        self.coalition_delivery = 0.0168        self.dist = 0.0169        self.unchanged_time = 0.0170        self.idle_wait_steps = 0171172        self.resupply_ratio = 0.0173        self.resupplied = False174175        self.last_state = AgentLastState()176177    def act(self, cid: int):178        # 只有联盟编号变化时才清空路径；保持同一联盟会累积 unchanged_time。179        if cid != self.coalition:180            self.coalition = cid181            self.coalition_changed = True182            self.path = []183            self.unchanged_time = 0.0184        else:185            self.unchanged_time += 1.0186187        if cid == EnvCore.idle_task_id:188            self.idle = True189        else:190            self.idle_wait_steps = 0191192    def consume(self):193        # 当前模型按步扣电，电量耗尽后变为死亡，死亡事件只在本步奖励中使用。194        if self.living:195            self.battery -= 1196            if self.battery == 0:197                self.living = False198                self.dead = True199200    def deliver(self, demand):201        # 交付量受当前载荷和任务剩余需求共同限制；无货交付会给负事件信号。202        delivery = np.min([self.load, demand])203        self.load -= delivery204        self.delivery = delivery if delivery > 0 else -10205        self.idle = True206207        if self.delivery < 0:208            pass209            210        return delivery211212    def complete(self):213        # 当前智能体完成自己负责的当前动作，runner 会据此决定是否再次激活该 agent。214        self.current_task_completed = True215        self.idle = True216217    def resupply(self):218        # 补给恢复电量和载荷，并记录本次补给比例用于奖励。219        self.living = True220        self.dead = False221        self.idle = True222        self.battery = int(self.capbilities[3] * 500) + 500223        self.resupply_ratio = (self.init_load - self.load) / self.init_load224        self.resupplied = True225        self.load = self.init_load226227    def step(self):228        # 先把当前状态拷贝到 last_state，再清空“本步事件”字段。229        # 这样奖励函数可以同时访问上一步状态和本步事件。230        for key in self.last_state.__slots__:231            attr = getattr(self, key)232            setattr(self.last_state, key, attr)233234        self.dead = False235        self.new_task_found = False236        self.current_task_completed = False237        self.any_task_completed = False238        self.coalition_completed = False239        self.coalition_changed = False240        self.discovery = 0.0241        self.delivery = 0.0242        self.coalition_delivery = 0.0243        self.resupply_ratio = 0.0244        self.resupplied = False245        self.idle = False246247248class Task:249    """环境内部任务实体。250251    Deliver 任务初始不可见；Explore/Idle/Resupply 是辅助任务槽，始终可见。252    """253254    __slots__ = [255        'id', 'cap_dim', 'position', 'demand', 'initial_demand', 'urgency',256        'req_vec', 'req_ratio', 'type', 'discovered', 'completed',257        'discovered_step', 'first_assigned_step', 'response_time'258    ]259260    def __init__(self, cid: int, pos: np.ndarray, demand: int, urgency: float, cap_dim: int, rng=None):261        self.id = cid262        self.cap_dim = cap_dim263        self.position = np.array(pos, dtype=float)264        self.demand = demand265        self.initial_demand = demand266        self.urgency = urgency267        self.req_vec = ((rng.rand(cap_dim) if rng is not None else np.random.rand(cap_dim)) * (demand / 100.0)).astype(float)268        self.req_ratio = np.zeros(cap_dim)269        self.type = TaskType.Deliver270        self.discovered = False271        self.completed = False272273        self.discovered_step = None274        self.first_assigned_step = None275        self.response_time = None276277    def calc_req_ratio(self, cap_now):278        # req_ratio 是当前联盟能力相对需求的比例差，只用于 info/诊断，不直接进入局部观测。279        if self.discovered and not self.completed:280            req_vec = self.req_vec281            for dim in range(self.cap_dim):282                if np.isnan(req_vec[dim]) or req_vec[dim] <= 0.0:283                    self.req_ratio[dim] = 0.0284                else:285                    self.req_ratio[dim] = (cap_now[dim] - req_vec[dim]) / req_vec[dim]286        else:287            self.req_ratio = np.zeros(self.cap_dim)288289290291class EnvCore:292    """MACF 环境核心。293294    负责连续地图、智能体/任务状态推进、局部 Dict 观测、全局 critic state、奖励和终止条件。295    Gym wrapper 只负责把这里的返回值包装成向量环境接口。296    """297298    explore_task_id = -1299    idle_task_id = -1300    resupply_task_id = -1301302    def __init__(self, config):303        # scenario_name 决定是否启用 coalition_delivery 专用任务特征和奖励。304        scenario_name = getattr(config, "scenario_name", None)305        if scenario_name in (None, "", "MyEnv", "coadel", "delivcoal"):306            scenario_name = "coalition_delivery"307        self.scenario_name = scenario_name308        self.use_coalition_delivery = self.scenario_name == "coalition_delivery"309        self.coalition_delivery_reward_mode = getattr(310            config,311            "reward_mode",312            getattr(config, "coalition_delivery_reward_mode", "baseline"),313        )314        if self.coalition_delivery_reward_mode in ("v1", "local", "local_progress"):315            self.coalition_delivery_reward_mode = "delivcoal_v1"316        elif self.coalition_delivery_reward_mode in ("v2", "local_shaped", "potential"):317            self.coalition_delivery_reward_mode = "delivcoal_v2"318        elif self.coalition_delivery_reward_mode in ("v3", "potential_small"):319            self.coalition_delivery_reward_mode = "delivcoal_v3"320        elif self.coalition_delivery_reward_mode in ("v4", "potential_medium"):321            self.coalition_delivery_reward_mode = "delivcoal_v4"322        elif self.coalition_delivery_reward_mode in ("v5", "potential_local"):323            self.coalition_delivery_reward_mode = "delivcoal_v5"324        if self.coalition_delivery_reward_mode not in (325            "baseline",326            "shaped",327            "delivcoal_v1",328            "delivcoal_v2",329            "delivcoal_v3",330            "delivcoal_v4",331            "delivcoal_v5",332        ):333            raise ValueError(334                "coalition_delivery_reward_mode must be 'baseline', 'shaped', "335                "'delivcoal_v1', 'delivcoal_v2', 'delivcoal_v3', "336                "'delivcoal_v4', or 'delivcoal_v5', "337                f"got {self.coalition_delivery_reward_mode!r}"338            )339340        deliver_target_num = config.target_num341        # 探索动作可以是单一 nearest，也可以拆成多个 sector/survey 动作槽。342        self.explore_action_mode = getattr(config, "explore_action_mode", "nearest")343        if self.explore_action_mode not in ("nearest", "sector", "survey"):344            raise ValueError(f"Unsupported explore_action_mode: {self.explore_action_mode}")345        self.num_explore_actions = (346            int(getattr(config, "num_explore_actions", 1))347            if self.explore_action_mode in ("sector", "survey")348            else 1349        )350        if self.num_explore_actions <= 0:351            raise ValueError("num_explore_actions must be positive.")352        self._configured_explore_step_size = getattr(config, "explore_step_size", None)353        self.explore_random_jitter = float(getattr(config, "explore_random_jitter", 0.0))354        self.sector_redecision_on_waypoint = bool(355            getattr(config, "sector_redecision_on_waypoint", True)356        )357        self.explore_event_redecision = bool(358            getattr(config, "explore_event_redecision", True)359        )360        self.sector_min_leg_mobility_steps = float(361            getattr(config, "sector_min_leg_mobility_steps", 1.0)362        )363        if self.sector_min_leg_mobility_steps < 0.0:364            raise ValueError("sector_min_leg_mobility_steps must be non-negative.")365        self.delivery_tasks_initially_discovered = bool(366            getattr(config, "delivery_tasks_initially_discovered", False)367        )368        self.mask_explore_actions = bool(369            getattr(config, "mask_explore_actions", False)370        )371        self.idle_time_cost_mode = str(372            getattr(config, "idle_time_cost_mode", "legacy")373        )374        if self.idle_time_cost_mode not in ("legacy", "uniform"):375            raise ValueError(376                "idle_time_cost_mode must be 'legacy' or 'uniform', got "377                f"{self.idle_time_cost_mode!r}"378            )379        self.explore_coverage_features = bool(380            getattr(config, "explore_coverage_features", False)381        )382        self.explore_observer_relative_geometry = bool(383            getattr(config, "explore_observer_relative_geometry", False)384        )385        self.explore_coverage_grid_resolution = int(386            getattr(config, "explore_coverage_grid_resolution", 20)387        )388        if self.explore_coverage_grid_resolution <= 0:389            raise ValueError(390                "explore_coverage_grid_resolution must be positive."391            )392        self.explore_task_start = deliver_target_num393        self.explore_task_end = deliver_target_num + self.num_explore_actions394        # 任务槽顺序：真实配送任务 [0, target_num)，探索槽，Idle 槽，Resupply 槽。395        self.__class__.explore_task_id = self.explore_task_start396        self.__class__.idle_task_id = self.explore_task_end397        self.__class__.resupply_task_id = self.explore_task_end + 1398        # 地图是连续二维空间，危险区域用圆心 + 半径表示。399        self.map_size = float(config.map_size)             # 连续地图边长 / m400        coverage_axis = (401            np.arange(self.explore_coverage_grid_resolution, dtype=np.float64)402            + 0.5403        ) * (self.map_size / self.explore_coverage_grid_resolution)404        coverage_x, coverage_y = np.meshgrid(405            coverage_axis, coverage_axis, indexing="xy"406        )407        self._coverage_points = np.stack(408            [coverage_x.reshape(-1), coverage_y.reshape(-1)], axis=-1409        )410        self._explored_cells = np.zeros(411            len(self._coverage_points), dtype=bool412        )413        self.explore_step_size = (414            float(self._configured_explore_step_size)415            if self._configured_explore_step_size is not None416            else self.map_size * 0.25417        )418        # 危险圆几何参数；clearance 是绕行路径额外安全距离。419        self.danger_radius = float(getattr(config, 'danger_radius', 1.0))420        self.clearance = float(getattr(config, 'clearance', 0.5))421        # 仍保留 UAV/UGV 数量字段；总智能体数由二者相加得到。422        self.uav_num = config.num_uav               # 无人机数量423        self.ugv_num = config.num_ugv               # 无人车数量424        self.uav_perception_rg = tuple(config.uav_perception_rg[0:2])425        self.ugv_perception_rg = tuple(config.ugv_perception_rg[0:2])426        self.delivery_task_num = config.target_num427        self.task_num = config.target_num + self.num_explore_actions + 2428        self.gat_dim = config.gat_dim               # GAT 输出维度429        self.max_steps = config.max_steps           # 最大步数430        configured_time_limit = getattr(config, "episode_time_limit", None)431        self.episode_time_limit = (432            self.max_steps433            if configured_time_limit is None434            else int(configured_time_limit)435        )436        if self.episode_time_limit <= 0:437            raise ValueError("episode_time_limit must be positive or null.")438        self.cap_dim = config.cap_dim               # 能力维度439        self.reconfiguration_mode = str(getattr(config, "reconfiguration_mode", "legacy"))440        valid_reconfiguration_modes = {"legacy", "always", "periodic", "event", "urgency", "learned"}441        if self.reconfiguration_mode not in valid_reconfiguration_modes:442            raise ValueError(f"Unsupported reconfiguration_mode: {self.reconfiguration_mode!r}")443        self.reconfiguration_interval = int(getattr(config, "reconfiguration_interval", 10))444        if self.reconfiguration_interval <= 0:445            raise ValueError("reconfiguration_interval must be positive.")446        self.reconfiguration_urgency_threshold = float(447            getattr(config, "reconfiguration_urgency_threshold", 0.7)448        )449        self.reconfiguration_urgency_aggregation = str(450            getattr(config, "reconfiguration_urgency_aggregation", "mean")451        )452        if self.reconfiguration_urgency_aggregation not in ("mean", "sum"):453            raise ValueError("reconfiguration_urgency_aggregation must be 'mean' or 'sum'.")454        self.reconfiguration_switch_cost = float(455            getattr(config, "reconfiguration_switch_cost", 0.0)456        )457        if self.reconfiguration_switch_cost < 0.0:458            raise ValueError("reconfiguration_switch_cost must be non-negative.")459        self.reconfiguration_context_features = bool(460            getattr(config, "reconfiguration_context_features", False)461        )462        self.reconfiguration_context_dim = int(463            getattr(config, "reconfiguration_context_dim", 9)464        )465        # 9 维 context 是最初暴露给 learned gate 的事件/时间特征；466        # 11 维在此基础上增加 all_still 和 mandatory，用于 hierarchical gate467        # 区分“可以选择是否重配”和“环境语义要求必须重新决策”。468        if self.reconfiguration_context_features and self.reconfiguration_context_dim not in (9, 11):469            raise ValueError("delivcoal reconfiguration_context_dim must be 9 or 11.")470471        self.reward_keys = ['switch_coalition', 'dead', 'discover', 'deliver', 'resupply', 'coalition_deliver', 'coalition_complete', 'dist', 'end']472473        # 总智能体数，后续 obs/buffer/policy 都以该值为 agent 维度。474        self.agent_num = self.uav_num + self.ugv_num475476        self.step_count = 0                 # 当前步数477        # ego 观测：位置(2)、能力(cap_dim)、当前联盟编号、当前联盟类型、478        # 载荷比例、已发现配送任务比例、电量比例。479        self.ego_feat_dim = self.cap_dim + 7480        if self.use_coalition_delivery:481            self.coalition_task_feature_mode = str(482                getattr(config, "coalition_task_feature_mode", "compact")483            )484            if self.coalition_task_feature_mode not in ("compact", "full"):485                raise ValueError(486                    "coalition_task_feature_mode must be 'compact' or 'full', got "487                    f"{self.coalition_task_feature_mode!r}"488                )489            if self.coalition_task_feature_mode == "full":490                self.per_task_feat = 2 + self.cap_dim + 1 + 1 + 3 + 1491            else:492                self.per_task_feat = 2 + 1 + 1 + 1 + 1493        else:494            self.coalition_task_feature_mode = "full"495            # 默认任务观测：位置、能力需求、剩余需求、任务类型。496            self.per_task_feat = 2 + self.cap_dim + 1 + 1497        # other 观测：位置、能力、载荷、电量、联盟编号、智能体类型、存活标志。498        self.other_agent_feat_dim = 2 + self.cap_dim + 1 + 1 + 1 + 1 + 1499        self.global_agent_feat_dim = self.other_agent_feat_dim500        # critic 的全局任务状态额外暴露 discovered/completed 标志。501        self.global_task_feat_dim = self.per_task_feat + 2502        self.global_feat_dim = 1503504        # flatten 后的 actor 观测维度；Dict 观测实际仍按 ego/tasks/others 三个 key 返回。505        self.obs_dim = (506            self.ego_feat_dim507            + self.task_num * self.per_task_feat508            + self.agent_num * self.other_agent_feat_dim509        )510511        # 动作维度等于任务槽数。Idle/Resupply/Explore 都已经是任务槽，不再额外增加动作。512        self.action_dim = self.task_num513514        # 事件标志只描述当前 step 内发生的关键事件，奖励和 active_mask 会读取它们。515        self.event_new_task_found       = 0516        self.event_task_completed       = [0 for i in range(self.agent_num)]517        self.event_any_task_completed   = 0518519        self.data_dir = 'data'520521        if config.rewards_config_file:522            with open(config.rewards_config_file, 'r', encoding='utf-8') as f:523                rewards_config = yaml.safe_load(f) or {}524        525            for key, value in rewards_config.items():526                setattr(self, key, value)527528        else:529530            self.pun_change_coalition_event = 0.001531            self.pun_change_coalition = 1532            self.w_short = 2.0   # 能力不足权重（重罚）533            self.w_surplus = 0.3 # 能力过多权重（轻罚）534            self.b_short = 8.0   # 能力不足权重（重罚）535            self.b_surplus = 20.0 # 能力过多权重（轻罚）536            self.rwd_ratio_discover_task = 5537            self.rwd_ratio_discover_dp = 1.0538            self.pun_ag_dead = 10539            self.rwd_ratio_send = 1540            self.rwd_coalition_ratio_send = 0.2541            self.rwd_task_completed  = 200542            self.rwd_coalition_task_completed = 50543            self.rwd_dist_ratio = 3.0544            self.pun_time_ratio = 0.1545546        # legacy GAT 的全局图状态占位；当前 HATT/MAPPO 主要使用 Dict state。547        self.global_state = np.zeros((self.agent_num, self.task_num, self.gat_dim))548549        # reset 相关缓存：jitter 模式复用基础场景，只扰动配送任务位置。550        self._seed = None551        self.rng = None552        self._base_initialized = False553        self._base_danger_points = None554        self._base_tasks = None555        self._base_agents = None556        self.task_position_jitter = float(getattr(config, 'task_position_jitter', 0.05))557        self.env_reset_mode = getattr(config, 'env_reset_mode', 'jitter')558559    def seed(self, seed):560        # 设置种子后清空基础场景缓存，保证下一次 reset 从该种子重新生成。561        self._seed = int(seed)562        self.rng = np.random.RandomState(self._seed)563        self._base_initialized = False564        self._base_danger_points = None565        self._base_tasks = None566        self._base_agents = None567        return [self._seed]568569    def _rand(self, *args):570        if self.rng is not None:571            return self.rng.rand(*args)572        return np.random.rand(*args)573574    def _randint(self, *args, **kwargs):575        if self.rng is not None:576            return self.rng.randint(*args, **kwargs)577        return np.random.randint(*args, **kwargs)578579    def _choice(self, a):580        if self.rng is not None:581            return self.rng.choice(a)582        return np.random.choice(a)583584    def _sector_min_leg_distance(self, ag):585        return min(586            self.explore_step_size,587            max(0.0, ag.mobility * self.sector_min_leg_mobility_steps),588        )589590    def _explore_sector_nominal_goal(self, ag, sector):591        angle, _ = self._explore_sector_angle(sector)592        direction = np.array([np.cos(angle), np.sin(angle)], dtype=float)593        return np.clip(594            ag.position + direction * self.explore_step_size,595            0.0,596            self.map_size,597        )598599    def _explore_sector_goal(self, ag):600        # sector 模式下，探索任务不是固定点，而是从当前位置朝某个扇区方向推进。601        sector = int(ag.coalition - self.explore_task_start)602        sector = max(0, min(self.num_explore_actions - 1, sector))603        angle, sector_width = self._explore_sector_angle(sector)604        if self.explore_random_jitter > 0.0:605            jitter = (float(self._rand()) * 2.0 - 1.0) * self.explore_random_jitter * sector_width606            angle += jitter607        direction = np.array([np.cos(angle), np.sin(angle)], dtype=float)608        goal = ag.position + direction * self.explore_step_size609        goal = np.clip(goal, 0.0, self.map_size)610611        # Jitter may turn a nominally valid direction into a very short clipped612        # boundary leg. Fall back to the nominal sector center in that case.613        min_leg_distance = self._sector_min_leg_distance(ag)614        if np.linalg.norm(goal - ag.position) + eps < min_leg_distance:615            nominal_goal = self._explore_sector_nominal_goal(ag, sector)616            if np.linalg.norm(nominal_goal - ag.position) + eps >= min_leg_distance:617                goal = nominal_goal618        return goal619620    def _explore_sector_angle(self, sector):621        sector_width = 2.0 * np.pi / float(self.num_explore_actions)622        return sector * sector_width, sector_width623624    def _explore_sector_anchor(self, sector):625        angle, _ = self._explore_sector_angle(sector)626        direction = np.array([np.cos(angle), np.sin(angle)], dtype=float)627        center = np.array([0.5 * self.map_size, 0.5 * self.map_size], dtype=float)628        return np.clip(center + direction * (0.45 * self.map_size), 0.0, self.map_size)629630    @staticmethod631    def _point_segment_distances(points, a, b):632        points = np.asarray(points, dtype=np.float64)633        if len(points) == 0:634            return np.empty((0,), dtype=np.float64)635        a = np.asarray(a, dtype=np.float64)636        b = np.asarray(b, dtype=np.float64)637        segment = b - a638        denominator = float(np.dot(segment, segment))639        if denominator <= eps:640            return np.linalg.norm(points - a[None, :], axis=1)641        position = ((points - a[None, :]) @ segment) / denominator642        position = np.clip(position, 0.0, 1.0)643        projection = a[None, :] + position[:, None] * segment[None, :]644        return np.linalg.norm(points - projection, axis=1)645646    def _mark_explored_segment(self, agent, start, end):647        if not self.explore_coverage_features:648            return649        sensed = self._point_segment_distances(650            self._coverage_points, start, end651        ) <= float(agent.perception)652        self._explored_cells[sensed] = True653654    def _candidate_novel_coverage(self, agent, goal):655        if not self.explore_coverage_features:656            return 0.0657        sensed = self._point_segment_distances(658            self._coverage_points, agent.position, goal659        ) <= float(agent.perception)660        sensed_count = int(np.count_nonzero(sensed))661        if sensed_count <= 0:662            return 0.0663        return float(np.count_nonzero(sensed & ~self._explored_cells) / sensed_count)664665    def _reset_exploration_coverage(self):666        self._explored_cells.fill(False)667        if not self.explore_coverage_features:668            return669        for agent in self.agents:670            self._mark_explored_segment(671                agent, agent.position, agent.position672            )673674    def _survey_anchor(self, survey_idx):675        # survey 模式将地图划分成近似网格，每个探索动作对应一个固定巡查锚点。676        cols = int(np.ceil(np.sqrt(self.num_explore_actions)))677        rows = int(np.ceil(self.num_explore_actions / cols))678        col = survey_idx % cols679        row = survey_idx // cols680        x = (col + 1.0) / (cols + 1.0) * self.map_size681        y = (row + 1.0) / (rows + 1.0) * self.map_size682        return np.array([x, y], dtype=float)683684    def _is_explore_task_id(self, task_id):685        # 判断某个任务槽是否属于探索动作槽。686        return self.explore_task_start <= int(task_id) < self.explore_task_end687688    def _save_base_env(self):689        # jitter reset 的基础场景快照，包括危险点、任务和智能体初始状态。690        self._base_danger_points = copy.deepcopy(self.danger_points)691        self._base_tasks = copy.deepcopy(self.tasks)692        self._base_agents = copy.deepcopy(self.agents)693694    def _restore_base_env(self):695        # 从基础场景快照恢复，避免每个 episode 完全重采样。696        self.danger_points = copy.deepcopy(self._base_danger_points)697        self.tasks = copy.deepcopy(self._base_tasks)698        self.agents = copy.deepcopy(self._base_agents)699700    def _jitter_task_positions(self):701        # 只扰动配送任务位置；探索/空闲/补给槽保持固定语义。702        if self.task_position_jitter <= 0.0:703            return704        scale = self.task_position_jitter * self.map_size705        for task in self.tasks:706            if task.type != TaskType.Deliver:707                continue708            base_pos = task.position.copy()709            for _ in range(20):710                offset = self._rand(2) * 2.0 * scale - scale711                new_pos = np.clip(base_pos + offset, 0.0, self.map_size)712                if not any(713                    np.linalg.norm(new_pos - np.array(dp['position'])) <= (self.danger_radius + self.clearance)714                    for dp in self.danger_points715                ):716                    task.position = new_pos717                    break718            else:719                task.position = np.clip(base_pos, 0.0, self.map_size)720721    def _init_map(self):722        # 连续地图：只记录危险点圆心723        self.danger_points = []724        # 危险点数量目前固定，历史代码里保留过按面积缩放的方案。725        num_danger = 10#max(1, int(self.map_size * self.map_size * 0.002))726        for _ in range(num_danger):727            p = (self._rand(2) * self.map_size).astype(float)728            self.danger_points.append({729                'position': p,730                'discovered': False731            })732733    def _init_agents(self):734        # 随机生成智能体能力和位置；位置不能落在危险圆安全范围内。735        self.agents: list[Agent] = []736        for i in range(self.agent_num):737            capbilities = self._rand(self.cap_dim)738            # 采样连续位置，避开危险点中心739            pos = self._rand(2) * self.map_size740            # 保证不在危险半径内741            tries = 0742            while any(np.linalg.norm(pos - np.array(dp['position'])) <= (self.danger_radius + self.clearance) for dp in self.danger_points) and tries < 50:743                pos = self._rand(2) * self.map_size744                tries += 1745746            agent = Agent(i, pos, capbilities, rng=self.rng)747748            self.agents.append(agent)749750    def _init_tasks(self):751        '''随机放置任务，并追加探索、空闲、补给三个类别的辅助任务槽。'''752        self.tasks: list[Task] = []753754        # 先生成所有任务槽，后面会覆盖探索/空闲/补给槽。755        for i in range(self.task_num):756            demand_val = int(self._randint(100, 500))  # 50–100757            pos = self._rand(2) * self.map_size758            tries = 0759            while any(np.linalg.norm(pos - np.array(dp['position'])) <= (self.danger_radius + self.clearance) for dp in self.danger_points) and tries < 200:760                pos = self._rand(2) * self.map_size761                tries += 1762            task = Task(i, np.array(pos, dtype=float), demand_val, self._rand(), self.cap_dim, rng=self.rng)763            if self.use_coalition_delivery and i < self.delivery_task_num:764                deadline_frac = 0.45 + 0.5 * float(self._rand())765                priority = 0.5 + 0.5 * float(self._rand())766                task.urgency = deadline_frac767                task.req_vec = np.array([priority, 0.0, 0.0, 0.0], dtype=float)768            self.tasks.append(task)769770        # 覆盖探索槽。nearest 模式下探索槽位置为 NaN，step 时动态寻找最近未发现任务。771        for sector, explore_id in enumerate(range(self.explore_task_start, self.explore_task_end)):772            if self.explore_action_mode == "survey":773                explore_pos = self._survey_anchor(sector)774            elif self.explore_action_mode == "sector":775                explore_pos = self._explore_sector_anchor(sector)776            else:777                explore_pos = np.array((np.nan, np.nan))778            exploring_task = Task(explore_id, explore_pos, 0, 0.5, self.cap_dim)779            exploring_task.req_vec = np.array([0.25, 0, 0, 0], dtype=float)780            exploring_task.type = TaskType.Explore781            exploring_task.discovered = True782            self.tasks[explore_id] = exploring_task783784        # Idle 槽始终可选，但没有空间目标。785        idle_task = Task(self.__class__.idle_task_id, np.array((np.nan, np.nan)), 0, 0, self.cap_dim)786        idle_task.req_vec = np.array([0, 0, 0, 0])787        idle_task.type = TaskType.Idle788        idle_task.discovered = True789        self.tasks[self.__class__.idle_task_id] = idle_task790791        # Resupply 槽固定在原点，智能体到达后恢复载荷和电量。792        resupply_task = Task(self.__class__.resupply_task_id, np.array((0, 0)), 0, 0, self.cap_dim)793        resupply_task.req_vec = np.array([0, 0, 0, 0])794        resupply_task.type = TaskType.Resupply795        resupply_task.discovered = True796        self.tasks[self.__class__.resupply_task_id] = resupply_task797798        # Delivery-only diagnostics keep the original task/action tensor shapes.799        # Revealing tasks here tests learned allocation and transport without800        # renumbering action slots or replacing the actor checkpoint.801        if self.delivery_tasks_initially_discovered:802            for task in self.tasks:803                if task.type == TaskType.Deliver:804                    task.discovered = True805                    task.discovered_step = 0806807808    def set_reset_mode(self, mode):809        """动态切换 reset 模式，课程学习会调用。"""810        assert mode in ("jitter", "full_random")811        self.env_reset_mode = mode812813    def reset(self, episode=-1):814        self.step_count = 0815        if self.env_reset_mode == "full_random":816            # 每个 episode 完全重采样地图、任务和智能体。817            self._init_map()818            self._init_tasks()819            self._init_agents()820        elif not self._base_initialized:821            # jitter 模式第一次 reset 先生成基础场景并缓存。822            self._init_map()823            self._init_tasks()824            self._init_agents()825            self._save_base_env()826            self._base_initialized = True827        else:828            # jitter 模式后续 episode 恢复基础场景，只扰动配送任务位置。829            self._restore_base_env()830            self._jitter_task_positions()831        self._reset_exploration_coverage()832        # sent_goal 包含所有任务槽的初始需求；辅助任务需求为 0，不影响总配送目标。833        self.sent_goal = np.sum([task.initial_demand for task in self.tasks])834        self.event_new_task_found = 0835        self.event_task_completed = [0 for i in range(self.agent_num)]836        self.event_any_task_completed = 0837        self.global_state = np.zeros((self.agent_num, self.task_num, self.gat_dim))838839        self.total_coalition_changes = 0840        self.total_reconfiguration_decisions = 0841        self._current_decision_mask = np.ones((self.agent_num,), dtype=np.float32)842        self._last_reconfiguration_decision_step = np.zeros(843            (self.agent_num,), dtype=np.int32844        )845        self._last_coalition_switch_step = np.zeros(846            (self.agent_num,), dtype=np.int32847        )848        self.total_alive_agent_steps = 0849        self.response_times = []850851        if episode != -1:852            self.episode = episode853        observations = self._get_observations()854855        active_mask = np.ones((self.agent_num,), dtype=np.float32)856        active_mask = active_mask[..., None]857858        available_actions = self._get_available_actions()859860        return observations, active_mask, available_actions#, info861862    def _get_available_actions(self):863        # 当前策略只能选择“已发现且未完成”的任务槽。864        # Explore/Idle/Resupply 在初始化时设置为 discovered=True，因此一般保持可选。865        task_mask = np.fromiter(866            (1.0 if task.discovered and not task.completed else 0.0 for task in self.tasks),867            dtype=np.float32,868            count=self.task_num,869        )870        available = np.repeat(task_mask[None, :], self.agent_num, axis=0)871        if self.mask_explore_actions:872            available[:, self.explore_task_start : self.explore_task_end] = 0.0873        if self.explore_action_mode == "sector":874            # A relative direction that immediately clips against the map edge875            # is not a meaningful search action. Mask legs shorter than one876            # configured mobility interval separately for every agent.877            for agent_idx, ag in enumerate(self.agents):878                min_leg_distance = self._sector_min_leg_distance(ag)879                for sector, action_id in enumerate(880                    range(self.explore_task_start, self.explore_task_end)881                ):882                    goal = self._explore_sector_nominal_goal(ag, sector)883                    if np.linalg.norm(goal - ag.position) + eps < min_leg_distance:884                        available[agent_idx, action_id] = 0.0885        return available886887    def _reconfiguration_urgency_pressure(self):888        time_ratio = self.step_count / max(1.0, float(self.max_steps))889        pressures = [890            1.0 - min(1.0, max(0.0, float(task.urgency) - time_ratio))891            for task in self.tasks892            if task.type == TaskType.Deliver and task.discovered and not task.completed893        ]894        if not pressures:895            return 0.0896        aggregate = np.sum if self.reconfiguration_urgency_aggregation == "sum" else np.mean897        return float(aggregate(pressures))898899    def _time_limit_reached(self):900        return self.step_count >= self.episode_time_limit901902    def _get_reconfiguration_active_mask(self, key_event, all_still):903        """决定本 step 哪些 agent 会被送入 actor 重新计算动作。904905        这里的 active 只是“有机会/需要被 actor 调用”，不等同于最终一定换联盟。906        learned/hierarchical gate 的职责是在这些机会中继续判断是否保持 current。907        """908909        living = np.array([ag.living for ag in self.agents], dtype=bool)910        completed = np.array(911            [ag.living and ag.current_task_completed for ag in self.agents], dtype=bool912        )913        if all_still:914            # 全体停住时必须让存活 agent 重新进 actor，否则 episode 会卡死。915            return living.astype(np.float32)916        if self.reconfiguration_mode == "legacy":917            return np.array(918                [919                    1.0 if ag.living and (920                        ag.current_task_completed921                        or (ag.idle and key_event)922                        or (923                            self.explore_event_redecision924                            and self.event_new_task_found == 1925                            and self.tasks[ag.coalition].type == TaskType.Explore926                        )927                    ) else 0.0928                    for ag in self.agents929                ],930                dtype=np.float32,931            )932        if self.reconfiguration_mode == "always":933            return living.astype(np.float32)934        interval_due = self.step_count % self.reconfiguration_interval == 0935        if self.reconfiguration_mode == "periodic":936            reconsider_all = interval_due937        elif self.reconfiguration_mode == "event":938            reconsider_all = key_event939        elif self.reconfiguration_mode == "urgency":940            reconsider_all = self._reconfiguration_urgency_pressure() >= self.reconfiguration_urgency_threshold941        else:942            reconsider_all = interval_due or key_event943        return np.where(reconsider_all, living, completed).astype(np.float32)944945    def _allocation_diagnostics(self):946        """Return load-based allocation diagnostics without changing dynamics.947948        This deliberately does not use the historical ``req_vec`` surplus:949        perception, mobility, payload and battery are not additive resources.950        The first diagnostic definition asks only whether the payload already951        committed to a discovered delivery task covers its remaining demand.952        """953        committed_load = np.zeros(self.task_num, dtype=np.float64)954        for ag in self.agents:955            if not ag.living or ag.coalition is None:956                continue957            cid = int(ag.coalition)958            if not (0 <= cid < self.task_num):959                continue960            task = self.tasks[cid]961            if task.type == TaskType.Deliver and task.discovered and not task.completed:962                committed_load[cid] += max(0.0, float(ag.load))963964        task_rows = []965        total_shortage = 0.0966        total_surplus = 0.0967        has_payload_deficit = False968        for task in self.tasks:969            if task.type != TaskType.Deliver or not task.discovered or task.completed:970                continue971            demand = max(0.0, float(task.demand))972            committed = float(committed_load[task.id])973            shortage = max(0.0, demand - committed)974            surplus = max(0.0, committed - demand)975            total_shortage += shortage976            total_surplus += surplus977            has_payload_deficit = has_payload_deficit or shortage > 1e-6978            task_rows.append({979                "task_id": int(task.id),980                "remaining_demand": demand,981                "committed_load": committed,982                "payload_shortage": shortage,983                "payload_surplus": surplus,984            })985986        idle_agents = [987            ag for ag in self.agents988            if ag.living and self.tasks[int(ag.coalition)].type == TaskType.Idle989        ]990        harmful_idle = sum(991            1 for ag in idle_agents992            if float(ag.load) > 1e-6 and has_payload_deficit993        )994        normalizer = max(1.0, float(self.sent_goal))995        return {996            "idle_agents": int(len(idle_agents)),997            "harmful_idle_agents": int(harmful_idle),998            "useful_idle_agents": int(len(idle_agents) - harmful_idle),999            "committed_payload_shortage_ratio": float(total_shortage / normalizer),1000            "committed_payload_surplus_ratio": float(total_surplus / normalizer),1001            "tasks": task_rows,1002        }10031004    def discover_along_segment(self, agent_idx, a, b):1005        # 沿移动线段检查感知范围，标记新发现的配送任务和危险点。1006        # 使用点到线段距离，而不是只看终点距离，避免高速移动穿过目标但漏检。1007        ag = self.agents[agent_idx]1008        per = float(ag.perception)1009        discovered_tasks = []1010        discovered_dp = 01011        self._mark_explored_segment(ag, a, b)10121013        a = np.asarray(a, dtype=float)1014        b = np.asarray(b, dtype=float)1015        ab = b - a1016        denom = float(np.dot(ab, ab))10171018        def point_segment_distances(points):1019            if len(points) == 0:1020                return np.empty((0,), dtype=float)1021            if denom <= eps:1022                return np.linalg.norm(points - a[None, :], axis=1)1023            t = ((points - a[None, :]) @ ab) / denom1024            t = np.clip(t, 0.0, 1.0)1025            proj = a[None, :] + t[:, None] * ab[None, :]1026            return np.linalg.norm(points - proj, axis=1)10271028        task_ids = [1029            idx1030            for idx, task in enumerate(self.tasks)1031            if (not task.discovered)1032            and task.type == TaskType.Deliver1033            and not np.any(np.isnan(task.position))1034        ]1035        if task_ids:1036            task_positions = np.stack([self.tasks[idx].position for idx in task_ids])1037            discovered_mask = point_segment_distances(task_positions) <= per1038            for idx in np.asarray(task_ids, dtype=np.int64)[discovered_mask]:1039                task = self.tasks[int(idx)]1040                task.discovered = True1041                task.discovered_step = self.step_count1042                discovered_tasks.append(int(idx))10431044        danger_ids = [1045            idx for idx, dp in enumerate(self.danger_points) if not dp["discovered"]1046        ]1047        if danger_ids:1048            danger_positions = np.stack(1049                [self.danger_points[idx]["position"] for idx in danger_ids]1050            )1051            discovered_mask = point_segment_distances(danger_positions) <= per1052            for idx in np.asarray(danger_ids, dtype=np.int64)[discovered_mask]:1053                self.danger_points[int(idx)]["discovered"] = True1054                discovered_dp += 11055        return discovered_tasks, discovered_dp10561057    def move_along_path(self, path_pts, start_pos, max_dist):1058        # path_pts 是从当前位置到目标的路点列表；max_dist 是本步最多可移动距离。1059        # 返回新位置和剩余路径。剩余路径首点始终是新位置。1060        pos = np.array(start_pos, dtype=float)1061        rem = []1062        # 保证路径从当前位置开始，避免上一帧路径首点和当前位置存在微小偏差。1063        pts = [tuple(pos)] + [tuple(p) for p in path_pts[1:]] if len(path_pts) > 0 else [tuple(pos)]1064        dist_left = float(max_dist)1065        i = 01066        while dist_left > 1e-8 and i < len(pts)-1:1067            a = np.array(pts[i], dtype=float)1068            b = np.array(pts[i+1], dtype=float)1069            seg_len = np.linalg.norm(b - a)1070            if seg_len <= 1e-8:1071                i += 11072                continue1073            need = min(seg_len - np.linalg.norm(pos - a), dist_left)1074            # 如果位置不在线段开头，计算剩下在线段上还有多长1075            to_end = np.linalg.norm(b - pos)1076            move = min(to_end, dist_left)1077            if move >= to_end - 1e-8:1078                # 到达当前线段终点，继续走下一段。1079                pos = b1080                dist_left -= to_end1081                i += 11082            else:1083                # 本步距离不足以到达线段终点，停在线段中间。1084                direction = (b - pos) / np.linalg.norm(b - pos)1085                pos = pos + direction * move1086                dist_left = 0.01087                break1088        # 剩余路径：从当前位置开始，然后接上未走完的后续路点。1089        remaining = [tuple(pos)]1090        for j in range(i+1, len(pts)):1091            remaining.append(tuple(pts[j]))1092        return pos, remaining109310941095    def step(self, actions):1096        '''1097        参数1098        -------1099    actions : List[int]  长度 = self.agent_num  1100        每个元素为智能体选中的任务 ID  1101        动作语义：0..(task_num-1) 对应任务 ID，task_num 表示 Idle（空闲，不选择任何任务）。1102        为兼容旧版，也接受 -1 表示 Idle。1103        返回1104        -------1105        observations : dict        {'ego':..., 'tasks':..., 'others':...}1106        rewards      : np.ndarray  [N_agents]1107        dones        : np.ndarray  [N_agents]  (全部任务完成或到 max_steps)1108        info         : dict        调试、评估和日志指标1109        '''1110        from datetime import datetime1111        d = d0 = datetime.now().timestamp()1112        self.step_count += 11113        step_reconfiguration_decisions = int(np.count_nonzero(self._current_decision_mask > 0.5))1114        self.total_reconfiguration_decisions += step_reconfiguration_decisions1115        self._last_reconfiguration_decision_step[1116            self._current_decision_mask > 0.51117        ] = self.step_count1118        self.total_alive_agent_steps += int(sum(ag.living for ag in self.agents))11191120        new_discovered = set()1121        explore_legs_planned = 01122        explore_planned_distance = 0.01123        short_explore_legs = 01124        explore_distance_moved = 0.011251126        self.t = np.array([0, 0, 0, 0, 0], float)11271128        # 1. 联盟决策：动作直接写入每个智能体的 coalition。1129        # available_actions 在 wrapper/policy 侧限制可选任务；这里仍容忍越界动作并忽略。1130        for i, raw_a in enumerate(actions):1131            # 兼容 raw_a 是 tensor 或普通 int 的情况1132            try:1133                a = int(raw_a.item())1134            except Exception:1135                a = int(raw_a)11361137            # 当前动作范围是 [0, task_num)，其中 Idle 已经是一个普通任务槽。1138            if a < 0 or a >= self.task_num:1139                # 非法动作，忽略1140                continue11411142            self.agents[i].act(a)1143            # 若选择非法任务 ID，忽略，保持原状态11441145        step_coalition_changes = int(sum(ag.coalition_changed for ag in self.agents))1146        self.total_coalition_changes += step_coalition_changes1147        for idx, ag in enumerate(self.agents):1148            if ag.coalition_changed:1149                self._last_coalition_switch_step[idx] = self.step_count11501151        # 2. 首次响应时间：任务被发现后，第一次有活着的智能体加入该任务联盟时记录。1152        for task in self.tasks:1153            if task.type != TaskType.Deliver:1154                continue1155            if not task.discovered or task.completed:1156                continue1157            if task.discovered_step is None:1158                continue1159            if task.first_assigned_step is not None:1160                continue11611162            assigned = any(1163                ag.living and ag.coalition == task.id1164                for ag in self.agents1165            )1166            if assigned:1167                task.first_assigned_step = self.step_count1168                task.response_time = task.first_assigned_step - task.discovered_step1169                self.response_times.append(task.response_time)11701171        # 3. 清空本步事件标志；后续发现、完成、死亡等逻辑会重新写入。1172        self.event_new_task_found      = 01173        self.event_task_completed      = [0 for i in range(self.agent_num)]1174        self.event_any_task_completed  = 011751176        task_caps = np.zeros((self.task_num, self.cap_dim))1177        coalition_sizes = np.zeros(self.task_num, dtype=int)1178        coalitions = [[] for i in range(self.task_num)]11791180        # 4. 依据当前 coalition 聚合每个任务槽上的联盟成员能力和成员列表。1181        # 这份聚合既用于任务完成逻辑，也用于奖励/诊断中的能力匹配度。1182        for i, ag in enumerate(self.agents):1183            c = ag.coalition1184            if c is None or c == -1 or c == self.task_num:1185                continue1186            task_caps[c] += ag.capbilities1187            coalition_sizes[c] += 11188            coalitions[c].append(ag.aid)11891190        # 5. 联盟-任务能力匹配度诊断。1191        # 当前惩罚分摊代码仍保留但未实际写入 rewards，用于后续恢复或分析。11921193        for j, task in enumerate(self.tasks):1194            # 不考虑 Idle 任务和补给任务1195            if j == self.__class__.idle_task_id or j == self.__class__.resupply_task_id:1196                continue1197            # 只对 active 的、未完成的任务算匹配度1198            if coalition_sizes[j] == 0:1199                continue1200            if not task.completed:1201                continue12021203            req = task.req_vec1204            if req is None:1205                continue12061207            cap = task_caps[j]1208            min_val = 0.21209            # 避免除零1210            denom = np.where((cap > -min_val) & (cap < min_val), min_val, cap)12111212            diff = cap - req1213            shortage = np.clip(-diff, 0.0, None)   # cap < req 的部分1214            surplus  = np.clip( diff, 0.0, None)   # cap > req 的部分12151216            # 按平方惩罚，让严重不匹配的惩罚更大1217            penalty_j = (1218                self.w_short  * (float(np.sum(shortage ** 2)) - self.b_short) +1219                self.w_surplus * (float(np.sum(surplus ** 2)) - self.b_surplus)1220            )12211222            if penalty_j <= 0:1223                continue12241225            # 把惩罚均摊给联盟内所有 agent1226            share = penalty_j / float(coalition_sizes[j])12271228            share = np.clip(share, -100, 2)12291230            # for i in coalitions[j]:1231            #     rewards[i] -= share12321233        d = datetime.now().timestamp()1234        self.t[0] = d - d01235        d0 = d1236        1237        # rewards_by_step[1] = rewards - rewards_by_step[0]12381239        # 6. 为没有剩余路径的非空闲智能体生成目标点和路径。1240        # 已有路径的智能体继续执行旧路径，避免每步重规划造成抖动。12411242        danger_positions = [dp['position'] for dp in self.danger_points]12431244        for i, ag in enumerate(self.agents):12451246            # 死亡智能体不规划路径。1247            if not ag.living:1248                continue12491250            # 未走完旧路径时不重新分配路径。1251            if len(ag.path) != 0:1252                continue12531254            task = self.tasks[ag.coalition]12551256            # Idle 槽没有空间目标。1257            if task.type == TaskType.Idle:1258                continue12591260            # 根据任务类型确定目标点：探索任务可动态选择，配送/补给使用任务位置。1261            if task.type == TaskType.Explore:1262                if self.explore_action_mode == "survey":1263                    goal = np.array(task.position, dtype=float)1264                elif self.explore_action_mode == "sector":1265                    goal = self._explore_sector_goal(ag)1266                else:1267                    # nearest 探索：优先朝最近的未发现配送任务移动。1268                    cand_tasks = [task for task in self.tasks if (not task.discovered) and (task.type == TaskType.Deliver)]1269                    if cand_tasks:1270                        # 按欧氏距离排序，选择最近未发现配送任务。1271                        cand_tasks.sort(key=lambda task: np.linalg.norm(task.position - ag.position))1272                        goal = tuple(cand_tasks[0].position)1273                    else:1274                        # 若没有未发现配送任务，则随机探索一个地图点。1275                        goal = tuple((self._rand(2) * self.map_size))12761277                    goal = np.array(goal, dtype=float)12781279            else: # elif coalition.type == TaskType.Deliver or coalition.type == TaskType.Resupply:12801281                goal = task.position12821283            if task.type == TaskType.Explore and self.explore_action_mode == "sector":1284                planned_distance = float(np.linalg.norm(goal - ag.position))1285                explore_legs_planned += 11286                explore_planned_distance += planned_distance1287                if planned_distance + eps < self._sector_min_leg_distance(ag):1288                    short_explore_legs += 112891290            # 根据危险点生成绕行路径。1291            ag.path = fast_path(ag.position, goal, danger_positions, danger_radius=self.danger_radius, clearance=self.clearance)12921293            # 理论上 fast_path 很少返回空；若失败，本步该智能体不移动。1294            if len(ag.path) == 0:1295                print(f'step {self.step_count}, agent {i}: cannot find path for exploration agent')12961297        # 7. 沿路径移动，并在移动线段上触发探索、交付和补给事件。12981299        sent = 01300        completed = 013011302        for i, ag in enumerate(self.agents):13031304            # 死亡智能体不移动。1305            if not ag.living:1306                continue13071308            # 没有路径说明没有空间目标或规划失败。1309            if len(ag.path) == 0:1310                continue13111312            task = self.tasks[ag.coalition]13131314            # Idle 智能体保持当前位置。1315            if task.type == TaskType.Idle:1316                continue13171318            # 只要执行移动就消耗电量。1319            ag.consume()13201321            # mobility 决定本步沿路径最多前进多远。1322            start = ag.position1323            new_pos, remaining = self.move_along_path(ag.path, start, ag.mobility)1324            # 移动过程中可能发现新任务或危险点。1325            discovered_tasks, discovered_dp = self.discover_along_segment(i, start, new_pos)13261327            # 新发现任务会激活探索奖励，并触发 active_mask 让部分智能体重新决策。1328            if len(discovered_tasks) > 0:1329                ag.discovery = len(discovered_tasks) + 0.1 * discovered_dp1330                self.event_new_task_found = 11331                for task_id in discovered_tasks:1332                    new_discovered.add(task_id)13331334            ag.position = np.array(new_pos, dtype=float)1335            if task.type == TaskType.Explore:1336                explore_distance_moved += float(np.linalg.norm(ag.position - start))13371338            # 如果路径已经走到终点，则清空路径。1339            ag.path = remaining if len(remaining) > 1 else []13401341            # 配送/补给任务记录到目标的距离，距离奖励依赖 last_state.dist - dist。1342            if task.type in [TaskType.Deliver, TaskType.Resupply]:1343                dist = np.linalg.norm(ag.position - task.position)1344                ag.dist = float(dist)13451346            if task.type == TaskType.Explore:1347                if self.explore_action_mode == "survey" and len(ag.path) == 0:1348                    task.completed = True1349                    ag.current_task_completed = True1350                elif (1351                    self.explore_action_mode == "sector"1352                    and self.sector_redecision_on_waypoint1353                    and len(ag.path) == 01354                ):1355                    # Sector actions denote one finite search leg, not a permanent1356                    # assignment. Reaching the waypoint should request the next1357                    # policy decision without globally completing the Explore slot.1358                    ag.current_task_completed = True1359                elif len(discovered_tasks) > 0:#ag.unchanged_time >= 10:1360                    ag.current_task_completed = True13611362            # 配送智能体到达任务点后消耗载荷并减少任务 demand。1363            elif task.type == TaskType.Deliver:1364                if not np.any(np.isnan(task.position)) and ag.dist < 1e-3:1365                    ag.complete()1366                    send_amount = ag.deliver(task.demand)1367                    sent += send_amount1368                    task.demand -= send_amount1369                    # 交付量作为联盟贡献同步给同联盟成员，用于联盟运输奖励。1370                    for i in coalitions[task.id]:1371                        self.agents[i].coalition_delivery += ag.delivery13721373                    if task.demand <= 0:1374                        task.completed = True1375                        completed += 11376                        1377                        # 任务完成时，同联盟成员都获得完成事件。1378                        for i in coalitions[task.id]:1379                            self.event_task_completed[i] = 11380                            self.agents[i].coalition_completed = True1381                            self.agents[i].complete()13821383                            self.event_any_task_completed = 113841385            # 补给智能体到达补给点后恢复载荷和电量。1386            elif task.type == TaskType.Resupply:1387                if not np.any(np.isnan(task.position)) and ag.dist < 1e-3:1388                    ag.complete()1389                    ag.resupply()13901391        d = datetime.now().timestamp()1392        self.t[3] = d - d01393        d0 = d13941395        if len(new_discovered) > 0:1396            # 探索发现新任务后，清空探索智能体路径，让它们可以尽快重新决策。1397            for ag in self.agents:1398                if self.tasks[ag.coalition].type == TaskType.Explore:1399                    ag.path = []14001401        # 历史时间惩罚位置保留在这里；当前实际奖励在 _get_rewards 中统一计算。1402        # for i, ag in enumerate(self.agents):1403        #     if ag['living']:1404        #         # 时间惩罚1405        #         rewards[i] -= self.pun_time_ratio * (self.step_count / self.max_steps) # 时间惩罚1406            1407        # 8. 生成下一时刻局部观测。奖励在后面算，但观测已经反映移动/任务状态更新。1408        observations = self._get_observations()14091410        # 9. 终止条件与 active_mask。1411        # done 表示环境回合终止或单个智能体死亡；bad_mask 区分自然终止和时间截断。1412        1413        # key_event 会触发空闲且存活的智能体重新决策。1414        key_event = (self.event_new_task_found == 1 or1415                     self.event_any_task_completed == 1)1416        1417        all_task_completed = all(task.completed for task in self.tasks if task.type == TaskType.Deliver)1418        all_still = all([not ag.living or ag.idle for ag in self.agents])14191420        all_agents_dead = all(not ag.living for ag in self.agents)1421        natural_done = all_task_completed or all_agents_dead1422        time_limit = self._time_limit_reached()14231424        episode_done = natural_done or time_limit1425        dones = np.array([episode_done or not ag.living for ag in self.agents])1426        bad_mask = np.array([0.0 if time_limit and not natural_done else 1.0 for ag in self.agents])1427        bad_mask = bad_mask[..., None]14281429        if episode_done:1430            # episode 结束时所有 agent 都激活，方便 runner flush 最后一帧。1431            active_mask = np.ones((len(self.agents),), dtype=np.float32)1432        else:1433            active_mask = self._get_reconfiguration_active_mask(key_event, all_still)1434        active_mask = active_mask[..., None]1435        self._current_decision_mask = active_mask[:, 0].copy()14361437        available_actions = self._get_available_actions()1438        if self.explore_action_mode == "sector":1439            sector_available = available_actions[1440                :, self.explore_task_start : self.explore_task_end1441            ]1442            unavailable_sector_ratio = float(1443                np.mean(sector_available <= 0.0)1444            )1445        else:1446            unavailable_sector_ratio = 0.014471448        d = datetime.now().timestamp()1449        self.t[4] = d - d01450        d0 = d14511452        # 10. 奖励和任务能力比例诊断。14531454        if self.event_any_task_completed == 1:1455            for ag in self.agents:1456                ag.any_task_completed = True14571458        for i, task in enumerate(self.tasks):1459            # req_ratio 反映当前联盟能力相对需求的富余/缺口，写入任务对象供诊断使用。1460            task.calc_req_ratio(task_caps[i,:])14611462        rewards_ret, rewards_info = self._get_rewards(sent, completed, new_discovered, all_task_completed, all_still)1463        if self.reconfiguration_switch_cost > 0.0:1464            switched = np.array(1465                [1466                    ag.coalition_changed1467                    and not ag.last_state.idle1468                    and not self.tasks[int(ag.last_state.coalition)].completed1469                    for ag in self.agents1470                ],1471                dtype=np.float32,1472            )1473            penalty = self.reconfiguration_switch_cost * switched1474            rewards_ret[:, 0] -= penalty1475            rewards_info[self.reward_keys.index("switch_coalition")] -= penalty14761477        if key_event:1478            pass14791480        # 11. info 主要供日志、评估和离线分析；不进入策略网络。1481        allocation_diagnostics = self._allocation_diagnostics()1482        info = {1483            'agents': [{1484                'position': ag.position,1485                'coalition': ag.coalition,1486                'living': ag.living,1487                'capabilities': ag.capbilities,1488                'load': ag.load,1489                'unchanged': ag.unchanged_time,1490                'idle_wait_steps': ag.idle_wait_steps,1491                } for ag in self.agents],1492            'tasks': [{1493                'position': task.position,1494                'req_vec': task.req_vec,1495                'demand': task.demand,1496                'urgency': task.urgency,1497                'initial_demand': task.initial_demand,1498                'type': int(task.type),1499                'discovered': task.discovered,1500                'completed': task.completed1501                } for task in self.tasks],1502            'danger_points': self.danger_points,1503            't': self.t,1504            'rewards': rewards_info,1505            'reward_keys': self.reward_keys,1506            'sent': sent,1507            'sent_goal': self.sent_goal,1508            'agent_caps': np.stack([ag.capbilities for ag in self.agents]),1509            'task_reqs': np.stack([task.req_vec for task in self.tasks]),1510            'allocation_diagnostics': allocation_diagnostics,1511            'reconfiguration': {1512                'mode': self.reconfiguration_mode,1513                'decisions': step_reconfiguration_decisions,1514                'coalition_changes': step_coalition_changes,1515                'total_decisions': self.total_reconfiguration_decisions,1516                'total_coalition_changes': self.total_coalition_changes,1517                'total_alive_agent_steps': self.total_alive_agent_steps,1518                'urgency_pressure': self._reconfiguration_urgency_pressure(),1519                'mean_response_time': float(np.mean(self.response_times))1520                if self.response_times else 0.0,1521            },1522            'exploration_diagnostics': {1523                'legs_planned': int(explore_legs_planned),1524                'planned_distance': float(explore_planned_distance),1525                'short_legs': int(short_explore_legs),1526                'distance_moved': float(explore_distance_moved),1527                'unavailable_sector_ratio': unavailable_sector_ratio,1528                'explored_area_ratio': float(np.mean(self._explored_cells))1529                if self.explore_coverage_features1530                else 0.0,1531            },1532            'end': self.step_count + 1 if episode_done else 01533        }15341535        for ag in self.agents:1536            if ag.living and self.tasks[int(ag.coalition)].type == TaskType.Idle:1537                ag.idle_wait_steps += 11538            else:1539                ag.idle_wait_steps = 01540            # 最后更新 last_state，并清空本步事件，准备下一次 step。1541            ag.step()154215431544        return observations, rewards_ret, dones, info, active_mask, bad_mask, available_actions154515461547    def _task_feature_matrix(self, include_flags=False, local_visible=False):1548        """构造任务特征矩阵。15491550        local_visible=True 时，未发现或已完成任务在局部观测中置零；1551        include_flags=True 时，为 critic 额外添加 discovered/completed 标志。1552        """15531554        extra_dim = 2 if include_flags else 01555        features = np.zeros((self.task_num, self.per_task_feat + extra_dim), dtype=np.float32)1556        for idx, task in enumerate(self.tasks):1557            if local_visible and (not task.discovered or task.completed):1558                continue1559            pos = task.position1560            if np.isnan(pos[0]) or np.isnan(pos[1]):1561                # Idle 或 nearest Explore 这类无固定空间目标的任务用 0 位置占位。1562                pos = np.array([0.0, 0.0])1563            if self.use_coalition_delivery:1564                features[idx, 0:2] = pos / self.map_size1565                if self.coalition_task_feature_mode == "full":1566                    features[idx, 2:2 + self.cap_dim] = task.req_vec / 5.01567                    features[idx, 2 + self.cap_dim] = task.demand / 500.01568                    deadline_idx = 3 + self.cap_dim1569                    type_idx = 7 + self.cap_dim1570                else:1571                    features[idx, 2] = task.demand / 500.01572                    features[idx, 4] = (1573                        float(task.req_vec[0]) if task.req_vec is not None else 0.01574                    )1575                    deadline_idx = 31576                    type_idx = 51577                if task.type == TaskType.Deliver:1578                    features[idx, deadline_idx] = max(1579                        0.0,1580                        float(task.urgency)1581                        - self.step_count / max(1.0, self.max_steps),1582                    )1583                else:1584                    features[idx, deadline_idx] = 1.01585                features[idx, type_idx] = float(task.type) / float(max(TaskType))1586            else:1587                # 默认 MACF 任务特征：位置、能力需求、剩余需求、任务类型。1588                # req_vec / 5.0 与 HATT residual 中的 hatt_capability_scale 对应。1589                features[idx, 0:2] = pos / self.map_size1590                features[idx, 2:2 + self.cap_dim] = task.req_vec / 5.01591                features[idx, 2 + self.cap_dim] = task.demand / 500.01592                features[idx, 3 + self.cap_dim] = float(task.type) / float(max(TaskType))1593            if include_flags:1594                # 这两个 flag 只给 critic 全局状态，不给 actor 局部任务观测。1595                features[idx, self.per_task_feat] = 1.0 if task.discovered else 0.01596                features[idx, self.per_task_feat + 1] = 1.0 if task.completed else 0.01597        return features15981599    def _agent_feature_matrix(self):1600        """构造所有智能体的状态矩阵，用于 other 观测和 critic 全局状态。"""16011602        features = np.zeros((self.agent_num, self.other_agent_feat_dim), dtype=np.float32)1603        max_coalition = float(max(1, self.task_num - 1))1604        max_agent_type = float(max(AgentType))1605        for idx, ag in enumerate(self.agents):1606            # 注意：这里保留 agent 自己；_get_observations 会在 others 中把对角线置零。1607            features[idx, 0:2] = ag.position / self.map_size1608            features[idx, 2:2 + self.cap_dim] = ag.capbilities1609            features[idx, 2 + self.cap_dim] = float(ag.load) / float(ag.init_load)1610            features[idx, 3 + self.cap_dim] = ag.battery / 1000.01611            features[idx, 4 + self.cap_dim] = float(ag.coalition) / max_coalition1612            features[idx, 5 + self.cap_dim] = float(ag.type) / max_agent_type1613            features[idx, 6 + self.cap_dim] = 1.0 if ag.living else 0.01614        return features16151616    def _task_observation_matrix(self):1617        """Return observer-specific task features.16181619        Ordinary delivery/control slots remain shared.  Observer-relative1620        geometry makes each sector slot describe the executable goal and leg1621        for the corresponding agent.  Coverage additionally places the1622        candidate's novel searched-area ratio and the remaining global1623        unexplored ratio in type-conditioned scalar fields.1624        """16251626        shared = self._task_feature_matrix(local_visible=True)1627        task_obs = np.repeat(shared[None, :, :], self.agent_num, axis=0)1628        if not (1629            (1630                self.explore_coverage_features1631                or self.explore_observer_relative_geometry1632            )1633            and self.explore_action_mode == "sector"1634            and self.use_coalition_delivery1635        ):1636            return task_obs16371638        remaining_unexplored = (1639            float(1.0 - np.mean(self._explored_cells))1640            if self.explore_coverage_features1641            else 0.01642        )1643        for agent_idx, agent in enumerate(self.agents):1644            for sector, task_id in enumerate(1645                range(self.explore_task_start, self.explore_task_end)1646            ):1647                goal = self._explore_sector_nominal_goal(agent, sector)1648                leg_distance = float(np.linalg.norm(goal - agent.position))1649                task_obs[agent_idx, task_id, 0:2] = goal / self.map_size1650                if self.coalition_task_feature_mode == "full":1651                    geometry_start = 4 + self.cap_dim1652                else:1653                    geometry_start = 21654                task_obs[agent_idx, task_id, geometry_start] = (1655                    self._candidate_novel_coverage(agent, goal)1656                )1657                task_obs[agent_idx, task_id, geometry_start + 1] = (1658                    leg_distance / max(eps, self.explore_step_size)1659                )1660                task_obs[agent_idx, task_id, geometry_start + 2] = (1661                    remaining_unexplored1662                )1663        return task_obs16641665    def _get_observations(self):1666        """返回 actor 使用的局部 Dict 观测。16671668        ego:    每个 agent 自己的状态，shape [A, ego_dim]1669        tasks:  每个 agent 看到的任务槽，shape [A, T, task_dim]1670        others: 每个 agent 看到的其他智能体，shape [A, A, other_dim]，自身位置置零1671        """16721673        task_obs = self._task_observation_matrix()16741675        agent_features = self._agent_feature_matrix()1676        other_obs = np.repeat(agent_features[None, :, :], self.agent_num, axis=0)1677        self_indices = np.arange(self.agent_num)1678        # others 的第 idx 个 observer 不应看到“另一个自己”，对角线置零。1679        other_obs[self_indices, self_indices, :] = 0.016801681        discovered_deliver = sum(1682            1 for task in self.tasks1683            if task.discovered and task.type == TaskType.Deliver1684        )1685        discovered_ratio = float(discovered_deliver) / float(self.delivery_task_num)1686        max_coalition = float(max(1, self.task_num - 1))1687        max_task_type = float(max(TaskType))1688        ego_obs = np.zeros((self.agent_num, self.ego_feat_dim), dtype=np.float32)1689        for idx, ag in enumerate(self.agents):1690            coalition = self.tasks[ag.coalition]1691            # ego 中保留当前联盟编号和联盟类型，让 actor 知道当前正在执行的任务语义。1692            ego_obs[idx, 0:2] = ag.position / self.map_size1693            ego_obs[idx, 2:2 + self.cap_dim] = ag.capbilities1694            ego_obs[idx, 2 + self.cap_dim] = float(ag.coalition) / max_coalition1695            ego_obs[idx, 3 + self.cap_dim] = float(coalition.type) / max_task_type1696            ego_obs[idx, 4 + self.cap_dim] = float(ag.load) / float(ag.init_load)1697            ego_obs[idx, 5 + self.cap_dim] = discovered_ratio1698            ego_obs[idx, 6 + self.cap_dim] = ag.battery / 1000.016991700        observations = {1701            "ego": ego_obs,1702            "tasks": task_obs.astype(np.float32, copy=False),1703            "others": other_obs.astype(np.float32, copy=False),1704        }1705        if self.reconfiguration_context_features:1706            observations["reconfiguration"] = self._get_reconfiguration_context()1707        return observations17081709    def _get_reconfiguration_context(self):1710        """Return explicit event/time features used only by the optional gate.17111712        Columns are new-task event, any-task-completed event, periodic trigger,1713        focal current-task-completed, focal current-task-available, normalized1714        time since focal decision/switch, episode progress, and urgency pressure.1715        """17161717        context = np.zeros(1718            (self.agent_num, self.reconfiguration_context_dim), dtype=np.float321719        )1720        context[:, 0] = float(self.event_new_task_found == 1)1721        context[:, 1] = float(self.event_any_task_completed == 1)1722        context[:, 2] = float(1723            self.step_count % self.reconfiguration_interval == 01724        )1725        episode_scale = max(1.0, float(self.max_steps))1726        decision_scale = max(1.0, float(self.reconfiguration_interval))1727        for idx, ag in enumerate(self.agents):1728            current = self.tasks[int(ag.coalition)]1729            context[idx, 3] = float(ag.current_task_completed)1730            context[idx, 4] = float(current.discovered and not current.completed)1731            context[idx, 5] = min(1732                1.0,1733                (self.step_count - self._last_reconfiguration_decision_step[idx])1734                / decision_scale,1735            )1736            context[idx, 6] = min(1737                1.0,1738                (self.step_count - self._last_coalition_switch_step[idx])1739                / episode_scale,1740            )1741        context[:, 7] = min(1.0, self.step_count / episode_scale)1742        context[:, 8] = self._reconfiguration_urgency_pressure()1743        if self.reconfiguration_context_dim >= 11:1744            all_still = all(not ag.living or ag.idle for ag in self.agents)1745            # all_still 是全局“不能再靠旧动作推进”的信号；mandatory 是逐 agent1746            # “当前动作已经无效或完成”的信号。二者给 gate 做强制开门判断。1747            context[:, 9] = float(all_still)1748            for idx, ag in enumerate(self.agents):1749                current = self.tasks[int(ag.coalition)]1750                current_invalid = not current.discovered or current.completed1751                context[idx, 10] = float(1752                    all_still or ag.current_task_completed or current_invalid1753                )1754        return context17551756    def get_global_state(self):1757        """返回 critic 使用的特权全局状态。17581759        actor 只能看到局部可见 tasks；critic 可以看到所有任务及 discovered/completed 标志。1760        """17611762        return {1763            "agents": self._agent_feature_matrix(),1764            "tasks": self._task_feature_matrix(include_flags=True),1765            "global": np.array([self.step_count / self.max_steps], dtype=np.float32),1766        }17671768    def _get_rewards(self, sent, completed, new_discovered, all_task_completed, all_still):1769        """选择当前场景的奖励函数，并返回每个 agent 的总奖励和分项奖励。"""17701771        if self.use_coalition_delivery:1772            if self.coalition_delivery_reward_mode == "shaped":1773                return self._get_coalition_delivery_rewards_shaped(1774                    sent, completed, new_discovered, all_task_completed, all_still1775                )1776            if self.coalition_delivery_reward_mode == "delivcoal_v1":1777                return self._get_coalition_delivery_rewards_v1(1778                    sent, completed, new_discovered, all_task_completed, all_still1779                )1780            if self.coalition_delivery_reward_mode == "delivcoal_v2":1781                return self._get_coalition_delivery_rewards_v2(1782                    sent, completed, new_discovered, all_task_completed, all_still1783                )1784            if self.coalition_delivery_reward_mode == "delivcoal_v3":1785                return self._get_coalition_delivery_rewards_potential(1786                    sent,1787                    completed,1788                    new_discovered,1789                    all_task_completed,1790                    all_still,1791                    scale=2.0,1792                    local=False,1793                )1794            if self.coalition_delivery_reward_mode == "delivcoal_v4":1795                return self._get_coalition_delivery_rewards_potential(1796                    sent,1797                    completed,1798                    new_discovered,1799                    all_task_completed,1800                    all_still,1801                    scale=5.0,1802                    local=False,1803                )1804            if self.coalition_delivery_reward_mode == "delivcoal_v5":1805                return self._get_coalition_delivery_rewards_potential(1806                    sent,1807                    completed,1808                    new_discovered,1809                    all_task_completed,1810                    all_still,1811                    scale=2.0,1812                    local=True,1813                )1814            return self._get_coalition_delivery_rewards(1815                sent, completed, new_discovered, all_task_completed, all_still1816            )18171818        # 默认奖励：保留早期探索/配送/补给/完成/距离塑形逻辑。1819        any_explore = any(self._is_explore_task_id(ag.coalition) and ag.living for ag in self.agents)1820        # explore_decay = max(0.1, 1 - self.step_count / (0.5 * self.max_steps))1821        # 已发现任务比例用于衰减探索奖励，前期鼓励发现，后期避免一直探索。1822        discovered_ratio = float([task.discovered and task.type == TaskType.Deliver for task in self.tasks].count(True)) / self.delivery_task_num1823        ep_check_1 = 2001824        ep_check_2 = 6001825        # if self.episode < ep_check_1:1826        #     ep_decay = 0.01827        # elif self.episode < ep_check_2:1828        #     ep_decay = float(self.episode - ep_check_1) / (ep_check_2 - ep_check_1)1829        # else:1830        #     ep_decay = 1.01831        ep_decay = 1.018321833        step_decay = self.step_count / self.max_steps1834        # rewards_1/rewards_2 是历史上两套奖励方案的混合接口；当前 ep_decay=1，只使用 rewards_2。1835        rewards = [{key: 0.0 for key in self.reward_keys} for _ in range(self.agent_num)]1836        rewards_1 = [{key: 0.0 for key in self.reward_keys} for _ in range(self.agent_num)]1837        rewards_2 = [{key: 0.0 for key in self.reward_keys} for _ in range(self.agent_num)]183818391840        for i, ag in enumerate(self.agents):18411842            if not ag.dead and not ag.living:1843                ag.coalition = self.idle_task_id1844                continue18451846            coalition_type = self.tasks[ag.coalition].type1847            last_coalition_type = self.tasks[ag.last_state.coalition].type1848            # 切换联盟，且上回合不空闲，惩罚1849            if ag.coalition_changed and not ag.last_state.idle:1850                # 切换到探索（第一回合不罚）1851                if self.step_count > 0 and coalition_type == TaskType.Explore:1852                    # 从运输、补给切换到探索，罚1853                    if last_coalition_type in [TaskType.Deliver, TaskType.Resupply]:1854                        rewards_2[i]['switch_coalition'] -= 1.018551856                # 从运输或补给切换1857                if last_coalition_type in [TaskType.Deliver, TaskType.Resupply]:1858                    rewards_2[i]['switch_coalition'] -= 1.518591860            if ag.coalition_changed and ag.last_state.idle:1861                # 从空闲状态出发时，用载荷比例决定补给/执行任务是否合理。1862                load_ratio = ag.load / int(ag.init_load)1863                rewards_2[i]['switch_coalition'] += (-1.0 if coalition_type == TaskType.Resupply else 1.0) \1864                                                * np.sinh((load_ratio - 0.5) * 2) / np.sinh(1.0) * 2.01865                # if coalition_type == TaskType.Resupply:1866                    # if load_ratio < 0.4:1867                    #     rewards_2[i]['switch_coalition'] += ((0.4 - load_ratio) ** 2) * 518681869                # else:1870                #     rewards_2[i]['switch_coalition'] += load_ratio - 0.518711872            # 死亡惩罚1873            if ag.dead:1874                rewards_2[i]['dead'] -= 5.018751876            # 探索奖励1877            # 发现事件奖励按当前发现比例衰减；没有任何探索者时额外惩罚，避免全体过早放弃探索。1878            rewards_2[i]['discover'] = ag.discovery * (0.1/(discovered_ratio + 0.1) if discovered_ratio<0.25 else 0.1/0.35)1879            rewards_2[i]['discover'] -= 0.2 if not any_explore and discovered_ratio <= 0.9 else 0.018801881            # 运输奖励1882            # deliver 是本步真实交付量，直接转成稠密奖励。1883            rewards_2[i]['deliver'] = ag.delivery * 0.118841885            # 补给奖励1886            # 补给越充分奖励越高；无效补给会通过 -1 基线体现代价。1887            rewards_2[i]['resupply'] += ag.resupply_ratio * 2.0 - 1.0 if ag.resupplied else 0.018881889            # 联盟运输奖励1890            # rewards_2[i]['coalition_deliver']  = ag.coalition_delivery * 0.0718911892            # 联盟完成任务奖励1893            # rewards_2[i]['coalition_complete'] += ag.coalition_completed * 4.018941895            base = 20.01896            time_coef = 20.01897            # 单任务完成奖励预留变量；当前没有写入分项，只保留计算位置。1898            completion_reward = (base + time_coef * (1.0 - self.step_count / self.max_steps)) if ag.coalition_completed else 018991900            if all_task_completed:1901                # 全部配送任务完成时给全体终局奖励，越早完成奖励越高。1902                rewards_2[i]['coalition_complete'] += (1 - self.step_count / self.max_steps) * 20 + 2019031904            # 距离奖励1905            if not ag.coalition_changed \1906                    and coalition_type != TaskType.Idle \1907                    and coalition_type != TaskType.Explore:1908                unchanged_decay = (ag.unchanged_time if ag.unchanged_time < 5.0 else 5.0) / 5.01909                rewards_2[i]['dist'] = 0.002 * (ag.last_state.dist - ag.dist)19101911            # 时间惩罚1912            # 每步固定惩罚起到时间成本作用，所有 agent 都会承担。1913            rewards_2[i]['dist'] -= 0.219141915            if all_still:1916                # 全体静止通常表示策略陷入空转，额外惩罚。1917                rewards_2[i]['dist'] -= 1.019181919            # 回合结束平均奖励1920            # end 分项按当前总交付量和完成数给所有 agent 共享进度信号。1921            rewards_2[i]['end'] += sent / self.sent_goal * 101922            rewards_2[i]['end'] += completed / self.delivery_task_num * 501923            rewards_2[i]['end'] /= 2.019241925        for i in range(self.agent_num):1926            for key in self.reward_keys:1927                # 当前 ep_decay=1，因此 rewards_1 不影响实际奖励。1928                rewards[i][key] = (1 - ep_decay) * rewards_1[i][key] + ep_decay * rewards_2[i][key]19291930        rewards_ret = np.zeros(self.agent_num, dtype=float)1931        rewards_info = np.zeros((len(self.reward_keys), self.agent_num), dtype=float)19321933        for i, rewards_ag in enumerate(rewards):1934            for key, rw in rewards_ag.items():1935                # rewards_ret 给 PPO 使用；rewards_info 保留每个分项，供日志和评估分析。1936                rewards_ret[i] += rw1937                rewards_info[self.reward_keys.index(key)][i] = rw19381939        rewards_ret = rewards_ret[..., None]19401941        # rewards_info = np.concatenate([rewards_info, np.array([ag.coalition for ag in self.agents]).reshape(1, -1)])19421943        return rewards_ret, rewards_info19441945    def _get_coalition_delivery_rewards(self, sent, completed, new_discovered, all_task_completed, all_still):1946        """coalition_delivery 场景的基础奖励版本。19471948        强调配送进度、按时完成、缺货补给和未完成任务的终局惩罚。1949        """19501951        rewards = [{key: 0.0 for key in self.reward_keys} for _ in range(self.agent_num)]1952        discovered_ratio = float(1953            [task.discovered and task.type == TaskType.Deliver for task in self.tasks].count(True)1954        ) / self.delivery_task_num1955        any_explore = any(self._is_explore_task_id(ag.coalition) and ag.living for ag in self.agents)1956        remaining_demand = sum(1957            max(0.0, float(task.demand))1958            for task in self.tasks1959            if task.type == TaskType.Deliver and not task.completed1960        )1961        unfinished_tasks = sum(1962            11963            for task in self.tasks1964            if task.type == TaskType.Deliver and not task.completed1965        )1966        has_remaining_delivery = remaining_demand > 1e-619671968        for i, ag in enumerate(self.agents):1969            if not ag.dead and not ag.living:1970                ag.coalition = self.idle_task_id1971                continue19721973            task = self.tasks[ag.coalition]1974            coalition_type = task.type1975            last_coalition_type = self.tasks[ag.last_state.coalition].type19761977            if ag.coalition_changed and not ag.last_state.idle:1978                rewards[i]["switch_coalition"] -= 0.051979                if last_coalition_type == TaskType.Deliver and coalition_type != TaskType.Deliver:1980                    rewards[i]["switch_coalition"] -= 0.0519811982            if ag.dead:1983                rewards[i]["dead"] -= 5.019841985            rewards[i]["discover"] += ag.discovery * 1.01986            if not any_explore and discovered_ratio < 0.9:1987                rewards[i]["discover"] -= 0.0519881989            if ag.delivery > 0:1990                rewards[i]["deliver"] += 0.05 * ag.delivery1991            elif ag.delivery < 0:1992                rewards[i]["deliver"] -= 0.519931994            if ag.resupplied:1995                rewards[i]["resupply"] += 0.8 * ag.resupply_ratio19961997            if ag.coalition_completed:1998                completed_task = self.tasks[ag.coalition]1999                priority = float(completed_task.req_vec[0]) if completed_task.req_vec is not None else 1.02000                deadline_step = float(completed_task.urgency) * self.max_steps2001                on_time_bonus = max(0.0, (deadline_step - self.step_count) / max(1.0, self.max_steps))2002                rewards[i]["coalition_complete"] += 10.0 * priority + 10.0 * on_time_bonus20032004            if has_remaining_delivery:2005                rewards[i]["dist"] -= 0.0052006                if coalition_type == TaskType.Idle:2007                    rewards[i]["dist"] -= 0.032008                if coalition_type == TaskType.Explore and discovered_ratio >= 0.99:2009                    rewards[i]["discover"] -= 0.052010                if ag.load <= 0 and coalition_type != TaskType.Resupply:2011                    rewards[i]["dist"] -= 0.042012            if coalition_type != TaskType.Idle:2013                rewards[i]["dist"] -= 0.0152014            if coalition_type in (TaskType.Deliver, TaskType.Resupply):2015                rewards[i]["dist"] += 0.001 * (ag.last_state.dist - ag.dist)2016            if all_still:2017                rewards[i]["dist"] -= 0.220182019        if self._time_limit_reached() or all_task_completed:2020            shortage_penalty = (2021                12.0 * remaining_demand / max(1.0, float(self.sent_goal))2022                + 4.0 * unfinished_tasks / max(1.0, float(self.delivery_task_num))2023            )2024            completion_bonus = 15.0 if all_task_completed else 0.02025            for item in rewards:2026                item["end"] += completion_bonus - shortage_penalty20272028        rewards_ret = np.zeros(self.agent_num, dtype=float)2029        rewards_info = np.zeros((len(self.reward_keys), self.agent_num), dtype=float)2030        for i, rewards_ag in enumerate(rewards):2031            for key, rw in rewards_ag.items():2032                rewards_ret[i] += rw2033                rewards_info[self.reward_keys.index(key)][i] = rw2034        return rewards_ret[..., None], rewards_info20352036    def _get_coalition_delivery_rewards_shaped(self, sent, completed, new_discovered, all_task_completed, all_still):2037        """coalition_delivery 场景的 shaped 奖励版本。20382039        在基础版本上加强全局配送进度、剩余 backlog 和 deadline pressure 的连续塑形。2040        """20412042        rewards = [{key: 0.0 for key in self.reward_keys} for _ in range(self.agent_num)]2043        delivery_tasks = [task for task in self.tasks if task.type == TaskType.Deliver]2044        discovered_ratio = float(2045            [task.discovered for task in delivery_tasks].count(True)2046        ) / max(1, self.delivery_task_num)2047        any_explore = any(self._is_explore_task_id(ag.coalition) and ag.living for ag in self.agents)2048        remaining_demand = sum(2049            max(0.0, float(task.demand))2050            for task in delivery_tasks2051            if not task.completed2052        )2053        unfinished_tasks = sum(1 for task in delivery_tasks if not task.completed)2054        has_remaining_delivery = remaining_demand > 1e-62055        time_ratio = self.step_count / max(1.0, float(self.max_steps))2056        sent_goal = max(1.0, float(self.sent_goal))20572058        progress_reward = 25.0 * float(sent) / sent_goal2059        backlog_cost = 0.025 * (remaining_demand / sent_goal) * (0.5 + time_ratio)2060        urgency_pressure = 0.02061        for task in delivery_tasks:2062            if task.completed:2063                continue2064            remaining_ratio = max(0.0, float(task.demand)) / max(1.0, float(task.initial_demand))2065            priority = float(task.req_vec[0]) if task.req_vec is not None else 1.02066            deadline_overrun = max(0.0, time_ratio - float(task.urgency))2067            urgency_pressure += priority * remaining_ratio * (0.25 * time_ratio + deadline_overrun)2068        urgency_cost = 0.02 * urgency_pressure / max(1, self.delivery_task_num)20692070        for i, ag in enumerate(self.agents):2071            if not ag.dead and not ag.living:2072                ag.coalition = self.idle_task_id2073                continue20742075            task = self.tasks[ag.coalition]2076            coalition_type = task.type2077            last_coalition_type = self.tasks[ag.last_state.coalition].type20782079            if ag.coalition_changed and not ag.last_state.idle:2080                rewards[i]["switch_coalition"] -= 0.052081                if last_coalition_type == TaskType.Deliver and coalition_type != TaskType.Deliver:2082                    rewards[i]["switch_coalition"] -= 0.0520832084            if ag.dead:2085                rewards[i]["dead"] -= 5.020862087            rewards[i]["discover"] += ag.discovery * 1.02088            if not any_explore and discovered_ratio < 0.9:2089                rewards[i]["discover"] -= 0.052090            if has_remaining_delivery and coalition_type == TaskType.Explore and discovered_ratio >= 0.99:2091                rewards[i]["discover"] -= 0.0820922093            if ag.delivery > 0:2094                rewards[i]["deliver"] += 0.04 * ag.delivery2095            elif ag.delivery < 0:2096                rewards[i]["deliver"] -= 1.020972098            if ag.resupplied:2099                resource_gain = float(ag.resupply_ratio)2100                rewards[i]["resupply"] += 1.0 * resource_gain2101                if resource_gain <= 0.05:2102                    rewards[i]["resupply"] -= 0.5 * (1.0 - resource_gain / 0.05)21032104            if ag.coalition_completed:2105                completed_task = self.tasks[ag.coalition]2106                priority = float(completed_task.req_vec[0]) if completed_task.req_vec is not None else 1.02107                deadline_step = float(completed_task.urgency) * self.max_steps2108                on_time_bonus = max(0.0, (deadline_step - self.step_count) / max(1.0, self.max_steps))2109                rewards[i]["coalition_complete"] += 8.0 * priority + 8.0 * on_time_bonus21102111            if progress_reward != 0.0:2112                rewards[i]["coalition_deliver"] += progress_reward21132114            if has_remaining_delivery:2115                rewards[i]["dist"] -= backlog_cost + urgency_cost2116                if (2117                    self.idle_time_cost_mode == "legacy"2118                    and coalition_type == TaskType.Idle2119                ):2120                    rewards[i]["dist"] -= 0.042121                if ag.load <= 0 and coalition_type != TaskType.Resupply:2122                    rewards[i]["dist"] -= 0.042123                if self.idle_time_cost_mode == "uniform":2124                    rewards[i]["dist"] -= 0.022125            if (2126                self.idle_time_cost_mode == "legacy"2127                and coalition_type != TaskType.Idle2128            ):2129                rewards[i]["dist"] -= 0.022130            if coalition_type in (TaskType.Deliver, TaskType.Resupply):2131                rewards[i]["dist"] += 0.001 * (ag.last_state.dist - ag.dist)2132            if all_still:2133                rewards[i]["dist"] -= 0.221342135        if self._time_limit_reached() or all_task_completed:2136            shortage_penalty = (2137                14.0 * remaining_demand / sent_goal2138                + 4.0 * unfinished_tasks / max(1.0, float(self.delivery_task_num))2139            )2140            completion_bonus = 15.0 if all_task_completed else 0.02141            for item in rewards:2142                item["end"] += completion_bonus - shortage_penalty21432144        rewards_ret = np.zeros(self.agent_num, dtype=float)2145        rewards_info = np.zeros((len(self.reward_keys), self.agent_num), dtype=float)2146        for i, rewards_ag in enumerate(rewards):2147            for key, rw in rewards_ag.items():2148                rewards_ret[i] += rw2149                rewards_info[self.reward_keys.index(key)][i] = rw2150        return rewards_ret[..., None], rewards_info21512152    def _delivery_task_weight(self, task, urgency_gain=0.5, priority_gain=0.5):2153        priority = float(task.req_vec[0]) if task.req_vec is not None else 1.02154        deadline_left = max(2155            0.0,2156            float(task.urgency) - self.step_count / max(1.0, float(self.max_steps)),2157        )2158        urgency = 1.0 - deadline_left2159        return 1.0 + priority_gain * priority + urgency_gain * urgency21602161    def _coalition_sizes_now(self):2162        sizes = np.zeros(self.task_num, dtype=np.int32)2163        for ag in self.agents:2164            if ag.living and ag.coalition is not None and 0 <= ag.coalition < self.task_num:2165                sizes[ag.coalition] += 12166        return sizes21672168    def _get_coalition_delivery_rewards_potential(2169        self,2170        sent,2171        completed,2172        new_discovered,2173        all_task_completed,2174        all_still,2175        scale,2176        local,2177    ):2178        """delivcoal_v3/v4/v5：保留 shaped 奖励，只追加小幅配送进度势函数项。21792180        v1/v2 重写了 credit assignment，会显著改变 MAPPO 的目标；这里刻意不改原奖励，2181        只把本步真实交付量转成额外进度项，用于检验更稠密的配送进度信号是否能帮助 HATT。2182        """21832184        rewards_ret, rewards_info = self._get_coalition_delivery_rewards_shaped(2185            sent, completed, new_discovered, all_task_completed, all_still2186        )2187        bonus = np.zeros(self.agent_num, dtype=float)21882189        if local:2190            # 局部版本：只给当前配送联盟成员，保留 HATT task-slot credit 的诊断价值。2191            for i, ag in enumerate(self.agents):2192                if not ag.living or ag.coalition is None or not (0 <= ag.coalition < self.task_num):2193                    continue2194                task = self.tasks[ag.coalition]2195                if task.type != TaskType.Deliver or ag.coalition_delivery <= 0:2196                    continue2197                progress = float(ag.coalition_delivery) / max(1.0, float(task.initial_demand))2198                task_weight = self._delivery_task_weight(task, urgency_gain=0.25, priority_gain=0.25)2199                bonus[i] += float(scale) * task_weight * progress2200        else:2201            # 全局版本：所有 agent 共享本步新增配送进度，最大限度保持 shaped 的原始团队目标。2202            step_progress = 0.02203            for ag in self.agents:2204                if not ag.living or ag.coalition is None or not (0 <= ag.coalition < self.task_num):2205                    continue2206                task = self.tasks[ag.coalition]2207                if task.type != TaskType.Deliver or ag.delivery <= 0:2208                    continue2209                progress = float(ag.delivery) / max(1.0, float(task.initial_demand))2210                task_weight = self._delivery_task_weight(task, urgency_gain=0.25, priority_gain=0.25)2211                step_progress += task_weight * progress2212            bonus += float(scale) * step_progress22132214        rewards_ret[:, 0] += bonus2215        rewards_info[self.reward_keys.index("coalition_deliver")] += bonus2216        return rewards_ret, rewards_info22172218    def _get_coalition_delivery_rewards_v1(self, sent, completed, new_discovered, all_task_completed, all_still):2219        """delivcoal_v1：把配送进度奖励从全体共享改为当前任务联盟局部 credit。22202221        目标是验证 HATT 的 pair-wise / coalition scorer 是否需要更直接的任务槽奖励信号。2222        v1 不引入 deadline/priority 过程塑形，只修正 credit assignment。2223        """22242225        rewards = [{key: 0.0 for key in self.reward_keys} for _ in range(self.agent_num)]2226        delivery_tasks = [task for task in self.tasks if task.type == TaskType.Deliver]2227        discovered_ratio = float([task.discovered for task in delivery_tasks].count(True)) / max(1, self.delivery_task_num)2228        any_explore = any(self._is_explore_task_id(ag.coalition) and ag.living for ag in self.agents)2229        remaining_demand = sum(max(0.0, float(task.demand)) for task in delivery_tasks if not task.completed)2230        unfinished_tasks = sum(1 for task in delivery_tasks if not task.completed)2231        has_remaining_delivery = remaining_demand > 1e-622322233        for i, ag in enumerate(self.agents):2234            if not ag.dead and not ag.living:2235                ag.coalition = self.idle_task_id2236                continue22372238            task = self.tasks[ag.coalition]2239            coalition_type = task.type2240            last_coalition_type = self.tasks[ag.last_state.coalition].type22412242            if ag.coalition_changed and not ag.last_state.idle:2243                rewards[i]["switch_coalition"] -= 0.042244                if last_coalition_type == TaskType.Deliver and coalition_type != TaskType.Deliver:2245                    rewards[i]["switch_coalition"] -= 0.0422462247            if ag.dead:2248                rewards[i]["dead"] -= 5.022492250            rewards[i]["discover"] += ag.discovery * 0.82251            if not any_explore and discovered_ratio < 0.9 and coalition_type == TaskType.Idle:2252                rewards[i]["discover"] -= 0.0422532254            if ag.delivery > 0:2255                rewards[i]["deliver"] += 0.03 * ag.delivery2256            elif ag.delivery < 0:2257                rewards[i]["deliver"] -= 0.622582259            if ag.resupplied:2260                rewards[i]["resupply"] += 0.8 * ag.resupply_ratio22612262            if coalition_type == TaskType.Deliver and ag.coalition_delivery > 0:2263                progress = float(ag.coalition_delivery) / max(1.0, float(task.initial_demand))2264                rewards[i]["coalition_deliver"] += 25.0 * progress22652266            if ag.coalition_completed:2267                rewards[i]["coalition_complete"] += 6.022682269            if has_remaining_delivery:2270                if coalition_type == TaskType.Idle:2271                    rewards[i]["dist"] -= 0.0252272                if coalition_type == TaskType.Explore and discovered_ratio >= 0.99:2273                    rewards[i]["discover"] -= 0.042274                if ag.load <= 0 and coalition_type != TaskType.Resupply:2275                    rewards[i]["dist"] -= 0.042276            if coalition_type != TaskType.Idle:2277                rewards[i]["dist"] -= 0.012278            if coalition_type in (TaskType.Deliver, TaskType.Resupply):2279                rewards[i]["dist"] += 0.0015 * (ag.last_state.dist - ag.dist)2280            if all_still:2281                rewards[i]["dist"] -= 0.222822283        if self._time_limit_reached() or all_task_completed:2284            shortage_penalty = (2285                10.0 * remaining_demand / max(1.0, float(self.sent_goal))2286                + 3.0 * unfinished_tasks / max(1.0, float(self.delivery_task_num))2287            )2288            completion_bonus = 15.0 if all_task_completed else 0.02289            for item in rewards:2290                item["end"] += completion_bonus - shortage_penalty22912292        rewards_ret = np.zeros(self.agent_num, dtype=float)2293        rewards_info = np.zeros((len(self.reward_keys), self.agent_num), dtype=float)2294        for i, rewards_ag in enumerate(rewards):2295            for key, rw in rewards_ag.items():2296                rewards_ret[i] += rw2297                rewards_info[self.reward_keys.index(key)][i] = rw2298        return rewards_ret[..., None], rewards_info22992300    def _get_coalition_delivery_rewards_v2(self, sent, completed, new_discovered, all_task_completed, all_still):2301        """delivcoal_v2：v1 + priority/deadline 加权的局部进度奖励和轻量联盟规模约束。23022303        该版本仍只改环境 reward，不改变动作集合、观测结构或 HATT 网络。2304        """23052306        rewards = [{key: 0.0 for key in self.reward_keys} for _ in range(self.agent_num)]2307        delivery_tasks = [task for task in self.tasks if task.type == TaskType.Deliver]2308        discovered_ratio = float([task.discovered for task in delivery_tasks].count(True)) / max(1, self.delivery_task_num)2309        any_explore = any(self._is_explore_task_id(ag.coalition) and ag.living for ag in self.agents)2310        remaining_demand = sum(max(0.0, float(task.demand)) for task in delivery_tasks if not task.completed)2311        unfinished_tasks = sum(1 for task in delivery_tasks if not task.completed)2312        has_remaining_delivery = remaining_demand > 1e-62313        coalition_sizes = self._coalition_sizes_now()23142315        for i, ag in enumerate(self.agents):2316            if not ag.dead and not ag.living:2317                ag.coalition = self.idle_task_id2318                continue23192320            task = self.tasks[ag.coalition]2321            coalition_type = task.type2322            last_coalition_type = self.tasks[ag.last_state.coalition].type23232324            if ag.coalition_changed and not ag.last_state.idle:2325                rewards[i]["switch_coalition"] -= 0.042326                if last_coalition_type == TaskType.Deliver and coalition_type != TaskType.Deliver:2327                    rewards[i]["switch_coalition"] -= 0.0423282329            if ag.dead:2330                rewards[i]["dead"] -= 5.023312332            rewards[i]["discover"] += ag.discovery * 0.82333            if not any_explore and discovered_ratio < 0.9 and coalition_type in (TaskType.Idle, TaskType.Resupply):2334                rewards[i]["discover"] -= 0.0352335            if has_remaining_delivery and coalition_type == TaskType.Explore and discovered_ratio >= 0.99:2336                rewards[i]["discover"] -= 0.0523372338            if ag.delivery > 0:2339                rewards[i]["deliver"] += 0.025 * ag.delivery2340            elif ag.delivery < 0:2341                rewards[i]["deliver"] -= 0.823422343            if ag.resupplied:2344                resource_gain = float(ag.resupply_ratio)2345                rewards[i]["resupply"] += 0.9 * resource_gain2346                if resource_gain <= 0.05:2347                    rewards[i]["resupply"] -= 0.25 * (1.0 - resource_gain / 0.05)23482349            if coalition_type == TaskType.Deliver:2350                task_weight = self._delivery_task_weight(task)2351                if ag.coalition_delivery > 0:2352                    progress = float(ag.coalition_delivery) / max(1.0, float(task.initial_demand))2353                    rewards[i]["coalition_deliver"] += 25.0 * task_weight * progress2354                elif not task.completed and coalition_sizes[task.id] > 2:2355                    rewards[i]["coalition_deliver"] -= 0.003 * float(coalition_sizes[task.id] - 2)23562357            if ag.coalition_completed:2358                completed_task = self.tasks[ag.coalition]2359                task_weight = self._delivery_task_weight(completed_task)2360                rewards[i]["coalition_complete"] += 5.0 * task_weight23612362            if has_remaining_delivery:2363                if coalition_type == TaskType.Idle:2364                    rewards[i]["dist"] -= 0.0252365                if ag.load <= 0 and coalition_type != TaskType.Resupply:2366                    rewards[i]["dist"] -= 0.042367            if coalition_type != TaskType.Idle:2368                rewards[i]["dist"] -= 0.0082369            if coalition_type in (TaskType.Deliver, TaskType.Resupply):2370                rewards[i]["dist"] += 0.0015 * (ag.last_state.dist - ag.dist)2371            if all_still:2372                rewards[i]["dist"] -= 0.223732374        if self._time_limit_reached() or all_task_completed:2375            shortage_penalty = (2376                10.0 * remaining_demand / max(1.0, float(self.sent_goal))2377                + 3.0 * unfinished_tasks / max(1.0, float(self.delivery_task_num))2378            )2379            completion_bonus = 15.0 if all_task_completed else 0.02380            for item in rewards:2381                item["end"] += completion_bonus - shortage_penalty23822383        rewards_ret = np.zeros(self.agent_num, dtype=float)2384        rewards_info = np.zeros((len(self.reward_keys), self.agent_num), dtype=float)2385        for i, rewards_ag in enumerate(rewards):2386            for key, rw in rewards_ag.items():2387                rewards_ret[i] += rw2388                rewards_info[self.reward_keys.index(key)][i] = rw2389        return rewards_ret[..., None], rewards_info