env_discrete.py
Gym Dict/Discrete 空间与环境封装
原文件 · 198 行 · 7373 字节
envs/delivcoal/env_discrete.py
SHA-256: 6c8d15b5133f40d1278e2d7cf84a58842c12c685dea0fab2d2b9ea41049124a6
1"""2# @Time : 2021/7/2 5:22 下午3# @Author : hezhiqiang4# @Email : tinyzqh@163.com5# @File : env_discrete.py6"""78import gym9from gym import spaces10import numpy as np11from .env_core import EnvCore121314class DiscreteActionEnv(object):15 """16 对于离散动作环境的封装17 Wrapper for discrete action environment.18 """1920 def __init__(self, config):21 self.env = EnvCore(config)22 self.num_agent = self.env.agent_num2324 self.signal_obs_dim = self.env.obs_dim25 self.signal_action_dim = self.env.action_dim26 self.cap_dim = self.env.cap_dim27 self.task_num = self.env.task_num28 self.per_task_feat = self.env.per_task_feat29 self.ego_dim = self.env.ego_feat_dim30 self.other_agent_feat_dim = self.env.other_agent_feat_dim31 self.global_agent_feat_dim = self.env.global_agent_feat_dim32 self.global_task_feat_dim = self.env.global_task_feat_dim33 self.global_feat_dim = self.env.global_feat_dim3435 # if true, action is a number 0...N, otherwise action is a one-hot N-dimensional vector36 # Edited37 self.discrete_action_input = True3839 self.movable = True4041 # configure spaces42 self.action_space = []43 self.observation_space = []44 self.share_observation_space = []4546 total_action_space = []47 for agent_idx in range(self.num_agent):48 # physical action space49 u_action_space = spaces.Discrete(self.signal_action_dim) # signal_action_dim 个离散的动作5051 total_action_space.append(u_action_space)52 self.action_space.append(total_action_space[agent_idx])5354 # observation space: policy keys plus structured graph features.55 observation_spaces = {56 "ego": spaces.Box(57 low=-np.inf,58 high=+np.inf,59 shape=(self.ego_dim,),60 dtype=np.float32,61 ),62 "tasks": spaces.Box(63 low=-np.inf,64 high=+np.inf,65 shape=(self.task_num, self.per_task_feat),66 dtype=np.float32,67 ),68 "others": spaces.Box(69 low=-np.inf,70 high=+np.inf,71 shape=(self.num_agent, self.other_agent_feat_dim),72 dtype=np.float32,73 ),74 }75 if self.env.reconfiguration_context_features:76 observation_spaces["reconfiguration"] = spaces.Box(77 low=-np.inf,78 high=+np.inf,79 shape=(self.env.reconfiguration_context_dim,),80 dtype=np.float32,81 )82 self.observation_space.append(spaces.Dict(observation_spaces))8384 state_space = spaces.Dict({85 "agents": spaces.Box(86 low=-np.inf,87 high=+np.inf,88 shape=(self.num_agent, self.global_agent_feat_dim),89 dtype=np.float32,90 ),91 "tasks": spaces.Box(92 low=-np.inf,93 high=+np.inf,94 shape=(self.task_num, self.global_task_feat_dim),95 dtype=np.float32,96 ),97 "global": spaces.Box(98 low=-np.inf,99 high=+np.inf,100 shape=(self.global_feat_dim,),101 dtype=np.float32,102 ),103 "focal": spaces.Box(104 low=0.0,105 high=1.0,106 shape=(self.num_agent, 1),107 dtype=np.float32,108 ),109 })110 self.share_observation_space = [state_space for _ in range(self.num_agent)]111112 def step(self, actions):113 """114 输入actions维度假设：115 # actions shape = (5, 2, 5)116 # 5个线程的环境，里面有2个智能体，每个智能体的动作是一个one_hot的5维编码117 Input actions dimension assumption:118 # actions shape = (5, 2, 5)119 # 5 threads of the environment, with 2 intelligent agents inside, and each intelligent agent's action is a 5-dimensional one_hot encoding120 """121 obs, reward, dones, info, active_mask, bad_mask, available_actions = self.env.step(actions)122 obs = self._format_obs_dict(obs)123 return obs, reward, dones, info, active_mask, bad_mask, available_actions124125 def reset(self, episode):126 obs, active_mask, available_actions = self.env.reset(episode)127 obs = self._format_obs_dict(obs)128 return obs, active_mask, available_actions129130 def _format_obs_dict(self, obs):131 return {k: v.astype(np.float32) for k, v in obs.items()}132133 def get_global_state(self):134 return self.env.get_global_state()135136 def close(self):137 pass138139 def render(self, mode="rgb_array"):140 pass141142 def set_reset_mode(self, mode):143 self.env.set_reset_mode(mode)144145 def seed(self, seed):146 return self.env.seed(seed)147148149class MultiDiscrete:150 """151 - The multi-discrete action space consists of a series of discrete action spaces with different parameters152 - It can be adapted to both a Discrete action space or a continuous (Box) action space153 - It is useful to represent game controllers or keyboards where each key can be represented as a discrete action space154 - It is parametrized by passing an array of arrays containing [min, max] for each discrete action space155 where the discrete action space can take any integers from `min` to `max` (both inclusive)156 Note: A value of 0 always need to represent the NOOP action.157 e.g. Nintendo Game Controller158 - Can be conceptualized as 3 discrete action spaces:159 1) Arrow Keys: Discrete 5 - NOOP[0], UP[1], RIGHT[2], DOWN[3], LEFT[4] - params: min: 0, max: 4160 2) Button A: Discrete 2 - NOOP[0], Pressed[1] - params: min: 0, max: 1161 3) Button B: Discrete 2 - NOOP[0], Pressed[1] - params: min: 0, max: 1162 - Can be initialized as163 MultiDiscrete([ [0,4], [0,1], [0,1] ])164 """165166 def __init__(self, array_of_param_array):167 super().__init__()168 self.low = np.array([x[0] for x in array_of_param_array])169 self.high = np.array([x[1] for x in array_of_param_array])170 self.num_discrete_space = self.low.shape[0]171 self.n = np.sum(self.high) + 2172173 def sample(self):174 """Returns a array with one sample from each discrete action space"""175 # For each row: round(random .* (max - min) + min, 0)176 random_array = np.random.rand(self.num_discrete_space)177 return [int(x) for x in np.floor(np.multiply((self.high - self.low + 1.0), random_array) + self.low)]178179 def contains(self, x):180 return (181 len(x) == self.num_discrete_space182 and (np.array(x) >= self.low).all()183 and (np.array(x) <= self.high).all()184 )185186 @property187 def shape(self):188 return self.num_discrete_space189190 def __repr__(self):191 return "MultiDiscrete" + str(self.num_discrete_space)192193 def __eq__(self, other):194 return np.array_equal(self.low, other.low) and np.array_equal(self.high, other.high)195196197if __name__ == "__main__":198 DiscreteActionEnv().step(actions=None)