import math import operator from functools import reduce import numpy as np import gym from gym import error, spaces, utils from .minigrid import OBJECT_TO_IDX class ActionBonus(gym.core.Wrapper): """ Wrapper which adds an exploration bonus. This is a reward to encourage exploration of less visited (state,action) pairs. """ def __init__(self, env): self.__dict__.update(vars(env)) # Pass values to super wrapper super().__init__(env) self.counts = {} def step(self, action): obs, reward, done, info = self.env.step(action) env = self.unwrapped tup = (tuple(env.agent_pos), env.agent_dir, action) # Get the count for this (s,a) pair pre_count = 0 if tup in self.counts: pre_count = self.counts[tup] # Update the count for this (s,a) pair new_count = pre_count + 1 self.counts[tup] = new_count bonus = 1 / math.sqrt(new_count) reward += bonus return obs, reward, done, info def reset(self, **kwargs): return self.env.reset(**kwargs) class StateBonus(gym.core.Wrapper): """ Adds an exploration bonus based on which positions are visited on the grid. """ def __init__(self, env): self.__dict__.update(vars(env)) # Pass values to super wrapper super().__init__(env) self.counts = {} def step(self, action): obs, reward, done, info = self.env.step(action) # Tuple based on which we index the counts # We use the position after an update env = self.unwrapped tup = (tuple(env.agent_pos)) # Get the count for this key pre_count = 0 if tup in self.counts: pre_count = self.counts[tup] # Update the count for this key new_count = pre_count + 1 self.counts[tup] = new_count bonus = 1 / math.sqrt(new_count) reward += bonus return obs, reward, done, info def reset(self, **kwargs): return self.env.reset(**kwargs) class ImgObsWrapper(gym.core.ObservationWrapper): """ Use the image as the only observation output, no language/mission. """ def __init__(self, env): self.__dict__.update(vars(env)) # Pass values to super wrapper super().__init__(env) self.observation_space = env.observation_space.spaces['image'] def observation(self, obs): return obs['image'] class FullyObsWrapper(gym.core.ObservationWrapper): """ Fully observable gridworld using a compact grid encoding """ def __init__(self, env): self.__dict__.update(vars(env)) # Pass values to super wrapper super().__init__(env) self.observation_space = spaces.Box( low=0, high=255, shape=(self.env.width, self.env.height, 3), # number of cells dtype='uint8' ) def observation(self, obs): env = self.unwrapped full_grid = env.grid.encode() full_grid[env.agent_pos[0]][env.agent_pos[1]] = np.array([ OBJECT_TO_IDX['agent'], env.agent_dir, 0 ]) return full_grid class FlatObsWrapper(gym.core.ObservationWrapper): """ Encode mission strings using a one-hot scheme, and combine these with observed images into one flat array """ def __init__(self, env, maxStrLen=96): self.__dict__.update(vars(env)) # Pass values to super wrapper super().__init__(env) self.maxStrLen = maxStrLen self.numCharCodes = 27 imgSpace = env.observation_space.spaces['image'] imgSize = reduce(operator.mul, imgSpace.shape, 1) self.observation_space = spaces.Box( low=0, high=255, shape=(1, imgSize + self.numCharCodes * self.maxStrLen), dtype='uint8' ) self.cachedStr = None self.cachedArray = None def observation(self, obs): image = obs['image'] mission = obs['mission'] # Cache the last-encoded mission string if mission != self.cachedStr: assert len(mission) <= self.maxStrLen, 'mission string too long ({} chars)'.format(len(mission)) mission = mission.lower() strArray = np.zeros(shape=(self.maxStrLen, self.numCharCodes), dtype='float32') for idx, ch in enumerate(mission): if ch >= 'a' and ch <= 'z': chNo = ord(ch) - ord('a') elif ch == ' ': chNo = ord('z') - ord('a') + 1 assert chNo < self.numCharCodes, '%s : %d' % (ch, chNo) strArray[idx, chNo] = 1 self.cachedStr = mission self.cachedArray = strArray obs = np.concatenate((image.flatten(), self.cachedArray.flatten())) return obs class AgentViewWrapper(gym.core.Wrapper): """ Wrapper to customize the agent's field of view. """ def __init__(self, env, agent_view_size=7): self.__dict__.update(vars(env)) # Pass values to super wrapper super(AgentViewWrapper, self).__init__(env) # Override default view size env.unwrapped.agent_view_size = agent_view_size # Compute observation space with specified view size observation_space = gym.spaces.Box( low=0, high=255, shape=(agent_view_size, agent_view_size, 3), dtype='uint8' ) # Override the environment's observation space self.observation_space = spaces.Dict({ 'image': observation_space }) def reset(self, **kwargs): return self.env.reset(**kwargs) def step(self, action): return self.env.step(action)