Vectorized environments (#1513)

* Initial version of vectorized environments

* Raise an exception in the main process if child process raises an exception

* Add list of exposed functions in vector module

* Use deepcopy instead of np.copy

* Add documentation for vector utils

* Add tests for copy in AsyncVectorEnv

* Add example in documentation for batch_space

* Add cloudpickle dependency in setup.py

* Fix __del__ in VectorEnv

* Check if all observation spaces are equal in AsyncVectorEnv

* Check if all observation spaces are equal in SyncVectorEnv

* Fix spaces non equality in SyncVectorEnv for Python 2

* Handle None parameter in create_empty_array

* Fix check_observation_space with spaces equality

* Raise an exception when operations are out of order in AsyncVectorEnv

* Add version requirement for cloudpickle

* Use a state instead of binary flags in AsyncVectorEnv

* Use numpy.zeros when initializing observations in vectorized environments

* Remove poll from public API in AsyncVectorEnv

* Remove close_extras from VectorEnv

* Add test between AsyncVectorEnv and SyncVectorEnv

* Remove close in check_observation_space

* Add documentation for seed and close

* Refactor exceptions for AsyncVectorEnv

* Close pipes if the environment raises an error

* Add tests for out of order operations

* Change default argument in create_empty_array to np.zeros

* Add get_attr and set_attr methods to VectorEnv

* Improve consistency in SyncVectorEnv

gym/__init__.py

@@ -10,5 +10,6 @@
 from gym.spaces import Space
 from gym.envs import make, spec, register
 from gym import logger
+from gym import vector
 
 __all__ = ["Env", "Space", "Wrapper", "make", "spec", "register"]

gym/error.py

@@ -137,3 +137,31 @@ class WrapAfterConfigureError(Error):
 
 class RetriesExceededError(Error):
     pass
+
+# Vectorized environments errors
+
+class AlreadyPendingCallError(Exception):
+    """
+    Raised when `reset`, or `step` is called asynchronously (e.g. with
+    `reset_async`, or `step_async` respectively), and `reset_async`, or
+    `step_async` (respectively) is called again (without a complete call to
+    `reset_wait`, or `step_wait` respectively).
+    """
+    def __init__(self, message, name):
+        super(AlreadyPendingCallError, self).__init__(message)
+        self.name = name
+
+class NoAsyncCallError(Exception):
+    """
+    Raised when an asynchronous `reset`, or `step` is not running, but
+    `reset_wait`, or `step_wait` (respectively) is called.
+    """
+    def __init__(self, message, name):
+        super(NoAsyncCallError, self).__init__(message)
+        self.name = name
+
+class ClosedEnvironmentError(Exception):
+    """
+    Trying to call `reset`, or `step`, while the environment is closed.
+    """
+    pass

gym/vector/__init__.py

@@ -0,0 +1,47 @@
+from gym.vector.async_vector_env import AsyncVectorEnv
+from gym.vector.sync_vector_env import SyncVectorEnv
+from gym.vector.vector_env import VectorEnv
+
+__all__ = ['AsyncVectorEnv', 'SyncVectorEnv', 'VectorEnv', 'make']
+
+def make(id, num_envs=1, asynchronous=True, **kwargs):
+    """Create a vectorized environment from multiple copies of an environment,
+    from its id
+
+    Parameters
+    ----------
+    id : str
+        The environment ID. This must be a valid ID from the registry.
+
+    num_envs : int
+        Number of copies of the environment. If `1`, then it returns an
+        unwrapped (i.e. non-vectorized) environment.
+
+    asynchronous : bool (default: `True`)
+        If `True`, wraps the environments in an `AsyncVectorEnv` (which uses 
+        `multiprocessing` to run the environments in parallel). If `False`,
+        wraps the environments in a `SyncVectorEnv`.
+
+    Returns
+    -------
+    env : `gym.vector.VectorEnv` instance
+        The vectorized environment.
+
+    Example
+    -------
+    >>> import gym
+    >>> env = gym.vector.make('CartPole-v1', 3)
+    >>> env.reset()
+    array([[-0.04456399,  0.04653909,  0.01326909, -0.02099827],
+           [ 0.03073904,  0.00145001, -0.03088818, -0.03131252],
+           [ 0.03468829,  0.01500225,  0.01230312,  0.01825218]],
+          dtype=float32)
+    """
+    from gym.envs import make as make_
+    def _make_env():
+        return make_(id, **kwargs)
+    if num_envs == 1:
+        return _make_env()
+    env_fns = [_make_env for _ in range(num_envs)]
+
+    return AsyncVectorEnv(env_fns) if asynchronous else SyncVectorEnv(env_fns)