added the pypiwin32 package

docs/Installation-Windows.md

@@ -153,6 +153,14 @@ pip install -e .
 This will complete the installation of all the required Python packages to run
 the ML-Agents toolkit.
 
+Sometimes on Windows, when you use pip to install certain python packages, the pip will get stuck when trying to read the cache of the package. If you see this, you can try:
+
+```sh
+pip install -e . --no-cache-dir
+```
+
+This `--no-cache-dir` tells the pip to disable the cache.  
+
 ## (Optional) Step 4: GPU Training using The ML-Agents Toolkit
 
 GPU is not required for the ML-Agents toolkit and won't speed up the PPO

ml-agents/setup.py

@@ -37,7 +37,8 @@
         'docopt',
         'pyyaml',
         'protobuf>=3.6,<3.7',
-        'grpcio>=1.11.0,<1.12.0'],
+        'grpcio>=1.11.0,<1.12.0',
+        'pypiwin32==223;platform_system=="Windows"'],
 
     python_requires=">=3.6,<3.7",
 

ml-agents/tests/trainers/test_ppo.py

@@ -42,15 +42,15 @@ def dummy_config():
 
 @mock.patch('mlagents.envs.UnityEnvironment.executable_launcher')
 @mock.patch('mlagents.envs.UnityEnvironment.get_communicator')
-def test_ppo_policy_evaluate(mock_communicator, mock_launcher):
+def test_ppo_policy_evaluate(mock_communicator, mock_launcher, dummy_config):
     tf.reset_default_graph()
     mock_communicator.return_value = MockCommunicator(
         discrete_action=False, visual_inputs=0)
     env = UnityEnvironment(' ')
     brain_infos = env.reset()
     brain_info = brain_infos[env.brain_names[0]]
 
-    trainer_parameters = dummy_config()
+    trainer_parameters = dummy_config
     model_path = env.brain_names[0]
     trainer_parameters['model_path'] = model_path
     trainer_parameters['keep_checkpoints'] = 3