ap_rllib_experiment_analysis/notebooks/victim_opponent_contact.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /scratch/sergei/miniconda3/envs/adv-tf2/lib/python3.8/site-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "non-resource variables are not supported in the long term\n"
     ]
    }
   ],
   "source": [
    "from ap_rllib.config import get_trainer, get_config_by_name\n",
    "from ap_rllib.helpers import ray_init\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'node_ip_address': '128.32.175.10',\n",
       " 'raylet_ip_address': '128.32.175.10',\n",
       " 'redis_address': '128.32.175.10:16394',\n",
       " 'object_store_address': '/tmp/session_2020-12-06_06-11-01_945050_37844/sockets/plasma_store',\n",
       " 'raylet_socket_name': '/tmp/session_2020-12-06_06-11-01_945050_37844/sockets/raylet',\n",
       " 'webui_url': '127.0.0.1:8265',\n",
       " 'session_dir': '/tmp/session_2020-12-06_06-11-01_945050_37844',\n",
       " 'metrics_export_port': 55710,\n",
       " 'node_id': 'f689179b8e82c8dfcab3f9900513706f9aa02759'}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ray_init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating agent humanoid_blocker\n",
      "Reading agent XML from: /scratch/sergei/better-adversarial-defenses/multiagent-competition/gym_compete/new_envs/assets/humanoid_body.xml\n",
      "Creating agent humanoid\n",
      "Reading agent XML from: /scratch/sergei/better-adversarial-defenses/multiagent-competition/gym_compete/new_envs/assets/humanoid_body.xml\n",
      "Scene XML path: /scratch/sergei/better-adversarial-defenses/multiagent-competition/gym_compete/new_envs/assets/world_body.humanoid_body.humanoid_body.xml\n",
      "Created Scene with agents\n",
      "Config:\n",
      "batch_mode: complete_episodes\n",
      "callbacks: <class 'ap_rllib.config_info_callbacks.InfoCallbacks'>\n",
      "env: multicomp\n",
      "env_config:\n",
      "  SingleAgentToMultiAgent: false\n",
      "  env_name: multicomp/YouShallNotPassHumans-v0\n",
      "  with_video: false\n",
      "framework: tfe\n",
      "http_remote_port: http://127.0.0.1:50001\n",
      "kl_coeff: 1.0\n",
      "local_tf_session_args:\n",
      "  inter_op_parallelism_threads: 4\n",
      "  intra_op_parallelism_threads: 4\n",
      "lr: 0.0003\n",
      "multiagent:\n",
      "  policies:\n",
      "    player_1:\n",
      "    - <class 'ray.rllib.policy.tf_policy_template.PPOTFPolicy'>\n",
      "    - Box(380,)\n",
      "    - Box(17,)\n",
      "    - framework: tfe\n",
      "      model:\n",
      "        custom_model: GymCompetePretrainedModel\n",
      "        custom_model_config:\n",
      "          agent_id: 0\n",
      "          env_name: multicomp/YouShallNotPassHumans-v0\n",
      "          load_weights: true\n",
      "          model_config: {}\n",
      "          name: model_0\n",
      "    player_2:\n",
      "    - <class 'ray.rllib.policy.tf_policy_template.PPOTFPolicy'>\n",
      "    - Box(380,)\n",
      "    - Box(17,)\n",
      "    - framework: tfe\n",
      "      model:\n",
      "        custom_model: GymCompetePretrainedModel\n",
      "        custom_model_config:\n",
      "          agent_id: 1\n",
      "          env_name: multicomp/YouShallNotPassHumans-v0\n",
      "          load_weights: true\n",
      "          model_config: {}\n",
      "          name: model_1\n",
      "  policies_to_train:\n",
      "  - player_1\n",
      "  - player_2\n",
      "  policy_mapping_fn: 'functools.partial(<function select_policy_default at 0x7fda0b05f3a0>,\n",
      "    config={''kl_coeff'': 1.0, ''_num_workers_tf'': 4, ''use_gae'': True, ''num_gpus'':\n",
      "    0, ''_env_name_rllib'': ''multicomp'', ''_env_fcn'': <function create_env at 0x7fda0c62b700>,\n",
      "    ''_policies'': [None, ''pretrained'', ''pretrained''], ''_env'': {''with_video'':\n",
      "    False, ''SingleAgentToMultiAgent'': False, ''env_name'': ''multicomp/YouShallNotPassHumans-v0''},\n",
      "    ''framework'': ''tfe'', ''_train_policies'': [''player_1'', ''player_2''], ''_call'':\n",
      "    {''checkpoint_freq'': 0, ''num_samples'': 2, ''name'': ''external_test_normal''},\n",
      "    ''_trainer'': ''External'', ''_policy'': ''PPO'', ''_train_steps'': 1, ''_update_config'':\n",
      "    None, ''_run_inline'': False, ''_postprocess'': None, ''num_envs_per_worker'':\n",
      "    2, ''_log_error'': True, ''_model_params'': {''use_lstm'': False, ''fcnet_hiddens'':\n",
      "    [64, 64], ''fcnet_activation'': ''tanh'', ''free_log_std'': True}, ''_select_policy'':\n",
      "    <function select_policy_default at 0x7fda0b05f3a0>, ''_get_policies'': <function\n",
      "    get_policies_default at 0x7fda0b05f310>, ''_do_not_train_policies'': [], ''_update_withpolicies'':\n",
      "    None, ''callbacks'': <class ''ap_rllib.config_info_callbacks.InfoCallbacks''>,\n",
      "    ''train_batch_size'': 1024, ''lr'': 0.0003, ''sgd_minibatch_size'': 1024, ''num_sgd_iter'':\n",
      "    2, ''rollout_fragment_length'': 100, ''run_uid'': ''_setme'', ''batch_mode'':\n",
      "    ''complete_episodes'', ''http_remote_port'': ''http://127.0.0.1:50001'', ''num_workers'':\n",
      "    0, ''_verbose'': True, ''_all_policies'': [''player_1'', ''player_2''], ''multiagent'':\n",
      "    {''policies'': {''player_1'': (<class ''ray.rllib.policy.tf_policy_template.PPOTFPolicy''>,\n",
      "    Box(380,), Box(17,), {''model'': {''custom_model'': ''GymCompetePretrainedModel'',\n",
      "    ''custom_model_config'': {''agent_id'': 0, ''env_name'': ''multicomp/YouShallNotPassHumans-v0'',\n",
      "    ''model_config'': {}, ''name'': ''model_0'', ''load_weights'': True}}, ''framework'':\n",
      "    ''tfe''}), ''player_2'': (<class ''ray.rllib.policy.tf_policy_template.PPOTFPolicy''>,\n",
      "    Box(380,), Box(17,), {''model'': {''custom_model'': ''GymCompetePretrainedModel'',\n",
      "    ''custom_model_config'': {''agent_id'': 1, ''env_name'': ''multicomp/YouShallNotPassHumans-v0'',\n",
      "    ''model_config'': {}, ''name'': ''model_1'', ''load_weights'': True}}, ''framework'':\n",
      "    ''tfe''})}}})'\n",
      "num_envs_per_worker: 2\n",
      "num_gpus: 0\n",
      "num_sgd_iter: 2\n",
      "num_workers: 0\n",
      "rollout_fragment_length: 100\n",
      "run_uid: _setme\n",
      "sgd_minibatch_size: 1024\n",
      "tf_session_args:\n",
      "  allow_soft_placement: true\n",
      "  device_count:\n",
      "    CPU: 4\n",
      "  gpu_options:\n",
      "    allow_growth: true\n",
      "  inter_op_parallelism_threads: 4\n",
      "  intra_op_parallelism_threads: 4\n",
      "  log_device_placement: true\n",
      "train_batch_size: 1024\n",
      "use_gae: true\n",
      "\n",
      "Creating agent humanoid_blocker\n",
      "Reading agent XML from: /scratch/sergei/better-adversarial-defenses/multiagent-competition/gym_compete/new_envs/assets/humanoid_body.xml\n",
      "Creating agent humanoid\n",
      "Reading agent XML from: /scratch/sergei/better-adversarial-defenses/multiagent-competition/gym_compete/new_envs/assets/humanoid_body.xml\n",
      "Scene XML path: /scratch/sergei/better-adversarial-defenses/multiagent-competition/gym_compete/new_envs/assets/world_body.humanoid_body.humanoid_body.xml\n",
      "Created Scene with agents\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-12-06 06:11:05,518\tWARNING catalog.py:333 -- Custom ModelV2 should accept all custom options as **kwargs, instead of expecting them in config['custom_model_config']!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting NN weights\n",
      "Loading normalization\n",
      "Setting NN weights\n",
      "Loading normalization\n",
      "It looks like variables [\"<tf.Varaible dense/bias:0 shape=(64,) <dtype: 'float32'>>\", \"<tf.Varaible player_1/param/clip:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible player_1/sequential/h2/kernel:0 shape=(64, 64) <dtype: 'float32'>>\", \"<tf.Varaible player_1/param/std:0 shape=(380,) <dtype: 'float32'>>\", \"<tf.Varaible player_1/param/clip:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible dense_1/bias:0 shape=(64,) <dtype: 'float32'>>\", \"<tf.Varaible player_1/var/std:0 shape=(1, 17) <dtype: 'float32'>>\", \"<tf.Varaible player_1/param/std:0 shape=(380,) <dtype: 'float32'>>\", \"<tf.Varaible player_1/param/mean:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible mean/bias:0 shape=(17,) <dtype: 'float32'>>\", \"<tf.Varaible mean/kernel:0 shape=(64, 17) <dtype: 'float32'>>\", \"<tf.Varaible player_1/sequential/h1/kernel:0 shape=(380, 64) <dtype: 'float32'>>\", \"<tf.Varaible player_1/sequential/h1/bias:0 shape=(64,) <dtype: 'float32'>>\", \"<tf.Varaible player_1/sequential/value/bias:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible player_1/param/mean:0 shape=(380,) <dtype: 'float32'>>\", \"<tf.Varaible dense_1/kernel:0 shape=(64, 64) <dtype: 'float32'>>\", \"<tf.Varaible player_1/sequential/h2/bias:0 shape=(64,) <dtype: 'float32'>>\", \"<tf.Varaible player_1/param/mean:0 shape=(380,) <dtype: 'float32'>>\", \"<tf.Varaible dense/kernel:0 shape=(380, 64) <dtype: 'float32'>>\", \"<tf.Varaible player_1/param/std:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible player_1/sequential/value/kernel:0 shape=(64, 1) <dtype: 'float32'>>\"] were created as part of <gym_compete_rllib.gym_compete_to_rllib.GymCompetePretrainedModel object at 0x7fc9a80bb880> but does not appear in model.variables() ([\"<tf.Varaible player_1/param/clip:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible player_1/sequential_1/h1/kernel:0 shape=(380, 64) <dtype: 'float32'>>\", \"<tf.Varaible player_1/param/std:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible player_1/sequential_1/h1/bias:0 shape=(64,) <dtype: 'float32'>>\", \"<tf.Varaible player_1/param/mean:0 shape=(380,) <dtype: 'float32'>>\", \"<tf.Varaible dense_2/bias:0 shape=(64,) <dtype: 'float32'>>\", \"<tf.Varaible dense_3/bias:0 shape=(64,) <dtype: 'float32'>>\", \"<tf.Varaible player_1/sequential_1/value/kernel:0 shape=(64, 1) <dtype: 'float32'>>\", \"<tf.Varaible player_1/param/std:0 shape=(380,) <dtype: 'float32'>>\", \"<tf.Varaible player_1/param/mean:0 shape=(380,) <dtype: 'float32'>>\", \"<tf.Varaible player_1/sequential_1/h2/kernel:0 shape=(64, 64) <dtype: 'float32'>>\", \"<tf.Varaible player_1/sequential_1/value/bias:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible mean_1/kernel:0 shape=(64, 17) <dtype: 'float32'>>\", \"<tf.Varaible player_1/param/clip:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible player_1/sequential_1/h2/bias:0 shape=(64,) <dtype: 'float32'>>\", \"<tf.Varaible player_1/param/mean:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible player_1/var/std:0 shape=(1, 17) <dtype: 'float32'>>\", \"<tf.Varaible player_1/param/std:0 shape=(380,) <dtype: 'float32'>>\", \"<tf.Varaible dense_2/kernel:0 shape=(380, 64) <dtype: 'float32'>>\", \"<tf.Varaible dense_3/kernel:0 shape=(64, 64) <dtype: 'float32'>>\", \"<tf.Varaible mean_1/bias:0 shape=(17,) <dtype: 'float32'>>\"]). Did you forget to call model.register_variables() on the variables in question?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-12-06 06:11:06,129\tWARNING catalog.py:333 -- Custom ModelV2 should accept all custom options as **kwargs, instead of expecting them in config['custom_model_config']!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting NN weights\n",
      "Loading normalization\n",
      "Setting NN weights\n",
      "Loading normalization\n",
      "It looks like variables [\"<tf.Varaible dense_5/kernel:0 shape=(64, 64) <dtype: 'float32'>>\", \"<tf.Varaible player_2/sequential_2/h2/kernel:0 shape=(64, 64) <dtype: 'float32'>>\", \"<tf.Varaible mean_2/kernel:0 shape=(64, 17) <dtype: 'float32'>>\", \"<tf.Varaible mean_2/bias:0 shape=(17,) <dtype: 'float32'>>\", \"<tf.Varaible dense_5/bias:0 shape=(64,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/param/mean:0 shape=(380,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/param/mean:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/sequential_2/h2/bias:0 shape=(64,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/sequential_2/h1/kernel:0 shape=(380, 64) <dtype: 'float32'>>\", \"<tf.Varaible player_2/param/std:0 shape=(380,) <dtype: 'float32'>>\", \"<tf.Varaible dense_4/bias:0 shape=(64,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/sequential_2/value/kernel:0 shape=(64, 1) <dtype: 'float32'>>\", \"<tf.Varaible player_2/param/mean:0 shape=(380,) <dtype: 'float32'>>\", \"<tf.Varaible dense_4/kernel:0 shape=(380, 64) <dtype: 'float32'>>\", \"<tf.Varaible player_2/var/std:0 shape=(1, 17) <dtype: 'float32'>>\", \"<tf.Varaible player_2/sequential_2/value/bias:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/param/std:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/param/clip:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/param/clip:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/param/std:0 shape=(380,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/sequential_2/h1/bias:0 shape=(64,) <dtype: 'float32'>>\"] were created as part of <gym_compete_rllib.gym_compete_to_rllib.GymCompetePretrainedModel object at 0x7fc9907e7520> but does not appear in model.variables() ([\"<tf.Varaible dense_7/bias:0 shape=(64,) <dtype: 'float32'>>\", \"<tf.Varaible mean_3/kernel:0 shape=(64, 17) <dtype: 'float32'>>\", \"<tf.Varaible player_2/param/std:0 shape=(380,) <dtype: 'float32'>>\", \"<tf.Varaible mean_3/bias:0 shape=(17,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/var/std:0 shape=(1, 17) <dtype: 'float32'>>\", \"<tf.Varaible player_2/param/clip:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/param/mean:0 shape=(380,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/sequential_3/h2/kernel:0 shape=(64, 64) <dtype: 'float32'>>\", \"<tf.Varaible dense_6/bias:0 shape=(64,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/param/mean:0 shape=(380,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/sequential_3/h1/kernel:0 shape=(380, 64) <dtype: 'float32'>>\", \"<tf.Varaible player_2/sequential_3/h2/bias:0 shape=(64,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/param/mean:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible dense_6/kernel:0 shape=(380, 64) <dtype: 'float32'>>\", \"<tf.Varaible player_2/param/std:0 shape=(380,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/sequential_3/h1/bias:0 shape=(64,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/sequential_3/value/kernel:0 shape=(64, 1) <dtype: 'float32'>>\", \"<tf.Varaible player_2/param/clip:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/param/std:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible player_2/sequential_3/value/bias:0 shape=(1,) <dtype: 'float32'>>\", \"<tf.Varaible dense_7/kernel:0 shape=(64, 64) <dtype: 'float32'>>\"]). Did you forget to call model.register_variables() on the variables in question?\n",
      "Creating agent humanoid_blocker\n",
      "Reading agent XML from: /scratch/sergei/better-adversarial-defenses/multiagent-competition/gym_compete/new_envs/assets/humanoid_body.xml\n",
      "Creating agent humanoid\n",
      "Reading agent XML from: /scratch/sergei/better-adversarial-defenses/multiagent-competition/gym_compete/new_envs/assets/humanoid_body.xml\n",
      "Scene XML path: /scratch/sergei/better-adversarial-defenses/multiagent-competition/gym_compete/new_envs/assets/world_body.humanoid_body.humanoid_body.xml\n",
      "Created Scene with agents\n"
     ]
    }
   ],
   "source": [
    "config = get_config_by_name(\"external_test_normal\")\n",
    "config['_verbose'] = True\n",
    "trainer = get_trainer(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'player_1': (<class 'ray.rllib.policy.tf_policy_template.PPOTFPolicy'>, Box(380,), Box(17,), {'model': {'custom_model': 'GymCompetePretrainedModel', 'custom_model_config': {'agent_id': 0, 'env_name': 'multicomp/YouShallNotPassHumans-v0', 'model_config': {}, 'name': 'model_0', 'load_weights': True}}, 'framework': 'tfe'}), 'player_2': (<class 'ray.rllib.policy.tf_policy_template.PPOTFPolicy'>, Box(380,), Box(17,), {'model': {'custom_model': 'GymCompetePretrainedModel', 'custom_model_config': {'agent_id': 1, 'env_name': 'multicomp/YouShallNotPassHumans-v0', 'model_config': {}, 'name': 'model_1', 'load_weights': True}}, 'framework': 'tfe'})}\n",
      "Setting NN weights\n",
      "Loading normalization\n",
      "Setting NN weights\n",
      "Loading normalization\n",
      "WARNING:tensorflow:From /scratch/sergei/miniconda3/envs/adv-tf2/lib/python3.8/site-packages/ray/rllib/policy/tf_policy.py:875: Variable.load (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Prefer Variable.assign which has equivalent behavior in 2.X.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'episode_reward_max': 6.714535920480557,\n",
       " 'episode_reward_min': -2.5237779254675843,\n",
       " 'episode_reward_mean': -0.4037645710236575,\n",
       " 'episode_len_mean': 139.75,\n",
       " 'episodes_this_iter': 8,\n",
       " 'policy_reward_min': {'player_1': -10.0, 'player_2': -12.523777925467584},\n",
       " 'policy_reward_max': {'player_1': 10.0, 'player_2': 16.714535920480557},\n",
       " 'policy_reward_mean': {'player_1': 7.5, 'player_2': -7.9037645710236575},\n",
       " 'custom_metrics': {'contacts_mean': 3.875,\n",
       "  'contacts_min': 3,\n",
       "  'contacts_max': 6},\n",
       " 'hist_stats': {'episode_reward': [-1.2458420806033246,\n",
       "   6.714535920480557,\n",
       "   -2.5237779254675843,\n",
       "   -0.9334178798599169,\n",
       "   -1.5543348569844966,\n",
       "   -0.405577100264054,\n",
       "   -1.1074954450195946,\n",
       "   -2.174207200470846],\n",
       "  'episode_lengths': [138, 203, 145, 110, 114, 113, 126, 169],\n",
       "  'policy_player_1_reward': [10.0, -10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0],\n",
       "  'policy_player_2_reward': [-11.245842080603325,\n",
       "   16.714535920480557,\n",
       "   -12.523777925467584,\n",
       "   -10.933417879859917,\n",
       "   -11.554334856984497,\n",
       "   -10.405577100264054,\n",
       "   -11.107495445019595,\n",
       "   -12.174207200470846]},\n",
       " 'sampler_perf': {'mean_env_wait_ms': 5.9740195338358015,\n",
       "  'mean_raw_obs_processing_ms': 0.2214932601723895,\n",
       "  'mean_inference_ms': 4.482342892845205,\n",
       "  'mean_action_processing_ms': 0.1548768689968442},\n",
       " 'off_policy_estimator': {},\n",
       " 'num_healthy_workers': 0,\n",
       " 'timesteps_total': 1118,\n",
       " 'timers': {'sample_time_ms': 6474.789,\n",
       "  'sample_throughput': 172.67,\n",
       "  'load_time_ms': 0.0,\n",
       "  'load_throughput': 0.0,\n",
       "  'learn_time_ms': 0.0,\n",
       "  'learn_throughput': 0.0},\n",
       " 'info': {'learner': {'player_1': {'serial_timesteps': 1111,\n",
       "    'n_updates': 1,\n",
       "    'total_timesteps': 0,\n",
       "    'fps': 15466,\n",
       "    'explained_variance': 0.5691617429256439,\n",
       "    'time_elapsed': 3.170967102050781e-05,\n",
       "    'policy_loss': -0.007818365,\n",
       "    'value_loss': 10.258049,\n",
       "    'policy_entropy': 4.980152,\n",
       "    'approxkl': 0.00039974062,\n",
       "    'clipfrac': 0.00045004502},\n",
       "   'player_2': {'serial_timesteps': 1111,\n",
       "    'n_updates': 1,\n",
       "    'total_timesteps': 0,\n",
       "    'fps': 15905,\n",
       "    'explained_variance': 0.6273690462112427,\n",
       "    'time_elapsed': 2.7179718017578125e-05,\n",
       "    'policy_loss': -0.008410335,\n",
       "    'value_loss': 21.054031,\n",
       "    'policy_entropy': 2.6639924,\n",
       "    'approxkl': 0.00043446154,\n",
       "    'clipfrac': 0.00045004502}},\n",
       "  'num_steps_sampled': 1118,\n",
       "  'num_steps_trained': 1118},\n",
       " 'done': False,\n",
       " 'episodes_total': 8,\n",
       " 'training_iteration': 1,\n",
       " 'experiment_id': '6fb13c7105c34bfdbd840afb20a0e74d',\n",
       " 'date': '2020-12-06_06-11-13',\n",
       " 'timestamp': 1607263873,\n",
       " 'time_this_iter_s': 6.844105005264282,\n",
       " 'time_total_s': 6.844105005264282,\n",
       " 'pid': 37844,\n",
       " 'hostname': 'svm',\n",
       " 'node_ip': '128.32.175.10',\n",
       " 'config': {'num_workers': 0,\n",
       "  'num_envs_per_worker': 2,\n",
       "  'create_env_on_driver': False,\n",
       "  'rollout_fragment_length': 100,\n",
       "  'batch_mode': 'complete_episodes',\n",
       "  'num_gpus': 0,\n",
       "  'train_batch_size': 1024,\n",
       "  'model': {'fcnet_hiddens': [256, 256],\n",
       "   'fcnet_activation': 'tanh',\n",
       "   'conv_filters': None,\n",
       "   'conv_activation': 'relu',\n",
       "   'free_log_std': False,\n",
       "   'no_final_linear': False,\n",
       "   'vf_share_layers': True,\n",
       "   'use_lstm': False,\n",
       "   'max_seq_len': 20,\n",
       "   'lstm_cell_size': 256,\n",
       "   'lstm_use_prev_action_reward': False,\n",
       "   '_time_major': False,\n",
       "   'framestack': True,\n",
       "   'dim': 84,\n",
       "   'grayscale': False,\n",
       "   'zero_mean': True,\n",
       "   'custom_model': None,\n",
       "   'custom_model_config': {},\n",
       "   'custom_action_dist': None,\n",
       "   'custom_preprocessor': None},\n",
       "  'optimizer': {},\n",
       "  'gamma': 0.99,\n",
       "  'horizon': None,\n",
       "  'soft_horizon': False,\n",
       "  'no_done_at_end': False,\n",
       "  'env_config': {'with_video': False,\n",
       "   'SingleAgentToMultiAgent': False,\n",
       "   'env_name': 'multicomp/YouShallNotPassHumans-v0'},\n",
       "  'env': 'multicomp',\n",
       "  'normalize_actions': False,\n",
       "  'clip_rewards': None,\n",
       "  'clip_actions': True,\n",
       "  'preprocessor_pref': 'deepmind',\n",
       "  'lr': 0.0003,\n",
       "  'monitor': False,\n",
       "  'log_level': 'WARN',\n",
       "  'callbacks': ap_rllib.config_info_callbacks.InfoCallbacks,\n",
       "  'ignore_worker_failures': False,\n",
       "  'log_sys_usage': True,\n",
       "  'fake_sampler': False,\n",
       "  'framework': 'tfe',\n",
       "  'eager_tracing': False,\n",
       "  'explore': True,\n",
       "  'exploration_config': {'type': 'StochasticSampling'},\n",
       "  'evaluation_interval': None,\n",
       "  'evaluation_num_episodes': 10,\n",
       "  'in_evaluation': False,\n",
       "  'evaluation_config': {},\n",
       "  'evaluation_num_workers': 0,\n",
       "  'custom_eval_function': None,\n",
       "  'sample_async': False,\n",
       "  '_use_trajectory_view_api': False,\n",
       "  'observation_filter': 'NoFilter',\n",
       "  'synchronize_filters': True,\n",
       "  'tf_session_args': {'intra_op_parallelism_threads': 4,\n",
       "   'inter_op_parallelism_threads': 4,\n",
       "   'gpu_options': {'allow_growth': True},\n",
       "   'log_device_placement': True,\n",
       "   'device_count': {'CPU': 4},\n",
       "   'allow_soft_placement': True},\n",
       "  'local_tf_session_args': {'intra_op_parallelism_threads': 4,\n",
       "   'inter_op_parallelism_threads': 4},\n",
       "  'compress_observations': False,\n",
       "  'collect_metrics_timeout': 180,\n",
       "  'metrics_smoothing_episodes': 100,\n",
       "  'remote_worker_envs': False,\n",
       "  'remote_env_batch_wait_ms': 0,\n",
       "  'min_iter_time_s': 0,\n",
       "  'timesteps_per_iteration': 0,\n",
       "  'seed': None,\n",
       "  'extra_python_environs_for_driver': {},\n",
       "  'extra_python_environs_for_worker': {},\n",
       "  'num_cpus_per_worker': 1,\n",
       "  'num_gpus_per_worker': 0,\n",
       "  'custom_resources_per_worker': {},\n",
       "  'num_cpus_for_driver': 1,\n",
       "  'memory': 0,\n",
       "  'object_store_memory': 0,\n",
       "  'memory_per_worker': 0,\n",
       "  'object_store_memory_per_worker': 0,\n",
       "  'input': 'sampler',\n",
       "  'input_evaluation': ['is', 'wis'],\n",
       "  'postprocess_inputs': False,\n",
       "  'shuffle_buffer_size': 0,\n",
       "  'output': None,\n",
       "  'output_compress_columns': ['obs', 'new_obs'],\n",
       "  'output_max_file_size': 67108864,\n",
       "  'multiagent': {'policies': {'player_1': (ray.rllib.policy.tf_policy_template.PPOTFPolicy,\n",
       "     Box(380,),\n",
       "     Box(17,),\n",
       "     {'model': {'custom_model': 'GymCompetePretrainedModel',\n",
       "       'custom_model_config': {'agent_id': 0,\n",
       "        'env_name': 'multicomp/YouShallNotPassHumans-v0',\n",
       "        'model_config': {},\n",
       "        'name': 'model_0',\n",
       "        'load_weights': True}},\n",
       "      'framework': 'tfe'}),\n",
       "    'player_2': (ray.rllib.policy.tf_policy_template.PPOTFPolicy,\n",
       "     Box(380,),\n",
       "     Box(17,),\n",
       "     {'model': {'custom_model': 'GymCompetePretrainedModel',\n",
       "       'custom_model_config': {'agent_id': 1,\n",
       "        'env_name': 'multicomp/YouShallNotPassHumans-v0',\n",
       "        'model_config': {},\n",
       "        'name': 'model_1',\n",
       "        'load_weights': True}},\n",
       "      'framework': 'tfe'})},\n",
       "   'policy_mapping_fn': functools.partial(<function select_policy_default at 0x7fda0b05f3a0>, config={'kl_coeff': 1.0, '_num_workers_tf': 4, 'use_gae': True, 'num_gpus': 0, '_env_name_rllib': 'multicomp', '_env_fcn': <function create_env at 0x7fda0c62b700>, '_policies': [None, 'pretrained', 'pretrained'], '_env': {'with_video': False, 'SingleAgentToMultiAgent': False, 'env_name': 'multicomp/YouShallNotPassHumans-v0'}, 'framework': 'tfe', '_train_policies': ['player_1', 'player_2'], '_call': {'checkpoint_freq': 0, 'num_samples': 2, 'name': 'external_test_normal'}, '_trainer': 'External', '_policy': 'PPO', '_train_steps': 1, '_update_config': None, '_run_inline': False, '_postprocess': None, 'num_envs_per_worker': 2, '_log_error': True, '_model_params': {'use_lstm': False, 'fcnet_hiddens': [64, 64], 'fcnet_activation': 'tanh', 'free_log_std': True}, '_select_policy': <function select_policy_default at 0x7fda0b05f3a0>, '_get_policies': <function get_policies_default at 0x7fda0b05f310>, '_do_not_train_policies': [], '_update_withpolicies': None, 'callbacks': <class 'ap_rllib.config_info_callbacks.InfoCallbacks'>, 'train_batch_size': 1024, 'lr': 0.0003, 'sgd_minibatch_size': 1024, 'num_sgd_iter': 2, 'rollout_fragment_length': 100, 'run_uid': '_setme', 'batch_mode': 'complete_episodes', 'http_remote_port': 'http://127.0.0.1:50001', 'num_workers': 0, '_verbose': True, '_all_policies': ['player_1', 'player_2'], 'multiagent': {'policies': {'player_1': (<class 'ray.rllib.policy.tf_policy_template.PPOTFPolicy'>, Box(380,), Box(17,), {'model': {'custom_model': 'GymCompetePretrainedModel', 'custom_model_config': {'agent_id': 0, 'env_name': 'multicomp/YouShallNotPassHumans-v0', 'model_config': {}, 'name': 'model_0', 'load_weights': True}}, 'framework': 'tfe'}), 'player_2': (<class 'ray.rllib.policy.tf_policy_template.PPOTFPolicy'>, Box(380,), Box(17,), {'model': {'custom_model': 'GymCompetePretrainedModel', 'custom_model_config': {'agent_id': 1, 'env_name': 'multicomp/YouShallNotPassHumans-v0', 'model_config': {}, 'name': 'model_1', 'load_weights': True}}, 'framework': 'tfe'})}}}),\n",
       "   'policies_to_train': ['player_1', 'player_2'],\n",
       "   'observation_fn': None,\n",
       "   'replay_mode': 'independent'},\n",
       "  'logger_config': None,\n",
       "  'replay_sequence_length': 1,\n",
       "  'use_critic': True,\n",
       "  'use_gae': True,\n",
       "  'lambda': 1.0,\n",
       "  'kl_coeff': 1.0,\n",
       "  'sgd_minibatch_size': 1024,\n",
       "  'shuffle_sequences': True,\n",
       "  'num_sgd_iter': 2,\n",
       "  'lr_schedule': None,\n",
       "  'vf_share_layers': False,\n",
       "  'vf_loss_coeff': 1.0,\n",
       "  'entropy_coeff': 0.0,\n",
       "  'entropy_coeff_schedule': None,\n",
       "  'clip_param': 0.3,\n",
       "  'vf_clip_param': 10.0,\n",
       "  'grad_clip': None,\n",
       "  'kl_target': 0.01,\n",
       "  'simple_optimizer': False,\n",
       "  '_fake_gpus': False,\n",
       "  'http_remote_port': 'http://127.0.0.1:50001',\n",
       "  'run_uid': '_setme',\n",
       "  'tmp_dir': '/tmp/'},\n",
       " 'time_since_restore': 6.844105005264282,\n",
       " 'timesteps_since_restore': 0,\n",
       " 'iterations_since_restore': 1,\n",
       " 'perf': {'cpu_util_percent': 5.77, 'ram_util_percent': 16.6}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating agent humanoid_blocker\n",
      "Reading agent XML from: /scratch/sergei/better-adversarial-defenses/multiagent-competition/gym_compete/new_envs/assets/humanoid_body.xml\n",
      "Creating agent humanoid\n",
      "Reading agent XML from: /scratch/sergei/better-adversarial-defenses/multiagent-competition/gym_compete/new_envs/assets/humanoid_body.xml\n",
      "Scene XML path: /scratch/sergei/better-adversarial-defenses/multiagent-competition/gym_compete/new_envs/assets/world_body.humanoid_body.humanoid_body.xml\n",
      "Created Scene with agents\n"
     ]
    }
   ],
   "source": [
    "env_rllib = trainer.env_creator(config=config['_env'])\n",
    "env = env_rllib._env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "p1 = trainer.get_policy('player_1')\n",
    "p2 = trainer.get_policy('player_2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 1 [(b'agent0/head', b'agent1/left_uarm1', -0.02856779159299743)]\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 1 [(b'agent0/head', b'agent1/left_uarm1', -0.030826327961423657)]\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 6 [(b'agent0/head', b'agent1/head', -0.004791843197400364), (b'agent0/head', b'agent1/left_uarm1', -0.018502689419600656), (b'agent0/left_uarm1', b'agent1/right_uarm1', -0.007209018556424557), (b'agent0/left_uarm1', b'agent1/right_larm', -0.011258003785089503), (b'agent0/left_larm', b'agent1/right_uarm1', -0.001352341748708609), (b'agent0/left_larm', b'agent1/right_larm', -0.013471852146583432)]\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 9 [(b'agent0/head', b'agent1/head', -0.007295457887646711), (b'agent0/head', b'agent1/left_uarm1', -0.008305901855659013), (b'agent0/right_larm', b'agent1/left_larm', -0.01359971910231813), (b'agent0/right_hand', b'agent1/left_larm', -0.013964256131554359), (b'agent0/left_uarm1', b'agent1/torso1', -0.0018965621558573176), (b'agent0/left_uarm1', b'agent1/right_uarm1', -0.016017511486812144), (b'agent0/left_uarm1', b'agent1/right_larm', -0.023956464128704807), (b'agent0/left_larm', b'agent1/right_uarm1', -0.002457225814464871), (b'agent0/left_larm', b'agent1/right_larm', -0.02548028846789782)]\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 9 [(b'agent0/head', b'agent1/head', -0.0031635540710462373), (b'agent0/torso1', b'agent1/left_uarm1', -0.010120193355406225), (b'agent0/head', b'agent1/left_uarm1', -0.003098324534673201), (b'agent0/right_larm', b'agent1/left_larm', -0.017024208985568133), (b'agent0/right_hand', b'agent1/left_larm', -0.01690716262079516), (b'agent0/left_uarm1', b'agent1/torso1', -0.01536004078541376), (b'agent0/left_uarm1', b'agent1/right_uarm1', -0.012581789179336722), (b'agent0/left_uarm1', b'agent1/right_larm', -0.0196928884198568), (b'agent0/left_larm', b'agent1/right_larm', -0.02009241430088702)]\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 9 [(b'agent0/head', b'agent1/head', -0.0008469677183224056), (b'agent0/torso1', b'agent1/left_uarm1', -0.010786322347100467), (b'agent0/head', b'agent1/left_uarm1', -0.0013092447524617398), (b'agent0/right_larm', b'agent1/left_larm', -0.010625633554749825), (b'agent0/right_hand', b'agent1/left_larm', -0.012751991110987411), (b'agent0/left_uarm1', b'agent1/torso1', -0.013272927094187599), (b'agent0/left_uarm1', b'agent1/right_uarm1', -0.006135798043239354), (b'agent0/left_uarm1', b'agent1/right_larm', -0.011884248334987788), (b'agent0/left_larm', b'agent1/right_larm', -0.011034986401076835)]\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 6 [(b'agent0/torso1', b'agent1/left_uarm1', -0.00794943372258592), (b'agent0/head', b'agent1/left_uarm1', -0.0006341712185296486), (b'agent0/right_larm', b'agent1/left_larm', -0.004835676372065098), (b'agent0/right_hand', b'agent1/left_larm', -0.007865659893469316), (b'agent0/left_uarm1', b'agent1/torso1', -0.006569510458251687), (b'agent0/left_uarm1', b'agent1/right_larm', -0.0008921447224562659)]\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 3 [(b'agent0/torso1', b'agent1/left_uarm1', -0.004921561767085435), (b'agent0/head', b'agent1/left_uarm1', -0.0003796597366096874), (b'agent0/right_hand', b'agent1/left_larm', -0.004507904680246123)]\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 3 [(b'agent0/torso1', b'agent1/left_uarm1', -0.00017259610709565582), (b'agent0/head', b'agent1/left_uarm1', -0.0005250951777242482), (b'agent0/right_hand', b'agent1/left_larm', -0.0007853666155498637)]\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 2 [(b'agent0/head', b'agent1/left_uarm1', -0.0007273997315753108), (b'agent0/right_larm', b'agent1/left_larm', -0.0025311595279453245)]\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 2 [(b'agent0/head', b'agent1/left_uarm1', -0.0004331854974061397), (b'agent0/right_larm', b'agent1/left_larm', -0.002647133057397999)]\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 1 [(b'agent1/right_hand', b'agent0/left_uarm1', -0.0007311036136538329)]\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 1 [(b'agent1/right_hand', b'agent0/left_uarm1', -0.0018955327726117993)]\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 1 [(b'agent1/right_hand', b'agent0/left_uarm1', -0.0012192795933963357)]\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 1 [(b'agent1/right_hand', b'agent0/left_uarm1', -0.0015684281037782605)]\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': False, 'player_2': False, '__all__': False} 0 []\n",
      "{'player_1': True, 'player_2': True, '__all__': True} 0 []\n"
     ]
    }
   ],
   "source": [
    "obs = env_rllib.reset()\n",
    "done = {'__all__': False}\n",
    "\n",
    "while not done['__all__']:\n",
    "    a1 = p1.compute_single_action(obs['player_1'])[0]\n",
    "    a2 = p2.compute_single_action(obs['player_2'])[0]\n",
    "\n",
    "    contacts = env_rllib.get_agent_contacts()\n",
    "    \n",
    "    obs, rew, done, info = env_rllib.step({'player_1': a1, 'player_2': a2})\n",
    "    print(done, len(contacts), contacts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}