doesn't train property

Author: bic4907

agent.py

@@ -31,28 +31,37 @@ def __init__(self, s_dim, a_dim, n_agents, **kwargs):
         hard_update(self.policy, self.policy_target)
         hard_update(self.critic, self.critic_target)
 
-        self.random_process = OrnsteinUhlenbeckProcess(size=self.a_dim, theta=self.config.ou_theta, mu=self.config.ou_mu, sigma=self.config.ou_sigma)
+        self.random_process = OrnsteinUhlenbeckProcess(size=self.a_dim,
+                                                       theta=self.config.ou_theta,
+                                                       mu=self.config.ou_mu,
+                                                       sigma=self.config.ou_sigma)
         self.replay_buffer = list()
         self.epsilon = 1.
         self.depsilon = self.epsilon / self.config.epsilon_decay
 
         self.c_loss = None
         self.a_loss = None
+        self.action_log = list()
 
     def choose_action(self, obs, noisy=True):
         obs = torch.Tensor([obs]).to(self.device)
 
         action = self.policy(obs).cpu().detach().numpy()[0]
+        self.action_log.append(action)
+
         if noisy:
             for agent_idx in range(self.n_agents):
-                action[agent_idx] += max(self.epsilon, 0.001) * self.random_process.sample()
+                pass
+                # action[agent_idx] += self.epsilon * self.random_process.sample()
             self.epsilon -= self.depsilon
+            self.epsilon = max(self.epsilon, 0.001)
         np.clip(action, -1., 1.)
 
         return action
 
     def reset(self):
         self.random_process.reset_states()
+        self.action_log.clear()
 
     def prep_train(self):
         self.policy.train()
@@ -66,7 +75,6 @@ def prep_eval(self):
         self.policy_target.eval()
         self.critic_target.eval()
 
-
     def random_action(self):
         return np.random.uniform(low=-1, high=1, size=(self.n_agents, 2))
 
@@ -85,9 +93,11 @@ def get_batches(self):
         next_state_batches = np.array([_[3] for _ in experiences])
         done_batches = np.array([_[4] for _ in experiences])
 
+
         return state_batches, action_batches, reward_batches, next_state_batches, done_batches
 
     def train(self):
+
         state_batches, action_batches, reward_batches, next_state_batches, done_batches = self.get_batches()
 
         state_batches = torch.Tensor(state_batches).to(self.device)
@@ -97,23 +107,36 @@ def train(self):
         done_batches = torch.Tensor((done_batches == False) * 1).view(-1, self.n_agents, 1).to(self.device)
 
         target_next_actions = self.policy_target.forward(next_state_batches).detach()
-        target_next_q = self.critic_target.forward(next_state_batches, target_next_actions).detach()
-
+        target_next_q = self.critic_target.forward(next_state_batches, target_next_actions)
         main_q = self.critic(state_batches, action_batches)
 
+        '''
+        How to concat each agent's Q value?
+        '''
+        #target_next_q = target_next_q
+        #main_q = main_q.mean(dim=1)
+
+
+        '''
+        Reward Norm
+        '''
+        # reward_batches = (reward_batches - reward_batches.mean(dim=0)) / reward_batches.std(dim=0) / 1024
+
         # Critic Loss
         self.critic.zero_grad()
         baselines = reward_batches + done_batches * self.config.gamma * target_next_q
-        loss_critic = torch.nn.MSELoss()(main_q, baselines.cuda())
+        loss_critic = torch.nn.MSELoss()(main_q, baselines.detach())
         loss_critic.backward()
+        torch.nn.utils.clip_grad_norm_(self.critic.parameters(), 0.5)
         self.critic_optimizer.step()
 
-        # TODO Edit Actor Loss
         # Actor Loss
         self.policy.zero_grad()
         clear_action_batches = self.policy.forward(state_batches)
         loss_actor = (-self.critic.forward(state_batches, clear_action_batches)).mean()
+        loss_actor += (clear_action_batches ** 2).mean() * 1e-3
         loss_actor.backward()
+        torch.nn.utils.clip_grad_norm_(self.policy.parameters(), 0.5)
         self.policy_optimizer.step()
 
         # This is for logging
@@ -124,4 +147,7 @@ def train(self):
         soft_update(self.critic, self.critic_target, self.config.tau)
 
     def get_loss(self):
-        return self.c_loss, self.a_loss
+        return self.c_loss, self.a_loss
+
+    def get_action_std(self):
+        return np.array(self.action_log).std(axis=-1).mean()

network.py

@@ -1,6 +1,10 @@
 import torch
 import torch.nn as nn
 
+from utils import weight_init
+
+HIDDEN_DIM = 200
+
 
 class Actor(nn.Module):
     def __init__(self, s_dim, a_dim, n_agents):
@@ -11,19 +15,19 @@ def __init__(self, s_dim, a_dim, n_agents):
         self.n_agents = n_agents
 
         # input (batch, s_dim) output (batch, 300)
-        self.prev_dense = DenseNet(s_dim, 200, 200, output_activation=None, norm_in=True)
+
+        self.prev_dense = DenseNet(s_dim, HIDDEN_DIM // 2, HIDDEN_DIM, output_activation=None, norm_in=True)
         # input (num_agents, batch, 200) output (num_agents, batch, num_agents * 2)\
-        self.comm_net = LSTMNet(200, n_agents, num_layers=1)
+        self.comm_net = LSTMNet(HIDDEN_DIM, HIDDEN_DIM, num_layers=1)
         # input (batch, 2) output (batch, a_dim)
-        self.post_dense = DenseNet(6, 32, a_dim, output_activation=nn.Tanh)
+        self.post_dense = DenseNet(HIDDEN_DIM * 2, HIDDEN_DIM, a_dim, output_activation=nn.Tanh)
 
     def forward(self, x):
-
         x = x.view(-1, self.s_dim)
         x = self.prev_dense(x)
-        x = x.reshape(-1, self.n_agents, 200)
+        x = x.reshape(-1, self.n_agents, HIDDEN_DIM)
         x = self.comm_net(x)
-        x = x.reshape(-1, 6)
+        x = x.reshape(-1, HIDDEN_DIM * 2)
         x = self.post_dense(x)
         x = x.view(-1, self.n_agents, self.a_dim)
         return x
@@ -38,19 +42,19 @@ def __init__(self, s_dim, a_dim, n_agents):
         self.n_agents = n_agents
 
         # input (batch, s_dim) output (batch, 300)
-        self.prev_dense = DenseNet((s_dim + a_dim), 200, 200, output_activation=None, norm_in=True)
+        self.prev_dense = DenseNet((s_dim + a_dim), HIDDEN_DIM // 2, HIDDEN_DIM, output_activation=None, norm_in=True)
         # input (num_agents, batch, 200) output (num_agents, batch, num_agents * 2)\
-        self.comm_net = LSTMNet(200, n_agents, num_layers=1)
+        self.comm_net = LSTMNet(HIDDEN_DIM, HIDDEN_DIM, num_layers=1)
         # input (batch, 2) output (batch, a_dim)
-        self.post_dense = DenseNet(6, 32, 1, output_activation=None)
+        self.post_dense = DenseNet(HIDDEN_DIM * 2, HIDDEN_DIM, 1, output_activation=None)
 
     def forward(self, x_n, a_n):
         x = torch.cat((x_n, a_n), dim=-1)
         x = x.view(-1, (self.s_dim + self.a_dim))
         x = self.prev_dense(x)
-        x = x.reshape(-1, self.n_agents, 200)
+        x = x.reshape(-1, self.n_agents, HIDDEN_DIM)
         x = self.comm_net(x)
-        x = x.reshape(-1, 6)
+        x = x.reshape(-1, HIDDEN_DIM * 2)
         x = self.post_dense(x)
         x = x.view(-1, self.n_agents, 1)
         return x
@@ -65,10 +69,10 @@ def __init__(self, s_dim, hidden_dim, a_dim, norm_in=False, hidden_activation=nn
         if self._norm_in:
             self.norm1 = nn.BatchNorm1d(s_dim)
 
-        self.dense1 = nn.Linear(s_dim, hidden_dim)
-        self.dense2 = nn.Linear(hidden_dim, hidden_dim)
-        self.dense3 = nn.Linear(hidden_dim, hidden_dim)
-        self.dense4 = nn.Linear(hidden_dim, a_dim)
+        self.dense1 = nn.Linear(s_dim, hidden_dim // 2)
+        self.dense2 = nn.Linear(hidden_dim // 2, hidden_dim)
+        self.dense3 = nn.Linear(hidden_dim, hidden_dim // 2)
+        self.dense4 = nn.Linear(hidden_dim // 2, a_dim)
 
         if hidden_activation:
             self.hidden_activation = hidden_activation()

normalized_env.py

@@ -0,0 +1,36 @@
+'''
+Implemented by ghliu
+https://github.com/ghliu/pytorch-ddpg/blob/master/normalized_env.py
+'''
+
+import gym
+import numpy as np
+
+# https://github.com/openai/gym/blob/master/gym/core.py
+class ActionNormalizedEnv(gym.ActionWrapper):
+    """ Wrap action """
+    def __init__(self, env):
+        super(ActionNormalizedEnv, self).__init__(env=env)
+        self.action_high = 1.
+        self.action_low = -1.
+
+    def action(self, action):
+        act_k = (self.action_high - self.action_low)/ 2.
+        act_b = (self.action_high + self.action_low)/ 2.
+        return act_k * action + act_b
+
+    def reverse_action(self, action):
+        act_k_inv = 2./(self.action_high - self.action_low)
+        act_b = (self.action_high + self.action_low)/ 2.
+        return act_k_inv * (action - act_b)
+
+class ObsNormalizedEnv(gym.ObservationWrapper):
+    """ Wrap action """
+    def __init__(self, env):
+        super(ObsNormalizedEnv, self).__init__(env=env)
+        self.action_high = 1.
+        self.action_low = -1.
+
+    def observation(self, observation):
+        obs = np.array([[observation[0][2] - observation[0][0], observation[0][3] - observation[0][1]]])
+        return obs

train.py

@@ -2,18 +2,25 @@
 import argparse, datetime
 from tensorboardX import SummaryWriter
 import numpy as np
+import torch
 
 from agent import BiCNet
+from normalized_env import ActionNormalizedEnv, ObsNormalizedEnv
 
 def main(args):
 
     env = make_env('simple_spread')
+    # env = make_env('simple')
+    env = ActionNormalizedEnv(env)
+    # env = ObsNormalizedEnv(env)
+
     kwargs = dict()
     kwargs['config'] = args
-
+    torch.manual_seed(args.seed)
 
     if args.tensorboard:
         writer = SummaryWriter(log_dir='runs/'+args.log_dir)
+    #model = BiCNet(18, 2, 3, **kwargs)
     model = BiCNet(18, 2, 3, **kwargs)
 
     episode = 0
@@ -22,45 +29,52 @@ def main(args):
     while episode < args.max_episodes:
 
         state = env.reset()
+
         episode += 1
         step = 0
         accum_reward = 0
+        prev_reward = np.zeros((3), dtype=np.float)
 
         while True:
 
             # action = agent.random_action()
             action = model.choose_action(state, noisy=True)
 
             next_state, reward, done, info = env.step(action)
-
             step += 1
             total_step += 1
             accum_reward += sum(reward)
             state = next_state
+            reward = np.array(reward)
 
-            if args.render:
+            if args.render and episode % 100 == 0:
                 env.render(mode='rgb_array')
+            model.memory(state, action, reward - prev_reward, next_state, done)
 
-            model.memory(state, action, reward, next_state, done)
-
+            prev_reward = reward
             if len(model.replay_buffer) >= args.batch_size and total_step % args.steps_per_update == 0:
                 model.prep_train()
                 model.train()
                 model.prep_eval()
 
             if args.episode_length < step or (True in done):
                 c_loss, a_loss = model.get_loss()
-
+                action_std = model.get_action_std()
                 print("[Episode %05d] reward %6.4f eps %.4f" % (episode, accum_reward, model.epsilon), end='')
                 if args.tensorboard:
                     writer.add_scalar(tag='agent/reward', global_step=episode, scalar_value=accum_reward.item())
                     writer.add_scalar(tag='agent/epsilon', global_step=episode, scalar_value=model.epsilon)
                     if c_loss and a_loss:
                         writer.add_scalars('agent/loss', global_step=episode, tag_scalar_dict={'actor':a_loss, 'critic':c_loss})
+                    if action_std:
+                        writer.add_scalar(tag='agent/action_std', global_step=episode, scalar_value=action_std)
                 if c_loss and a_loss:
-                    print(" a_loss %3.2f c_loss %3.2f" % (a_loss, c_loss))
-                else:
-                    print()
+                    print(" a_loss %3.2f c_loss %3.2f" % (a_loss, c_loss), end='')
+                if action_std:
+                    print(" action_std %3.2f" % (action_std), end='')
+
+
+                print()
                 env.reset()
                 model.reset()
                 break
@@ -75,22 +89,22 @@ def main(args):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument('--max_episodes', default=50000, type=int)
+    parser.add_argument('--max_episodes', default=1000000, type=int)
     parser.add_argument('--episode_length', default=25, type=int)
-    parser.add_argument('--memory_length', default=int(1e6), type=int)
+    parser.add_argument('--memory_length', default=int(1e5), type=int)
     parser.add_argument("--steps_per_update", default=100, type=int)
     parser.add_argument('--tau', default=0.01, type=float)
-    parser.add_argument('--gamma', default=0.95, type=float)
+    parser.add_argument('--gamma', default=0.99, type=float)
     parser.add_argument('--use_cuda', default=True, type=bool)
     parser.add_argument('--seed', default=777, type=int)
-    parser.add_argument('--a_lr', default=0.05, type=float)
-    parser.add_argument('--c_lr', default=0.05, type=float)
-    parser.add_argument('--batch_size', default=1024, type=int)
-    parser.add_argument('--render', default=False, type=bool)
+    parser.add_argument('--a_lr', default=0.001, type=float)
+    parser.add_argument('--c_lr', default=0.001, type=float)
+    parser.add_argument('--batch_size', default=512, type=int)
+    parser.add_argument('--render', action="store_true")
     parser.add_argument('--ou_theta', default=0.15, type=float)
     parser.add_argument('--ou_mu', default=0.0, type=float)
     parser.add_argument('--ou_sigma', default=0.2, type=float)
-    parser.add_argument('--epsilon_decay', default=600000, type=int)
+    parser.add_argument('--epsilon_decay', default=1000000, type=int)
     parser.add_argument('--reward_coef', default=1, type=float)
     parser.add_argument('--tensorboard', action="store_true")
     parser.add_argument("--save_interval", default=1000, type=int)

utils.py

@@ -1,6 +1,6 @@
 import torch
 import numpy as np
-
+import torch.nn as nn
 
 def to_torch(np_array):
     return torch.from_numpy(np_array)
@@ -21,3 +21,7 @@ def fanin_init(size, fanin=None):
     v = 1. / np.sqrt(fanin)
     return torch.Tensor(size).uniform_(-v, v)
 
+def weight_init(m):
+    if isinstance(m, torch.nn.Conv2d) or isinstance(m, torch.nn.Linear):
+        m.weight.data.fill_(0.)
+        m.bias.data.fill_(0.)