prevent fast network update from learner, configuratble discount, upd…

…ate agentlace version

Signed-off-by: youliangtan <[email protected]>

examples/async_bin_relocation_fwbw_drq/async_drq_randomized.py

@@ -56,7 +56,7 @@
 
 flags.DEFINE_integer("random_steps", 300, "Sample random actions for this many steps.")
 flags.DEFINE_integer("training_starts", 300, "Training starts after this step.")
-flags.DEFINE_integer("steps_per_update", 10, "Number of steps per update the server.")
+flags.DEFINE_integer("steps_per_update", 50, "Number of steps per update the server.")
 
 flags.DEFINE_integer("log_period", 10, "Logging period.")
 flags.DEFINE_integer("eval_period", 2000, "Evaluation period.")

examples/async_cable_route_drq/async_drq_randomized.py

@@ -56,7 +56,7 @@
 
 flags.DEFINE_integer("random_steps", 300, "Sample random actions for this many steps.")
 flags.DEFINE_integer("training_starts", 300, "Training starts after this step.")
-flags.DEFINE_integer("steps_per_update", 10, "Number of steps per update the server.")
+flags.DEFINE_integer("steps_per_update", 50, "Number of steps per update the server.")
 
 flags.DEFINE_integer("log_period", 10, "Logging period.")
 flags.DEFINE_integer("eval_period", 2000, "Evaluation period.")

examples/async_drq_sim/async_drq_sim.py

@@ -52,7 +52,7 @@
 
 flags.DEFINE_integer("random_steps", 300, "Sample random actions for this many steps.")
 flags.DEFINE_integer("training_starts", 300, "Training starts after this step.")
-flags.DEFINE_integer("steps_per_update", 10, "Number of steps per update the server.")
+flags.DEFINE_integer("steps_per_update", 50, "Number of steps per update the server.")
 
 flags.DEFINE_integer("log_period", 10, "Logging period.")
 flags.DEFINE_integer("eval_period", 2000, "Evaluation period.")

examples/async_pcb_insert_drq/async_drq_randomized.py

@@ -57,7 +57,7 @@
 
 flags.DEFINE_integer("random_steps", 300, "Sample random actions for this many steps.")
 flags.DEFINE_integer("training_starts", 300, "Training starts after this step.")
-flags.DEFINE_integer("steps_per_update", 10, "Number of steps per update the server.")
+flags.DEFINE_integer("steps_per_update", 50, "Number of steps per update the server.")
 
 flags.DEFINE_integer("log_period", 10, "Logging period.")
 flags.DEFINE_integer("eval_period", 2000, "Evaluation period.")

examples/async_peg_insert_drq/async_drq_randomized.py

@@ -53,7 +53,7 @@
 
 flags.DEFINE_integer("random_steps", 300, "Sample random actions for this many steps.")
 flags.DEFINE_integer("training_starts", 300, "Training starts after this step.")
-flags.DEFINE_integer("steps_per_update", 10, "Number of steps per update the server.")
+flags.DEFINE_integer("steps_per_update", 50, "Number of steps per update the server.")
 
 flags.DEFINE_integer("log_period", 10, "Logging period.")
 flags.DEFINE_integer("eval_period", 2000, "Evaluation period.")

examples/async_sac_state_sim/async_sac_state_sim.py

@@ -43,7 +43,7 @@
 
 flags.DEFINE_integer("random_steps", 300, "Sample random actions for this many steps.")
 flags.DEFINE_integer("training_starts", 300, "Training starts after this step.")
-flags.DEFINE_integer("steps_per_update", 10, "Number of steps per update the server.")
+flags.DEFINE_integer("steps_per_update", 50, "Number of steps per update the server.")
 
 flags.DEFINE_integer("log_period", 10, "Logging period.")
 flags.DEFINE_integer("eval_period", 2000, "Evaluation period.")

serl_launcher/serl_launcher/utils/launcher.py

@@ -47,7 +47,7 @@ def make_bc_agent(
     )
 
 
-def make_sac_agent(seed, sample_obs, sample_action):
+def make_sac_agent(seed, sample_obs, sample_action, discount=0.99):
     return SACAgent.create_states(
         jax.random.PRNGKey(seed),
         sample_obs,
@@ -69,15 +69,20 @@ def make_sac_agent(seed, sample_obs, sample_action):
             "hidden_dims": [256, 256],
         },
         temperature_init=1e-2,
-        discount=0.99,
+        discount=discount,
         backup_entropy=False,
         critic_ensemble_size=10,
         critic_subsample_size=2,
     )
 
 
 def make_drq_agent(
-    seed, sample_obs, sample_action, image_keys=("image",), encoder_type="small"
+    seed,
+    sample_obs,
+    sample_action,
+    image_keys=("image",),
+    encoder_type="small",
+    discount=0.96,
 ):
     agent = DrQAgent.create_drq(
         jax.random.PRNGKey(seed),
@@ -103,7 +108,7 @@ def make_drq_agent(
             "hidden_dims": [256, 256],
         },
         temperature_init=1e-2,
-        discount=0.96,  # 0.99
+        discount=discount,
         backup_entropy=False,
         critic_ensemble_size=10,
         critic_subsample_size=2,
@@ -119,6 +124,7 @@ def make_vice_agent(
     image_keys=("image",),
     vice_image_keys=("image",),
     encoder_type="small",
+    discount=0.96,
 ):
     agent = VICEAgent.create_vice(
         jax.random.PRNGKey(seed),
@@ -154,7 +160,7 @@ def make_vice_agent(
             "hidden_dims": [256, 256],
         },
         temperature_init=1e-2,
-        discount=0.96,  # 0.99
+        discount=discount,
         backup_entropy=False,
         critic_ensemble_size=10,
         critic_subsample_size=2,

serl_launcher/setup.py

@@ -13,7 +13,7 @@
         "typing_extensions",
         "opencv-python",
         "lz4",
-        "agentlace@git+https://github.com/youliangtan/agentlace.git@892d1557264d7bb1d5df04b37638c850c9d36f35",
+        "agentlace@git+https://github.com/youliangtan/agentlace.git@b9be677d5d20772fca98c8be44777ecb7111bc59",
     ],
     packages=find_packages(),
     zip_safe=False,