drone release fixes

config/drone.ini

@@ -31,6 +31,7 @@ use_rk2 = 0
 alpha_vel = 0.0
 alpha_omega = 0.002
 alpha_action = 0.0
+oob_penalty = 0
 
 # hover
 sphere_radius = 4.0
@@ -71,37 +72,36 @@ metric = perf
 distribution = log_normal
 min = 3e7
 max = 2e8
-mean = 8e7
 scale = auto
 
-# hover
-[sweep.env.hover_alpha_dist]
+[sweep.env.alpha_hover]
 distribution = log_normal
-min = 0.001
-max = 10.0
-mean = 1.0
+min = 0.0001
+max = 1.0
 scale = auto
 
-[sweep.env.alpha_hover]
+[sweep.env.hover_alpha_dist]
 distribution = log_normal
-min = 0.0001
+min = 0.001
 max = 1.0
-mean = 0.01
 scale = auto
 
-# race
 [sweep.env.race_alpha_dist]
 distribution = log_normal
 min = 0.001
-max = 10.0
-mean = 1.0
+max = 1.0
 scale = auto
 
 [sweep.env.ring_reward]
-distribution = log_normal
-min = 0.1
-max = 100.0
-mean = 1.0
+distribution = uniform
+min = 0.0
+max = 1.0
+scale = auto
+
+[sweep.env.oob_penalty]
+distribution = uniform
+min = 0.0
+max = 1.0
 scale = auto
 
 # fracs

ocean/drone/drone.h

@@ -88,6 +88,7 @@ struct Env {
 
     // Domain randomisation
     float dr;
+    float oob_penalty; // subtracted from the task reward on the step that ends an episode out of bounds
 
     // Physics integrator (0=RK4, 1=RK2)
     int integrator;
@@ -283,6 +284,7 @@ void puf_init(Env* env, Dict* kwargs) {
     env->alpha_omega = dict_get(kwargs, "alpha_omega");
     env->alpha_action = dict_get(kwargs, "alpha_action");
     env->dr = dict_get(kwargs, "dr");
+    env->oob_penalty = dict_get(kwargs, "oob_penalty");
     env->integrator = dict_get(kwargs, "use_rk2");
 
     task_fracs[TASK_HOVER] = dict_get(kwargs, "hover_frac");

ocean/drone/dronelib.h

@@ -42,7 +42,7 @@
 #define RING_RADIUS 0.5f
 #define V_TARGET 0.05f
 
-#define DRONE_OBS_SIZE 21
+#define DRONE_OBS_SIZE 23
 
 // Core Parameters
 #define DT 0.002f // 500 Hz
@@ -254,10 +254,14 @@ void compute_drone_observations(Drone* agent, float* observations, bool is_race)
     observations[idx++] = agent->state.omega.y / agent->params.max_omega;
     observations[idx++] = agent->state.omega.z / agent->params.max_omega;
 
-    observations[idx++] = q.w;
-    observations[idx++] = q.x;
-    observations[idx++] = q.y;
-    observations[idx++] = q.z;
+    Vec3 down_body = quat_rotate(q_inv, (Vec3){0.0f, 0.0f, -1.0f});
+    observations[idx++] = down_body.x;
+    observations[idx++] = down_body.y;
+    observations[idx++] = down_body.z;
+    Vec3 fwd_body = quat_rotate(q_inv, (Vec3){1.0f, 0.0f, 0.0f});
+    observations[idx++] = fwd_body.x;
+    observations[idx++] = fwd_body.y;
+    observations[idx++] = fwd_body.z;
 
     // this is body frame so we have to be careful about scaling
     // because distances are relative to the drone orientation

ocean/drone/task_hover.h

@@ -156,6 +156,7 @@ static float hover_reward(DroneEnv* env, Drone* agent, int idx, StepCache* cache
     state->ema_dist[idx] = 0.99f * state->ema_dist[idx] + 0.01f * cache->dist;
     state->ema_vel[idx] = 0.99f * state->ema_vel[idx] + 0.01f * cache->vel;
     state->ema_omega[idx] = 0.99f * state->ema_omega[idx] + 0.01f * cache->omega;
+    if (cache->dist > cfg->target_dist + 1.0f) reward -= env->oob_penalty; // sphere exit ends the episode
     return reward;
 }
 

ocean/drone/task_race.h

@@ -6,6 +6,7 @@
 #define RACE_RING_MAX_DIST 8.0f
 #define RACE_RING_SEPARATION (3.0f * RING_RADIUS)
 #define RACE_MAX_PLACE_ATTEMPTS 100
+#define RACE_MAX_TRACK_ATTEMPTS 16
 
 // types
 
@@ -124,10 +125,15 @@ static void race_env_reset(DroneEnv* env) {
     RaceConfig* cfg = (RaceConfig*)env->task_config;
     RaceState* state = (RaceState*)env->task_state;
 
-    state->ring_buffer[0] = rndring(&env->rng, RING_RADIUS);
-    for (int i = 1; i < cfg->max_rings; i++) {
-        const Target* close = (i == cfg->max_rings - 1) ? &state->ring_buffer[0] : NULL;
-        state->ring_buffer[i] = gen_next_ring(&env->rng, state->ring_buffer, i, close);
+    // regenerate the track when the closing segment (last ring back to ring 0) falls outside the gap band,
+    // which the per-ring fallback allowed for about one track in nine
+    for (int attempt = 0; attempt < RACE_MAX_TRACK_ATTEMPTS; attempt++) {
+        state->ring_buffer[0] = rndring(&env->rng, RING_RADIUS);
+        for (int i = 1; i < cfg->max_rings; i++) {
+            const Target* close = (i == cfg->max_rings - 1) ? &state->ring_buffer[0] : NULL;
+            state->ring_buffer[i] = gen_next_ring(&env->rng, state->ring_buffer, i, close);
+        }
+        if (in_gap_band(state->ring_buffer[cfg->max_rings - 1].pos, state->ring_buffer[0].pos)) break;
     }
 
     center_rings(state->ring_buffer, cfg->max_rings);
@@ -175,6 +181,7 @@ static float race_reward(DroneEnv* env, Drone* agent, int idx, StepCache* cache)
     } else if (result == -1) {
         state->collisions[idx] += 1.0f;
     }
+    if (out_of_bounds(agent->state.pos, RACE_OOB_SCALE)) reward -= env->oob_penalty; // the death step
 
     return reward;
 }