diff --git a/config/drone.ini b/config/drone.ini index 78e11f4006..7df97fed89 100644 --- a/config/drone.ini +++ b/config/drone.ini @@ -31,6 +31,7 @@ use_rk2 = 0 alpha_vel = 0.0 alpha_omega = 0.002 alpha_action = 0.0 +oob_penalty = 0 # hover sphere_radius = 4.0 @@ -71,37 +72,36 @@ metric = perf distribution = log_normal min = 3e7 max = 2e8 -mean = 8e7 scale = auto -# hover -[sweep.env.hover_alpha_dist] +[sweep.env.alpha_hover] distribution = log_normal -min = 0.001 -max = 10.0 -mean = 1.0 +min = 0.0001 +max = 1.0 scale = auto -[sweep.env.alpha_hover] +[sweep.env.hover_alpha_dist] distribution = log_normal -min = 0.0001 +min = 0.001 max = 1.0 -mean = 0.01 scale = auto -# race [sweep.env.race_alpha_dist] distribution = log_normal min = 0.001 -max = 10.0 -mean = 1.0 +max = 1.0 scale = auto [sweep.env.ring_reward] -distribution = log_normal -min = 0.1 -max = 100.0 -mean = 1.0 +distribution = uniform +min = 0.0 +max = 1.0 +scale = auto + +[sweep.env.oob_penalty] +distribution = uniform +min = 0.0 +max = 1.0 scale = auto # fracs diff --git a/ocean/drone/drone.h b/ocean/drone/drone.h index cf5f3d6f25..44b1afff9d 100644 --- a/ocean/drone/drone.h +++ b/ocean/drone/drone.h @@ -88,6 +88,7 @@ struct Env { // Domain randomisation float dr; + float oob_penalty; // subtracted from the task reward on the step that ends an episode out of bounds // Physics integrator (0=RK4, 1=RK2) int integrator; @@ -283,6 +284,7 @@ void puf_init(Env* env, Dict* kwargs) { env->alpha_omega = dict_get(kwargs, "alpha_omega"); env->alpha_action = dict_get(kwargs, "alpha_action"); env->dr = dict_get(kwargs, "dr"); + env->oob_penalty = dict_get(kwargs, "oob_penalty"); env->integrator = dict_get(kwargs, "use_rk2"); task_fracs[TASK_HOVER] = dict_get(kwargs, "hover_frac"); diff --git a/ocean/drone/dronelib.h b/ocean/drone/dronelib.h index 4ae9580535..5861a935ed 100644 --- a/ocean/drone/dronelib.h +++ b/ocean/drone/dronelib.h @@ -42,7 +42,7 @@ #define RING_RADIUS 0.5f #define V_TARGET 0.05f -#define DRONE_OBS_SIZE 21 +#define DRONE_OBS_SIZE 23 // Core Parameters #define DT 0.002f // 500 Hz @@ -254,10 +254,14 @@ void compute_drone_observations(Drone* agent, float* observations, bool is_race) observations[idx++] = agent->state.omega.y / agent->params.max_omega; observations[idx++] = agent->state.omega.z / agent->params.max_omega; - observations[idx++] = q.w; - observations[idx++] = q.x; - observations[idx++] = q.y; - observations[idx++] = q.z; + Vec3 down_body = quat_rotate(q_inv, (Vec3){0.0f, 0.0f, -1.0f}); + observations[idx++] = down_body.x; + observations[idx++] = down_body.y; + observations[idx++] = down_body.z; + Vec3 fwd_body = quat_rotate(q_inv, (Vec3){1.0f, 0.0f, 0.0f}); + observations[idx++] = fwd_body.x; + observations[idx++] = fwd_body.y; + observations[idx++] = fwd_body.z; // this is body frame so we have to be careful about scaling // because distances are relative to the drone orientation diff --git a/ocean/drone/task_hover.h b/ocean/drone/task_hover.h index b9ef75dbb0..291773afee 100644 --- a/ocean/drone/task_hover.h +++ b/ocean/drone/task_hover.h @@ -156,6 +156,7 @@ static float hover_reward(DroneEnv* env, Drone* agent, int idx, StepCache* cache state->ema_dist[idx] = 0.99f * state->ema_dist[idx] + 0.01f * cache->dist; state->ema_vel[idx] = 0.99f * state->ema_vel[idx] + 0.01f * cache->vel; state->ema_omega[idx] = 0.99f * state->ema_omega[idx] + 0.01f * cache->omega; + if (cache->dist > cfg->target_dist + 1.0f) reward -= env->oob_penalty; // sphere exit ends the episode return reward; } diff --git a/ocean/drone/task_race.h b/ocean/drone/task_race.h index 49b2fbdfa2..48e5ceabab 100644 --- a/ocean/drone/task_race.h +++ b/ocean/drone/task_race.h @@ -6,6 +6,7 @@ #define RACE_RING_MAX_DIST 8.0f #define RACE_RING_SEPARATION (3.0f * RING_RADIUS) #define RACE_MAX_PLACE_ATTEMPTS 100 +#define RACE_MAX_TRACK_ATTEMPTS 16 // types @@ -124,10 +125,15 @@ static void race_env_reset(DroneEnv* env) { RaceConfig* cfg = (RaceConfig*)env->task_config; RaceState* state = (RaceState*)env->task_state; - state->ring_buffer[0] = rndring(&env->rng, RING_RADIUS); - for (int i = 1; i < cfg->max_rings; i++) { - const Target* close = (i == cfg->max_rings - 1) ? &state->ring_buffer[0] : NULL; - state->ring_buffer[i] = gen_next_ring(&env->rng, state->ring_buffer, i, close); + // regenerate the track when the closing segment (last ring back to ring 0) falls outside the gap band, + // which the per-ring fallback allowed for about one track in nine + for (int attempt = 0; attempt < RACE_MAX_TRACK_ATTEMPTS; attempt++) { + state->ring_buffer[0] = rndring(&env->rng, RING_RADIUS); + for (int i = 1; i < cfg->max_rings; i++) { + const Target* close = (i == cfg->max_rings - 1) ? &state->ring_buffer[0] : NULL; + state->ring_buffer[i] = gen_next_ring(&env->rng, state->ring_buffer, i, close); + } + if (in_gap_band(state->ring_buffer[cfg->max_rings - 1].pos, state->ring_buffer[0].pos)) break; } center_rings(state->ring_buffer, cfg->max_rings); @@ -175,6 +181,7 @@ static float race_reward(DroneEnv* env, Drone* agent, int idx, StepCache* cache) } else if (result == -1) { state->collisions[idx] += 1.0f; } + if (out_of_bounds(agent->state.pos, RACE_OOB_SCALE)) reward -= env->oob_penalty; // the death step return reward; } diff --git a/resources/drone/drone_weights.bin b/resources/drone/drone_weights.bin deleted file mode 100755 index be7bda79f0..0000000000 Binary files a/resources/drone/drone_weights.bin and /dev/null differ