diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group0/group0_summary_reasoning_evals_cap2048.json b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group0/group0_summary_reasoning_evals_cap2048.json new file mode 100644 index 00000000..1933936e --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group0/group0_summary_reasoning_evals_cap2048.json @@ -0,0 +1,44 @@ +{ + "group0_mmlu_test_cap2048": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-125", + "accuracy": 0.04987531172069826, + "total": 401, + "correct": 20, + "num_truncated": 155 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-250", + "accuracy": 0.6084788029925187, + "total": 401, + "correct": 244, + "num_truncated": 16 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-500", + "accuracy": 0.6134663341645885, + "total": 401, + "correct": 246, + "num_truncated": 2 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-875", + "accuracy": 0.628428927680798, + "total": 401, + "correct": 252, + "num_truncated": 0 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1250", + "accuracy": 0.6334164588528678, + "total": 401, + "correct": 254, + "num_truncated": 0 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group0/group0_summary_reasoning_evals_cap4096.json b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group0/group0_summary_reasoning_evals_cap4096.json new file mode 100644 index 00000000..0573e6ae --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group0/group0_summary_reasoning_evals_cap4096.json @@ -0,0 +1,44 @@ +{ + "group0_mmlu_test_cap4096": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-125", + "accuracy": 0.059850374064837904, + "total": 401, + "correct": 24, + "num_truncated": 202 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-250", + "accuracy": 0.6134663341645885, + "total": 401, + "correct": 246, + "num_truncated": 33 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-500", + "accuracy": 0.6059850374064838, + "total": 401, + "correct": 243, + "num_truncated": 14 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-875", + "accuracy": 0.6234413965087282, + "total": 401, + "correct": 250, + "num_truncated": 3 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1250", + "accuracy": 0.6408977556109726, + "total": 401, + "correct": 257, + "num_truncated": 0 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group0/train_0_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group0/train_0_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet new file mode 100644 index 00000000..61a6ed30 Binary files /dev/null and b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group0/train_0_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet differ diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group1/group1_summary_reasoning_evals_cap2048.json b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group1/group1_summary_reasoning_evals_cap2048.json new file mode 100644 index 00000000..ed6e48c6 --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group1/group1_summary_reasoning_evals_cap2048.json @@ -0,0 +1,44 @@ +{ + "group1_mmlu_test_cap2048": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-120", + "accuracy": 0.029925187032418952, + "total": 401, + "correct": 12, + "num_truncated": 99 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-240", + "accuracy": 0.2793017456359102, + "total": 401, + "correct": 112, + "num_truncated": 32 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-480", + "accuracy": 0.3940149625935162, + "total": 401, + "correct": 158, + "num_truncated": 6 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-840", + "accuracy": 0.36159600997506236, + "total": 401, + "correct": 145, + "num_truncated": 2 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1200", + "accuracy": 0.4089775561097257, + "total": 401, + "correct": 164, + "num_truncated": 5 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group1/group1_summary_reasoning_evals_cap4096.json b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group1/group1_summary_reasoning_evals_cap4096.json new file mode 100644 index 00000000..9eeb3911 --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group1/group1_summary_reasoning_evals_cap4096.json @@ -0,0 +1,44 @@ +{ + "group1_mmlu_test_cap4096": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-120", + "accuracy": 0.04987531172069826, + "total": 401, + "correct": 20, + "num_truncated": 277 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-240", + "accuracy": 0.2518703241895262, + "total": 401, + "correct": 101, + "num_truncated": 116 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-480", + "accuracy": 0.35910224438902744, + "total": 401, + "correct": 144, + "num_truncated": 115 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-840", + "accuracy": 0.3740648379052369, + "total": 401, + "correct": 150, + "num_truncated": 7 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1200", + "accuracy": 0.41895261845386533, + "total": 401, + "correct": 168, + "num_truncated": 5 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group1/train_1_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group1/train_1_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet new file mode 100644 index 00000000..8ce5df09 Binary files /dev/null and b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group1/train_1_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet differ diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group2/group2_summary_reasoning_evals_cap2048.json b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group2/group2_summary_reasoning_evals_cap2048.json new file mode 100644 index 00000000..05aa058f --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group2/group2_summary_reasoning_evals_cap2048.json @@ -0,0 +1,44 @@ +{ + "group2_mmlu_test_cap2048": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-130", + "accuracy": 0.0024937655860349127, + "total": 401, + "correct": 1, + "num_truncated": 67 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-260", + "accuracy": 0.26683291770573564, + "total": 401, + "correct": 107, + "num_truncated": 8 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-520", + "accuracy": 0.3092269326683292, + "total": 401, + "correct": 124, + "num_truncated": 2 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-910", + "accuracy": 0.3541147132169576, + "total": 401, + "correct": 142, + "num_truncated": 2 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1300", + "accuracy": 0.3516209476309227, + "total": 401, + "correct": 141, + "num_truncated": 1 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group2/group2_summary_reasoning_evals_cap4096.json b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group2/group2_summary_reasoning_evals_cap4096.json new file mode 100644 index 00000000..ac8d93e5 --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group2/group2_summary_reasoning_evals_cap4096.json @@ -0,0 +1,44 @@ +{ + "group2_mmlu_test_cap4096": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-130", + "accuracy": 0.014962593516209476, + "total": 401, + "correct": 6, + "num_truncated": 179 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-260", + "accuracy": 0.24189526184538654, + "total": 401, + "correct": 97, + "num_truncated": 121 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-520", + "accuracy": 0.2967581047381546, + "total": 401, + "correct": 119, + "num_truncated": 32 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-910", + "accuracy": 0.3640897755610973, + "total": 401, + "correct": 146, + "num_truncated": 19 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1300", + "accuracy": 0.34663341645885287, + "total": 401, + "correct": 139, + "num_truncated": 14 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group2/train_2_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group2/train_2_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet new file mode 100644 index 00000000..6ebd99ad Binary files /dev/null and b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group2/train_2_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet differ diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group3/group3_summary_reasoning_evals_cap2048.json b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group3/group3_summary_reasoning_evals_cap2048.json new file mode 100644 index 00000000..285cef0c --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group3/group3_summary_reasoning_evals_cap2048.json @@ -0,0 +1,44 @@ +{ + "group3_mmlu_test_cap2048": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-130", + "accuracy": 0.0024937655860349127, + "total": 401, + "correct": 1, + "num_truncated": 46 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-260", + "accuracy": 0.20199501246882792, + "total": 401, + "correct": 81, + "num_truncated": 16 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-520", + "accuracy": 0.26683291770573564, + "total": 401, + "correct": 107, + "num_truncated": 3 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-910", + "accuracy": 0.26184538653366585, + "total": 401, + "correct": 105, + "num_truncated": 0 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1300", + "accuracy": 0.2793017456359102, + "total": 401, + "correct": 112, + "num_truncated": 7 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group3/group3_summary_reasoning_evals_cap4096.json b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group3/group3_summary_reasoning_evals_cap4096.json new file mode 100644 index 00000000..98d17c29 --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group3/group3_summary_reasoning_evals_cap4096.json @@ -0,0 +1,44 @@ +{ + "group3_mmlu_test_cap4096": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-130", + "accuracy": 0.014962593516209476, + "total": 401, + "correct": 6, + "num_truncated": 247 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-260", + "accuracy": 0.16209476309226933, + "total": 401, + "correct": 65, + "num_truncated": 232 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-520", + "accuracy": 0.22942643391521197, + "total": 401, + "correct": 92, + "num_truncated": 51 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-910", + "accuracy": 0.24688279301745636, + "total": 401, + "correct": 99, + "num_truncated": 35 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1300", + "accuracy": 0.27680798004987534, + "total": 401, + "correct": 111, + "num_truncated": 34 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group3/train_3_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group3/train_3_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet new file mode 100644 index 00000000..fc528c32 Binary files /dev/null and b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group3/train_3_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet differ diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group4/group4_summary_reasoning_evals_cap2048.json b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group4/group4_summary_reasoning_evals_cap2048.json new file mode 100644 index 00000000..9cd0d806 --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group4/group4_summary_reasoning_evals_cap2048.json @@ -0,0 +1,44 @@ +{ + "group4_mmlu_test_cap2048": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-130", + "accuracy": 0.0024937655860349127, + "total": 401, + "correct": 1, + "num_truncated": 36 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-260", + "accuracy": 0.26433915211970077, + "total": 401, + "correct": 106, + "num_truncated": 3 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-520", + "accuracy": 0.32418952618453867, + "total": 401, + "correct": 130, + "num_truncated": 0 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-910", + "accuracy": 0.32418952618453867, + "total": 401, + "correct": 130, + "num_truncated": 0 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1300", + "accuracy": 0.33665835411471323, + "total": 401, + "correct": 135, + "num_truncated": 2 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group4/group4_summary_reasoning_evals_cap4096.json b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group4/group4_summary_reasoning_evals_cap4096.json new file mode 100644 index 00000000..f53d5e2b --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group4/group4_summary_reasoning_evals_cap4096.json @@ -0,0 +1,44 @@ +{ + "group4_mmlu_test_cap4096": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-130", + "accuracy": 0.012468827930174564, + "total": 401, + "correct": 5, + "num_truncated": 245 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-260", + "accuracy": 0.23690773067331672, + "total": 401, + "correct": 95, + "num_truncated": 184 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-520", + "accuracy": 0.2892768079800499, + "total": 401, + "correct": 116, + "num_truncated": 151 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-910", + "accuracy": 0.29177057356608477, + "total": 401, + "correct": 117, + "num_truncated": 114 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1300", + "accuracy": 0.3117206982543641, + "total": 401, + "correct": 125, + "num_truncated": 115 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group4/train_4_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group4/train_4_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet new file mode 100644 index 00000000..b402a93c Binary files /dev/null and b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group4/train_4_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet differ diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group5/group5_summary_reasoning_evals_cap2048.json b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group5/group5_summary_reasoning_evals_cap2048.json new file mode 100644 index 00000000..661a6349 --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group5/group5_summary_reasoning_evals_cap2048.json @@ -0,0 +1,44 @@ +{ + "group5_mmlu_test_cap2048": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-125", + "accuracy": 0.004987531172069825, + "total": 401, + "correct": 2, + "num_truncated": 64 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-250", + "accuracy": 0.2169576059850374, + "total": 401, + "correct": 87, + "num_truncated": 13 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-500", + "accuracy": 0.26184538653366585, + "total": 401, + "correct": 105, + "num_truncated": 3 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-875", + "accuracy": 0.30423940149625933, + "total": 401, + "correct": 122, + "num_truncated": 1 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1250", + "accuracy": 0.28428927680798005, + "total": 401, + "correct": 114, + "num_truncated": 1 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group5/group5_summary_reasoning_evals_cap4096.json b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group5/group5_summary_reasoning_evals_cap4096.json new file mode 100644 index 00000000..b10f4ddb --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group5/group5_summary_reasoning_evals_cap4096.json @@ -0,0 +1,44 @@ +{ + "group5_mmlu_test_cap4096": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-125", + "accuracy": 0.017456359102244388, + "total": 401, + "correct": 7, + "num_truncated": 286 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-250", + "accuracy": 0.14962593516209477, + "total": 401, + "correct": 60, + "num_truncated": 269 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-500", + "accuracy": 0.2144638403990025, + "total": 401, + "correct": 86, + "num_truncated": 162 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-875", + "accuracy": 0.2593516209476309, + "total": 401, + "correct": 104, + "num_truncated": 139 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1250", + "accuracy": 0.24937655860349128, + "total": 401, + "correct": 100, + "num_truncated": 86 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group5/train_5_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group5/train_5_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet new file mode 100644 index 00000000..14025880 Binary files /dev/null and b/artifacts/distillation_by_complexity_splits/mmlu/Phi-4-mini-instruct/group5/train_5_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet differ diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group0/group0_summary_reasoning_evals_cap2048.json b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group0/group0_summary_reasoning_evals_cap2048.json new file mode 100644 index 00000000..43478c85 --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group0/group0_summary_reasoning_evals_cap2048.json @@ -0,0 +1,44 @@ +{ + "group0_mmlu_test_cap2048": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-135", + "accuracy": 0.6034912718204489, + "total": 401, + "correct": 242, + "num_truncated": 0 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-270", + "accuracy": 0.600997506234414, + "total": 401, + "correct": 241, + "num_truncated": 2 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-540", + "accuracy": 0.5960099750623441, + "total": 401, + "correct": 239, + "num_truncated": 0 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-945", + "accuracy": 0.5760598503740648, + "total": 401, + "correct": 231, + "num_truncated": 2 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1350", + "accuracy": 0.6209476309226932, + "total": 401, + "correct": 249, + "num_truncated": 3 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group0/group0_summary_reasoning_evals_cap4096.json b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group0/group0_summary_reasoning_evals_cap4096.json new file mode 100644 index 00000000..c89d35fb --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group0/group0_summary_reasoning_evals_cap4096.json @@ -0,0 +1,44 @@ +{ + "group0_mmlu_test_cap4096": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-135", + "accuracy": 0.5985037406483791, + "total": 401, + "correct": 240, + "num_truncated": 0 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-270", + "accuracy": 0.5985037406483791, + "total": 401, + "correct": 240, + "num_truncated": 3 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-540", + "accuracy": 0.5985037406483791, + "total": 401, + "correct": 240, + "num_truncated": 0 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-945", + "accuracy": 0.5835411471321695, + "total": 401, + "correct": 234, + "num_truncated": 3 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1350", + "accuracy": 0.628428927680798, + "total": 401, + "correct": 252, + "num_truncated": 0 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group0/train_0_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group0/train_0_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet new file mode 100644 index 00000000..5723067f Binary files /dev/null and b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group0/train_0_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet differ diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group1/group1_summary_reasoning_evals_cap2048.json b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group1/group1_summary_reasoning_evals_cap2048.json new file mode 100644 index 00000000..4f8d0acd --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group1/group1_summary_reasoning_evals_cap2048.json @@ -0,0 +1,44 @@ +{ + "group1_mmlu_test_cap2048": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-120", + "accuracy": 0.3491271820448878, + "total": 401, + "correct": 140, + "num_truncated": 0 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-240", + "accuracy": 0.39650872817955113, + "total": 401, + "correct": 159, + "num_truncated": 0 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-480", + "accuracy": 0.42643391521197005, + "total": 401, + "correct": 171, + "num_truncated": 2 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-840", + "accuracy": 0.38154613466334164, + "total": 401, + "correct": 153, + "num_truncated": 0 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1200", + "accuracy": 0.39900249376558605, + "total": 401, + "correct": 160, + "num_truncated": 0 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group1/group1_summary_reasoning_evals_cap4096.json b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group1/group1_summary_reasoning_evals_cap4096.json new file mode 100644 index 00000000..f4bb35a7 --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group1/group1_summary_reasoning_evals_cap4096.json @@ -0,0 +1,44 @@ +{ + "group1_mmlu_test_cap4096": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-120", + "accuracy": 0.34413965087281795, + "total": 401, + "correct": 138, + "num_truncated": 0 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-240", + "accuracy": 0.4014962593516209, + "total": 401, + "correct": 161, + "num_truncated": 0 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-480", + "accuracy": 0.43640897755610975, + "total": 401, + "correct": 175, + "num_truncated": 0 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-840", + "accuracy": 0.38154613466334164, + "total": 401, + "correct": 153, + "num_truncated": 2 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1200", + "accuracy": 0.40648379052369077, + "total": 401, + "correct": 163, + "num_truncated": 1 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group1/train_1_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group1/train_1_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet new file mode 100644 index 00000000..a655cbcc Binary files /dev/null and b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group1/train_1_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet differ diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group2/group2_summary_reasoning_evals_cap2048.json b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group2/group2_summary_reasoning_evals_cap2048.json new file mode 100644 index 00000000..8486ca4d --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group2/group2_summary_reasoning_evals_cap2048.json @@ -0,0 +1,44 @@ +{ + "group2_mmlu_test_cap2048": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-125", + "accuracy": 0.3341645885286783, + "total": 401, + "correct": 134, + "num_truncated": 3 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-250", + "accuracy": 0.3690773067331671, + "total": 401, + "correct": 148, + "num_truncated": 0 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-500", + "accuracy": 0.3566084788029925, + "total": 401, + "correct": 143, + "num_truncated": 1 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-875", + "accuracy": 0.4014962593516209, + "total": 401, + "correct": 161, + "num_truncated": 0 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1250", + "accuracy": 0.3640897755610973, + "total": 401, + "correct": 146, + "num_truncated": 0 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group2/group2_summary_reasoning_evals_cap4096.json b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group2/group2_summary_reasoning_evals_cap4096.json new file mode 100644 index 00000000..56d1b860 --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group2/group2_summary_reasoning_evals_cap4096.json @@ -0,0 +1,44 @@ +{ + "group2_mmlu_test_cap4096": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-125", + "accuracy": 0.33915211970074816, + "total": 401, + "correct": 136, + "num_truncated": 1 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-250", + "accuracy": 0.3865336658354115, + "total": 401, + "correct": 155, + "num_truncated": 0 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-500", + "accuracy": 0.3690773067331671, + "total": 401, + "correct": 148, + "num_truncated": 1 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-875", + "accuracy": 0.3940149625935162, + "total": 401, + "correct": 158, + "num_truncated": 0 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1250", + "accuracy": 0.3865336658354115, + "total": 401, + "correct": 155, + "num_truncated": 0 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group2/train_2_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group2/train_2_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet new file mode 100644 index 00000000..0929ed55 Binary files /dev/null and b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group2/train_2_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet differ diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group3/group3_summary_reasoning_evals_cap2048.json b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group3/group3_summary_reasoning_evals_cap2048.json new file mode 100644 index 00000000..ff88c1c6 --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group3/group3_summary_reasoning_evals_cap2048.json @@ -0,0 +1,44 @@ +{ + "group3_mmlu_test_cap2048": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-120", + "accuracy": 0.30174563591022446, + "total": 401, + "correct": 121, + "num_truncated": 0 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-240", + "accuracy": 0.32917705735660846, + "total": 401, + "correct": 132, + "num_truncated": 0 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-480", + "accuracy": 0.28428927680798005, + "total": 401, + "correct": 114, + "num_truncated": 2 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-840", + "accuracy": 0.3266832917705736, + "total": 401, + "correct": 131, + "num_truncated": 7 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1200", + "accuracy": 0.2967581047381546, + "total": 401, + "correct": 119, + "num_truncated": 2 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group3/group3_summary_reasoning_evals_cap4096.json b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group3/group3_summary_reasoning_evals_cap4096.json new file mode 100644 index 00000000..dc799823 --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group3/group3_summary_reasoning_evals_cap4096.json @@ -0,0 +1,44 @@ +{ + "group3_mmlu_test_cap4096": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-120", + "accuracy": 0.29925187032418954, + "total": 401, + "correct": 120, + "num_truncated": 0 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-240", + "accuracy": 0.3341645885286783, + "total": 401, + "correct": 134, + "num_truncated": 0 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-480", + "accuracy": 0.286783042394015, + "total": 401, + "correct": 115, + "num_truncated": 1 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-840", + "accuracy": 0.3192019950124688, + "total": 401, + "correct": 128, + "num_truncated": 2 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1200", + "accuracy": 0.2892768079800499, + "total": 401, + "correct": 116, + "num_truncated": 0 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group3/train_3_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group3/train_3_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet new file mode 100644 index 00000000..507f49b1 Binary files /dev/null and b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group3/train_3_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet differ diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group4/group4_summary_reasoning_evals_cap2048.json b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group4/group4_summary_reasoning_evals_cap2048.json new file mode 100644 index 00000000..64737f96 --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group4/group4_summary_reasoning_evals_cap2048.json @@ -0,0 +1,44 @@ +{ + "group4_mmlu_test_cap2048": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-130", + "accuracy": 0.2817955112219451, + "total": 401, + "correct": 113, + "num_truncated": 2 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-260", + "accuracy": 0.2892768079800499, + "total": 401, + "correct": 116, + "num_truncated": 0 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-520", + "accuracy": 0.2817955112219451, + "total": 401, + "correct": 113, + "num_truncated": 2 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-910", + "accuracy": 0.2942643391521197, + "total": 401, + "correct": 118, + "num_truncated": 2 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1300", + "accuracy": 0.30673316708229426, + "total": 401, + "correct": 123, + "num_truncated": 2 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group4/group4_summary_reasoning_evals_cap4096.json b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group4/group4_summary_reasoning_evals_cap4096.json new file mode 100644 index 00000000..c4cc6880 --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group4/group4_summary_reasoning_evals_cap4096.json @@ -0,0 +1,44 @@ +{ + "group4_mmlu_test_cap4096": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-130", + "accuracy": 0.26683291770573564, + "total": 401, + "correct": 107, + "num_truncated": 2 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-260", + "accuracy": 0.30423940149625933, + "total": 401, + "correct": 122, + "num_truncated": 1 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-520", + "accuracy": 0.30174563591022446, + "total": 401, + "correct": 121, + "num_truncated": 1 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-910", + "accuracy": 0.32169576059850374, + "total": 401, + "correct": 129, + "num_truncated": 2 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1300", + "accuracy": 0.3117206982543641, + "total": 401, + "correct": 125, + "num_truncated": 3 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group4/train_4_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group4/train_4_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet new file mode 100644 index 00000000..ee3dcef6 Binary files /dev/null and b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group4/train_4_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet differ diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group5/group5_summary_reasoning_evals_cap2048.json b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group5/group5_summary_reasoning_evals_cap2048.json new file mode 100644 index 00000000..d72ecc6d --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group5/group5_summary_reasoning_evals_cap2048.json @@ -0,0 +1,44 @@ +{ + "group5_mmlu_test_cap2048": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-125", + "accuracy": 0.2194513715710723, + "total": 401, + "correct": 88, + "num_truncated": 1 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-250", + "accuracy": 0.23690773067331672, + "total": 401, + "correct": 95, + "num_truncated": 0 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-500", + "accuracy": 0.22942643391521197, + "total": 401, + "correct": 92, + "num_truncated": 0 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-875", + "accuracy": 0.20698254364089774, + "total": 401, + "correct": 83, + "num_truncated": 1 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1250", + "accuracy": 0.20448877805486285, + "total": 401, + "correct": 82, + "num_truncated": 2 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group5/group5_summary_reasoning_evals_cap4096.json b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group5/group5_summary_reasoning_evals_cap4096.json new file mode 100644 index 00000000..c5fe773a --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group5/group5_summary_reasoning_evals_cap4096.json @@ -0,0 +1,44 @@ +{ + "group5_mmlu_test_cap4096": [ + { + "epoch": 5.0, + "checkpoint": "checkpoint-125", + "accuracy": 0.20698254364089774, + "total": 401, + "correct": 83, + "num_truncated": 1 + }, + { + "epoch": 10.0, + "checkpoint": "checkpoint-250", + "accuracy": 0.22693266832917705, + "total": 401, + "correct": 91, + "num_truncated": 0 + }, + { + "epoch": 20.0, + "checkpoint": "checkpoint-500", + "accuracy": 0.27680798004987534, + "total": 401, + "correct": 111, + "num_truncated": 0 + }, + { + "epoch": 35.0, + "checkpoint": "checkpoint-875", + "accuracy": 0.2194513715710723, + "total": 401, + "correct": 88, + "num_truncated": 2 + }, + { + "epoch": 50.0, + "checkpoint": "checkpoint-1250", + "accuracy": 0.20199501246882792, + "total": 401, + "correct": 81, + "num_truncated": 2 + } + ] +} \ No newline at end of file diff --git a/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group5/train_5_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group5/train_5_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet new file mode 100644 index 00000000..2e6ae64e Binary files /dev/null and b/artifacts/distillation_by_complexity_splits/mmlu/Qwen2.5-3B-Instruct/group5/train_5_distilled_deepseek_v4_flash_regenerate_incorrect_w_large.parquet differ diff --git a/artifacts/distillation_by_complexity_splits/mmlu/readme.md b/artifacts/distillation_by_complexity_splits/mmlu/readme.md new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/artifacts/distillation_by_complexity_splits/mmlu/readme.md @@ -0,0 +1 @@ +