发布 IRL/RLHF/DPO，并加入 Imitation Learning 系列文件夹

assets/home.js

@@ -0,0 +1,14 @@
+document.querySelectorAll('.series-folder').forEach(folder => {
+  const summary = folder.querySelector('summary');
+  if (!summary) return;
+  summary.addEventListener('click', () => {
+    window.requestAnimationFrame(() => {
+      if (folder.open) {
+        folder.classList.add('is-open');
+      } else {
+        folder.classList.remove('is-open');
+      }
+    });
+  });
+  if (folder.open) folder.classList.add('is-open');
+});

assets/style.css

@@ -78,13 +78,217 @@ main { flex: 1; }
   50% { transform: scale(1.035); filter: saturate(1.12); }
 }
 .post-list { display: grid; gap: 18px; padding-bottom: 64px; }
-.post-list-compact {
-  gap: 10px;
+.post-list:empty { padding: 0; }
+.home-feed {
+  display: grid;
+  gap: 12px;
   max-width: 560px;
   margin: 0 auto;
   padding-bottom: 56px;
 }
-.post-list:empty { padding: 0; }
+.series-folder {
+  border: 1px solid var(--bg-300);
+  border-radius: 16px;
+  background:
+    linear-gradient(180deg, color-mix(in srgb, var(--primary-100) 55%, var(--bg-100)), var(--bg-100) 42%);
+  box-shadow: 0 10px 28px color-mix(in srgb, var(--primary-300) 5%, transparent);
+  overflow: hidden;
+  transition: border-color .2s ease, box-shadow .2s ease, transform .2s ease;
+}
+.series-folder[open],
+.series-folder.is-open {
+  border-color: var(--primary-200);
+  box-shadow: 0 16px 34px color-mix(in srgb, var(--accent-200) 12%, transparent);
+}
+.series-folder-summary {
+  list-style: none;
+  display: grid;
+  grid-template-columns: auto 1fr auto;
+  gap: 14px;
+  align-items: center;
+  cursor: pointer;
+  padding: 16px 18px;
+  user-select: none;
+}
+.series-folder-summary::-webkit-details-marker { display: none; }
+.series-folder-mark {
+  position: relative;
+  width: 42px;
+  height: 34px;
+  flex-shrink: 0;
+}
+.series-folder-tab,
+.series-folder-body,
+.series-folder-stack {
+  position: absolute;
+  display: block;
+  border: 1.5px solid color-mix(in srgb, var(--accent-200) 55%, var(--primary-200));
+  background: color-mix(in srgb, var(--primary-100) 70%, var(--bg-100));
+}
+.series-folder-tab {
+  left: 2px;
+  top: 2px;
+  width: 18px;
+  height: 10px;
+  border-radius: 6px 8px 0 0;
+  border-bottom: 0;
+}
+.series-folder-body {
+  left: 0;
+  top: 10px;
+  width: 100%;
+  height: 22px;
+  border-radius: 8px;
+  background:
+    linear-gradient(180deg, color-mix(in srgb, var(--accent-100) 35%, var(--bg-100)), var(--bg-100));
+  box-shadow: inset 0 1px 0 color-mix(in srgb, #fff 55%, transparent);
+  transition: transform .28s ease;
+}
+.series-folder-stack {
+  left: 6px;
+  top: 16px;
+  width: calc(100% - 12px);
+  height: 14px;
+  border-radius: 5px;
+  opacity: .55;
+  transform: translateY(0);
+  transition: transform .28s ease, opacity .28s ease;
+}
+.series-folder[open] .series-folder-body,
+.series-folder.is-open .series-folder-body {
+  transform: translateY(-2px) rotate(-2deg);
+}
+.series-folder[open] .series-folder-stack,
+.series-folder.is-open .series-folder-stack {
+  opacity: .9;
+  transform: translateY(4px);
+}
+.series-folder-copy { display: grid; gap: 2px; min-width: 0; }
+.series-folder-kicker,
+.series-page-kicker {
+  font-size: 11px;
+  letter-spacing: .08em;
+  text-transform: uppercase;
+  color: var(--accent-200);
+}
+.series-folder-title {
+  font: normal 22px/1.35 var(--serif);
+  letter-spacing: -.02em;
+  color: var(--text-100);
+}
+.series-folder-meta {
+  font-size: 12px;
+  color: var(--text-200);
+  font-variant-numeric: tabular-nums;
+}
+.series-folder-chevron {
+  width: 10px;
+  height: 10px;
+  border-right: 2px solid var(--accent-200);
+  border-bottom: 2px solid var(--accent-200);
+  transform: rotate(45deg);
+  transition: transform .28s ease;
+  margin-right: 4px;
+}
+.series-folder[open] .series-folder-chevron,
+.series-folder.is-open .series-folder-chevron {
+  transform: rotate(225deg);
+}
+.series-folder-panel {
+  display: grid;
+  grid-template-rows: 0fr;
+  transition: grid-template-rows .36s ease;
+}
+.series-folder[open] .series-folder-panel,
+.series-folder.is-open .series-folder-panel {
+  grid-template-rows: 1fr;
+}
+.series-folder-panel-inner {
+  overflow: hidden;
+  min-height: 0;
+}
+.series-folder-blurb {
+  margin: 0;
+  padding: 0 18px 12px;
+  color: var(--text-200);
+  font-size: 13px;
+  line-height: 1.7;
+}
+.series-folder-posts {
+  display: grid;
+  gap: 8px;
+  padding: 0 12px 12px;
+  position: relative;
+}
+.series-folder-posts::before {
+  content: "";
+  position: absolute;
+  left: 28px;
+  top: 4px;
+  bottom: 18px;
+  width: 2px;
+  border-radius: 2px;
+  background: color-mix(in srgb, var(--accent-100) 70%, var(--primary-200));
+}
+.series-folder-posts .post-card-compact {
+  margin-left: 8px;
+  display: grid;
+  grid-template-columns: auto 1fr;
+  gap: 12px;
+  align-items: start;
+  background: color-mix(in srgb, var(--bg-100) 92%, var(--primary-100));
+  animation: series-card-in .34s ease both;
+}
+.series-folder[open] .series-folder-posts .post-card-compact:nth-child(1) { animation-delay: .04s; }
+.series-folder[open] .series-folder-posts .post-card-compact:nth-child(2) { animation-delay: .1s; }
+.series-folder[open] .series-folder-posts .post-card-compact:nth-child(3) { animation-delay: .16s; }
+.series-folder[open] .series-folder-posts .post-card-compact:nth-child(4) { animation-delay: .22s; }
+@keyframes series-card-in {
+  from { opacity: 0; transform: translateY(8px); }
+  to { opacity: 1; transform: translateY(0); }
+}
+.series-index {
+  display: grid;
+  place-items: center;
+  width: 28px;
+  height: 28px;
+  margin-top: 2px;
+  border-radius: 999px;
+  background: var(--primary-100);
+  color: var(--accent-200);
+  font-size: 12px;
+  font-variant-numeric: tabular-nums;
+  position: relative;
+  z-index: 1;
+}
+.compact-copy { min-width: 0; }
+.series-folder-link {
+  display: inline-flex;
+  align-items: center;
+  gap: 6px;
+  margin: 0 18px 16px;
+  font-size: 13px;
+  color: var(--accent-200);
+}
+.series-page-summary {
+  margin: 12px 0 0;
+  max-width: 40rem;
+  color: var(--text-200);
+  font-size: 15px;
+  line-height: 1.75;
+}
+.series-page-list {
+  display: grid;
+  gap: 10px;
+  max-width: 640px;
+  padding-bottom: 64px;
+}
+.series-page-list .post-card-compact {
+  display: grid;
+  grid-template-columns: auto 1fr;
+  gap: 14px;
+  align-items: start;
+}
 .post-card {
   display: block;
   padding: 28px 32px 26px;
@@ -241,6 +445,12 @@ main { flex: 1; }
   .post-card:hover { transform: none; }
   .post-card-compact .card-summary { transition: none; }
   .geometry-glow { animation: none; }
+  .series-folder,
+  .series-folder-body,
+  .series-folder-stack,
+  .series-folder-chevron,
+  .series-folder-panel { transition: none; }
+  .series-folder-posts .post-card-compact { animation: none; }
   .prose h2, .prose h3 { transition: none; }
   .prose h2.heading-flash,
   .prose h3.heading-flash { animation: none; }

blog/active-learning/index.html

@@ -150,7 +150,7 @@ <h2 id="section-12">6. 两类不确定性</h2>
 <p>实际使用还需要留意三点。首先，BALD 的质量取决于后验或其近似是否可信；少数偏差相似的模型，并不一定能代表完整的参数不确定性。其次，信息增益不等于测试误差下降，三种方法都不保证选中带来最大性能提升的样本。最后，如果一次选择多个样本，直接取单点分数最高的若干个，可能获得大量重复信息；批量选样还需要考虑样本之间的关系。</p>
 <p>这三种方法关注的对象不同：Uncertainty Sampling 看当前预测有多模糊，QBC 看委员会成员有多不一致，BALD 看标签预计能为参数带来多少信息。理解这个差别，比单独记住某个采集公式更有助于判断方法适合什么场景。</p>
 <h2 id="section-13">参考资料</h2>
-<ul><li><a href="https://github.com/enalisnick/enalisnick.github.io/blob/master/ML_with_Humans_Course_Notes.pdf">课程讲义</a></li><li><a href="https://arxiv.org/abs/1112.5745">BALD 论文</a></li><li>系列后续：<a href="/blog/behavior-cloning-plid/">Behavior Cloning 与 PLID</a>、<a href="/blog/distribution-matching-gail/">Distribution Matching 与 GAIL</a></li></ul></div></article><aside class="desktop-toc"><div class="toc-title">目录</div><nav class="toc-links" aria-label="文章目录"><a class="toc-level-2" href="#section-1">1. 主动学习循环</a><a class="toc-level-2" href="#section-2">2. 熵</a><a class="toc-level-3" href="#section-3">2.1 信息量与熵</a><a class="toc-level-3" href="#section-4">2.2 不确定性采样</a><a class="toc-level-2" href="#section-5">3. KL 散度</a><a class="toc-level-2" href="#section-6">4. Query-by-Committee</a><a class="toc-level-2" href="#section-7">5. BALD</a><a class="toc-level-3" href="#section-8">5.1 先验与后验</a><a class="toc-level-3" href="#section-9">5.2 条件互信息</a><a class="toc-level-3" href="#section-10">5.3 熵差形式</a><a class="toc-level-3" href="#section-11">5.4 数值示例</a><a class="toc-level-2" href="#section-12">6. 两类不确定性</a><a class="toc-level-2" href="#section-13">参考资料</a></nav></aside></div></main>
+<ul><li><a href="https://github.com/enalisnick/enalisnick.github.io/blob/master/ML_with_Humans_Course_Notes.pdf">课程讲义</a></li><li><a href="https://arxiv.org/abs/1112.5745">BALD 论文</a></li><li>系列后续：<a href="/series/imitation-learning/">Imitation Learning</a></li></ul></div></article><aside class="desktop-toc"><div class="toc-title">目录</div><nav class="toc-links" aria-label="文章目录"><a class="toc-level-2" href="#section-1">1. 主动学习循环</a><a class="toc-level-2" href="#section-2">2. 熵</a><a class="toc-level-3" href="#section-3">2.1 信息量与熵</a><a class="toc-level-3" href="#section-4">2.2 不确定性采样</a><a class="toc-level-2" href="#section-5">3. KL 散度</a><a class="toc-level-2" href="#section-6">4. Query-by-Committee</a><a class="toc-level-2" href="#section-7">5. BALD</a><a class="toc-level-3" href="#section-8">5.1 先验与后验</a><a class="toc-level-3" href="#section-9">5.2 条件互信息</a><a class="toc-level-3" href="#section-10">5.3 熵差形式</a><a class="toc-level-3" href="#section-11">5.4 数值示例</a><a class="toc-level-2" href="#section-12">6. 两类不确定性</a><a class="toc-level-2" href="#section-13">参考资料</a></nav></aside></div></main>
     <footer class="footer"><span>© 2026 Musicaldg</span><a href="/feed.xml">RSS</a></footer>
   </div>
 </body>

blog/behavior-cloning-plid/index.html

@@ -203,7 +203,7 @@ <h2 id="section-21">9. 总结</h2>
 <div class="math math-block">\text{PLID：}\quad s\sim d_{\pi_m}</div>
 <p>这也是从 Behavior Cloning 走向 Distribution Matching 的起点：当专家持续标注成本过高时，下一步是不再逐个纠正动作，而是直接让模型产生与专家相似的行为分布。下一篇笔记讨论 <a href="/blog/distribution-matching-gail/">Distribution Matching 与 GAIL</a>。</p>
 <h2 id="section-22">参考资料</h2>
-<ul><li><a href="https://github.com/enalisnick/enalisnick.github.io/blob/master/ML_with_Humans_Course_Notes.pdf">课程讲义</a></li><li>系列上一篇：<a href="/blog/active-learning/">Active Learning</a></li><li>系列下一篇：<a href="/blog/distribution-matching-gail/">Distribution Matching 与 GAIL</a></li></ul></div></article><aside class="desktop-toc"><div class="toc-title">目录</div><nav class="toc-links" aria-label="文章目录"><a class="toc-level-2" href="#section-1">1. 问题设定</a><a class="toc-level-2" href="#section-2">2. 形式化</a><a class="toc-level-2" href="#section-3">3. Behavior Cloning</a><a class="toc-level-3" href="#section-4">3.1 训练目标</a><a class="toc-level-3" href="#section-5">3.2 状态分布</a><a class="toc-level-2" href="#section-6">4. 误差累积</a><a class="toc-level-3" href="#section-7">4.1 状态偏移</a><a class="toc-level-3" href="#section-8">4.2 状态递推</a><a class="toc-level-3" href="#section-9">4.3 误差连锁</a><a class="toc-level-2" href="#section-10">5. PLID</a><a class="toc-level-3" href="#section-11">5.1 核心直觉</a><a class="toc-level-3" href="#section-12">5.2 交互标注</a><a class="toc-level-3" href="#section-13">5.3 数据聚合</a><a class="toc-level-2" href="#section-14">6. 数学目标</a><a class="toc-level-2" href="#section-15">7. 训练流程</a><a class="toc-level-2" href="#section-16">8. 适用条件</a><a class="toc-level-3" href="#section-17">8.1 专家反馈成本</a><a class="toc-level-3" href="#section-18">8.2 偏离状态</a><a class="toc-level-3" href="#section-19">8.3 状态信息</a><a class="toc-level-3" href="#section-20">8.4 评估指标</a><a class="toc-level-2" href="#section-21">9. 总结</a><a class="toc-level-2" href="#section-22">参考资料</a></nav></aside></div></main>
+<ul><li><a href="https://github.com/enalisnick/enalisnick.github.io/blob/master/ML_with_Humans_Course_Notes.pdf">课程讲义</a></li><li>系列：<a href="/series/imitation-learning/">Imitation Learning</a></li><li>系列下一篇：<a href="/blog/distribution-matching-gail/">Distribution Matching 与 GAIL</a></li></ul></div></article><aside class="desktop-toc"><div class="toc-title">目录</div><nav class="toc-links" aria-label="文章目录"><a class="toc-level-2" href="#section-1">1. 问题设定</a><a class="toc-level-2" href="#section-2">2. 形式化</a><a class="toc-level-2" href="#section-3">3. Behavior Cloning</a><a class="toc-level-3" href="#section-4">3.1 训练目标</a><a class="toc-level-3" href="#section-5">3.2 状态分布</a><a class="toc-level-2" href="#section-6">4. 误差累积</a><a class="toc-level-3" href="#section-7">4.1 状态偏移</a><a class="toc-level-3" href="#section-8">4.2 状态递推</a><a class="toc-level-3" href="#section-9">4.3 误差连锁</a><a class="toc-level-2" href="#section-10">5. PLID</a><a class="toc-level-3" href="#section-11">5.1 核心直觉</a><a class="toc-level-3" href="#section-12">5.2 交互标注</a><a class="toc-level-3" href="#section-13">5.3 数据聚合</a><a class="toc-level-2" href="#section-14">6. 数学目标</a><a class="toc-level-2" href="#section-15">7. 训练流程</a><a class="toc-level-2" href="#section-16">8. 适用条件</a><a class="toc-level-3" href="#section-17">8.1 专家反馈成本</a><a class="toc-level-3" href="#section-18">8.2 偏离状态</a><a class="toc-level-3" href="#section-19">8.3 状态信息</a><a class="toc-level-3" href="#section-20">8.4 评估指标</a><a class="toc-level-2" href="#section-21">9. 总结</a><a class="toc-level-2" href="#section-22">参考资料</a></nav></aside></div></main>
     <footer class="footer"><span>© 2026 Musicaldg</span><a href="/feed.xml">RSS</a></footer>
   </div>
 </body>

blog/distribution-matching-gail/index.html

@@ -207,8 +207,9 @@ <h2 id="section-16">8. 总结</h2>
 <p>GAIL 用一个二分类器区分专家与模型，再把判别器信号转化为策略奖励。它的关键数学事实是：最优判别器的 log-odds 等于两个分布的 log density ratio；在理想条件下，对抗目标与 JS divergence 相关。</p>
 <p>如果说 BC 是“模仿专家在这个状态下做什么”，PLID 是“让专家纠正模型实际遇到的状态”，那么 GAIL 更接近于：</p>
 <blockquote><p>让模型产生整体上无法被判别器与专家区分的行为。</p></blockquote>
+<p>这也是通向目标学习的一步：当分布匹配仍不够时，下一篇讨论 <a href="/blog/irl-rlhf-dpo/">IRL、RLHF 与 DPO</a>。</p>
 <h2 id="section-17">参考资料</h2>
-<ul><li><a href="https://github.com/enalisnick/enalisnick.github.io/blob/master/ML_with_Humans_Course_Notes.pdf">课程讲义</a></li><li>系列上一篇：<a href="/blog/behavior-cloning-plid/">Behavior Cloning 与 PLID</a></li></ul></div></article><aside class="desktop-toc"><div class="toc-title">目录</div><nav class="toc-links" aria-label="文章目录"><a class="toc-level-2" href="#section-1">1. 专家反馈成本</a><a class="toc-level-2" href="#section-2">2. 轨迹分布</a><a class="toc-level-2" href="#section-3">3. Forward KL</a><a class="toc-level-3" href="#section-4">3.1 最大似然</a><a class="toc-level-3" href="#section-5">3.2 与 BC 的关系</a><a class="toc-level-2" href="#section-6">4. 状态—动作分布</a><a class="toc-level-3" href="#section-7">4.1 联合分布</a><a class="toc-level-3" href="#section-8">4.2 Occupancy measure</a><a class="toc-level-2" href="#section-9">5. KL 的方向</a><a class="toc-level-2" href="#section-10">6. GAIL</a><a class="toc-level-3" href="#section-11">6.1 二分类目标</a><a class="toc-level-3" href="#section-12">6.2 策略奖励</a><a class="toc-level-3" href="#section-13">6.3 密度比</a><a class="toc-level-3" href="#section-14">6.4 JS 散度</a><a class="toc-level-2" href="#section-15">7. 训练边界</a><a class="toc-level-2" href="#section-16">8. 总结</a><a class="toc-level-2" href="#section-17">参考资料</a></nav></aside></div></main>
+<ul><li><a href="https://github.com/enalisnick/enalisnick.github.io/blob/master/ML_with_Humans_Course_Notes.pdf">课程讲义</a></li><li>系列：<a href="/series/imitation-learning/">Imitation Learning</a></li><li>系列上一篇：<a href="/blog/behavior-cloning-plid/">Behavior Cloning 与 PLID</a></li><li>系列下一篇：<a href="/blog/irl-rlhf-dpo/">IRL、RLHF 与 DPO</a></li></ul></div></article><aside class="desktop-toc"><div class="toc-title">目录</div><nav class="toc-links" aria-label="文章目录"><a class="toc-level-2" href="#section-1">1. 专家反馈成本</a><a class="toc-level-2" href="#section-2">2. 轨迹分布</a><a class="toc-level-2" href="#section-3">3. Forward KL</a><a class="toc-level-3" href="#section-4">3.1 最大似然</a><a class="toc-level-3" href="#section-5">3.2 与 BC 的关系</a><a class="toc-level-2" href="#section-6">4. 状态—动作分布</a><a class="toc-level-3" href="#section-7">4.1 联合分布</a><a class="toc-level-3" href="#section-8">4.2 Occupancy measure</a><a class="toc-level-2" href="#section-9">5. KL 的方向</a><a class="toc-level-2" href="#section-10">6. GAIL</a><a class="toc-level-3" href="#section-11">6.1 二分类目标</a><a class="toc-level-3" href="#section-12">6.2 策略奖励</a><a class="toc-level-3" href="#section-13">6.3 密度比</a><a class="toc-level-3" href="#section-14">6.4 JS 散度</a><a class="toc-level-2" href="#section-15">7. 训练边界</a><a class="toc-level-2" href="#section-16">8. 总结</a><a class="toc-level-2" href="#section-17">参考资料</a></nav></aside></div></main>
     <footer class="footer"><span>© 2026 Musicaldg</span><a href="/feed.xml">RSS</a></footer>
   </div>
 </body>