[{"data":1,"prerenderedAt":3136},["ShallowReactive",2],{"post-\u002Fposts\u002Fcycleresearcher-post-training":3,"archive-public-posts":2622},{"id":4,"title":5,"body":6,"categories":2601,"cover":2603,"coverAlt":2604,"date":2605,"description":2606,"draft":2607,"extension":2608,"haloPublished":2609,"haloSync":2609,"hidden":2607,"legacy":2607,"meta":2610,"navigation":2609,"path":2611,"published":2607,"readingTime":2612,"seo":2613,"series":2612,"seriesOrder":2612,"stem":2614,"tags":2615,"__hash__":2621},"posts\u002Fposts\u002Fcycleresearcher-post-training.md","从 DeepReviewer 2.0 到 CycleResearcher：科研 Agent 如何真正做 Post-training",{"type":7,"value":8,"toc":2581},"minimark",[9,13,20,25,92,108,111,115,118,121,131,134,138,145,159,162,166,169,188,191,411,415,418,421,432,508,511,515,522,528,870,876,879,883,886,889,1229,1263,1610,1613,1617,1620,2012,2015,2018,2022,2025,2028,2390,2393,2399,2402,2405,2410,2413,2416,2419,2422,2425,2428,2431,2471,2478,2482,2485,2514,2517,2522,2525],[10,11,12],"p",{},"看到一个系统会检索、规划、反思和调用工具，不能据此判断它做过 Post-training。Agent 描述的是模型在运行时如何完成任务；Post-training 描述的是预训练之后，是否继续用监督数据、偏好数据或奖励信号更新模型参数。两者可以结合，也可以完全分离。",[10,14,15,16],{},"这篇文章以西湖大学 NLP 团队的两个科研 Agent 为对照：DeepReviewer 2.0 主要依靠推理时流程控制，CycleResearcher 则包含完整的领域 SFT、生成式 Reward Model 和迭代偏好优化。重点不是复述系统功能，而是回答一个求职和面试中更有价值的问题：",[17,18,19],"strong",{},"一个项目究竟在哪些环节真正更新了模型，训练信号从哪里来，为什么这样设计？",[21,22,24],"h2",{"id":23},"先给结论agent-不等于-post-training","先给结论：Agent 不等于 Post-training",[26,27,28,47],"table",{},[29,30,31],"thead",{},[32,33,34,38,41,44],"tr",{},[35,36,37],"th",{},"系统",[35,39,40],{},"底层模型是否针对任务继续训练",[35,42,43],{},"核心方法",[35,45,46],{},"更准确的定位",[48,49,50,65,79],"tbody",{},[32,51,52,56,59,62],{},[53,54,55],"td",{},"DeepReviewer 2.0",[53,57,58],{},"否",[53,60,61],{},"分阶段审稿、检索、证据锚定、导出门控",[53,63,64],{},"推理时 Agent 系统",[32,66,67,70,73,76],{},[53,68,69],{},"CycleResearcher",[53,71,72],{},"是",[53,74,75],{},"领域 SFT、CycleReviewer、Iterative SimPO",[53,77,78],{},"Agent + Post-training",[32,80,81,84,86,89],{},[53,82,83],{},"FPO \u002F Pre-DPO",[53,85,72],{},[53,87,88],{},"DPO 目标与参考约束改进",[53,90,91],{},"偏好优化算法",[10,93,94,95,99,100,103,104,107],{},"DeepReviewer 2.0 的论文表格注明其主系统运行在 ",[96,97,98],"code",{},"Step-3.5-Flash 196B"," 上，并强调模型未经审稿任务专项微调。这里的 ",[96,101,102],{},"196B"," 是总参数量口径；StepFun 模型卡给出的单 Token 激活参数约为 ",[96,105,106],{},"11B","。它的提升来自 claim-evidence-risk ledger、检索议程、段落锚定和 export gate，而不是更新底层模型权重。",[10,109,110],{},"CycleResearcher 不同。它先训练科研写作 Policy，再训练能够模拟同行评审的 Reviewer，最后让 Reviewer 对 Policy 的多个输出打分，构造偏好对并继续优化 Policy。论文把这套闭环称为 Research-Review-Refinement。",[21,112,114],{"id":113},"instruct-模型是后训练的起点","Instruct 模型是后训练的起点",[10,116,117],{},"CycleResearcher 没有从 Base Model 开始。它使用的底座包括 Mistral-Nemo-Instruct-2407、Qwen2.5-72B-Instruct 和 Mistral-Large-2。",[10,119,120],{},"Base Model 主要通过 next-token prediction 学习语言、知识和代码模式，但不一定稳定服从用户指令。Instruct Model 已经过通用指令微调，学会把输入识别为任务并按要求回答。可以把常见路径写成：",[122,123,129],"pre",{"className":124,"code":126,"language":127,"meta":128},[125],"language-text","Base Model\n  -> 通用指令 SFT\n  -> Instruct Model\n  -> 领域 SFT\n  -> 偏好优化\n  -> 领域模型或 Agent Policy\n","text","",[96,130,126],{"__ignoreMap":128},[10,132,133],{},"因此，CycleResearcher 做的是 Instruct 模型之上的二次领域 Post-training，而不是从头教模型如何对话。",[21,135,137],{"id":136},"cycleresearcher-的训练闭环","CycleResearcher 的训练闭环",[10,139,140],{},[141,142],"img",{"alt":143,"src":144},"CycleResearcher 迭代训练框架","\u002Fimages\u002Fposts\u002Fcycleresearcher-post-training\u002Fcycleresearcher-figure-2-original.png",[10,146,147],{},[148,149,150,151,158],"em",{},"CycleResearcher 论文 Figure 2：Iterative Training Framework。图片来自 ",[152,153,157],"a",{"href":154,"rel":155},"https:\u002F\u002Fproceedings.iclr.cc\u002Fpaper_files\u002Fpaper\u002F2025\u002Ffile\u002F0a48036026dc7946ef6033ae14719cc5-Paper-Conference.pdf",[156],"nofollow","CycleResearcher 论文","。",[10,160,161],{},"整个 Post-training 可以拆成三个阶段：领域 SFT、生成式 Reward Model 训练、Iterative SimPO。",[21,163,165],{"id":164},"阶段一用-research-14k-做领域-sft","阶段一：用 Research-14K 做领域 SFT",[10,167,168],{},"Research-14K 从 2022 至 2024 年 ICLR、NeurIPS、ICML、ACL、EMNLP、CVPR 和 ICCV 等会议论文中构建。作者先收集 14,911 篇论文，再过滤无效内容、补充参考文献摘要，并把正文拆成结构化 outline 与 section。最终得到：",[170,171,172,176,179,182,185],"ul",{},[173,174,175],"li",{},"12,696 个训练样本；",[173,177,178],{},"802 个按时间划分的测试样本；",[173,180,181],{},"输入是参考文献、BibTeX 和摘要；",[173,183,184],{},"输出是研究动机、主要想法、方法、实验设计、结果分析和完整 LaTeX 正文；",[173,186,187],{},"平均输出长度约 28K tokens。",[10,189,190],{},"这不是普通短指令 SFT，而是长上下文、长输出的领域行为学习。模型需要学习科研写作的内容，还要学习先规划 outline、再生成正文的输出协议。",[10,192,193,194,328,329,158],{},"Policy 训练使用 8 张 H100、DeepSpeed ZeRO-2，12B 模型训练上下文设为 32K，72B 和 123B 设为 24K；学习率为 ",[195,196,199,240],"span",{"className":197},[198],"katex",[195,200,203],{"className":201},[202],"katex-mathml",[204,205,207],"math",{"xmlns":206},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[208,209,210,235],"semantics",{},[211,212,213,217,221],"mrow",{},[214,215,216],"mn",{},"4",[218,219,220],"mo",{},"×",[222,223,224,227],"msup",{},[214,225,226],{},"10",[211,228,229,232],{},[218,230,231],{},"−",[214,233,234],{},"5",[236,237,239],"annotation",{"encoding":238},"application\u002Fx-tex","4\\times10^{-5}",[195,241,245,270],{"className":242,"ariaHidden":244},[243],"katex-html","true",[195,246,249,254,258,263,267],{"className":247},[248],"base",[195,250],{"className":251,"style":253},[252],"strut","height:0.7278em;vertical-align:-0.0833em;",[195,255,216],{"className":256},[257],"mord",[195,259],{"className":260,"style":262},[261],"mspace","margin-right:0.2222em;",[195,264,220],{"className":265},[266],"mbin",[195,268],{"className":269,"style":262},[261],[195,271,273,277,281],{"className":272},[248],[195,274],{"className":275,"style":276},[252],"height:0.8141em;",[195,278,280],{"className":279},[257],"1",[195,282,284,288],{"className":283},[257],[195,285,287],{"className":286},[257],"0",[195,289,292],{"className":290},[291],"msupsub",[195,293,296],{"className":294},[295],"vlist-t",[195,297,300],{"className":298},[299],"vlist-r",[195,301,304],{"className":302,"style":276},[303],"vlist",[195,305,307,312],{"style":306},"top:-3.063em;margin-right:0.05em;",[195,308],{"className":309,"style":311},[310],"pstrut","height:2.7em;",[195,313,319],{"className":314},[315,316,317,318],"sizing","reset-size6","size3","mtight",[195,320,322,325],{"className":321},[257,318],[195,323,231],{"className":324},[257,318],[195,326,234],{"className":327},[257,318],"，训练 12,000 steps。该阶段得到初始 CycleResearcher，也就是后续偏好优化的初始策略 ",[195,330,332,353],{"className":331},[198],[195,333,335],{"className":334},[202],[204,336,337],{"xmlns":206},[208,338,339,350],{},[211,340,341],{},[342,343,344,348],"msub",{},[345,346,347],"mi",{},"P",[214,349,280],{},[236,351,352],{"encoding":238},"P_1",[195,354,356],{"className":355,"ariaHidden":244},[243],[195,357,359,363],{"className":358},[248],[195,360],{"className":361,"style":362},[252],"height:0.8333em;vertical-align:-0.15em;",[195,364,366,371],{"className":365},[257],[195,367,347],{"className":368,"style":370},[257,369],"mathnormal","margin-right:0.1389em;",[195,372,374],{"className":373},[291],[195,375,378,402],{"className":376},[295,377],"vlist-t2",[195,379,381,397],{"className":380},[299],[195,382,385],{"className":383,"style":384},[303],"height:0.3011em;",[195,386,388,391],{"style":387},"top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;",[195,389],{"className":390,"style":311},[310],[195,392,394],{"className":393},[315,316,317,318],[195,395,280],{"className":396},[257,318],[195,398,401],{"className":399},[400],"vlist-s","​",[195,403,405],{"className":404},[299],[195,406,409],{"className":407,"style":408},[303],"height:0.15em;",[195,410],{},[21,412,414],{"id":413},"阶段二训练生成式-cyclereviewer","阶段二：训练生成式 CycleReviewer",[10,416,417],{},"传统 Reward Model 常在 Transformer 顶部增加标量打分头，输入 prompt-response 后直接输出一个 reward。CycleReviewer 更接近 Generative Reward Model：它先生成完整评审，再在评审中给出结构化分数和最终建议。",[10,419,420],{},"Review-5K 来自 ICLR 2024 投稿与 OpenReview 评审数据，包含 4,991 篇论文和超过 16,000 条评审意见。其中 4,189 篇用于训练，782 篇用于测试。每个样本保留：",[170,422,423,426,429],{},[173,424,425],{},"多位 Reviewer 的 summary、strengths、weaknesses 和 questions；",[173,427,428],{},"soundness、presentation、contribution 与 overall score；",[173,430,431],{},"Meta Reviewer 的汇总意见和最终决定。",[10,433,434,435,507],{},"CycleReviewer 使用 Mistral-Large-2 和 LoRA-GA，在 8 张 H100 80GB 上训练 12 epochs，学习率为 ",[195,436,438,460],{"className":437},[198],[195,439,441],{"className":440},[202],[204,442,443],{"xmlns":206},[208,444,445,457],{},[211,446,447],{},[222,448,449,451],{},[214,450,226],{},[211,452,453,455],{},[218,454,231],{},[214,456,234],{},[236,458,459],{"encoding":238},"10^{-5}",[195,461,463],{"className":462,"ariaHidden":244},[243],[195,464,466,469,472],{"className":465},[248],[195,467],{"className":468,"style":276},[252],[195,470,280],{"className":471},[257],[195,473,475,478],{"className":474},[257],[195,476,287],{"className":477},[257],[195,479,481],{"className":480},[291],[195,482,484],{"className":483},[295],[195,485,487],{"className":486},[299],[195,488,490],{"className":489,"style":276},[303],[195,491,492,495],{"style":306},[195,493],{"className":494,"style":311},[310],[195,496,498],{"className":497},[315,316,317,318],[195,499,501,504],{"className":500},[257,318],[195,502,231],{"className":503},[257,318],[195,505,234],{"className":506},[257,318],"。推理时，它会模拟多名不同严格程度的 Reviewer，最后由 Senior Reviewer 汇总，得到自然语言反馈与平均分数。",[10,509,510],{},"这种设计的优点是 reward 可解释：可以看到模型为什么认为一篇论文更好。代价是它比标量 Reward Head 更慢，而且生成式评分更容易受到提示格式、文本长度和语言风格影响。",[21,512,514],{"id":513},"阶段三从评分构造偏好对","阶段三：从评分构造偏好对",[10,516,517,518,521],{},"作者另外收集了 4,152 篇近期 arXiv 机器学习论文的参考文献部分，将其作为知识输入。对于每个输入，当前 CycleResearcher 以 ",[96,519,520],{},"temperature=0.4"," 采样三个候选结果：",[122,523,526],{"className":524,"code":525,"language":127,"meta":128},[125],"x -> M1, M2, M3\n",[96,527,525],{"__ignoreMap":128},[10,529,530,531,718,719,795,796,869],{},"CycleReviewer 分别模拟多名评审，计算候选结果的平均得分 ",[195,532,534,572],{"className":533},[198],[195,535,537],{"className":536},[202],[204,538,539],{"xmlns":206},[208,540,541,569],{},[211,542,543,550,553,560,562],{},[342,544,545,548],{},[345,546,547],{},"r",[214,549,280],{},[218,551,552],{"separator":244},",",[342,554,555,557],{},[345,556,547],{},[214,558,559],{},"2",[218,561,552],{"separator":244},[342,563,564,566],{},[345,565,547],{},[214,567,568],{},"3",[236,570,571],{"encoding":238},"r_1,r_2,r_3",[195,573,575],{"className":574,"ariaHidden":244},[243],[195,576,578,582,624,628,632,672,675,678],{"className":577},[248],[195,579],{"className":580,"style":581},[252],"height:0.625em;vertical-align:-0.1944em;",[195,583,585,589],{"className":584},[257],[195,586,547],{"className":587,"style":588},[257,369],"margin-right:0.0278em;",[195,590,592],{"className":591},[291],[195,593,595,616],{"className":594},[295,377],[195,596,598,613],{"className":597},[299],[195,599,601],{"className":600,"style":384},[303],[195,602,604,607],{"style":603},"top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;",[195,605],{"className":606,"style":311},[310],[195,608,610],{"className":609},[315,316,317,318],[195,611,280],{"className":612},[257,318],[195,614,401],{"className":615},[400],[195,617,619],{"className":618},[299],[195,620,622],{"className":621,"style":408},[303],[195,623],{},[195,625,552],{"className":626},[627],"mpunct",[195,629],{"className":630,"style":631},[261],"margin-right:0.1667em;",[195,633,635,638],{"className":634},[257],[195,636,547],{"className":637,"style":588},[257,369],[195,639,641],{"className":640},[291],[195,642,644,664],{"className":643},[295,377],[195,645,647,661],{"className":646},[299],[195,648,650],{"className":649,"style":384},[303],[195,651,652,655],{"style":603},[195,653],{"className":654,"style":311},[310],[195,656,658],{"className":657},[315,316,317,318],[195,659,559],{"className":660},[257,318],[195,662,401],{"className":663},[400],[195,665,667],{"className":666},[299],[195,668,670],{"className":669,"style":408},[303],[195,671],{},[195,673,552],{"className":674},[627],[195,676],{"className":677,"style":631},[261],[195,679,681,684],{"className":680},[257],[195,682,547],{"className":683,"style":588},[257,369],[195,685,687],{"className":686},[291],[195,688,690,710],{"className":689},[295,377],[195,691,693,707],{"className":692},[299],[195,694,696],{"className":695,"style":384},[303],[195,697,698,701],{"style":603},[195,699],{"className":700,"style":311},[310],[195,702,704],{"className":703},[315,316,317,318],[195,705,568],{"className":706},[257,318],[195,708,401],{"className":709},[400],[195,711,713],{"className":712},[299],[195,714,716],{"className":715,"style":408},[303],[195,717],{},"。最高分结果作为 chosen ",[195,720,722,742],{"className":721},[198],[195,723,725],{"className":724},[202],[204,726,727],{"xmlns":206},[208,728,729,739],{},[211,730,731],{},[342,732,733,736],{},[345,734,735],{},"y",[345,737,738],{},"w",[236,740,741],{"encoding":238},"y_w",[195,743,745],{"className":744,"ariaHidden":244},[243],[195,746,748,751],{"className":747},[248],[195,749],{"className":750,"style":581},[252],[195,752,754,758],{"className":753},[257],[195,755,735],{"className":756,"style":757},[257,369],"margin-right:0.0359em;",[195,759,761],{"className":760},[291],[195,762,764,787],{"className":763},[295,377],[195,765,767,784],{"className":766},[299],[195,768,771],{"className":769,"style":770},[303],"height:0.1514em;",[195,772,774,777],{"style":773},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[195,775],{"className":776,"style":311},[310],[195,778,780],{"className":779},[315,316,317,318],[195,781,738],{"className":782,"style":783},[257,369,318],"margin-right:0.0269em;",[195,785,401],{"className":786},[400],[195,788,790],{"className":789},[299],[195,791,793],{"className":792,"style":408},[303],[195,794],{},"，最低分结果作为 rejected ",[195,797,799,818],{"className":798},[198],[195,800,802],{"className":801},[202],[204,803,804],{"xmlns":206},[208,805,806,815],{},[211,807,808],{},[342,809,810,812],{},[345,811,735],{},[345,813,814],{},"l",[236,816,817],{"encoding":238},"y_l",[195,819,821],{"className":820,"ariaHidden":244},[243],[195,822,824,827],{"className":823},[248],[195,825],{"className":826,"style":581},[252],[195,828,830,833],{"className":829},[257],[195,831,735],{"className":832,"style":757},[257,369],[195,834,836],{"className":835},[291],[195,837,839,861],{"className":838},[295,377],[195,840,842,858],{"className":841},[299],[195,843,846],{"className":844,"style":845},[303],"height:0.3361em;",[195,847,848,851],{"style":773},[195,849],{"className":850,"style":311},[310],[195,852,854],{"className":853},[315,316,317,318],[195,855,814],{"className":856,"style":857},[257,369,318],"margin-right:0.0197em;",[195,859,401],{"className":860},[400],[195,862,864],{"className":863},[299],[195,865,867],{"className":866,"style":408},[303],[195,868],{},"：",[122,871,874],{"className":872,"code":873,"language":127,"meta":128},[125],"(x, y_w, y_l)\n",[96,875,873],{"__ignoreMap":128},[10,877,878],{},"这一步把连续评分转换成 pairwise preference。偏好来自 AI Reviewer 而不是逐条人工标注，因此也可以视为一种 RLAIF 数据生产方式。",[21,880,882],{"id":881},"为什么使用-simpo而不是-ppo","为什么使用 SimPO，而不是 PPO",[10,884,885],{},"CycleResearcher 没有使用 PPO，也没有直接把 Reviewer 分数接入 policy gradient。它使用 SimPO 对 chosen 与 rejected 的生成概率进行比较。",[10,887,888],{},"SimPO 对一个回答定义长度归一化的隐式奖励：",[195,890,893],{"className":891},[892],"katex-display",[195,894,896,974],{"className":895},[198],[195,897,899],{"className":898},[202],[204,900,902],{"xmlns":206,"display":901},"block",[208,903,904,971],{},[211,905,906,914,918,921,923,925,928,931,947,950,953,961,963,965,967,969],{},[342,907,908,910],{},[345,909,547],{},[911,912,913],"mtext",{},"SimPO",[218,915,917],{"stretchy":916},"false","(",[345,919,920],{},"x",[218,922,552],{"separator":244},[345,924,735],{},[218,926,927],{"stretchy":916},")",[218,929,930],{},"=",[932,933,934,937],"mfrac",{},[345,935,936],{},"β",[211,938,939,943,945],{},[345,940,942],{"mathvariant":941},"normal","∣",[345,944,735],{},[345,946,942],{"mathvariant":941},[345,948,949],{},"log",[218,951,952],{},"⁡",[342,954,955,958],{},[345,956,957],{},"π",[345,959,960],{},"θ",[218,962,917],{"stretchy":916},[345,964,735],{},[218,966,942],{},[345,968,920],{},[218,970,927],{"stretchy":916},[236,972,973],{"encoding":238},"r_{\\text{SimPO}}(x,y)=\\frac{\\beta}{|y|}\\log \\pi_\\theta(y\\mid x)",[195,975,977,1062,1217],{"className":976,"ariaHidden":244},[243],[195,978,980,984,1031,1035,1038,1041,1044,1047,1051,1055,1059],{"className":979},[248],[195,981],{"className":982,"style":983},[252],"height:1em;vertical-align:-0.25em;",[195,985,987,990],{"className":986},[257],[195,988,547],{"className":989,"style":588},[257,369],[195,991,993],{"className":992},[291],[195,994,996,1023],{"className":995},[295,377],[195,997,999,1020],{"className":998},[299],[195,1000,1003],{"className":1001,"style":1002},[303],"height:0.3283em;",[195,1004,1005,1008],{"style":603},[195,1006],{"className":1007,"style":311},[310],[195,1009,1011],{"className":1010},[315,316,317,318],[195,1012,1014],{"className":1013},[257,318],[195,1015,1017],{"className":1016},[257,127,318],[195,1018,913],{"className":1019},[257,318],[195,1021,401],{"className":1022},[400],[195,1024,1026],{"className":1025},[299],[195,1027,1029],{"className":1028,"style":408},[303],[195,1030],{},[195,1032,917],{"className":1033},[1034],"mopen",[195,1036,920],{"className":1037},[257,369],[195,1039,552],{"className":1040},[627],[195,1042],{"className":1043,"style":631},[261],[195,1045,735],{"className":1046,"style":757},[257,369],[195,1048,927],{"className":1049},[1050],"mclose",[195,1052],{"className":1053,"style":1054},[261],"margin-right:0.2778em;",[195,1056,930],{"className":1057},[1058],"mrel",[195,1060],{"className":1061,"style":1054},[261],[195,1063,1065,1069,1147,1150,1159,1162,1202,1205,1208,1211,1214],{"className":1064},[248],[195,1066],{"className":1067,"style":1068},[252],"height:2.3074em;vertical-align:-0.936em;",[195,1070,1072,1076,1144],{"className":1071},[257],[195,1073],{"className":1074},[1034,1075],"nulldelimiter",[195,1077,1079],{"className":1078},[932],[195,1080,1082,1135],{"className":1081},[295,377],[195,1083,1085,1132],{"className":1084},[299],[195,1086,1089,1108,1119],{"className":1087,"style":1088},[303],"height:1.3714em;",[195,1090,1092,1096],{"style":1091},"top:-2.314em;",[195,1093],{"className":1094,"style":1095},[310],"height:3em;",[195,1097,1099,1102,1105],{"className":1098},[257],[195,1100,942],{"className":1101},[257],[195,1103,735],{"className":1104,"style":757},[257,369],[195,1106,942],{"className":1107},[257],[195,1109,1111,1114],{"style":1110},"top:-3.23em;",[195,1112],{"className":1113,"style":1095},[310],[195,1115],{"className":1116,"style":1118},[1117],"frac-line","border-bottom-width:0.04em;",[195,1120,1122,1125],{"style":1121},"top:-3.677em;",[195,1123],{"className":1124,"style":1095},[310],[195,1126,1128],{"className":1127},[257],[195,1129,936],{"className":1130,"style":1131},[257,369],"margin-right:0.0528em;",[195,1133,401],{"className":1134},[400],[195,1136,1138],{"className":1137},[299],[195,1139,1142],{"className":1140,"style":1141},[303],"height:0.936em;",[195,1143],{},[195,1145],{"className":1146},[1050,1075],[195,1148],{"className":1149,"style":631},[261],[195,1151,1154,1155],{"className":1152},[1153],"mop","lo",[195,1156,1158],{"style":1157},"margin-right:0.0139em;","g",[195,1160],{"className":1161,"style":631},[261],[195,1163,1165,1168],{"className":1164},[257],[195,1166,957],{"className":1167,"style":757},[257,369],[195,1169,1171],{"className":1170},[291],[195,1172,1174,1194],{"className":1173},[295,377],[195,1175,1177,1191],{"className":1176},[299],[195,1178,1180],{"className":1179,"style":845},[303],[195,1181,1182,1185],{"style":773},[195,1183],{"className":1184,"style":311},[310],[195,1186,1188],{"className":1187},[315,316,317,318],[195,1189,960],{"className":1190,"style":588},[257,369,318],[195,1192,401],{"className":1193},[400],[195,1195,1197],{"className":1196},[299],[195,1198,1200],{"className":1199,"style":408},[303],[195,1201],{},[195,1203,917],{"className":1204},[1034],[195,1206,735],{"className":1207,"style":757},[257,369],[195,1209],{"className":1210,"style":1054},[261],[195,1212,942],{"className":1213},[1058],[195,1215],{"className":1216,"style":1054},[261],[195,1218,1220,1223,1226],{"className":1219},[248],[195,1221],{"className":1222,"style":983},[252],[195,1224,920],{"className":1225},[257,369],[195,1227,927],{"className":1228},[1050],[10,1230,1231,1232,869],{},"训练目标要求 chosen 的隐式奖励比 rejected 至少高出 margin ",[195,1233,1235,1250],{"className":1234},[198],[195,1236,1238],{"className":1237},[202],[204,1239,1240],{"xmlns":206},[208,1241,1242,1247],{},[211,1243,1244],{},[345,1245,1246],{},"γ",[236,1248,1249],{"encoding":238},"\\gamma",[195,1251,1253],{"className":1252,"ariaHidden":244},[243],[195,1254,1256,1259],{"className":1255},[248],[195,1257],{"className":1258,"style":581},[252],[195,1260,1246],{"className":1261,"style":1262},[257,369],"margin-right:0.0556em;",[195,1264,1266],{"className":1265},[892],[195,1267,1269,1356],{"className":1268},[198],[195,1270,1272],{"className":1271},[202],[204,1273,1274],{"xmlns":206,"display":901},[208,1275,1276,1353],{},[211,1277,1278,1286,1288,1290,1294],{},[342,1279,1280,1284],{},[345,1281,1283],{"mathvariant":1282},"script","L",[911,1285,913],{},[218,1287,930],{},[218,1289,231],{},[345,1291,1293],{"mathvariant":1292},"double-struck","E",[211,1295,1296,1299,1301,1303,1306,1350],{},[218,1297,1298],{"fence":244},"[",[345,1300,949],{},[218,1302,952],{},[345,1304,1305],{},"σ",[211,1307,1308,1310,1312,1314,1316,1318,1324,1326,1328,1330,1332,1334,1336,1342,1344,1346,1348],{},[218,1309,917],{"fence":244},[345,1311,547],{},[218,1313,917],{"stretchy":916},[345,1315,920],{},[218,1317,552],{"separator":244},[342,1319,1320,1322],{},[345,1321,735],{},[345,1323,738],{},[218,1325,927],{"stretchy":916},[218,1327,231],{},[345,1329,547],{},[218,1331,917],{"stretchy":916},[345,1333,920],{},[218,1335,552],{"separator":244},[342,1337,1338,1340],{},[345,1339,735],{},[345,1341,814],{},[218,1343,927],{"stretchy":916},[218,1345,231],{},[345,1347,1246],{},[218,1349,927],{"fence":244},[218,1351,1352],{"fence":244},"]",[236,1354,1355],{"encoding":238},"\\mathcal{L}_{\\text{SimPO}}\n=-\\mathbb{E}\\left[\\log\\sigma\\left(\nr(x,y_w)-r(x,y_l)-\\gamma\n\\right)\\right]",[195,1357,1359,1422],{"className":1358,"ariaHidden":244},[243],[195,1360,1362,1365,1413,1416,1419],{"className":1361},[248],[195,1363],{"className":1364,"style":362},[252],[195,1366,1368,1372],{"className":1367},[257],[195,1369,1283],{"className":1370},[257,1371],"mathcal",[195,1373,1375],{"className":1374},[291],[195,1376,1378,1405],{"className":1377},[295,377],[195,1379,1381,1402],{"className":1380},[299],[195,1382,1384],{"className":1383,"style":1002},[303],[195,1385,1387,1390],{"style":1386},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[195,1388],{"className":1389,"style":311},[310],[195,1391,1393],{"className":1392},[315,316,317,318],[195,1394,1396],{"className":1395},[257,318],[195,1397,1399],{"className":1398},[257,127,318],[195,1400,913],{"className":1401},[257,318],[195,1403,401],{"className":1404},[400],[195,1406,1408],{"className":1407},[299],[195,1409,1411],{"className":1410,"style":408},[303],[195,1412],{},[195,1414],{"className":1415,"style":1054},[261],[195,1417,930],{"className":1418},[1058],[195,1420],{"className":1421,"style":1054},[261],[195,1423,1425,1428,1431,1435,1438],{"className":1424},[248],[195,1426],{"className":1427,"style":983},[252],[195,1429,231],{"className":1430},[257],[195,1432,1293],{"className":1433},[257,1434],"mathbb",[195,1436],{"className":1437,"style":631},[261],[195,1439,1442,1447,1452,1455,1458,1461,1607],{"className":1440},[1441],"minner",[195,1443,1298],{"className":1444,"style":1446},[1034,1445],"delimcenter","top:0em;",[195,1448,1154,1450],{"className":1449},[1153],[195,1451,1158],{"style":1157},[195,1453],{"className":1454,"style":631},[261],[195,1456,1305],{"className":1457,"style":757},[257,369],[195,1459],{"className":1460,"style":631},[261],[195,1462,1464,1467,1470,1473,1476,1479,1482,1522,1525,1528,1531,1534,1537,1540,1543,1546,1549,1589,1592,1595,1598,1601,1604],{"className":1463},[1441],[195,1465,917],{"className":1466,"style":1446},[1034,1445],[195,1468,547],{"className":1469,"style":588},[257,369],[195,1471,917],{"className":1472},[1034],[195,1474,920],{"className":1475},[257,369],[195,1477,552],{"className":1478},[627],[195,1480],{"className":1481,"style":631},[261],[195,1483,1485,1488],{"className":1484},[257],[195,1486,735],{"className":1487,"style":757},[257,369],[195,1489,1491],{"className":1490},[291],[195,1492,1494,1514],{"className":1493},[295,377],[195,1495,1497,1511],{"className":1496},[299],[195,1498,1500],{"className":1499,"style":770},[303],[195,1501,1502,1505],{"style":773},[195,1503],{"className":1504,"style":311},[310],[195,1506,1508],{"className":1507},[315,316,317,318],[195,1509,738],{"className":1510,"style":783},[257,369,318],[195,1512,401],{"className":1513},[400],[195,1515,1517],{"className":1516},[299],[195,1518,1520],{"className":1519,"style":408},[303],[195,1521],{},[195,1523,927],{"className":1524},[1050],[195,1526],{"className":1527,"style":262},[261],[195,1529,231],{"className":1530},[266],[195,1532],{"className":1533,"style":262},[261],[195,1535,547],{"className":1536,"style":588},[257,369],[195,1538,917],{"className":1539},[1034],[195,1541,920],{"className":1542},[257,369],[195,1544,552],{"className":1545},[627],[195,1547],{"className":1548,"style":631},[261],[195,1550,1552,1555],{"className":1551},[257],[195,1553,735],{"className":1554,"style":757},[257,369],[195,1556,1558],{"className":1557},[291],[195,1559,1561,1581],{"className":1560},[295,377],[195,1562,1564,1578],{"className":1563},[299],[195,1565,1567],{"className":1566,"style":845},[303],[195,1568,1569,1572],{"style":773},[195,1570],{"className":1571,"style":311},[310],[195,1573,1575],{"className":1574},[315,316,317,318],[195,1576,814],{"className":1577,"style":857},[257,369,318],[195,1579,401],{"className":1580},[400],[195,1582,1584],{"className":1583},[299],[195,1585,1587],{"className":1586,"style":408},[303],[195,1588],{},[195,1590,927],{"className":1591},[1050],[195,1593],{"className":1594,"style":262},[261],[195,1596,231],{"className":1597},[266],[195,1599],{"className":1600,"style":262},[261],[195,1602,1246],{"className":1603,"style":1262},[257,369],[195,1605,927],{"className":1606,"style":1446},[1050,1445],[195,1608,1352],{"className":1609,"style":1446},[1050,1445],[10,1611,1612],{},"与标准 DPO 相比，SimPO 不需要在训练时常驻 reference model，并且对序列 log probability 做长度归一化。对于 24K 甚至更长的输出，这能减少显存和计算压力，也避免模型仅通过改变回答长度获得更高序列概率。",[21,1614,1616],{"id":1615},"nll-不是附属项而是稳定器","NLL 不是附属项，而是稳定器",[10,1618,1619],{},"论文没有只优化 SimPO，而是为 chosen response 增加 Negative Log-Likelihood：",[195,1621,1623],{"className":1622},[892],[195,1624,1626,1700],{"className":1625},[198],[195,1627,1629],{"className":1628},[202],[204,1630,1631],{"xmlns":206,"display":901},[208,1632,1633,1697],{},[211,1634,1635,1637,1639,1645,1647,1650,1653,1673,1675,1677,1683,1685,1691,1693,1695],{},[345,1636,1283],{"mathvariant":1282},[218,1638,930],{},[342,1640,1641,1643],{},[345,1642,1283],{"mathvariant":1282},[911,1644,913],{},[218,1646,231],{},[345,1648,1649],{},"λ",[911,1651,1652],{}," ",[342,1654,1655,1657],{},[345,1656,1293],{"mathvariant":1292},[211,1658,1659,1661,1663,1665,1671],{},[218,1660,917],{"stretchy":916},[345,1662,920],{},[218,1664,552],{"separator":244},[342,1666,1667,1669],{},[345,1668,735],{},[345,1670,738],{},[218,1672,927],{"stretchy":916},[345,1674,949],{},[218,1676,952],{},[342,1678,1679,1681],{},[345,1680,957],{},[345,1682,960],{},[218,1684,917],{"stretchy":916},[342,1686,1687,1689],{},[345,1688,735],{},[345,1690,738],{},[218,1692,942],{},[345,1694,920],{},[218,1696,927],{"stretchy":916},[236,1698,1699],{"encoding":238},"\\mathcal{L}=\\mathcal{L}_{\\text{SimPO}}\n-\\lambda\\,\\mathbb{E}_{(x,y_w)}\\log\\pi_\\theta(y_w\\mid x)",[195,1701,1703,1722,1783,2000],{"className":1702,"ariaHidden":244},[243],[195,1704,1706,1710,1713,1716,1719],{"className":1705},[248],[195,1707],{"className":1708,"style":1709},[252],"height:0.6833em;",[195,1711,1283],{"className":1712},[257,1371],[195,1714],{"className":1715,"style":1054},[261],[195,1717,930],{"className":1718},[1058],[195,1720],{"className":1721,"style":1054},[261],[195,1723,1725,1728,1774,1777,1780],{"className":1724},[248],[195,1726],{"className":1727,"style":362},[252],[195,1729,1731,1734],{"className":1730},[257],[195,1732,1283],{"className":1733},[257,1371],[195,1735,1737],{"className":1736},[291],[195,1738,1740,1766],{"className":1739},[295,377],[195,1741,1743,1763],{"className":1742},[299],[195,1744,1746],{"className":1745,"style":1002},[303],[195,1747,1748,1751],{"style":1386},[195,1749],{"className":1750,"style":311},[310],[195,1752,1754],{"className":1753},[315,316,317,318],[195,1755,1757],{"className":1756},[257,318],[195,1758,1760],{"className":1759},[257,127,318],[195,1761,913],{"className":1762},[257,318],[195,1764,401],{"className":1765},[400],[195,1767,1769],{"className":1768},[299],[195,1770,1772],{"className":1771,"style":408},[303],[195,1773],{},[195,1775],{"className":1776,"style":262},[261],[195,1778,231],{"className":1779},[266],[195,1781],{"className":1782,"style":262},[261],[195,1784,1786,1790,1793,1796,1897,1900,1905,1908,1948,1951,1991,1994,1997],{"className":1785},[248],[195,1787],{"className":1788,"style":1789},[252],"height:1.1052em;vertical-align:-0.3552em;",[195,1791,1649],{"className":1792},[257,369],[195,1794],{"className":1795,"style":631},[261],[195,1797,1799,1802],{"className":1798},[257],[195,1800,1293],{"className":1801},[257,1434],[195,1803,1805],{"className":1804},[291],[195,1806,1808,1888],{"className":1807},[295,377],[195,1809,1811,1885],{"className":1810},[299],[195,1812,1815],{"className":1813,"style":1814},[303],"height:0.3448em;",[195,1816,1818,1821],{"style":1817},"top:-2.5198em;margin-left:0em;margin-right:0.05em;",[195,1819],{"className":1820,"style":311},[310],[195,1822,1824],{"className":1823},[315,316,317,318],[195,1825,1827,1830,1833,1836,1882],{"className":1826},[257,318],[195,1828,917],{"className":1829},[1034,318],[195,1831,920],{"className":1832},[257,369,318],[195,1834,552],{"className":1835},[627,318],[195,1837,1839,1842],{"className":1838},[257,318],[195,1840,735],{"className":1841,"style":757},[257,369,318],[195,1843,1845],{"className":1844},[291],[195,1846,1848,1873],{"className":1847},[295,377],[195,1849,1851,1870],{"className":1850},[299],[195,1852,1855],{"className":1853,"style":1854},[303],"height:0.1645em;",[195,1856,1858,1862],{"style":1857},"top:-2.357em;margin-left:-0.0359em;margin-right:0.0714em;",[195,1859],{"className":1860,"style":1861},[310],"height:2.5em;",[195,1863,1867],{"className":1864},[315,1865,1866,318],"reset-size3","size1",[195,1868,738],{"className":1869,"style":783},[257,369,318],[195,1871,401],{"className":1872},[400],[195,1874,1876],{"className":1875},[299],[195,1877,1880],{"className":1878,"style":1879},[303],"height:0.143em;",[195,1881],{},[195,1883,927],{"className":1884},[1050,318],[195,1886,401],{"className":1887},[400],[195,1889,1891],{"className":1890},[299],[195,1892,1895],{"className":1893,"style":1894},[303],"height:0.3552em;",[195,1896],{},[195,1898],{"className":1899,"style":631},[261],[195,1901,1154,1903],{"className":1902},[1153],[195,1904,1158],{"style":1157},[195,1906],{"className":1907,"style":631},[261],[195,1909,1911,1914],{"className":1910},[257],[195,1912,957],{"className":1913,"style":757},[257,369],[195,1915,1917],{"className":1916},[291],[195,1918,1920,1940],{"className":1919},[295,377],[195,1921,1923,1937],{"className":1922},[299],[195,1924,1926],{"className":1925,"style":845},[303],[195,1927,1928,1931],{"style":773},[195,1929],{"className":1930,"style":311},[310],[195,1932,1934],{"className":1933},[315,316,317,318],[195,1935,960],{"className":1936,"style":588},[257,369,318],[195,1938,401],{"className":1939},[400],[195,1941,1943],{"className":1942},[299],[195,1944,1946],{"className":1945,"style":408},[303],[195,1947],{},[195,1949,917],{"className":1950},[1034],[195,1952,1954,1957],{"className":1953},[257],[195,1955,735],{"className":1956,"style":757},[257,369],[195,1958,1960],{"className":1959},[291],[195,1961,1963,1983],{"className":1962},[295,377],[195,1964,1966,1980],{"className":1965},[299],[195,1967,1969],{"className":1968,"style":770},[303],[195,1970,1971,1974],{"style":773},[195,1972],{"className":1973,"style":311},[310],[195,1975,1977],{"className":1976},[315,316,317,318],[195,1978,738],{"className":1979,"style":783},[257,369,318],[195,1981,401],{"className":1982},[400],[195,1984,1986],{"className":1985},[299],[195,1987,1989],{"className":1988,"style":408},[303],[195,1990],{},[195,1992],{"className":1993,"style":1054},[261],[195,1995,942],{"className":1996},[1058],[195,1998],{"className":1999,"style":1054},[261],[195,2001,2003,2006,2009],{"className":2002},[248],[195,2004],{"className":2005,"style":983},[252],[195,2007,920],{"className":2008},[257,369],[195,2010,927],{"className":2011},[1050],[10,2013,2014],{},"偏好损失只要求 chosen 相对 rejected 更好，不保证长文本仍然流畅、完整且不重复。NLL 则继续用标准语言建模目标拟合高质量 chosen，承担行为克隆和分布约束的作用。",[10,2016,2017],{},"消融实验很能说明问题：完整模型平均分为 5.36，去掉偏好训练后降至 5.12，去掉迭代采样后为 5.21；去掉 NLL 后直接降至 4.91，自动接收率从 35.14% 降至 12.03%。论文观察到无 NLL 时会出现重复文本和明显内容错误。",[21,2019,2021],{"id":2020},"iterative-simpo-到底算不算强化学习","Iterative SimPO 到底算不算强化学习",[10,2023,2024],{},"论文将方法描述为 iterative reinforcement learning framework，但面试时最好说得更精确。",[10,2026,2027],{},"从优化目标看，它不是 PPO\u002FGRPO 式在线 RL：没有对 Reviewer 标量分数做策略梯度，也不在每个 rollout 后直接更新策略。每一轮仍然先把评分转换成离线偏好对，再优化 SimPO loss。",[10,2029,2030,2031,2061,2062,2133,2134,2205,2206,2294,2295,158],{},"从数据闭环看，它又不是固定数据集上的一次性离线 DPO。第 ",[195,2032,2034,2048],{"className":2033},[198],[195,2035,2037],{"className":2036},[202],[204,2038,2039],{"xmlns":206},[208,2040,2041,2046],{},[211,2042,2043],{},[345,2044,2045],{},"t",[236,2047,2045],{"encoding":238},[195,2049,2051],{"className":2050,"ariaHidden":244},[243],[195,2052,2054,2058],{"className":2053},[248],[195,2055],{"className":2056,"style":2057},[252],"height:0.6151em;",[195,2059,2045],{"className":2060},[257,369]," 轮使用当前策略 ",[195,2063,2065,2083],{"className":2064},[198],[195,2066,2068],{"className":2067},[202],[204,2069,2070],{"xmlns":206},[208,2071,2072,2080],{},[211,2073,2074],{},[342,2075,2076,2078],{},[345,2077,347],{},[345,2079,2045],{},[236,2081,2082],{"encoding":238},"P_t",[195,2084,2086],{"className":2085,"ariaHidden":244},[243],[195,2087,2089,2092],{"className":2088},[248],[195,2090],{"className":2091,"style":362},[252],[195,2093,2095,2098],{"className":2094},[257],[195,2096,347],{"className":2097,"style":370},[257,369],[195,2099,2101],{"className":2100},[291],[195,2102,2104,2125],{"className":2103},[295,377],[195,2105,2107,2122],{"className":2106},[299],[195,2108,2111],{"className":2109,"style":2110},[303],"height:0.2806em;",[195,2112,2113,2116],{"style":387},[195,2114],{"className":2115,"style":311},[310],[195,2117,2119],{"className":2118},[315,316,317,318],[195,2120,2045],{"className":2121},[257,369,318],[195,2123,401],{"className":2124},[400],[195,2126,2128],{"className":2127},[299],[195,2129,2131],{"className":2130,"style":408},[303],[195,2132],{}," 采样，Reviewer 构造 ",[195,2135,2137,2156],{"className":2136},[198],[195,2138,2140],{"className":2139},[202],[204,2141,2142],{"xmlns":206},[208,2143,2144,2153],{},[211,2145,2146],{},[342,2147,2148,2151],{},[345,2149,2150],{},"D",[345,2152,2045],{},[236,2154,2155],{"encoding":238},"D_t",[195,2157,2159],{"className":2158,"ariaHidden":244},[243],[195,2160,2162,2165],{"className":2161},[248],[195,2163],{"className":2164,"style":362},[252],[195,2166,2168,2171],{"className":2167},[257],[195,2169,2150],{"className":2170,"style":588},[257,369],[195,2172,2174],{"className":2173},[291],[195,2175,2177,2197],{"className":2176},[295,377],[195,2178,2180,2194],{"className":2179},[299],[195,2181,2183],{"className":2182,"style":2110},[303],[195,2184,2185,2188],{"style":603},[195,2186],{"className":2187,"style":311},[310],[195,2189,2191],{"className":2190},[315,316,317,318],[195,2192,2045],{"className":2193},[257,369,318],[195,2195,401],{"className":2196},[400],[195,2198,2200],{"className":2199},[299],[195,2201,2203],{"className":2202,"style":408},[303],[195,2204],{},"，训练得到 ",[195,2207,2209,2234],{"className":2208},[198],[195,2210,2212],{"className":2211},[202],[204,2213,2214],{"xmlns":206},[208,2215,2216,2231],{},[211,2217,2218],{},[342,2219,2220,2222],{},[345,2221,347],{},[211,2223,2224,2226,2229],{},[345,2225,2045],{},[218,2227,2228],{},"+",[214,2230,280],{},[236,2232,2233],{"encoding":238},"P_{t+1}",[195,2235,2237],{"className":2236,"ariaHidden":244},[243],[195,2238,2240,2244],{"className":2239},[248],[195,2241],{"className":2242,"style":2243},[252],"height:0.8917em;vertical-align:-0.2083em;",[195,2245,2247,2250],{"className":2246},[257],[195,2248,347],{"className":2249,"style":370},[257,369],[195,2251,2253],{"className":2252},[291],[195,2254,2256,2285],{"className":2255},[295,377],[195,2257,2259,2282],{"className":2258},[299],[195,2260,2262],{"className":2261,"style":384},[303],[195,2263,2264,2267],{"style":387},[195,2265],{"className":2266,"style":311},[310],[195,2268,2270],{"className":2269},[315,316,317,318],[195,2271,2273,2276,2279],{"className":2272},[257,318],[195,2274,2045],{"className":2275},[257,369,318],[195,2277,2228],{"className":2278},[266,318],[195,2280,280],{"className":2281},[257,318],[195,2283,401],{"className":2284},[400],[195,2286,2288],{"className":2287},[299],[195,2289,2292],{"className":2290,"style":2291},[303],"height:0.2083em;",[195,2293],{},"，下一轮再由新策略产生数据。作者每轮抽取约三分之一数据、训练一个 epoch，RL 阶段学习率为 ",[195,2296,2298,2325],{"className":2297},[198],[195,2299,2301],{"className":2300},[202],[204,2302,2303],{"xmlns":206},[208,2304,2305,2322],{},[211,2306,2307,2309,2311],{},[214,2308,234],{},[218,2310,220],{},[222,2312,2313,2315],{},[214,2314,226],{},[211,2316,2317,2319],{},[218,2318,231],{},[214,2320,2321],{},"7",[236,2323,2324],{"encoding":238},"5\\times10^{-7}",[195,2326,2328,2346],{"className":2327,"ariaHidden":244},[243],[195,2329,2331,2334,2337,2340,2343],{"className":2330},[248],[195,2332],{"className":2333,"style":253},[252],[195,2335,234],{"className":2336},[257],[195,2338],{"className":2339,"style":262},[261],[195,2341,220],{"className":2342},[266],[195,2344],{"className":2345,"style":262},[261],[195,2347,2349,2352,2355],{"className":2348},[248],[195,2350],{"className":2351,"style":276},[252],[195,2353,280],{"className":2354},[257],[195,2356,2358,2361],{"className":2357},[257],[195,2359,287],{"className":2360},[257],[195,2362,2364],{"className":2363},[291],[195,2365,2367],{"className":2366},[295],[195,2368,2370],{"className":2369},[299],[195,2371,2373],{"className":2372,"style":276},[303],[195,2374,2375,2378],{"style":306},[195,2376],{"className":2377,"style":311},[310],[195,2379,2381],{"className":2380},[315,316,317,318],[195,2382,2384,2387],{"className":2383},[257,318],[195,2385,231],{"className":2386},[257,318],[195,2388,2321],{"className":2389},[257,318],[10,2391,2392],{},"所以更准确的表述是：",[2394,2395,2396],"blockquote",{},[10,2397,2398],{},"CycleResearcher 使用带在线数据刷新的迭代离线偏好优化，形成近似在线的 Policy-Reward 数据闭环。",[10,2400,2401],{},"这个表述既承认论文的 RL 框架定位，也不会把 SimPO 错说成 PPO。",[21,2403,2404],{"id":2404},"结果背后的两个风险",[2406,2407,2409],"h3",{"id":2408},"reward-hacking","Reward hacking",[10,2411,2412],{},"Policy 可能学会迎合 CycleReviewer，例如偏好某种结构、措辞或篇幅，而没有真正提升研究质量。论文也明确指出，Policy 与 Reward Model 没有同步更新，长期迭代可能放大 Reviewer 的固定漏洞。",[10,2414,2415],{},"作者用独立 Reward Model 做了补充评估：在该附录实验的评估设置下，原 Reviewer 的平均分为 5.36，独立 Reviewer 下为 5.29；接收率从 31.07% 降到 28.65%。这里的 31.07% 与正文消融表中的 35.14% 来自不同表格和评估设置，不应直接混用。整体下降幅度不算剧烈，但只能缓解疑虑，不能证明不存在 reward exploitation。",[2406,2417,2418],{"id":2418},"训练环境不等于真实科研",[10,2420,2421],{},"训练阶段并不执行真实实验，论文中的虚拟实验结果由模型生成。系统主要训练文献理解、研究规划和论文写作，不应被理解成已经完成了从假设到实验验证的全自动科学发现。",[10,2423,2424],{},"因此，自动评分高于某个基线只能说明它更符合当前 Reviewer 的评价分布，不能直接等价为科学创新能力更强。",[21,2426,2427],{"id":2427},"西湖大学近两年的相关方向",[10,2429,2430],{},"如果关注 2024 至 2026 年的 Post-training，可以把相关工作分成三类：",[2432,2433,2434,2444,2460],"ol",{},[173,2435,2436,869,2439,2443],{},[17,2437,2438],{},"科研 Agent 的领域后训练",[152,2440,69],{"href":2441,"rel":2442},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2411.00816",[156]," 在 ICLR 2025 发表，将 SFT、生成式 Reviewer 和 Iterative SimPO 放进同一闭环。",[173,2445,2446,869,2448,2453,2454,2459],{},[17,2447,91],{},[152,2449,2452],{"href":2450,"rel":2451},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2411.07618",[156],"FPO"," 使用 Sparse Autoencoder 的特征级约束改进 DPO；",[152,2455,2458],{"href":2456,"rel":2457},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2504.15843",[156],"Pre-DPO"," 使用 guiding reference model 改善偏好数据利用率。",[173,2461,2462,869,2465,2470],{},[17,2463,2464],{},"多模态后训练",[152,2466,2469],{"href":2467,"rel":2468},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2510.23479",[156],"MergeMix"," 用图像混合构造偏好关系，并以 SimPO 训练多模态模型；它属于西湖大学其他团队，而不是 CycleResearcher 所在的 WestlakeNLP 工作线。",[10,2472,2473,2477],{},[152,2474,55],{"href":2475,"rel":2476},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2604.09590",[156]," 则适合作为反例：它是很新的科研 Agent，但论文强调底层模型未针对审稿任务微调，主要贡献在推理时流程和可审计输出协议，不应包装成 Post-training 项目。",[21,2479,2481],{"id":2480},"面向-post-training-岗位应该讲清什么","面向 Post-training 岗位，应该讲清什么",[10,2483,2484],{},"如果用 CycleResearcher 准备面试，重点不应停留在“做了一个科研 Agent”，而应能回答下面这些问题：",[2432,2486,2487,2490,2493,2496,2499,2502,2505,2508,2511],{},[173,2488,2489],{},"Research-14K 和 Review-5K 分别提供什么训练信号？",[173,2491,2492],{},"Generative Reward Model 与标量 Reward Model 有什么差异？",[173,2494,2495],{},"Reviewer 分数为什么要转成 chosen-rejected，而不是直接回归分数？",[173,2497,2498],{},"SimPO 为什么可以不加载 reference model？",[173,2500,2501],{},"长度归一化对长文本偏好学习有什么作用？",[173,2503,2504],{},"为什么仅使用 preference loss 会导致重复和能力漂移？",[173,2506,2507],{},"每轮重新采样如何改变训练分布？",[173,2509,2510],{},"如何用独立 Judge、规则奖励、Reward Ensemble 和人工评测降低 reward hacking？",[173,2512,2513],{},"如果改成 GRPO\u002FPPO，训练成本、稳定性和 reward 设计会发生什么变化？",[10,2515,2516],{},"一句适合面试的总结是：",[2394,2518,2519],{},[10,2520,2521],{},"CycleResearcher 以 Instruct 模型为起点，在 Research-14K 上进行长上下文领域 SFT，并在 Review-5K 上训练生成式 Reward Model。随后从当前 Policy 采样多个科研输出，由 Reviewer 评分并构造偏好对，再使用带 NLL 正则的 Iterative SimPO 更新 Policy，每轮重新生成训练数据，形成自动反馈的 Post-training 闭环。",[21,2523,2524],{"id":2524},"参考资料",[2432,2526,2527,2535,2543,2550,2558,2566,2574],{},[173,2528,2529,2530,2534],{},"Weng et al., ",[152,2531,2533],{"href":154,"rel":2532},[156],"CycleResearcher: Improving Automated Research via Automated Review",", ICLR 2025.",[173,2536,2537,2542],{},[152,2538,2541],{"href":2539,"rel":2540},"https:\u002F\u002Fgithub.com\u002Fzhu-minjun\u002FResearcher",[156],"CycleResearcher 项目与模型仓库",".",[173,2544,2529,2545,2549],{},[152,2546,2548],{"href":2475,"rel":2547},[156],"DeepReviewer 2.0: A Traceable Agentic System for Auditable Scientific Peer Review",", 2026.",[173,2551,2552,2553,2542],{},"StepFun, ",[152,2554,2557],{"href":2555,"rel":2556},"https:\u002F\u002Fhuggingface.co\u002Fstepfun-ai\u002FStep-3.5-Flash",[156],"Step-3.5-Flash Model Card",[173,2559,2560,2561,2565],{},"Yin et al., ",[152,2562,2564],{"href":2450,"rel":2563},[156],"Direct Preference Optimization Using Sparse Feature-Level Constraints",", 2024.",[173,2567,2568,2569,2573],{},"Pan et al., ",[152,2570,2572],{"href":2456,"rel":2571},[156],"Pre-DPO: Improving Data Utilization in Direct Preference Optimization Using a Guiding Reference Model",", 2025.",[173,2575,2576,2577,2573],{},"Jin et al., ",[152,2578,2580],{"href":2467,"rel":2579},[156],"MergeMix: A Unified Augmentation Paradigm for Visual and Multi-Modal Understanding",{"title":128,"searchDepth":2582,"depth":2583,"links":2584},2,3,[2585,2586,2587,2588,2589,2590,2591,2592,2593,2594,2598,2599,2600],{"id":23,"depth":2582,"text":24},{"id":113,"depth":2582,"text":114},{"id":136,"depth":2582,"text":137},{"id":164,"depth":2582,"text":165},{"id":413,"depth":2582,"text":414},{"id":513,"depth":2582,"text":514},{"id":881,"depth":2582,"text":882},{"id":1615,"depth":2582,"text":1616},{"id":2020,"depth":2582,"text":2021},{"id":2404,"depth":2582,"text":2404,"children":2595},[2596,2597],{"id":2408,"depth":2583,"text":2409},{"id":2418,"depth":2583,"text":2418},{"id":2427,"depth":2582,"text":2427},{"id":2480,"depth":2582,"text":2481},{"id":2524,"depth":2582,"text":2524},[2602],"AI","\u002Fimages\u002Fposts\u002Fcycleresearcher-post-training\u002Fcycleresearcher-cover-20261018.webp","策略生成模块产出多份候选文稿经多维评审棱镜排序后通过偏好梯度反馈弧闭环优化的等距技术插画","2026-08-07 17:33:05","区分推理时 Agent 编排与模型后训练，并拆解 CycleResearcher 的领域 SFT、生成式 Reward Model、Iterative SimPO 和评测风险。",false,"md",true,{"mathjax":2609},"\u002Fposts\u002Fcycleresearcher-post-training",null,{"title":5,"description":2606},"posts\u002Fcycleresearcher-post-training",[2616,2617,2618,913,2619,2620],"LLM","Post-training","Agent","Reward Model","ICLR","zUp-jobsDgykiHb-F76jA6uWGf9vZRLznVr-bMfACSY",[2623,2642,2658,2673,2686,2701,2716,2730,2743,2756,2771,2784,2798,2811,2824,2839,2856,2871,2882,2894,2909,2922,2936,2948,2962,2974,2989,3003,3017,3028,3041,3058,3073,3077,3089,3101,3114,3127],{"slug":2624,"path":2625,"title":2626,"date":2627,"tags":2628,"description":2634,"cover":2635,"coverAlt":2636,"excerpt":2637,"draft":2607,"hidden":2607,"published":2609,"readingTime":2638,"series":2639,"seriesOrder":2638,"codeUrl":2640,"attachmentUrl":2641},"cs336-cross-entropy-adamw-training-loop","\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop","Stanford CS336 学习记录 08：数值稳定交叉熵、AdamW、余弦调度与完整训练闭环","2026-10-27",[2629,2630,2631,2632,2633],"CS336","PyTorch","AdamW","CrossEntropy","训练闭环","用 LogSumExp 消去交叉熵中的指数溢出，手写解耦权重衰减的 AdamW 优化器、全局 L2 梯度裁剪、Warmup 余弦退火调度、memmap 数据采样、Checkpoint 序列化与 Top-p 解码，完成 CS336 Assignment 1 全部 48 项测试。","\u002Fimages\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop\u002Fcs336-cross-entropy-adamw-training-loop-cover.webp","滑动窗口采样、余弦退火曲线、双动量优化轨迹与检查点归档构成的训练闭环 AI 辅助概念插画","作为 Stanford CS336 Assignment 1 系列的收官篇，我们已经拥有了分词器（bpe.py、tokenizer.py）和完整的语言模型架构（transformer.py）。要让模型在语料上真正训练并生成文本，还需",8,"Stanford CS336","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fb5c93ad0cc73178b2b2a62e518a0900da47a0fb7","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-08-adamw-training-loop-20261027.zip",{"slug":2643,"path":2644,"title":2645,"date":2646,"tags":2647,"description":2654,"cover":2655,"coverAlt":2656,"excerpt":2657,"draft":2607,"hidden":2607,"published":2609,"readingTime":2638},"editcache-video-dit-inference-acceleration","\u002Fposts\u002Feditcache-video-dit-inference-acceleration","EditCache：视频 Diffusion Transformer (DiT) 局部编辑的跨步特征复用与区域重算加速","2026-10-25",[2648,2649,2650,2651,2652,2653],"扩散模型","视频生成","AI Infra","推理加速","GPU","论文解读","拆解视频扩散模型（Wan2.1-VACE）局部编辑中的三重算力冗余，通过可重入 Latent、跨步 Block 隐状态缓存复用、带 Halo 边界的局部区域重算与自适应推理起点，在单卡 H100 上实现 2.46× 模型计算加速与 1.49× 严格同步端到端加速。","\u002Fimages\u002Fposts\u002Feditcache-video-dit-inference-acceleration\u002Feditcache-dit-cover-20261025.webp","视频扩散 Transformer 时空三维体素网格中冻结背景缓存与暖橙色局部编辑重算通道的等距技术插画","在基于 Diffusion Transformer（DiT）的视频编辑与局部重绘（Video Inpainting \u002F Local Editing）任务中，用户的典型工作流并不是一次性生成最终成片，而是针对同一段源视频反复调整掩码区",{"slug":2659,"path":2660,"title":2661,"date":2646,"tags":2662,"description":2667,"cover":2668,"coverAlt":2669,"excerpt":2670,"draft":2607,"hidden":2607,"published":2609,"readingTime":2638,"series":2639,"seriesOrder":2671,"codeUrl":2640,"attachmentUrl":2672},"cs336-bpe-tokenizer-training-encoding","\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding","Stanford CS336 学习记录 07：Byte-Level BPE 分词器训练、倒排索引加速与流式编码",[2629,2663,2664,2665,2666],"Tokenizer","BPE","Tiktoken","算法优化","实现 GPT-2 正则预分词、特殊 token 硬边界切分、基于 pair-to-word 倒排索引的增量 BPE 合并算法（\u003C 0.25s 通过速度基准），以及与 tiktoken 完全对齐的 Tokenizer。","\u002Fimages\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding\u002Fcs336-bpe-tokenizer-training-encoding-cover.webp","UTF-8 字节块经过正则切分闸口后沿合并树聚合成子词单元的 AI 辅助概念插画","前面六篇完成了 Transformer 语言模型本身的全部神经网络组件。要让模型真正处理自然语言文本，还需要打通入口处的 ByteLevel BPE（BytePair Encoding）分词器。这一篇记录 cs336basics\u002Fb",7,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-07-bpe-tokenizer-20261025.zip",{"slug":2674,"path":2675,"title":2676,"date":2677,"tags":2678,"description":2682,"cover":2683,"coverAlt":2684,"excerpt":2685,"draft":2607,"hidden":2607,"published":2609,"readingTime":2638},"in-situ-autoguidance-editable-scientific-illustration","\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration","从原位自校正引导（In-situ Autoguidance）到可编辑科研插图生成（AutoFigure-Edit）","2026-10-24",[2648,2679,2618,2680,2681,2653],"多模态","ICML","ACL","串联两项生成式模型与多模态智能体研究：如何通过推理期局部随机微扰在单模型内构造免训练扩散自引导向量（ICML 2025 Workshop 第一作者，arXiv:2510.17136），以及如何通过五阶段视觉-符号混合流水线生成原生可编辑 SVG 科研插图（ACL 2026 Demo，FigureBench 83.0% 胜率）。","\u002Fimages\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration\u002Fautoguidance-autofigure-cover-20261024.webp","左侧扩散流形自校正轨迹矢量场与右侧五阶段可编辑 SVG 科研插图分层网格相衔接的等距技术插画","生成式视觉模型正在经历从“生成高保真像素（Pixel Synthesis）”向“可控结构化生成（Controllable & Editable Generation）”的演进。在这一进程中，我作为第一作者或核心贡献者参与了两项分别处",{"slug":2687,"path":2688,"title":2689,"date":2690,"tags":2691,"description":2695,"cover":2696,"coverAlt":2697,"excerpt":2698,"draft":2607,"hidden":2607,"published":2609,"readingTime":2638,"series":2639,"seriesOrder":2699,"codeUrl":2640,"attachmentUrl":2700},"cs336-pre-norm-transformer-block-lm","\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm","Stanford CS336 学习记录 06：Pre-Norm Transformer Block 与完整语言模型装配","2026-10-23",[2629,2630,2692,2693,2694],"Transformer","Pre-Norm","FLOPs","将 RMSNorm、带 RoPE 的因果多头自注意力和 SwiGLU 组装成 Pre-Norm Transformer Block，搭建完整 TransformerLM，并核算 GPT-2 XL 的参数量与前向 FLOPs。","\u002Fimages\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm\u002Fcs336-pre-norm-transformer-block-lm-cover.webp","干净残差流主干穿过多个 Pre-Norm 子层并汇聚到词表投影层的 AI 辅助概念插画","前五篇把 Linear、Embedding、RMSNorm、SwiGLU、RotaryPositionalEmbedding 和因果 MultiHeadSelfAttention 全部单独打通了。这一篇进入 Assignment 1",6,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-06-transformer-block-lm-20261023.zip",{"slug":2702,"path":2703,"title":2704,"date":2690,"tags":2705,"description":2711,"cover":2712,"coverAlt":2713,"excerpt":2714,"draft":2607,"hidden":2607,"published":2609,"readingTime":2715},"xgrammar-concurrent-cache-generation-fix","\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix","XGrammar 结构化解码缓存剖析：Unlimited 模式并发 Clear 字节记账竞态与代际修复（PR #1011）",[2650,2706,2707,2708,2709,2710],"C++","XGrammar","结构化解码","并发编程","源码分析","拆解 XGrammar GrammarCompiler 的双层缓存架构与 ThreadSafeLRUCache 锁分离实现，定位 Unlimited 模式下 Clear() 字节残留与在途编译（In-flight Computation）跨代污染根因，给出代际号 + 读写锁原子修复与零 sleep 确定性并发 GoogleTest 验证。","\u002Fimages\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix\u002Fxgrammar-cache-fix-cover-20261022.webp","双层磨砂玻璃流道经暖橙色代际闸门过滤后汇入晶格缓存矩阵的等距技术插画","在大语言模型服务（如 SGLang、vLLM）的结构化输出场景中，JSON Schema 或 EBNF 语法需要先编译为词表级掩码缓存（CompiledGrammar），才能在每个 Decode 步以微秒级开销生成合法 token ",14,{"slug":2717,"path":2718,"title":2719,"date":2720,"tags":2721,"description":2726,"cover":2727,"coverAlt":2728,"excerpt":2729,"draft":2607,"hidden":2607,"published":2609,"readingTime":2715},"multi-constraint-agent-routing-dag-eval","\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval","多约束生产级 Agent 系统实践：两阶段工具路由、DAG 局部重规划与轨迹级评测闭环","2026-10-22",[2618,2616,2722,2723,2724,2725],"工具调用","上下文工程","DPO","强化学习","围绕跨境酒旅等强耦合多约束业务场景，拆解向量 Top-K 召回 + LLM 精排 + 跨语言实体链接与 Schema 门控（96%+ 调用合法率）、基于状态机的 DAG 局部子图重规划、Observation 冗余裁剪（节省 40% Token），以及从线上失败恢复轨迹构造 SFT\u002FDPO 偏好对的评测与训练闭环。","\u002Fimages\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval\u002Fagent-dag-routing-cover-20261020.webp","候选模块经漏斗滤网进入双分支有向无环图轨道并在受阻节点触发暖橙色局部旁路重连的等距技术插画","在玩具级 Demo 里，几个工具加上一段 ReAct 循环就能跑通大多数单步问答。但当 Agent 进入跨境酒旅、复杂交易或研发沙箱等真实业务生产环境时，单靠线性 ReAct 循环很快会撞上四堵墙： 1. 工具空间与参数耦合爆炸：候",{"slug":2731,"path":2732,"title":2733,"date":2734,"tags":2735,"description":2738,"cover":2739,"coverAlt":2740,"excerpt":2741,"draft":2607,"hidden":2607,"published":2609,"readingTime":2742},"llm-mrd-distillation-auditable-review-agents","\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents","多视角推理蒸馏（LLM-MRD）与可审计生成式评审系统（DeepReviewer 2.0）深度拆解","2026-10-21",[2736,2679,2737,2618,2617,2653],"知识蒸馏","Qwen2.5-VL","对比大模型落地的两条互补路径：如何用 Qwen2.5-VL 离线多视角分析结合 MLP 残差校准与温度缩放 KL 蒸馏，将推理能力压入 358M 学生模型实现 17.2 ms 独立推理（LLM-MRD，DASFAA 2026 Oral）；以及如何通过主张-证据-风险台账、PASA 可比性门控与导出预算校验构建可审计的同行评审 Agent（DeepReviewer 2.0）。","\u002Fimages\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents\u002Fmrd-review-distill-cover-20261016.webp","三块半透明几何棱镜发出平行光束穿过中央校准环汇聚至右侧高密度晶体模块的等距技术插画","把大语言模型或视觉语言模型（VLM）的深度推理能力用到实际系统中，通常会面临一个两难选择： 如果场景受限于毫秒级在线延迟（如高吞吐多模态内容审核与虚假信息检测）：直接在线调用 7B～72B 的多模态大模型做长链条思维链（CoT）生成",18,{"slug":2744,"path":2745,"title":2746,"date":2734,"tags":2747,"description":2750,"cover":2751,"coverAlt":2752,"excerpt":2753,"draft":2607,"hidden":2607,"published":2609,"readingTime":2671,"series":2639,"seriesOrder":2754,"codeUrl":2640,"attachmentUrl":2755},"cs336-rope-rotary-position-embedding","\u002Fposts\u002Fcs336-rope-rotary-position-embedding","Stanford CS336 学习记录 05：RoPE 旋转位置编码与带位置的多头注意力",[2629,2630,2692,2748,2749],"RoPE","Attention","从二维平面正交旋转推导 RoPE 的奇偶特征配对公式，理解为什么内积只依赖相对位置差 (i - j)，并用 register_buffer 预计算 cos\u002Fsin 缓存接入因果多头自注意力。","\u002Fimages\u002Fposts\u002Fcs336-rope-rotary-position-embedding\u002Fcs336-rope-rotary-position-embedding-cover.webp","二维特征平面中的旋转向量对与预计算余弦正弦网格的 AI 辅助概念插画","上一篇完成了不含位置编码的因果多头自注意力。如果只看那一套矩阵乘法，把输入序列前两个 token 交换顺序（并在因果 mask 允许的范围内看同一个后缀 query 对它们的点积），点积本身并不知道哪个 key 离当前 query ",5,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-05-rope-mha-20261021.zip",{"slug":2757,"path":2758,"title":2759,"date":2760,"tags":2761,"description":2767,"cover":2768,"coverAlt":2769,"excerpt":2770,"draft":2607,"hidden":2607,"published":2609,"readingTime":2638},"llm-gateway-semantic-cache-dynamic-batching","\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching","高并发 LLM 推理网关架构实践：两级精确\u002F语义缓存、50ms 有界动态批处理与尾部延迟治理","2026-10-19",[2650,2762,2763,2764,2765,2766],"LLM 推理","分布式系统","Redis","FastAPI","后端","深度拆解生产级大模型推理网关 LLM-Gateway 的核心系统设计：基于 SHA-256 与 Redis 向量余弦相似度的两级精确\u002F语义缓存、50ms 有界等待动态批处理调度器（Dynamic Batching）、令牌桶多租户限流与 Prometheus 全链路可观测性，将命中请求 P99 延迟从 5,307 ms 压降至 17 ms。","\u002Fimages\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching\u002Fllm-gateway-cover-20261019.webp","高并发请求流依次穿过精确哈希滤网、向量球面语义缓存层与 50ms 动态批处理时间窗拼装器的等距系统架构插画","在将大语言模型接入高并发业务系统时，如果让上游微服务直接裸连底层推理引擎（vLLM \u002F SGLang 或外部模型 API），系统很快会暴露出三个典型的生产痛点： 1. 高频重复与近义查询消耗巨量算力：大量用户请求在语义上高度等价（例",{"slug":2772,"path":2773,"title":2774,"date":2760,"tags":2775,"description":2777,"cover":2778,"coverAlt":2779,"excerpt":2780,"draft":2607,"hidden":2607,"published":2609,"readingTime":2715,"series":2639,"seriesOrder":2781,"codeUrl":2782,"attachmentUrl":2783},"cs336-multihead-attention-shapes","\u002Fposts\u002Fcs336-multihead-attention-shapes","Stanford CS336 学习记录 04：多头注意力怎样拆头、计算和合头",[2629,2630,2692,2749,2776],"张量形状","沿着 (2, 5, 64) 的输入追踪 QKV 投影、reshape、transpose、共享因果 mask 和合头，理解每个头仍处理完整序列，以及最后为什么还需要输出 Linear。","\u002Fimages\u002Fposts\u002Fcs336-multihead-attention-shapes\u002Fcs336-multihead-attention-shapes-cover.webp","宽向量束分成多条并行特征轨道后重新合并的 AI 辅助概念插画","上一篇完成了缩放点积注意力。那个函数接收已经准备好的 Q、K、V，计算匹配分数，应用 mask，再用权重汇总 V。这一篇继续把它放进一个带参数的多头自注意力模块。 这次最需要看清的是“拆头”。代码里只是几行 reshape 和 tr",4,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002F31363223498fafd1528c550a51efbe0aed231736","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-04-multihead-attention-20261019.zip",{"slug":2785,"path":2786,"title":2787,"date":2788,"tags":2789,"description":2793,"cover":2794,"coverAlt":2795,"excerpt":2796,"draft":2607,"hidden":2607,"published":2609,"readingTime":2797},"qwen25-sft-dpo-alignment-tax-ablation","\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation","大模型后训练偏好对齐实录：Qwen2.5 SFT 多维消融、动态参考 DPO 与对齐税（Alignment Tax）量化分析","2026-10-18",[2617,2724,2790,2791,2725,2792],"SFT","LoRA","Qwen2.5","基于 Qwen2.5 完整复现 Instruct SFT → Bradley-Terry DPO 两阶段后训练管线：通过 3 随机种子消融 LoRA Rank、学习率与数据规模（GSM8K +12.73pp），实现免双倍权重的动态 Adapter 切换 DPO 训练，并定量揭示偏好对齐对确定性数学推理的对齐税效应。","\u002Fimages\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation\u002Fqwen25-sft-dpo-cover-20261018.webp","左侧监督微调轨迹流经中央 Bradley-Terry 偏好分流天平分离出正负样本通道并连接右侧对齐税监控仪表盘的等距技术插画","在大语言模型后训练（PostTraining）管线中，监督微调（SFT）负责教会模型任务格式与领域推理模式，而直接偏好优化（DPO, Direct Preference Optimization）则利用成对偏好数据 $(x, yw,",9,{"slug":2799,"path":2800,"title":2801,"date":2788,"tags":2802,"description":2804,"cover":2805,"coverAlt":2806,"excerpt":2807,"draft":2607,"hidden":2607,"published":2609,"readingTime":2808,"series":2639,"seriesOrder":2583,"codeUrl":2809,"attachmentUrl":2810},"cs336-softmax-attention-mask","\u002Fposts\u002Fcs336-softmax-attention-mask","Stanford CS336 学习记录 03：Softmax 的数值稳定与 Attention Mask",[2629,2630,2692,2803,2749],"Softmax","从减去最大值为何不改变 Softmax 开始，理清函数与有参数层的区别，再沿着 Q、K、V 的形状理解缩放点积注意力，以及 mask 为什么必须填负无穷。","\u002Fimages\u002Fposts\u002Fcs336-softmax-attention-mask\u002Fcs336-softmax-attention-mask-cover.webp","注意力分数、权重与因果遮罩的 AI 辅助概念插画","上一篇写到 SwiGLU，接下来进入 Attention。真正开始相乘之前，先做了 Softmax：它把一组分数变成总和为 1 的权重，后面用这些权重汇总不同 token 提供的信息。 这段学习里，有两个很具体的问题值得记下来。一个",11,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fa39eb684710817f7f389626423fcb26ca0f6d629","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-03-softmax-sdpa-20261018.zip",{"slug":2812,"path":2813,"title":2814,"date":2815,"tags":2816,"description":2819,"cover":2820,"coverAlt":2821,"excerpt":2822,"draft":2607,"hidden":2607,"published":2609,"readingTime":2797,"series":2639,"seriesOrder":2582,"codeUrl":2809,"attachmentUrl":2823},"cs336-rmsnorm-swiglu","\u002Fposts\u002Fcs336-rmsnorm-swiglu","Stanford CS336 学习记录 02：RMSNorm 的广播与 SwiGLU 的门控","2026-10-16",[2629,2630,2692,2817,2818],"RMSNorm","SwiGLU","用一个 token 向量理解均方根归一化，拆开 dim=-1 与 keepdim=True 的作用，再沿着两条分支读懂 SiLU、逐元素乘法和 SwiGLU。","\u002Fimages\u002Fposts\u002Fcs336-rmsnorm-swiglu\u002Fcs336-rmsnorm-swiglu-cover.webp","特征尺度整理和双分支门控的 AI 辅助概念插画","上一篇把 Linear 和 Embedding 的参数形状理顺了。继续往下，最先遇到的是 RMSNorm：公式不算长，但把它落到一批 token 上时，dim=1、keepdim=True 和广播必须一起理解。 然后是 SiLU 和","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-02-rmsnorm-swiglu-20261016.zip",{"slug":2825,"path":2826,"title":2827,"date":2828,"tags":2829,"description":2832,"cover":2833,"coverAlt":2834,"excerpt":2835,"draft":2607,"hidden":2607,"published":2609,"readingTime":2836,"series":2639,"seriesOrder":2837,"codeUrl":2809,"attachmentUrl":2838},"cs336-linear-embedding-shapes","\u002Fposts\u002Fcs336-linear-embedding-shapes","Stanford CS336 学习记录 01：从 Linear、Embedding 搞清张量形状","2026-10-14",[2629,2630,2692,2830,2831],"Linear","Embedding","从 PyTorch 类、参数与 forward 的关系开始，理解 Linear 为什么保存两行三列的权重、计算时为什么要转置，再区分 Embedding 查表与线性变换。","\u002Fimages\u002Fposts\u002Fcs336-linear-embedding-shapes\u002Fcs336-linear-embedding-shapes-cover.webp","token 方块、向量表与矩阵变换的 AI 辅助概念插画","开始沿着 Stanford CS336 的 Assignment 1 一步步实现。BPE 之前已经接触过，这轮先跳过分词器，从 Transformer 的基础层开始。这里记录的是当时写代码时真正卡住的问题，以及对应测试能确认到的范围",10,1,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-01-linear-embedding-20261014.zip",{"slug":2840,"path":2841,"title":2842,"date":2843,"tags":2844,"description":2849,"cover":2850,"coverAlt":2851,"excerpt":2852,"draft":2607,"hidden":2607,"published":2609,"readingTime":2853,"series":2845,"seriesOrder":2715,"codeUrl":2854,"attachmentUrl":2855},"cmu-15-779-kv-cache-granularity-study","\u002Fposts\u002Fcmu-15-779-kv-cache-granularity-study","CMU 15-779 学习记录 14：KV 页大小、共享边界与 CPU 实现成本","2026-10-12",[2845,2846,2847,2848],"CMU 15-779","KV cache","PagedAttention","系统实验","用 210 组 CPU 配置比较连续与分页 KV cache，观察 31\u002F32-token 共享前缀的页边界，分别记录活跃槽位、池容量、复制计数和完整调用耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling\u002Fcmu-serving-cover-20261019.webp","浅色手绘分页存储抽屉与共享前缀卡片，旁边连接不同长度的请求队列","个人课程项目中的 CPU 实验记录，延续记录 11 的分页 KV 实现。已完成 210 组配置与 1470 次计时，输出均通过连续存储参考对照；本篇不代表正式作业提交、整门课程或项目全部交付完成。 同样是共享 31 个 token，",12,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F200faf569eccf94fb125e65110bb5d14094d454a","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fkv-cache-granularity-20261019.zip",{"slug":2857,"path":2858,"title":2859,"date":2860,"tags":2861,"description":2864,"cover":2865,"coverAlt":2866,"excerpt":2867,"draft":2607,"hidden":2607,"published":2609,"readingTime":2754,"series":2845,"seriesOrder":2868,"codeUrl":2869,"attachmentUrl":2870},"cmu-15-779-moe-routing-capacity","\u002Fposts\u002Fcmu-15-779-moe-routing-capacity","CMU 15-779 学习记录 13：MoE 路由、稀疏分发与容量溢出","2026-10-09",[2845,2862,2863],"MoE","Routing","用 CPU top-2 MoE 检查稀疏分发的输出与梯度，统计 expert-token 行数，并保留容量丢弃会改变输出的反例。","\u002Fimages\u002Fposts\u002Fcmu-15-779-moe-routing-capacity\u002Fcmu-moe-cover-20261019.webp","浅色手绘分拣器将小珠分配到四个专家工作屋，队列呈现负载不均","第 18 讲的个人配套实验。36 号脚本串行运行 CPU 专家，检查 topk 路由后的分发与合并；没有训练 router，也没有执行 GPU grouped GEMM 或跨设备 alltoall。 上一节记录低秩适配，这次转向每个",13,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fc4851e0cedadf13af7c421867e83c8ff57244fa0","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-parallel-serving-lora-moe-20261019.zip",{"slug":2872,"path":2873,"title":2874,"date":2875,"tags":2876,"description":2878,"cover":2879,"coverAlt":2880,"excerpt":2881,"draft":2607,"hidden":2607,"published":2609,"readingTime":2754,"series":2845,"seriesOrder":2853,"codeUrl":2869,"attachmentUrl":2870},"cmu-15-779-lora-uniform-quantization","\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization","CMU 15-779 学习记录 12：LoRA 首步梯度、权重合并与均匀 4-bit 打包","2026-10-06",[2845,2791,2877],"Quantization","用一次 CPU SGD 更新检查 LoRA 初始化、冻结基座与合并等价，再分别记录均匀 4-bit 存储字节数和量化误差。","\u002Fimages\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization\u002Fcmu-lora-cover-20261019.webp","浅色手绘的大计算板、两块细长适配板和成对打包的小方块","第 17 讲的个人配套实验，35 号脚本实际执行一次 CPU LoRA 更新，并实现均匀 4bit 打包。实验检查机制与数值误差，不评估微调任务效果，也没有实现完整 QLoRA。 这次把两个问题分开记录：低秩分支是否能合并进原权重，",{"slug":2883,"path":2884,"title":2885,"date":2886,"tags":2887,"description":2891,"cover":2850,"coverAlt":2892,"excerpt":2893,"draft":2607,"hidden":2607,"published":2609,"readingTime":2671,"series":2845,"seriesOrder":2808,"codeUrl":2869,"attachmentUrl":2870},"cmu-15-779-kv-serving-speculative-sampling","\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling","CMU 15-779 学习记录 11：分页 KV、连续批处理与推测采样","2026-10-03",[2845,2888,2889,2890],"LLM Serving","KV Cache","Speculative Decoding","在 CPU 上验证 KV 页共享和写时复制，用单位成本模型检查连续批处理，再用条件概率小模型核对推测采样的接受与补采样。","浅色手绘的共享卡片抽屉与不断补入小珠子的请求托盘","第 15、16 讲的个人配套实验。33 号实现 CPU 分页 KV 存储，37 号模拟请求准入，34 号运行小概率模型的推测采样。它们分别验证存储、调度和分布问题，没有测量真实 LLM 服务吞吐或 GPU 延迟。 记录 08已经对照",{"slug":2895,"path":2896,"title":2897,"date":2898,"tags":2899,"description":2904,"cover":2905,"coverAlt":2906,"excerpt":2907,"draft":2607,"hidden":2607,"published":2609,"readingTime":2908},"qwen3-mid-post-train-mlx","\u002Fposts\u002Fqwen3-mid-post-train-mlx","在 M2 Pro 上完成 Qwen3 两阶段训练：Mid Train、SFT 与可复核评估","2026-09-30 11:19:10",[2616,2900,2901,2791,2902,2790,2903],"Qwen3","MLX","持续预训练","Apple Silicon","用 Qwen3 0.6B Base 和 MLX 完成领域持续训练与监督微调，记录数据划分、聊天模板、损失掩码、数值稳定性修复和独立测试结果。","\u002Fimages\u002Fposts\u002Fqwen3-mid-post-train-mlx\u002Fqwen3-mlx-cover-20261018.webp","Apple Silicon 统一内存芯片底座支撑两阶段持续预训练与带精确 Loss 掩码滤网的监督微调桥接插画","在一台 M2 Pro、16 GB 内存的 Mac 上，能否把一个 Base 语言模型接续训练成输出 JSON 的运维事件分级助手？这次项目完成了领域持续训练、监督微调、检查点选择、重新加载和独立测试，并保留了权重、日志和数据哈希。 ",16,{"slug":2910,"path":2911,"title":2912,"date":2913,"tags":2914,"description":2918,"cover":2919,"coverAlt":2920,"excerpt":2921,"draft":2607,"hidden":2607,"published":2609,"readingTime":2699,"series":2845,"seriesOrder":2836,"codeUrl":2869,"attachmentUrl":2870},"cmu-15-779-cpu-parallelism-pipeline","\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline","CMU 15-779 学习记录 10：加权梯度、状态分片与同步流水线","2026-09-30",[2845,2915,2916,2917],"Parallel Training","Gloo","Pipeline","用双 CPU 进程 Gloo 检查不等数据分片的梯度，推导持久状态字节数，再用依赖模型比较 GPipe 与同步 1F1B 的激活存活。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline\u002Fcmu-parallel-cover-20261019.webp","浅色手绘的四阶段桌面流水线与按数量汇总的托盘","第 12 至 14 讲的个人配套实验，脚本编号 31、32 不对应官方作业。Gloo 实际运行两个 CPU 进程；张量切分、状态字节数和流水线分别按下文的数学或成本模型解释。真实 GPU 实测按学习安排跳过。 上一节检查单个计算图的",{"slug":2923,"path":2924,"title":2925,"date":2926,"tags":2927,"description":2930,"cover":2931,"coverAlt":2932,"excerpt":2933,"draft":2607,"hidden":2607,"published":2609,"readingTime":2671,"series":2845,"seriesOrder":2797,"codeUrl":2934,"attachmentUrl":2935},"cmu-15-779-cpu-compiler-optimizations","\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations","CMU 15-779 学习记录 09：CPU 编译优化、合法重写与候选搜索","2026-09-27",[2845,2928,2929,2630],"ML Compiler","TorchInductor","用 CPU 实验验证 BatchNorm 折叠、RMSNorm 重排与非线性反例，测量分块候选，并运行 TorchInductor，分别记录首调用与预热后的时间。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations\u002Fcmu-cpu-compiler-cover-20261019.webp","淡紫计算积木经过薄荷绿折叠模具变成浅蓝积木，前方摆放四种候选托盘的浅色手绘插画","编译章节的个人配套实验，编号 30 不对应官方作业。图重写、分块候选和 TorchInductor 均在 CPU 上运行；真实 GPU 实验按学习安排跳过。 上一节记录了分块 Attention 和 KV cache。这次转到编译优","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F9261bea6aedf31cdccbbab2541ca693fc97b3743","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-compiler-optimizations-20261019.zip",{"slug":2937,"path":2938,"title":2939,"date":2940,"tags":2941,"description":2943,"cover":2944,"coverAlt":2945,"excerpt":2946,"draft":2607,"hidden":2607,"published":2609,"readingTime":2853,"series":2845,"seriesOrder":2638,"codeUrl":2854,"attachmentUrl":2947},"cmu-15-779-attention-online-softmax","\u002Fposts\u002Fcmu-15-779-attention-online-softmax","CMU 15-779 学习记录 08：分块 Attention、在线 Softmax 与 KV cache","2026-09-24",[2845,2749,2803,2942,2630],"FlashAttention","在 CPU 上对照完整 Attention、在线 Softmax 分块实现与 PyTorch SDPA，检查因果掩码、浮点误差和分数块大小，理解 FlashAttention 的一部分算法基础。","\u002Fimages\u002Fposts\u002Fcmu-15-779-attention-online-softmax\u002Fcmu-online-attention-cover-20261016.webp","浅紫色卡片分批进入绿色计算托盘，旁边三个状态容器保存累积量的浅色手绘插画","第 5 讲 Transformers & Attention 的个人配套实验。28\u002F29 号是 CPU 单头前向与 KV cache；后续补入 38 号 CUDA 多头合成张量的显式 SDPA MATH\u002FFLASH 对照。没有手写 ","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcuda-triton-sdpa-20261019.zip",{"slug":2949,"path":2950,"title":2951,"date":2952,"tags":2953,"description":2958,"cover":2959,"coverAlt":2960,"excerpt":2961,"draft":2607,"hidden":2607,"published":2609,"readingTime":2836,"series":2845,"seriesOrder":2671,"codeUrl":2854,"attachmentUrl":2947},"cmu-15-779-matmul-tiling-tree-reduction","\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction","CMU 15-779 学习记录 07：矩阵分块、数据复用与树形归约","2026-09-21",[2845,2954,2955,2956,2957],"CUDA","矩阵乘法","数据复用","归约","用 CPU 模型统计矩阵分块的输入读取次数，再比较串行求和和树形归约的依赖深度，记录补零、同步与浮点加法顺序的边界。","\u002Fimages\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction\u002Fcmu-matmul-reduction-cover-20261016.webp","矩阵积木在共享托盘中复用，旁边的小珠沿树形管道逐层合并的浅色手绘插画","GPU Architecture & CUDA Programming 的个人配套实验。首次记录用 CPU 模型检查数据复用与依赖深度；后续补入单卡 GPU 原生 CUDA 矩阵验证，以及 Triton 两阶段归约和矩阵乘法实测。 ",{"slug":2963,"path":2964,"title":2965,"date":2966,"tags":2967,"description":2970,"cover":2971,"coverAlt":2972,"excerpt":2973,"draft":2607,"hidden":2607,"published":2609,"readingTime":2836,"series":2845,"seriesOrder":2699,"codeUrl":2854,"attachmentUrl":2947},"cmu-15-779-cuda-thread-indexing","\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing","CMU 15-779 学习记录 06：CUDA 线程索引与边界检查","2026-09-18",[2845,2954,2652,2968,2969],"线程索引","边界检查","用 CPU 演示 CUDA 的 block、thread 与全局下标：10 个元素如何分给 3 个线程块，为什么要跳过越界线程，以及这些验证能说明什么。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing\u002Fcmu-cuda-indexing-cover-20261015.webp","分组的浅紫色计算方块与末端边界挡板组成线程分配教具的浅色插画","GPU Architecture & CUDA Programming 的个人配套练习。首次记录为 CPU 索引演示；后续在单卡 GPU 环境补入原生 CUDA 编译、运行与 memcheck，以及 Triton 向量加法实测。 上",{"slug":2975,"path":2976,"title":2977,"date":2978,"tags":2979,"description":2983,"cover":2984,"coverAlt":2985,"excerpt":2986,"draft":2607,"hidden":2607,"published":2609,"readingTime":2836,"series":2845,"seriesOrder":2754,"codeUrl":2987,"attachmentUrl":2988},"cmu-15-779-profiler-operator-batching","\u002Fposts\u002Fcmu-15-779-profiler-operator-batching","CMU 15-779 学习记录 05：Profiler、算子调用与批量推理","2026-09-15",[2845,2630,2980,2981,2982],"Profiler","批量推理","性能分析","对照 ML Systems 基础，用 Profiler 拆开两层网络的执行过程，再固定 1000 个样本扫描 batch size，观察算子调用次数、尾批和 CPU 耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-profiler-operator-batching\u002Fcmu-pytorch-profiling-cover-20261015.webp","放大镜观察线性层与 ReLU 模块，旁边面板呈现执行片段的浅色技术教具插画","本篇对照课程官网的 Introduction to ML Systems 整理，是个人配套 CPU 实验，不是课程官方作业。脚本编号是练习顺序，不对应课程讲次。 前几篇一直在补 PyTorch 基础。到了这一步，我在学习聊天里问：现","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fcecf5110d8a95161bb5763eadfca94cd7a0cc50f","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-profiler-batching-20261015.zip",{"slug":2990,"path":2991,"title":2992,"date":2993,"tags":2994,"description":2997,"cover":2998,"coverAlt":2999,"excerpt":3000,"draft":2607,"hidden":2607,"published":2609,"readingTime":2797,"series":2845,"seriesOrder":2781,"codeUrl":3001,"attachmentUrl":3002},"cmu-15-779-pytorch-linear-relu-absolute-value","\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value","CMU 15-779 学习记录 04：多输出线性层、ReLU 与绝对值网络","2026-09-12",[2845,2630,2955,2995,2996],"ReLU","神经网络","把 nn.Linear 对应到矩阵乘法，弄清多个输出怎样计算，再逐层拆开一个绝对值网络，比较手动设置参数与训练得到的结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value\u002Fcmu-pytorch-absolute-cover-20261015.webp","双分支经过折角模块后汇入紫色 V 形曲线面板的浅色技术教具插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的批量推理与矩阵乘法，我继续把 model(x) 拆开看：一层里有多个输出时，每个数是怎么算出来的？两","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F4cc63904a3a583da4a2122c246a14b43aac38528","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-linear-relu-abs-20261015.zip",{"slug":3004,"path":3005,"title":3006,"date":3007,"tags":3008,"description":3011,"cover":3012,"coverAlt":3013,"excerpt":3014,"draft":2607,"hidden":2607,"published":2609,"readingTime":2797,"series":2845,"seriesOrder":2583,"codeUrl":3015,"attachmentUrl":3016},"cmu-15-779-pytorch-dataloader-checkpoints-batching","\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching","CMU 15-779 学习记录 03：DataLoader、模型保存与批量推理","2026-09-09",[2845,2630,3009,2981,3010],"DataLoader","模型保存","用 DataLoader 分批训练平方函数，保存并加载模型参数，再比较 1000 个样本与 1 个样本的推理耗时，最后把批量计算对应到矩阵乘法。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching\u002Fcmu-pytorch-batching-cover-20261015.webp","样本卡片按批次进入计算模块，旁边配有参数存储盒和输出矩阵面板的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的平方函数实验，这一节继续练习分批训练、保存参数和独立预测。最后做了一次计时：同一个小网络，逐个处理 ","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-batching-20261015.zip",{"slug":3018,"path":3019,"title":3020,"date":3021,"tags":3022,"description":3023,"cover":3024,"coverAlt":3025,"excerpt":3026,"draft":2607,"hidden":2607,"published":2609,"readingTime":2797,"series":2845,"seriesOrder":2582,"codeUrl":3015,"attachmentUrl":3027},"cmu-15-779-pytorch-linear-module-relu","\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu","CMU 15-779 学习记录 02：nn.Linear、张量形状与 ReLU","2026-09-06",[2845,2630,2996,2995],"从带偏置的线性回归开始，弄清 nn.Linear 和 nn.Module 的分工，再用平方函数观察线性模型的限制、ReLU 的作用和测试点的选择。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu\u002Fcmu-pytorch-relu-cover-20261015.webp","输入经过两层神经网络和 ReLU 折角模块，输出曲线拟合结果的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 上一篇里，我用一个参数拟合了 $y=2x$。这一篇继续顺着往下写，先加上偏置，再把手写参数换成 nn.Linea","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-models-20261015.zip",{"slug":3029,"path":3030,"title":3031,"date":3032,"tags":3033,"description":3036,"cover":3037,"coverAlt":3038,"excerpt":3039,"draft":2607,"hidden":2607,"published":2609,"readingTime":2836,"series":2845,"seriesOrder":2837,"codeUrl":3015,"attachmentUrl":3040},"cmu-15-779-pytorch-autograd-linear-regression","\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression","CMU 15-779 学习记录 01：PyTorch 自动求导与线性回归","2026-09-03",[2845,2630,3034,3035],"自动求导","线性回归","从自动求导和手动梯度下降开始，比较学习率，再用 SGD 训练线性回归。记录 w * 4 写错后停在 1.25 的原因和修正结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression\u002Fcmu-pytorch-autograd-cover-20261015.webp","梯度曲面上的小球沿路径下降，前侧面板展示散点与拟合直线的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 开始学习 CMU 15779 时，我会 Python，但还没用过 PyTorch。这次先在 Mac 上用 CPU","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-basics-20261013.zip",{"slug":3042,"path":3043,"title":3044,"date":3045,"tags":3046,"description":3052,"cover":3053,"coverAlt":3054,"excerpt":3055,"draft":2607,"hidden":2607,"published":2609,"readingTime":2868,"codeUrl":3056,"attachmentUrl":3057},"embodied-pickplace-lab-robustness","\u002Fposts\u002Fembodied-pickplace-lab-robustness","Pickplace Lab 项目复盘：抓取成功率之外的时延与失败分析","2026-08-29",[3047,3048,3049,2630,3050,3051],"具身智能","模仿学习","MuJoCo","机器人","鲁棒性","从 MuJoCo 抓取搬运闭环出发，记录 22,100 个评测回合。动作平滑在自建任务的 50 ms 延迟下有效，换到 Panda Lift 却没有支持同样的收益。","\u002Fimages\u002Fposts\u002Fembodied-pickplace-lab-robustness\u002Fpickplace-grasp-cover-20261016.webp","MuJoCo 实际渲染的笛卡尔夹爪抓取方块场景，桌面上标出放置目标","Pickplace Lab 从一个简单任务开始：让夹爪抓起方块，搬到指定位置，放下，再退开。先把仿真、示范采集、训练和闭环评测接起来，随后再问：干净条件下成功的策略，换一种执行条件，还能不能完成任务？ 最值得讨论的结果出现在指令延迟","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Ftree\u002Fv0.1.0","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Freleases\u002Fdownload\u002Fv0.1.0\u002Fpickplace-lab-research-v1.zip",{"slug":3059,"path":3060,"title":3061,"date":3062,"tags":3063,"description":3068,"cover":3069,"coverAlt":3070,"excerpt":3071,"draft":2607,"hidden":2607,"published":2609,"readingTime":3072},"qwen25-vl-sglang-verl-single-gpu-lab","\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab","从 Mac 到单卡 GPU：Qwen2.5-VL 部署、GRPO 与计数 LoRA 实践","2026-08-19",[2737,2679,3064,3065,3066,3067,2901],"SGLang","verl","GRPO","推理优化","在 M2 Pro 和云端单卡上部署 Qwen2.5-VL-3B，实测连续批处理、前缀缓存与 SGLang 参数调优，再用 verl 完成多模态 GRPO 更新和同条件前后评测，记录有效优化、兼容性问题与未观察到提升的结果。","\u002Fimages\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab\u002Fqvl-lab-cover-20261010.webp","笔记本与单卡计算模块通过图像和 token 光轨连接的实验台插画","这次实验从一个很具体的问题开始：M2 Pro、16 GB 统一内存，能不能在本机跑 Qwen2.5VL？跑起来之后，我又想把图文问答、推理性能测量和强化学习训练接成一条完整流程。 本机的 MLX 4bit 服务和云端的 SGLang",29,{"slug":3074,"path":2611,"title":5,"date":2605,"tags":3075,"description":2606,"cover":2603,"coverAlt":2604,"excerpt":3076,"draft":2607,"hidden":2607,"published":2609,"readingTime":2868},"cycleresearcher-post-training",[2616,2617,2618,913,2619,2620],"看到一个系统会检索、规划、反思和调用工具，不能据此判断它做过 Posttraining。Agent 描述的是模型在运行时如何完成任务；Posttraining 描述的是预训练之后，是否继续用监督数据、偏好数据或奖励信号更新模型参数。",{"slug":3078,"path":3079,"title":3080,"date":3081,"tags":3082,"description":3084,"cover":3085,"coverAlt":3086,"excerpt":3087,"draft":2607,"hidden":2607,"published":2609,"readingTime":2715,"series":3088,"seriesOrder":2837},"cuda-fused-rmsnorm-kernel","\u002Fposts\u002Fcuda-fused-rmsnorm-kernel","从 PyTorch 到 CUDA Kernel：融合 RMSNorm 在 RTX 4090D 上加速 4.27 倍","2026-07-22 18:16:50",[2954,2650,2652,2630,2762,3083],"Kernel 优化","从零实现 PyTorch CUDA 扩展，拆解 RMSNorm 的访存瓶颈、warp 归约、向量化加载、算子融合与冷热缓存 benchmark。","\u002Fimages\u002Fposts\u002Fcuda-fused-rmsnorm-kernel\u002Fcuda-rmsnorm-cover-20261018.webp","融合残差加法与向量化 float4 访存经 Warp Shuffle 蝴蝶树归约输出归一化张量的等距硬件架构插画","为了准备 AI Infra 岗位，我做了一个尽量小而完整的 CUDA 项目：不追求堆很多算子，而是选一个真实的 LLM 推理路径，把实现、正确性、benchmark 和性能解释全部做扎实。 项目实现了两个 PyTorch CUDA ","AI Infra 实战",{"slug":3090,"path":3091,"title":3092,"date":3093,"tags":3094,"description":3096,"cover":3097,"coverAlt":3098,"excerpt":3099,"draft":2607,"hidden":2607,"published":2609,"readingTime":2638,"series":3100,"seriesOrder":2837},"agent-memory","\u002Fposts\u002Fagent-memory","Agent 上下文工程与分层记忆架构：从 Observation 剪枝、滚动摘要到向量召回闭环","2026-04-24",[2616,2618,3095,2650],"RAG","结合生产级多约束 Agent 与超 100K Token 科研智能体实践，系统拆解长程会话中的上下文膨胀与注意力稀释问题：涵盖 Observation 结构化剪枝（降低约 40% Token 开销）、滚动工作摘要、向量检索长期记忆及 Prefix Cache 友好的上下文组装。","\u002Fimages\u002Fposts\u002Fagent-memory\u002Fagent-memory-cover-20261018.webp","底层高噪数据经剪枝滤网进入中期滑动窗口与金色状态台账并由顶层向量星座按需召回的三层记忆架构插画","在构建多步工具调用（ToolUse）与长周期自主规划（Autonomous Planning）的 Agent 系统时，上下文管理（Context Engineering）往往比单纯改写 System Prompt 更决定线上稳定性。","Agent 上下文工程",{"slug":3102,"path":3103,"title":3104,"date":3105,"tags":3106,"description":3109,"cover":3110,"coverAlt":3111,"excerpt":3112,"draft":2607,"hidden":2607,"published":2609,"readingTime":2754,"series":3113,"seriesOrder":2582},"backend-stack-deep-dive","\u002Fposts\u002Fbackend-stack-deep-dive","高并发 AI 平台后端基础设施实践：FastAPI 异步网关、SQLAlchemy 2.0 Async、PostgreSQL 与 Docker 沙箱编排","2026-04-07",[2766,2763,2765,3107,3108,2650],"PostgreSQL","Docker","结合 DeepScientist 生产级 AI 科研平台实践，拆解 FastAPI ASGI 异步事件循环、SQLAlchemy 2.0 + AsyncPG 连接池调优、PostgreSQL + MinIO 存储分层与 Docker SDK 动态沙箱容器编排的职责边界与核心设计。","\u002Fimages\u002Fposts\u002Fbackend-stack-deep-dive\u002Fbackend-stack-cover-20261018.webp","ASGI 异步事件循环涡轮将非阻塞协程流分发至 PostgreSQL 连接池、MinIO 存储矩阵与动态 Docker 沙箱的等距架构插画","结合 DeepScientist 生产级 AI 科研平台（服务 7,000+ 用户）的实际工程落地，本文系统拆解支撑高并发流式推理与容器沙箱调度的后端基础设施栈——FastAPI（ASGI 异步网关 + SSE 流式推送）、SQLA","DeepScientist 工程复盘",{"slug":3115,"path":3116,"title":3117,"date":3118,"tags":3119,"description":3122,"cover":3123,"coverAlt":3124,"excerpt":3125,"draft":2607,"hidden":2607,"published":2609,"readingTime":2808,"series":3126,"seriesOrder":2837},"minicode-source-analysis","\u002Fposts\u002Fminicode-source-analysis","MiniCode 架构与工具评测深度拆解：开源终端 Agentic Coding 引擎（1k+ Stars）的 ReAct、MCP 与 30 项基准验证","2026-04-06",[2618,3120,3121,2710,2650],"TypeScript","MCP","深度剖析开源终端 AI 编程智能体 MiniCode（1k+ GitHub Stars）的核心架构：从零重型依赖的 ReAct 规划循环、声明式工具注册表与权限拦截层、MCP 协议帧自动协商，到 30 项可执行 Tool-Use 评测基准（86.7% 完成率、100% 错误恢复率）的失败分布归因与闭环优化。","\u002Fimages\u002Fposts\u002Fminicode-source-analysis\u002Fminicode-cover-20261018.webp","终端编程智能体控制台被 ReAct 状态环、Unified Diff 补丁预览门与 MCP 协议插槽环绕的等距技术插画","MiniCode 是我作为核心开发者参与构建的开源终端 Agentic Coding 引擎（目前已在 GitHub 获得 1,000+ Stars）。项目的核心设计目标是：在不引入庞大运行时框架的前提下，用约 5,000 行高内聚的","AI 项目源码导读",{"slug":3128,"path":3129,"title":3130,"date":3118,"tags":3131,"description":3132,"cover":3133,"coverAlt":3134,"excerpt":3135,"draft":2607,"hidden":2607,"published":2609,"readingTime":2836,"series":3113,"seriesOrder":2837},"deepscientist-tech-stack","\u002Fposts\u002Fdeepscientist-tech-stack","DeepScientist 架构与运行时全解析：超 100K 上下文科研智能体、MCP 沙箱与高并发流式系统",[2618,2650,3121,3108,2765,2763],"深度拆解服务 7,000+ 用户的科研自主智能体平台 DeepScientist：从一题一仓库（Quest）持久化状态机、MCP 工具栈（memory \u002F artifact \u002F bash_exec）、Docker SDK 动态容器编排，到 SSE + WebSocket 双流通信与 PostgreSQL \u002F Redis \u002F Prometheus 生产级基础设施。","\u002Fimages\u002Fposts\u002Fdeepscientist-tech-stack\u002Fdeepscientist-cover-20261018.webp","科研智能体编排核心通过 memory、artifact 与隔离容器沙箱三座协议桥连接双流工作台的等距架构插画","DeepScientist（deepscientist.cc）是我们在西湖大学自然语言处理实验室研发的自主科学发现与科研管理平台，目前累计服务 7,000+ 科研用户。与单轮对话或纯文本文献问答工具不同，科研智能体需要在数小时乃至数",1795088943997]