[{"data":1,"prerenderedAt":3537},["ShallowReactive",2],{"post-\u002Fposts\u002Fcs336-rmsnorm-swiglu":3,"archive-public-posts":3027},{"id":4,"title":5,"body":6,"categories":3006,"cover":3008,"coverAlt":3009,"date":3010,"description":3011,"draft":3012,"extension":3013,"haloPublished":3012,"haloSync":3012,"hidden":3012,"legacy":3012,"meta":3014,"navigation":3016,"path":3017,"published":3012,"readingTime":3018,"seo":3019,"series":3020,"seriesOrder":204,"stem":3021,"tags":3022,"__hash__":3026},"posts\u002Fposts\u002Fcs336-rmsnorm-swiglu.md","Stanford CS336 学习记录 02：RMSNorm 的广播与 SwiGLU 的门控",{"type":7,"value":8,"toc":2993},"minimark",[9,27,38,42,155,579,582,755,758,761,765,768,1348,1515,1551,1558,1562,1569,1572,1632,1640,1731,1838,1851,1855,1858,1861,1864,1868,1871,1874,2128,2176,2252,2259,2262,2266,2269,2284,2342,2349,2692,2729,2767,2786,2807,2811,2817,2823,2826,2829,2840,2848,2859,2881,2898,2906,2909,2928,2940,2943,2989],[10,11,12,17,18,22,23,26],"p",{},[13,14,16],"a",{"href":15},"\u002Fposts\u002Fcs336-linear-embedding-shapes\u002F","上一篇","把 Linear 和 Embedding 的参数形状理顺了。继续往下，最先遇到的是 RMSNorm：公式不算长，但把它落到一批 token 上时，",[19,20,21],"code",{},"dim=-1","、",[19,24,25],{},"keepdim=True"," 和广播必须一起理解。",[10,28,29,30,33,34,37],{},"然后是 SiLU 和 SwiGLU。这里开始有了两条并行分支。看代码时最容易忽略的是，同样写着“乘”，",[19,31,32],{},"@"," 与 ",[19,35,36],{},"*"," 做的事情完全不同。",[39,40,41],"h2",{"id":41},"先算一个向量的均方根",[10,43,44,45,154],{},"对一个 token 的特征向量 ",[46,47,50,95],"span",{"className":48},[49],"katex",[46,51,54],{"className":52},[53],"katex-mathml",[55,56,58],"math",{"xmlns":57},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[59,60,61,90],"semantics",{},[62,63,64,68,72,76,80,84,87],"mrow",{},[65,66,67],"mi",{},"x",[69,70,71],"mo",{},"=",[69,73,75],{"stretchy":74},"false","[",[77,78,79],"mn",{},"3",[69,81,83],{"separator":82},"true",",",[77,85,86],{},"4",[69,88,89],{"stretchy":74},"]",[91,92,94],"annotation",{"encoding":93},"application\u002Fx-tex","x=[3,4]",[46,96,99,125],{"className":97,"ariaHidden":82},[98],"katex-html",[46,100,103,108,113,118,122],{"className":101},[102],"base",[46,104],{"className":105,"style":107},[106],"strut","height:0.4306em;",[46,109,67],{"className":110},[111,112],"mord","mathnormal",[46,114],{"className":115,"style":117},[116],"mspace","margin-right:0.2778em;",[46,119,71],{"className":120},[121],"mrel",[46,123],{"className":124,"style":117},[116],[46,126,128,132,136,139,143,147,150],{"className":127},[102],[46,129],{"className":130,"style":131},[106],"height:1em;vertical-align:-0.25em;",[46,133,75],{"className":134},[135],"mopen",[46,137,79],{"className":138},[111],[46,140,83],{"className":141},[142],"mpunct",[46,144],{"className":145,"style":146},[116],"margin-right:0.1667em;",[46,148,86],{"className":149},[111],[46,151,89],{"className":152},[153],"mclose","，均方根 RMS 的计算就是三个动作：先平方，再平均，最后开根号。",[46,156,159],{"className":157},[158],"katex-display",[46,160,162,232],{"className":161},[49],[46,163,165],{"className":164},[53],[55,166,168],{"xmlns":57,"display":167},"block",[59,169,170,229],{},[62,171,172,176,179,182,184,187,189,216,218,223,226],{},[65,173,175],{"mathvariant":174},"normal","RMS",[69,177,178],{},"⁡",[69,180,181],{"stretchy":74},"(",[65,183,67],{},[69,185,186],{"stretchy":74},")",[69,188,71],{},[190,191,192],"msqrt",{},[193,194,195,214],"mfrac",{},[62,196,197,205,208],{},[198,199,200,202],"msup",{},[77,201,79],{},[77,203,204],{},"2",[69,206,207],{},"+",[198,209,210,212],{},[77,211,86],{},[77,213,204],{},[77,215,204],{},[69,217,71],{},[190,219,220],{},[77,221,222],{},"12.5",[69,224,225],{},"≈",[77,227,228],{},"3.536.",[91,230,231],{"encoding":93},"\\operatorname{RMS}(x)\n=\\sqrt{\\frac{3^2+4^2}{2}}\n=\\sqrt{12.5}\n\\approx3.536.",[46,233,235,267,497,569],{"className":234,"ariaHidden":82},[98],[46,236,238,241,249,252,255,258,261,264],{"className":237},[102],[46,239],{"className":240,"style":131},[106],[46,242,245],{"className":243},[244],"mop",[46,246,175],{"className":247},[111,248],"mathrm",[46,250,181],{"className":251},[135],[46,253,67],{"className":254},[111,112],[46,256,186],{"className":257},[153],[46,259],{"className":260,"style":117},[116],[46,262,71],{"className":263},[121],[46,265],{"className":266,"style":117},[116],[46,268,270,274,488,491,494],{"className":269},[102],[46,271],{"className":272,"style":273},[106],"height:2.44em;vertical-align:-0.7406em;",[46,275,278],{"className":276},[111,277],"sqrt",[46,279,283,479],{"className":280},[281,282],"vlist-t","vlist-t2",[46,284,287,476],{"className":285},[286],"vlist-r",[46,288,292,453],{"className":289,"style":291},[290],"vlist","height:1.6994em;",[46,293,297,302],{"className":294,"style":296},[295],"svg-align","top:-4.4em;",[46,298],{"className":299,"style":301},[300],"pstrut","height:4.4em;",[46,303,306],{"className":304,"style":305},[111],"padding-left:1em;",[46,307,309,313,450],{"className":308},[111],[46,310],{"className":311},[135,312],"nulldelimiter",[46,314,316],{"className":315},[193],[46,317,319,441],{"className":318},[281,282],[46,320,322,436],{"className":321},[286],[46,323,326,339,350],{"className":324,"style":325},[290],"height:1.4171em;",[46,327,329,333],{"style":328},"top:-2.314em;",[46,330],{"className":331,"style":332},[300],"height:3em;",[46,334,336],{"className":335},[111],[46,337,204],{"className":338},[111],[46,340,342,345],{"style":341},"top:-3.23em;",[46,343],{"className":344,"style":332},[300],[46,346],{"className":347,"style":349},[348],"frac-line","border-bottom-width:0.04em;",[46,351,353,356],{"style":352},"top:-3.677em;",[46,354],{"className":355,"style":332},[300],[46,357,359,396,400,404,407],{"className":358},[111],[46,360,362,365],{"className":361},[111],[46,363,79],{"className":364},[111],[46,366,369],{"className":367},[368],"msupsub",[46,370,372],{"className":371},[281],[46,373,375],{"className":374},[286],[46,376,379],{"className":377,"style":378},[290],"height:0.7401em;",[46,380,382,386],{"style":381},"top:-2.989em;margin-right:0.05em;",[46,383],{"className":384,"style":385},[300],"height:2.7em;",[46,387,393],{"className":388},[389,390,391,392],"sizing","reset-size6","size3","mtight",[46,394,204],{"className":395},[111,392],[46,397],{"className":398,"style":399},[116],"margin-right:0.2222em;",[46,401,207],{"className":402},[403],"mbin",[46,405],{"className":406,"style":399},[116],[46,408,410,413],{"className":409},[111],[46,411,86],{"className":412},[111],[46,414,416],{"className":415},[368],[46,417,419],{"className":418},[281],[46,420,422],{"className":421},[286],[46,423,425],{"className":424,"style":378},[290],[46,426,427,430],{"style":381},[46,428],{"className":429,"style":385},[300],[46,431,433],{"className":432},[389,390,391,392],[46,434,204],{"className":435},[111,392],[46,437,440],{"className":438},[439],"vlist-s","​",[46,442,444],{"className":443},[286],[46,445,448],{"className":446,"style":447},[290],"height:0.686em;",[46,449],{},[46,451],{"className":452},[153,312],[46,454,456,459],{"style":455},"top:-3.6594em;",[46,457],{"className":458,"style":301},[300],[46,460,464],{"className":461,"style":463},[462],"hide-tail","min-width:1.02em;height:2.48em;",[465,466,472],"svg",{"xmlns":467,"width":468,"height":469,"viewBox":470,"preserveAspectRatio":471},"http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","400em","2.48em","0 0 400000 2592","xMinYMin slice",[473,474],"path",{"d":475},"M424,2478\nc-1.3,-0.7,-38.5,-172,-111.5,-514c-73,-342,-109.8,-513.3,-110.5,-514\nc0,-2,-10.7,14.3,-32,49c-4.7,7.3,-9.8,15.7,-15.5,25c-5.7,9.3,-9.8,16,-12.5,20\ns-5,7,-5,7c-4,-3.3,-8.3,-7.7,-13,-13s-13,-13,-13,-13s76,-122,76,-122s77,-121,77,-121\ns209,968,209,968c0,-2,84.7,-361.7,254,-1079c169.3,-717.3,254.7,-1077.7,256,-1081\nl0 -0c4,-6.7,10,-10,18,-10 H400000\nv40H1014.6\ns-87.3,378.7,-272.6,1166c-185.3,787.3,-279.3,1182.3,-282,1185\nc-2,6,-10,9,-24,9\nc-8,0,-12,-0.7,-12,-2z M1001 80\nh400000v40h-400000z",[46,477,440],{"className":478},[439],[46,480,482],{"className":481},[286],[46,483,486],{"className":484,"style":485},[290],"height:0.7406em;",[46,487],{},[46,489],{"className":490,"style":117},[116],[46,492,71],{"className":493},[121],[46,495],{"className":496,"style":117},[116],[46,498,500,504,560,563,566],{"className":499},[102],[46,501],{"className":502,"style":503},[106],"height:1.04em;vertical-align:-0.0839em;",[46,505,507],{"className":506},[111,277],[46,508,510,551],{"className":509},[281,282],[46,511,513,548],{"className":512},[286],[46,514,517,531],{"className":515,"style":516},[290],"height:0.9561em;",[46,518,521,524],{"className":519,"style":520},[295],"top:-3em;",[46,522],{"className":523,"style":332},[300],[46,525,528],{"className":526,"style":527},[111],"padding-left:0.833em;",[46,529,222],{"className":530},[111],[46,532,534,537],{"style":533},"top:-2.9161em;",[46,535],{"className":536,"style":332},[300],[46,538,541],{"className":539,"style":540},[462],"min-width:0.853em;height:1.08em;",[465,542,545],{"xmlns":467,"width":468,"height":543,"viewBox":544,"preserveAspectRatio":471},"1.08em","0 0 400000 1080",[473,546],{"d":547},"M95,702\nc-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14\nc0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54\nc44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10\ns173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429\nc69,-144,104.5,-217.7,106.5,-221\nl0 -0\nc5.3,-9.3,12,-14,20,-14\nH400000v40H845.2724\ns-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7\nc-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z\nM834 80h400000v40h-400000z",[46,549,440],{"className":550},[439],[46,552,554],{"className":553},[286],[46,555,558],{"className":556,"style":557},[290],"height:0.0839em;",[46,559],{},[46,561],{"className":562,"style":117},[116],[46,564,225],{"className":565},[121],[46,567],{"className":568,"style":117},[116],[46,570,572,576],{"className":571},[102],[46,573],{"className":574,"style":575},[106],"height:0.6444em;",[46,577,228],{"className":578},[111],[10,580,581],{},"让两个分量都除以这个共同的数：",[46,583,585],{"className":584},[158],[46,586,588,633],{"className":587},[49],[46,589,591],{"className":590},[53],[55,592,593],{"xmlns":57,"display":167},[59,594,595,630],{},[62,596,597,613,615,617,620,622,625,627],{},[193,598,599,601],{},[65,600,67],{},[62,602,603,605,607,609,611],{},[65,604,175],{"mathvariant":174},[69,606,178],{},[69,608,181],{"stretchy":74},[65,610,67],{},[69,612,186],{"stretchy":74},[69,614,225],{},[69,616,75],{"stretchy":74},[77,618,619],{},"0.849",[69,621,83],{"separator":82},[77,623,624],{},"1.131",[69,626,89],{"stretchy":74},[65,628,629],{"mathvariant":174},".",[91,631,632],{"encoding":93},"\\frac{x}{\\operatorname{RMS}(x)}\n\\approx[0.849,1.131].",[46,634,636,728],{"className":635,"ariaHidden":82},[98],[46,637,639,643,719,722,725],{"className":638},[102],[46,640],{"className":641,"style":642},[106],"height:2.0436em;vertical-align:-0.936em;",[46,644,646,649,716],{"className":645},[111],[46,647],{"className":648},[135,312],[46,650,652],{"className":651},[193],[46,653,655,707],{"className":654},[281,282],[46,656,658,704],{"className":657},[286],[46,659,662,685,693],{"className":660,"style":661},[290],"height:1.1076em;",[46,663,664,667],{"style":328},[46,665],{"className":666,"style":332},[300],[46,668,670,676,679,682],{"className":669},[111],[46,671,673],{"className":672},[244],[46,674,175],{"className":675},[111,248],[46,677,181],{"className":678},[135],[46,680,67],{"className":681},[111,112],[46,683,186],{"className":684},[153],[46,686,687,690],{"style":341},[46,688],{"className":689,"style":332},[300],[46,691],{"className":692,"style":349},[348],[46,694,695,698],{"style":352},[46,696],{"className":697,"style":332},[300],[46,699,701],{"className":700},[111],[46,702,67],{"className":703},[111,112],[46,705,440],{"className":706},[439],[46,708,710],{"className":709},[286],[46,711,714],{"className":712,"style":713},[290],"height:0.936em;",[46,715],{},[46,717],{"className":718},[153,312],[46,720],{"className":721,"style":117},[116],[46,723,225],{"className":724},[121],[46,726],{"className":727,"style":117},[116],[46,729,731,734,737,740,743,746,749,752],{"className":730},[102],[46,732],{"className":733,"style":131},[106],[46,735,75],{"className":736},[135],[46,738,619],{"className":739},[111],[46,741,83],{"className":742},[142],[46,744],{"className":745,"style":146},[116],[46,747,624],{"className":748},[111],[46,750,89],{"className":751},[153],[46,753,629],{"className":754},[111],[10,756,757],{},"归一化调整的是向量的整体数值规模。第二个分量仍大于 1，所以不能把“归一化”理解为把每个数都压到 0 到 1 之间。",[10,759,760],{},"这一步也没有减均值。RMSNorm 与 LayerNorm 的计算方式不同，不能只因为都带 Norm 就互换理解。",[39,762,764],{"id":763},"eps-和可训练权重分别解决什么问题","eps 和可训练权重分别解决什么问题",[10,766,767],{},"本次 RMSNorm 的公式是：",[46,769,771],{"className":770},[158],[46,772,774,861],{"className":773},[49],[46,775,777],{"className":776},[53],[55,778,779],{"xmlns":57,"display":167},[59,780,781,858],{},[62,782,783,792,794,845,849,856],{},[784,785,786,789],"msub",{},[65,787,788],{},"y",[65,790,791],{},"i",[69,793,71],{},[193,795,796,802],{},[784,797,798,800],{},[65,799,67],{},[65,801,791],{},[190,803,804],{},[62,805,806,814,831,840,842],{},[193,807,808,811],{},[77,809,810],{},"1",[65,812,813],{},"d",[815,816,817,820,829],"munderover",{},[69,818,819],{},"∑",[62,821,822,825,827],{},[65,823,824],{},"j",[69,826,71],{},[77,828,810],{},[65,830,813],{},[832,833,834,836,838],"msubsup",{},[65,835,67],{},[65,837,824],{},[77,839,204],{},[69,841,207],{},[65,843,844],{},"ε",[846,847,848],"mtext",{}," ",[784,850,851,854],{},[65,852,853],{},"g",[65,855,791],{},[65,857,629],{"mathvariant":174},[91,859,860],{"encoding":93},"y_i=\n\\frac{x_i}\n{\\sqrt{\\frac{1}{d}\\sum_{j=1}^{d}x_j^2+\\varepsilon}}\n\\,g_i.",[46,862,864,924],{"className":863,"ariaHidden":82},[98],[46,865,867,871,915,918,921],{"className":866},[102],[46,868],{"className":869,"style":870},[106],"height:0.625em;vertical-align:-0.1944em;",[46,872,874,878],{"className":873},[111],[46,875,788],{"className":876,"style":877},[111,112],"margin-right:0.0359em;",[46,879,881],{"className":880},[368],[46,882,884,906],{"className":883},[281,282],[46,885,887,903],{"className":886},[286],[46,888,891],{"className":889,"style":890},[290],"height:0.3117em;",[46,892,894,897],{"style":893},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[46,895],{"className":896,"style":385},[300],[46,898,900],{"className":899},[389,390,391,392],[46,901,791],{"className":902},[111,112,392],[46,904,440],{"className":905},[439],[46,907,909],{"className":908},[286],[46,910,913],{"className":911,"style":912},[290],"height:0.15em;",[46,914],{},[46,916],{"className":917,"style":117},[116],[46,919,71],{"className":920},[121],[46,922],{"className":923,"style":117},[116],[46,925,927,931,1302,1305,1345],{"className":926},[102],[46,928],{"className":929,"style":930},[106],"height:2.8376em;vertical-align:-1.73em;",[46,932,934,937,1299],{"className":933},[111],[46,935],{"className":936},[135,312],[46,938,940],{"className":939},[193],[46,941,943,1290],{"className":942},[281,282],[46,944,946,1287],{"className":945},[286],[46,947,949,1228,1237],{"className":948,"style":661},[290],[46,950,952,956],{"style":951},"top:-2.11em;",[46,953],{"className":954,"style":955},[300],"height:3.2616em;",[46,957,959],{"className":958},[111],[46,960,962],{"className":961},[111,277],[46,963,965,1219],{"className":964},[281,282],[46,966,968,1216],{"className":967},[286],[46,969,972,1199],{"className":970,"style":971},[290],"height:1.2616em;",[46,973,976,980],{"className":974,"style":975},[295],"top:-3.8em;",[46,977],{"className":978,"style":979},[300],"height:3.8em;",[46,981,983,1055,1058,1129,1132,1187,1190,1193,1196],{"className":982,"style":305},[111],[46,984,986,989,1052],{"className":985},[111],[46,987],{"className":988},[135,312],[46,990,992],{"className":991},[193],[46,993,995,1043],{"className":994},[281,282],[46,996,998,1040],{"className":997},[286],[46,999,1002,1017,1025],{"className":1000,"style":1001},[290],"height:0.8451em;",[46,1003,1005,1008],{"style":1004},"top:-2.655em;",[46,1006],{"className":1007,"style":332},[300],[46,1009,1011],{"className":1010},[389,390,391,392],[46,1012,1014],{"className":1013},[111,392],[46,1015,813],{"className":1016},[111,112,392],[46,1018,1019,1022],{"style":341},[46,1020],{"className":1021,"style":332},[300],[46,1023],{"className":1024,"style":349},[348],[46,1026,1028,1031],{"style":1027},"top:-3.394em;",[46,1029],{"className":1030,"style":332},[300],[46,1032,1034],{"className":1033},[389,390,391,392],[46,1035,1037],{"className":1036},[111,392],[46,1038,810],{"className":1039},[111,392],[46,1041,440],{"className":1042},[439],[46,1044,1046],{"className":1045},[286],[46,1047,1050],{"className":1048,"style":1049},[290],"height:0.345em;",[46,1051],{},[46,1053],{"className":1054},[153,312],[46,1056],{"className":1057,"style":146},[116],[46,1059,1061,1067],{"className":1060},[244],[46,1062,819],{"className":1063,"style":1066},[244,1064,1065],"op-symbol","small-op","position:relative;top:0em;",[46,1068,1070],{"className":1069},[368],[46,1071,1073,1120],{"className":1072},[281,282],[46,1074,1076,1117],{"className":1075},[286],[46,1077,1080,1102],{"className":1078,"style":1079},[290],"height:0.989em;",[46,1081,1083,1086],{"style":1082},"top:-2.4003em;margin-left:0em;margin-right:0.05em;",[46,1084],{"className":1085,"style":385},[300],[46,1087,1089],{"className":1088},[389,390,391,392],[46,1090,1092,1096,1099],{"className":1091},[111,392],[46,1093,824],{"className":1094,"style":1095},[111,112,392],"margin-right:0.0572em;",[46,1097,71],{"className":1098},[121,392],[46,1100,810],{"className":1101},[111,392],[46,1103,1105,1108],{"style":1104},"top:-3.2029em;margin-right:0.05em;",[46,1106],{"className":1107,"style":385},[300],[46,1109,1111],{"className":1110},[389,390,391,392],[46,1112,1114],{"className":1113},[111,392],[46,1115,813],{"className":1116},[111,112,392],[46,1118,440],{"className":1119},[439],[46,1121,1123],{"className":1122},[286],[46,1124,1127],{"className":1125,"style":1126},[290],"height:0.4358em;",[46,1128],{},[46,1130],{"className":1131,"style":146},[116],[46,1133,1135,1138],{"className":1134},[111],[46,1136,67],{"className":1137},[111,112],[46,1139,1141],{"className":1140},[368],[46,1142,1144,1178],{"className":1143},[281,282],[46,1145,1147,1175],{"className":1146},[286],[46,1148,1151,1163],{"className":1149,"style":1150},[290],"height:0.7959em;",[46,1152,1154,1157],{"style":1153},"top:-2.4231em;margin-left:0em;margin-right:0.05em;",[46,1155],{"className":1156,"style":385},[300],[46,1158,1160],{"className":1159},[389,390,391,392],[46,1161,824],{"className":1162,"style":1095},[111,112,392],[46,1164,1166,1169],{"style":1165},"top:-3.0448em;margin-right:0.05em;",[46,1167],{"className":1168,"style":385},[300],[46,1170,1172],{"className":1171},[389,390,391,392],[46,1173,204],{"className":1174},[111,392],[46,1176,440],{"className":1177},[439],[46,1179,1181],{"className":1180},[286],[46,1182,1185],{"className":1183,"style":1184},[290],"height:0.413em;",[46,1186],{},[46,1188],{"className":1189,"style":399},[116],[46,1191,207],{"className":1192},[403],[46,1194],{"className":1195,"style":399},[116],[46,1197,844],{"className":1198},[111,112],[46,1200,1202,1205],{"style":1201},"top:-3.2216em;",[46,1203],{"className":1204,"style":979},[300],[46,1206,1209],{"className":1207,"style":1208},[462],"min-width:1.02em;height:1.88em;",[465,1210,1213],{"xmlns":467,"width":468,"height":1211,"viewBox":1212,"preserveAspectRatio":471},"1.88em","0 0 400000 1944",[473,1214],{"d":1215},"M983 90\nl0 -0\nc4,-6.7,10,-10,18,-10 H400000v40\nH1013.1s-83.4,268,-264.1,840c-180.7,572,-277,876.3,-289,913c-4.7,4.7,-12.7,7,-24,7\ns-12,0,-12,0c-1.3,-3.3,-3.7,-11.7,-7,-25c-35.3,-125.3,-106.7,-373.3,-214,-744\nc-10,12,-21,25,-33,39s-32,39,-32,39c-6,-5.3,-15,-14,-27,-26s25,-30,25,-30\nc26.7,-32.7,52,-63,76,-91s52,-60,52,-60s208,722,208,722\nc56,-175.3,126.3,-397.3,211,-666c84.7,-268.7,153.8,-488.2,207.5,-658.5\nc53.7,-170.3,84.5,-266.8,92.5,-289.5z\nM1001 80h400000v40h-400000z",[46,1217,440],{"className":1218},[439],[46,1220,1222],{"className":1221},[286],[46,1223,1226],{"className":1224,"style":1225},[290],"height:0.5784em;",[46,1227],{},[46,1229,1231,1234],{"style":1230},"top:-3.4916em;",[46,1232],{"className":1233,"style":955},[300],[46,1235],{"className":1236,"style":349},[348],[46,1238,1240,1243],{"style":1239},"top:-3.9386em;",[46,1241],{"className":1242,"style":955},[300],[46,1244,1246],{"className":1245},[111],[46,1247,1249,1252],{"className":1248},[111],[46,1250,67],{"className":1251},[111,112],[46,1253,1255],{"className":1254},[368],[46,1256,1258,1279],{"className":1257},[281,282],[46,1259,1261,1276],{"className":1260},[286],[46,1262,1264],{"className":1263,"style":890},[290],[46,1265,1267,1270],{"style":1266},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[46,1268],{"className":1269,"style":385},[300],[46,1271,1273],{"className":1272},[389,390,391,392],[46,1274,791],{"className":1275},[111,112,392],[46,1277,440],{"className":1278},[439],[46,1280,1282],{"className":1281},[286],[46,1283,1285],{"className":1284,"style":912},[290],[46,1286],{},[46,1288,440],{"className":1289},[439],[46,1291,1293],{"className":1292},[286],[46,1294,1297],{"className":1295,"style":1296},[290],"height:1.73em;",[46,1298],{},[46,1300],{"className":1301},[153,312],[46,1303],{"className":1304,"style":146},[116],[46,1306,1308,1311],{"className":1307},[111],[46,1309,853],{"className":1310,"style":877},[111,112],[46,1312,1314],{"className":1313},[368],[46,1315,1317,1337],{"className":1316},[281,282],[46,1318,1320,1334],{"className":1319},[286],[46,1321,1323],{"className":1322,"style":890},[290],[46,1324,1325,1328],{"style":893},[46,1326],{"className":1327,"style":385},[300],[46,1329,1331],{"className":1330},[389,390,391,392],[46,1332,791],{"className":1333},[111,112,392],[46,1335,440],{"className":1336},[439],[46,1338,1340],{"className":1339},[286],[46,1341,1343],{"className":1342,"style":912},[290],[46,1344],{},[46,1346,629],{"className":1347},[111],[10,1349,1350,1379,1380,1409,1410,1480,1481,1510,1511,1514],{},[46,1351,1353,1366],{"className":1352},[49],[46,1354,1356],{"className":1355},[53],[55,1357,1358],{"xmlns":57},[59,1359,1360,1364],{},[62,1361,1362],{},[65,1363,813],{},[91,1365,813],{"encoding":93},[46,1367,1369],{"className":1368,"ariaHidden":82},[98],[46,1370,1372,1376],{"className":1371},[102],[46,1373],{"className":1374,"style":1375},[106],"height:0.6944em;",[46,1377,813],{"className":1378},[111,112]," 是特征维度；",[46,1381,1383,1397],{"className":1382},[49],[46,1384,1386],{"className":1385},[53],[55,1387,1388],{"xmlns":57},[59,1389,1390,1394],{},[62,1391,1392],{},[65,1393,844],{},[91,1395,1396],{"encoding":93},"\\varepsilon",[46,1398,1400],{"className":1399,"ariaHidden":82},[98],[46,1401,1403,1406],{"className":1402},[102],[46,1404],{"className":1405,"style":107},[106],[46,1407,844],{"className":1408},[111,112]," 是一个很小的正数；",[46,1411,1413,1431],{"className":1412},[49],[46,1414,1416],{"className":1415},[53],[55,1417,1418],{"xmlns":57},[59,1419,1420,1428],{},[62,1421,1422],{},[784,1423,1424,1426],{},[65,1425,853],{},[65,1427,791],{},[91,1429,1430],{"encoding":93},"g_i",[46,1432,1434],{"className":1433,"ariaHidden":82},[98],[46,1435,1437,1440],{"className":1436},[102],[46,1438],{"className":1439,"style":870},[106],[46,1441,1443,1446],{"className":1442},[111],[46,1444,853],{"className":1445,"style":877},[111,112],[46,1447,1449],{"className":1448},[368],[46,1450,1452,1472],{"className":1451},[281,282],[46,1453,1455,1469],{"className":1454},[286],[46,1456,1458],{"className":1457,"style":890},[290],[46,1459,1460,1463],{"style":893},[46,1461],{"className":1462,"style":385},[300],[46,1464,1466],{"className":1465},[389,390,391,392],[46,1467,791],{"className":1468},[111,112,392],[46,1470,440],{"className":1471},[439],[46,1473,1475],{"className":1474},[286],[46,1476,1478],{"className":1477,"style":912},[290],[46,1479],{}," 是第 ",[46,1482,1484,1497],{"className":1483},[49],[46,1485,1487],{"className":1486},[53],[55,1488,1489],{"xmlns":57},[59,1490,1491,1495],{},[62,1492,1493],{},[65,1494,791],{},[91,1496,791],{"encoding":93},[46,1498,1500],{"className":1499,"ariaHidden":82},[98],[46,1501,1503,1507],{"className":1502},[102],[46,1504],{"className":1505,"style":1506},[106],"height:0.6595em;",[46,1508,791],{"className":1509},[111,112]," 个特征对应的可训练缩放参数，代码里叫 ",[19,1512,1513],{},"weight","。",[10,1516,1517,1518,1546,1547,1550],{},"如果一个 token 的所有特征都是 0，没有 ",[46,1519,1521,1534],{"className":1520},[49],[46,1522,1524],{"className":1523},[53],[55,1525,1526],{"xmlns":57},[59,1527,1528,1532],{},[62,1529,1530],{},[65,1531,844],{},[91,1533,1396],{"encoding":93},[46,1535,1537],{"className":1536,"ariaHidden":82},[98],[46,1538,1540,1543],{"className":1539},[102],[46,1541],{"className":1542,"style":107},[106],[46,1544,844],{"className":1545},[111,112]," 就会除以 0。代码保存的 ",[19,1548,1549],{},"eps=1e-5","，即 0.00001，它加在开根号之前。",[10,1552,1553,1554,1557],{},"缩放权重的形状是 ",[19,1555,1556],{},"(d_model,)","，初始值全为 1。这样开始时它不额外改变归一化后的结果，训练之后再学习每个特征需要怎样缩放。每个 token 都有自己的 RMS，但同一个 RMSNorm 层的这组特征权重会在所有 token 之间共享。",[39,1559,1561],{"id":1560},"keepdimtrue-留下的是哪一维","keepdim=True 留下的是哪一维",[10,1563,1564,1565,1568],{},"假设输入形状为 ",[19,1566,1567],{},"(2, 5, 64)","，分别表示 batch、序列长度和特征维度。我们需要对每个 token 的 64 个特征求平方均值，不能把所有 batch 或所有 token 一起平均。",[10,1570,1571],{},"已有实现中，对应的是：",[1573,1574,1579],"pre",{"className":1575,"code":1576,"language":1577,"meta":1578,"style":1578},"language-python shiki shiki-themes github-dark-dimmed github-light houston","mean_square=(x**2).mean(dim=-1,keepdim=True)\n","python","",[19,1580,1581],{"__ignoreMap":1578},[46,1582,1585,1589,1592,1595,1598,1601,1604,1608,1610,1614,1617,1619,1621,1624,1626,1629],{"class":1583,"line":1584},"line",1,[46,1586,1588],{"class":1587},"sWURj","mean_square",[46,1590,71],{"class":1591},"sf2cM",[46,1593,1594],{"class":1587},"(x",[46,1596,1597],{"class":1591},"**",[46,1599,204],{"class":1600},"sgENW",[46,1602,1603],{"class":1587},").",[46,1605,1607],{"class":1606},"sq3c-","mean",[46,1609,181],{"class":1587},[46,1611,1613],{"class":1612},"sA9aF","dim",[46,1615,1616],{"class":1591},"=-",[46,1618,810],{"class":1600},[46,1620,83],{"class":1587},[46,1622,1623],{"class":1612},"keepdim",[46,1625,71],{"class":1591},[46,1627,1628],{"class":1600},"True",[46,1630,1631],{"class":1587},")\n",[10,1633,1634,1636,1637,1639],{},[19,1635,21],{}," 指最后一维。",[19,1638,25],{}," 保留这根轴，只把长度从 64 改成 1。",[1641,1642,1643,1659],"table",{},[1644,1645,1646],"thead",{},[1647,1648,1649,1653,1656],"tr",{},[1650,1651,1652],"th",{},"张量",[1650,1654,1655],{},"形状",[1650,1657,1658],{},"每个位置放什么",[1660,1661,1662,1677,1691,1705,1719],"tbody",{},[1647,1663,1664,1670,1674],{},[1665,1666,1667,1668],"td",{},"输入 ",[19,1669,67],{},[1665,1671,1672],{},[19,1673,1567],{},[1665,1675,1676],{},"每个 token 的 64 个特征",[1647,1678,1679,1683,1688],{},[1665,1680,1681],{},[19,1682,1588],{},[1665,1684,1685],{},[19,1686,1687],{},"(2, 5, 1)",[1665,1689,1690],{},"每个 token 的平方均值",[1647,1692,1693,1698,1702],{},[1665,1694,1695],{},[19,1696,1697],{},"rms",[1665,1699,1700],{},[19,1701,1687],{},[1665,1703,1704],{},"每个 token 对应的除数",[1647,1706,1707,1711,1716],{},[1665,1708,1709],{},[19,1710,1513],{},[1665,1712,1713],{},[19,1714,1715],{},"(64,)",[1665,1717,1718],{},"每个特征共享的缩放参数",[1647,1720,1721,1724,1728],{},[1665,1722,1723],{},"输出",[1665,1725,1726],{},[19,1727,1567],{},[1665,1729,1730],{},"归一化并缩放后的特征",[10,1732,1733,1734,1736,1737,1739,1740,1768,1769,1514],{},"广播从尾部对齐维度。",[19,1735,1687],{}," 的最后一维可以扩展到 64，于是一个 token 的 64 个特征都除以这个 token 自己的 RMS。",[19,1738,1715],{}," 也能沿前面两维广播，让各个 token 的第 ",[46,1741,1743,1756],{"className":1742},[49],[46,1744,1746],{"className":1745},[53],[55,1747,1748],{"xmlns":57},[59,1749,1750,1754],{},[62,1751,1752],{},[65,1753,791],{},[91,1755,791],{"encoding":93},[46,1757,1759],{"className":1758,"ariaHidden":82},[98],[46,1760,1762,1765],{"className":1761},[102],[46,1763],{"className":1764,"style":1506},[106],[46,1766,791],{"className":1767},[111,112]," 个特征都乘以同一个 ",[46,1770,1772,1789],{"className":1771},[49],[46,1773,1775],{"className":1774},[53],[55,1776,1777],{"xmlns":57},[59,1778,1779,1787],{},[62,1780,1781],{},[784,1782,1783,1785],{},[65,1784,853],{},[65,1786,791],{},[91,1788,1430],{"encoding":93},[46,1790,1792],{"className":1791,"ariaHidden":82},[98],[46,1793,1795,1798],{"className":1794},[102],[46,1796],{"className":1797,"style":870},[106],[46,1799,1801,1804],{"className":1800},[111],[46,1802,853],{"className":1803,"style":877},[111,112],[46,1805,1807],{"className":1806},[368],[46,1808,1810,1830],{"className":1809},[281,282],[46,1811,1813,1827],{"className":1812},[286],[46,1814,1816],{"className":1815,"style":890},[290],[46,1817,1818,1821],{"style":893},[46,1819],{"className":1820,"style":385},[300],[46,1822,1824],{"className":1823},[389,390,391,392],[46,1825,791],{"className":1826},[111,112,392],[46,1828,440],{"className":1829},[439],[46,1831,1833],{"className":1832},[286],[46,1834,1836],{"className":1835,"style":912},[290],[46,1837],{},[10,1839,1840,1841,1843,1844,1847,1848,1850],{},"如果把 ",[19,1842,1623],{}," 去掉，平方均值会变成 ",[19,1845,1846],{},"(2, 5)","。拿它与 ",[19,1849,1567],{}," 直接做除法，尾部就成了 5 与 64 对齐，形状无法匹配。保留长度为 1 的轴，既告诉 PyTorch 怎样广播，也保留了“这是特征轴上的一个统计量”这层含义。",[39,1852,1854],{"id":1853},"为什么计算前后还要转换-dtype","为什么计算前后还要转换 dtype",[10,1856,1857],{},"这份实现会先记住输入类型，将输入转为 float32，完成平方、平均与归一化后，再把结果转回输入类型。",[10,1859,1860],{},"原因在数值范围和精度。低精度输入的平方与求和可能更容易累积误差，float32 中间计算能降低这部分风险。但这不等于“转成 float32 后所有数值问题都会消失”，更不等于已经做过混合精度训练测试。",[10,1862,1863],{},"这一轮验证的是官方 RMSNorm 前向用例。理解类型转换的目的之后，仍要把结论限制在实际测试覆盖的范围里。",[39,1865,1867],{"id":1866},"silu-为什么要放在线性层后面","SiLU 为什么要放在线性层后面",[10,1869,1870],{},"只把多个无偏置 Linear 接起来，整体仍然能合并成一个线性变换。中间维度再大，也不能仅靠这种堆叠获得非线性。",[10,1872,1873],{},"SiLU 对每个数做：",[46,1875,1877],{"className":1876},[158],[46,1878,1880,1944],{"className":1879},[49],[46,1881,1883],{"className":1882},[53],[55,1884,1885],{"xmlns":57,"display":167},[59,1886,1887,1941],{},[62,1888,1889,1892,1894,1896,1898,1900,1902,1904,1906,1909,1911,1913,1915,1917,1939],{},[65,1890,1891],{"mathvariant":174},"SiLU",[69,1893,178],{},[69,1895,181],{"stretchy":74},[65,1897,67],{},[69,1899,186],{"stretchy":74},[69,1901,71],{},[65,1903,67],{},[846,1905,848],{},[65,1907,1908],{},"σ",[69,1910,181],{"stretchy":74},[65,1912,67],{},[69,1914,186],{"stretchy":74},[69,1916,71],{},[193,1918,1919,1921],{},[65,1920,67],{},[62,1922,1923,1925,1927],{},[77,1924,810],{},[69,1926,207],{},[198,1928,1929,1932],{},[65,1930,1931],{},"e",[62,1933,1934,1937],{},[69,1935,1936],{},"−",[65,1938,67],{},[65,1940,629],{"mathvariant":174},[91,1942,1943],{"encoding":93},"\\operatorname{SiLU}(x)=x\\,\\sigma(x)\n=\\frac{x}{1+e^{-x}}.",[46,1945,1947,1977,2010],{"className":1946,"ariaHidden":82},[98],[46,1948,1950,1953,1959,1962,1965,1968,1971,1974],{"className":1949},[102],[46,1951],{"className":1952,"style":131},[106],[46,1954,1956],{"className":1955},[244],[46,1957,1891],{"className":1958},[111,248],[46,1960,181],{"className":1961},[135],[46,1963,67],{"className":1964},[111,112],[46,1966,186],{"className":1967},[153],[46,1969],{"className":1970,"style":117},[116],[46,1972,71],{"className":1973},[121],[46,1975],{"className":1976,"style":117},[116],[46,1978,1980,1983,1986,1989,1992,1995,1998,2001,2004,2007],{"className":1979},[102],[46,1981],{"className":1982,"style":131},[106],[46,1984,67],{"className":1985},[111,112],[46,1987],{"className":1988,"style":146},[116],[46,1990,1908],{"className":1991,"style":877},[111,112],[46,1993,181],{"className":1994},[135],[46,1996,67],{"className":1997},[111,112],[46,1999,186],{"className":2000},[153],[46,2002],{"className":2003,"style":117},[116],[46,2005,71],{"className":2006},[121],[46,2008],{"className":2009,"style":117},[116],[46,2011,2013,2017,2125],{"className":2012},[102],[46,2014],{"className":2015,"style":2016},[106],"height:1.8769em;vertical-align:-0.7693em;",[46,2018,2020,2023,2122],{"className":2019},[111],[46,2021],{"className":2022},[135,312],[46,2024,2026],{"className":2025},[193],[46,2027,2029,2113],{"className":2028},[281,282],[46,2030,2032,2110],{"className":2031},[286],[46,2033,2035,2091,2099],{"className":2034,"style":661},[290],[46,2036,2037,2040],{"style":328},[46,2038],{"className":2039,"style":332},[300],[46,2041,2043,2046,2049,2052,2055],{"className":2042},[111],[46,2044,810],{"className":2045},[111],[46,2047],{"className":2048,"style":399},[116],[46,2050,207],{"className":2051},[403],[46,2053],{"className":2054,"style":399},[116],[46,2056,2058,2061],{"className":2057},[111],[46,2059,1931],{"className":2060},[111,112],[46,2062,2064],{"className":2063},[368],[46,2065,2067],{"className":2066},[281],[46,2068,2070],{"className":2069},[286],[46,2071,2074],{"className":2072,"style":2073},[290],"height:0.6973em;",[46,2075,2076,2079],{"style":381},[46,2077],{"className":2078,"style":385},[300],[46,2080,2082],{"className":2081},[389,390,391,392],[46,2083,2085,2088],{"className":2084},[111,392],[46,2086,1936],{"className":2087},[111,392],[46,2089,67],{"className":2090},[111,112,392],[46,2092,2093,2096],{"style":341},[46,2094],{"className":2095,"style":332},[300],[46,2097],{"className":2098,"style":349},[348],[46,2100,2101,2104],{"style":352},[46,2102],{"className":2103,"style":332},[300],[46,2105,2107],{"className":2106},[111],[46,2108,67],{"className":2109},[111,112],[46,2111,440],{"className":2112},[439],[46,2114,2116],{"className":2115},[286],[46,2117,2120],{"className":2118,"style":2119},[290],"height:0.7693em;",[46,2121],{},[46,2123],{"className":2124},[153,312],[46,2126,629],{"className":2127},[111],[10,2129,2130,2131,2175],{},"这里的 ",[46,2132,2134,2154],{"className":2133},[49],[46,2135,2137],{"className":2136},[53],[55,2138,2139],{"xmlns":57},[59,2140,2141,2151],{},[62,2142,2143,2145,2147,2149],{},[65,2144,1908],{},[69,2146,181],{"stretchy":74},[65,2148,67],{},[69,2150,186],{"stretchy":74},[91,2152,2153],{"encoding":93},"\\sigma(x)",[46,2155,2157],{"className":2156,"ariaHidden":82},[98],[46,2158,2160,2163,2166,2169,2172],{"className":2159},[102],[46,2161],{"className":2162,"style":131},[106],[46,2164,1908],{"className":2165,"style":877},[111,112],[46,2167,181],{"className":2168},[135],[46,2170,67],{"className":2171},[111,112],[46,2173,186],{"className":2174},[153]," 是 sigmoid。它的值在 0 到 1 之间，但 SiLU 的输出本身不受这个区间限制。",[1641,2177,2178,2219],{},[1644,2179,2180],{},[1647,2181,2182,2213,2216],{},[1650,2183,1667,2185],{"align":2184},"right",[46,2186,2188,2201],{"className":2187},[49],[46,2189,2191],{"className":2190},[53],[55,2192,2193],{"xmlns":57},[59,2194,2195,2199],{},[62,2196,2197],{},[65,2198,67],{},[91,2200,67],{"encoding":93},[46,2202,2204],{"className":2203,"ariaHidden":82},[98],[46,2205,2207,2210],{"className":2206},[102],[46,2208],{"className":2209,"style":107},[106],[46,2211,67],{"className":2212},[111,112],[1650,2214,2215],{"align":2184},"sigmoid 系数",[1650,2217,2218],{"align":2184},"SiLU 输出",[1660,2220,2221,2232,2242],{},[1647,2222,2223,2226,2229],{},[1665,2224,2225],{"align":2184},"-2",[1665,2227,2228],{"align":2184},"约 0.119",[1665,2230,2231],{"align":2184},"约 -0.238",[1647,2233,2234,2237,2240],{},[1665,2235,2236],{"align":2184},"0",[1665,2238,2239],{"align":2184},"0.5",[1665,2241,2236],{"align":2184},[1647,2243,2244,2246,2249],{},[1665,2245,204],{"align":2184},[1665,2247,2248],{"align":2184},"约 0.881",[1665,2250,2251],{"align":2184},"约 1.762",[10,2253,2254,2255,2258],{},"已有 ",[19,2256,2257],{},"activations.py"," 中的 SiLU 是一个普通函数。它对张量逐元素计算，不保存可训练权重，输入和输出形状相同。与 Linear 不同，这里不需要创建一张参数表。",[10,2260,2261],{},"这个朴素表达式已经通过对应官方测试；文章记录的是当前学习版本，没有据此声称它在所有极端输入或低精度场景下都达到最优的数值表现。",[39,2263,2265],{"id":2264},"swiglu-的三个-linear-怎样连接","SwiGLU 的三个 Linear 怎样连接",[10,2267,2268],{},"SwiGLU 复用前面写好的 Linear 和 SiLU。同一份输入先进入两条分支，一条经过线性变换再经过 SiLU，另一条只经过线性变换。两条结果逐元素相乘，再投影回原来的维度。",[10,2270,2271,2272,22,2275,22,2278,2281,2282,1514],{},"当前代码中，三个层的名字是 ",[19,2273,2274],{},"w1",[19,2276,2277],{},"w3",[19,2279,2280],{},"w2","。这个顺序很容易读错：最后工作的反而是 ",[19,2283,2280],{},[1641,2285,2286,2299],{},[1644,2287,2288],{},[1647,2289,2290,2293,2296],{},[1650,2291,2292],{},"层",[1650,2294,2295],{},"维度变化",[1650,2297,2298],{},"所在位置",[1660,2300,2301,2315,2328],{},[1647,2302,2303,2307,2312],{},[1665,2304,2305],{},[19,2306,2274],{},[1665,2308,2309],{},[19,2310,2311],{},"d_model → d_ff",[1665,2313,2314],{},"第一条分支，后接 SiLU",[1647,2316,2317,2321,2325],{},[1665,2318,2319],{},[19,2320,2277],{},[1665,2322,2323],{},[19,2324,2311],{},[1665,2326,2327],{},"第二条分支",[1647,2329,2330,2334,2339],{},[1665,2331,2332],{},[19,2333,2280],{},[1665,2335,2336],{},[19,2337,2338],{},"d_ff → d_model",[1665,2340,2341],{},"两条分支相乘后",[10,2343,2344,2345,2348],{},"考虑权重按 ",[19,2346,2347],{},"(输出维度, 输入维度)"," 保存，用行向量约定写出的公式为：",[46,2350,2352],{"className":2351},[158],[46,2353,2355,2434],{"className":2354},[49],[46,2356,2358],{"className":2357},[53],[55,2359,2360],{"xmlns":57,"display":167},[59,2361,2362,2431],{},[62,2363,2364,2367,2369,2371,2374,2376,2378,2421,2429],{},[65,2365,2366],{"mathvariant":174},"SwiGLU",[69,2368,178],{},[69,2370,181],{"stretchy":74},[65,2372,2373],{},"X",[69,2375,186],{"stretchy":74},[69,2377,71],{},[62,2379,2380,2382,2384,2386,2388,2390,2400,2402,2405,2407,2409,2417,2419],{},[69,2381,75],{"fence":82},[65,2383,1891],{"mathvariant":174},[69,2385,178],{},[69,2387,181],{"stretchy":74},[65,2389,2373],{},[832,2391,2392,2395,2397],{},[65,2393,2394],{},"W",[77,2396,810],{},[65,2398,2399],{"mathvariant":174},"⊤",[69,2401,186],{"stretchy":74},[69,2403,2404],{},"⊙",[69,2406,181],{"stretchy":74},[65,2408,2373],{},[832,2410,2411,2413,2415],{},[65,2412,2394],{},[77,2414,79],{},[65,2416,2399],{"mathvariant":174},[69,2418,186],{"stretchy":74},[69,2420,89],{"fence":82},[832,2422,2423,2425,2427],{},[65,2424,2394],{},[77,2426,204],{},[65,2428,2399],{"mathvariant":174},[65,2430,629],{"mathvariant":174},[91,2432,2433],{"encoding":93},"\\operatorname{SwiGLU}(X)\n=\\left[\\operatorname{SiLU}(XW_1^\\top)\n\\odot(XW_3^\\top)\\right]W_2^\\top.",[46,2435,2437,2468],{"className":2436,"ariaHidden":82},[98],[46,2438,2440,2443,2449,2452,2456,2459,2462,2465],{"className":2439},[102],[46,2441],{"className":2442,"style":131},[106],[46,2444,2446],{"className":2445},[244],[46,2447,2366],{"className":2448},[111,248],[46,2450,181],{"className":2451},[135],[46,2453,2373],{"className":2454,"style":2455},[111,112],"margin-right:0.0785em;",[46,2457,186],{"className":2458},[153],[46,2460],{"className":2461,"style":117},[116],[46,2463,71],{"className":2464},[121],[46,2466],{"className":2467,"style":117},[116],[46,2469,2471,2475,2635,2638,2689],{"className":2470},[102],[46,2472],{"className":2473,"style":2474},[106],"height:1.2491em;vertical-align:-0.35em;",[46,2476,2479,2489,2495,2498,2501,2557,2560,2563,2566,2569,2572,2575,2626,2629],{"className":2477},[2478],"minner",[46,2480,2484],{"className":2481,"style":2483},[135,2482],"delimcenter","top:0em;",[46,2485,75],{"className":2486},[2487,2488],"delimsizing","size1",[46,2490,2492],{"className":2491},[244],[46,2493,1891],{"className":2494},[111,248],[46,2496,181],{"className":2497},[135],[46,2499,2373],{"className":2500,"style":2455},[111,112],[46,2502,2504,2508],{"className":2503},[111],[46,2505,2394],{"className":2506,"style":2507},[111,112],"margin-right:0.1389em;",[46,2509,2511],{"className":2510},[368],[46,2512,2514,2548],{"className":2513},[281,282],[46,2515,2517,2545],{"className":2516},[286],[46,2518,2521,2533],{"className":2519,"style":2520},[290],"height:0.8991em;",[46,2522,2524,2527],{"style":2523},"top:-2.453em;margin-left:-0.1389em;margin-right:0.05em;",[46,2525],{"className":2526,"style":385},[300],[46,2528,2530],{"className":2529},[389,390,391,392],[46,2531,810],{"className":2532},[111,392],[46,2534,2536,2539],{"style":2535},"top:-3.113em;margin-right:0.05em;",[46,2537],{"className":2538,"style":385},[300],[46,2540,2542],{"className":2541},[389,390,391,392],[46,2543,2399],{"className":2544},[111,392],[46,2546,440],{"className":2547},[439],[46,2549,2551],{"className":2550},[286],[46,2552,2555],{"className":2553,"style":2554},[290],"height:0.247em;",[46,2556],{},[46,2558,186],{"className":2559},[153],[46,2561],{"className":2562,"style":399},[116],[46,2564,2404],{"className":2565},[403],[46,2567],{"className":2568,"style":399},[116],[46,2570,181],{"className":2571},[135],[46,2573,2373],{"className":2574,"style":2455},[111,112],[46,2576,2578,2581],{"className":2577},[111],[46,2579,2394],{"className":2580,"style":2507},[111,112],[46,2582,2584],{"className":2583},[368],[46,2585,2587,2618],{"className":2586},[281,282],[46,2588,2590,2615],{"className":2589},[286],[46,2591,2593,2604],{"className":2592,"style":2520},[290],[46,2594,2595,2598],{"style":2523},[46,2596],{"className":2597,"style":385},[300],[46,2599,2601],{"className":2600},[389,390,391,392],[46,2602,79],{"className":2603},[111,392],[46,2605,2606,2609],{"style":2535},[46,2607],{"className":2608,"style":385},[300],[46,2610,2612],{"className":2611},[389,390,391,392],[46,2613,2399],{"className":2614},[111,392],[46,2616,440],{"className":2617},[439],[46,2619,2621],{"className":2620},[286],[46,2622,2624],{"className":2623,"style":2554},[290],[46,2625],{},[46,2627,186],{"className":2628},[153],[46,2630,2632],{"className":2631,"style":2483},[153,2482],[46,2633,89],{"className":2634},[2487,2488],[46,2636],{"className":2637,"style":146},[116],[46,2639,2641,2644],{"className":2640},[111],[46,2642,2394],{"className":2643,"style":2507},[111,112],[46,2645,2647],{"className":2646},[368],[46,2648,2650,2681],{"className":2649},[281,282],[46,2651,2653,2678],{"className":2652},[286],[46,2654,2656,2667],{"className":2655,"style":2520},[290],[46,2657,2658,2661],{"style":2523},[46,2659],{"className":2660,"style":385},[300],[46,2662,2664],{"className":2663},[389,390,391,392],[46,2665,204],{"className":2666},[111,392],[46,2668,2669,2672],{"style":2535},[46,2670],{"className":2671,"style":385},[300],[46,2673,2675],{"className":2674},[389,390,391,392],[46,2676,2399],{"className":2677},[111,392],[46,2679,440],{"className":2680},[439],[46,2682,2684],{"className":2683},[286],[46,2685,2687],{"className":2686,"style":2554},[290],[46,2688],{},[46,2690,629],{"className":2691},[111],[10,2693,2694,2724,2725,2728],{},[46,2695,2697,2711],{"className":2696},[49],[46,2698,2700],{"className":2699},[53],[55,2701,2702],{"xmlns":57},[59,2703,2704,2708],{},[62,2705,2706],{},[69,2707,2404],{},[91,2709,2710],{"encoding":93},"\\odot",[46,2712,2714],{"className":2713,"ariaHidden":82},[98],[46,2715,2717,2721],{"className":2716},[102],[46,2718],{"className":2719,"style":2720},[106],"height:0.6667em;vertical-align:-0.0833em;",[46,2722,2404],{"className":2723},[111]," 是逐元素乘法。已有 ",[19,2726,2727],{},"forward"," 中最值得单独看的两行是：",[1573,2730,2732],{"className":1575,"code":2731,"language":1577,"meta":1578,"style":1578},"combined=gate*values\nreturn self.w2(combined)\n",[19,2733,2734,2749],{"__ignoreMap":1578},[46,2735,2736,2739,2741,2744,2746],{"class":1583,"line":1584},[46,2737,2738],{"class":1587},"combined",[46,2740,71],{"class":1591},[46,2742,2743],{"class":1587},"gate",[46,2745,36],{"class":1591},[46,2747,2748],{"class":1587},"values\n",[46,2750,2752,2756,2760,2762,2764],{"class":1583,"line":2751},2,[46,2753,2755],{"class":2754},"slEfO","return",[46,2757,2759],{"class":2758},"sRGqv"," self",[46,2761,629],{"class":1587},[46,2763,2280],{"class":1606},[46,2765,2766],{"class":1587},"(combined)\n",[10,2768,2769,2770,2772,2773,2776,2777,2780,2781,2783,2784,1514],{},"假设输入为 ",[19,2771,1567],{},"，",[19,2774,2775],{},"d_ff=128","，那么两条分支都是 ",[19,2778,2779],{},"(2, 5, 128)","，相乘后仍是这个形状，最后通过 ",[19,2782,2280],{}," 得到 ",[19,2785,1567],{},[10,2787,2130,2788,2790,2791,33,2794,2783,2797,2800,2801,2803,2804,2806],{},[19,2789,36],{}," 让对应位置相乘，例如 ",[19,2792,2793],{},"[2, 3]",[19,2795,2796],{},"[4, 5]",[19,2798,2799],{},"[8, 15]","。它没有像矩阵乘法那样沿某个维度求和。前面 Linear 中的 ",[19,2802,32],{}," 和这里的 ",[19,2805,36],{},"，不能因为都叫乘法就混用。",[39,2808,2810],{"id":2809},"门控不是一个-01-开关","门控不是一个 0\u002F1 开关",[10,2812,2813,2814,2816],{},"代码把 SiLU 分支叫 ",[19,2815,2743],{},"，于是容易想成每个位置只有“放行”和“关闭”两种状态。实际这里是连续的逐元素调节。",[10,2818,2819,2820,2822],{},"还需要区分 sigmoid 与 SiLU：虽然 sigmoid 系数在 0 到 1 之间，经过 SiLU 之后的 ",[19,2821,2743],{}," 可以大于 1，也可以小于 0。因此它既可能减弱某个分量，也可能放大或改变符号。把门控理解成“输入相关的调节系数”，比把它想成一排布尔开关更贴近这段计算。",[10,2824,2825],{},"这几个组件目前都在每个 token 内处理特征。它们不会因为输入带着序列维度，就自动让 token 之间交换信息。跨位置的信息汇总，要留给后面的 Attention。",[39,2827,2828],{"id":2828},"从导入失败到测试通过",[10,2830,2831,2832,2835,2836,2839],{},"RMSNorm 最初写过 ",[19,2833,2834],{},"import RMSNorm(nn.Module):","。这里是在定义新类，应该使用 ",[19,2837,2838],{},"class","。之后测试停在导入阶段，错误信息是：",[1573,2841,2846],{"className":2842,"code":2844,"language":2845,"meta":1578},[2843],"language-text","ModuleNotFoundError: No module named 'cs336_basics.RMSNorm'\n","text",[19,2847,2844],{"__ignoreMap":1578},[10,2849,2850,2851,2854,2855,2858],{},"文件名 ",[19,2852,2853],{},"rmsnorm.py"," 是小写，里面的类名 ",[19,2856,2857],{},"RMSNorm"," 保留大小写。导入路径的模块名和导入的类名是两个不同部分。",[10,2860,2861,2862,33,2865,22,2868,33,2871,22,2874,33,2877,2880],{},"到 SwiGLU，又遇到了 ",[19,2863,2864],{},"linear",[19,2866,2867],{},"Linear",[19,2869,2870],{},"super.__init__()",[19,2872,2873],{},"super().__init__()",[19,2875,2876],{},"Values",[19,2878,2879],{},"values"," 的区别。都是很短的符号，却会让程序在真正执行公式以前就停下。",[10,2882,2883,2884,22,2887,22,2890,2893,2894,2897],{},"修正以后，历史记录中 ",[19,2885,2886],{},"test_rmsnorm",[19,2888,2889],{},"test_silu_matches_pytorch",[19,2891,2892],{},"test_swiglu"," 分别得到 ",[19,2895,2896],{},"1 passed","。这些结果对应组件级的前向测试；当前还没有把它们组装成完整 Transformer block。",[10,2899,2900,2901,2905],{},"下一篇继续记录 ",[13,2902,2904],{"href":2903},"\u002Fposts\u002Fcs336-softmax-attention-mask\u002F","Softmax 和 Attention Mask","。这一次，前面反复遇到的“沿哪一维”和“保留哪一维”会再次出现。",[39,2907,2908],{"id":2908},"发布时的复核",[10,2910,2911,2912,2918,2919,2922,2923,1514],{},"发布前对",[13,2913,2917],{"href":2914,"rel":2915},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fa39eb684710817f7f389626423fcb26ca0f6d629",[2916],"nofollow","独立代码快照","重新运行了 8 项官方组件测试，结果为 ",[19,2920,2921],{},"8 passed","。完整清单和运行环境见",[13,2924,2927],{"href":2925,"rel":2926},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Fblob\u002Fa39eb684710817f7f389626423fcb26ca0f6d629\u002Fdocs\u002Fprogress.md",[2916],"进度记录",[10,2929,2930],{},[2931,2932,2933,2934,2939],"em",{},"封面使用 ",[13,2935,2938],{"href":2936,"rel":2937},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Faigc-image-generator",[2916],"aigc-image-generator"," 辅助生成，属于概念插画；准确的形状、公式与运算关系以正文为准。",[39,2941,2942],{"id":2942},"参考资料",[2944,2945,2946,2954,2961,2968,2975,2982],"ul",{},[2947,2948,2949],"li",{},[13,2950,2953],{"href":2951,"rel":2952},"https:\u002F\u002Fcs336.stanford.edu\u002F",[2916],"Stanford CS336 官方课程页面",[2947,2955,2956],{},[13,2957,2960],{"href":2958,"rel":2959},"https:\u002F\u002Fgithub.com\u002Fstanford-cs336\u002Fassignment1-basics",[2916],"Assignment 1 官方仓库与讲义",[2947,2962,2963],{},[13,2964,2967],{"href":2965,"rel":2966},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fnotes\u002Fbroadcasting.html",[2916],"PyTorch：广播语义",[2947,2969,2970],{},[13,2971,2974],{"href":2972,"rel":2973},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fgenerated\u002Ftorch.mean.html",[2916],"PyTorch：torch.mean",[2947,2976,2977],{},[13,2978,2981],{"href":2979,"rel":2980},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fgenerated\u002Ftorch.nn.RMSNorm.html",[2916],"PyTorch：nn.RMSNorm",[2947,2983,2984],{},[13,2985,2988],{"href":2986,"rel":2987},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fgenerated\u002Ftorch.nn.SiLU.html",[2916],"PyTorch：nn.SiLU",[2990,2991,2992],"style",{},"html pre.shiki code .sWURj, html code.shiki .sWURj{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#EEF0F9}html pre.shiki code .sf2cM, html code.shiki .sf2cM{--shiki-default:#F47067;--shiki-light:#D73A49;--shiki-cyber:#EEF0F9}html pre.shiki code .sgENW, html code.shiki .sgENW{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#FFD493}html pre.shiki code .sq3c-, html code.shiki .sq3c-{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#00DAEF}html pre.shiki code .sA9aF, html code.shiki .sA9aF{--shiki-default:#F69D50;--shiki-default-font-style:inherit;--shiki-light:#E36209;--shiki-light-font-style:inherit;--shiki-cyber:#4BF3C8;--shiki-cyber-font-style:italic}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html.cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html pre.shiki code .slEfO, html code.shiki .slEfO{--shiki-default:#F47067;--shiki-default-font-style:inherit;--shiki-light:#D73A49;--shiki-light-font-style:inherit;--shiki-cyber:#54B9FF;--shiki-cyber-font-style:italic}html pre.shiki code .sRGqv, html code.shiki .sRGqv{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#ACAFFF}",{"title":1578,"searchDepth":2751,"depth":2994,"links":2995},3,[2996,2997,2998,2999,3000,3001,3002,3003,3004,3005],{"id":41,"depth":2751,"text":41},{"id":763,"depth":2751,"text":764},{"id":1560,"depth":2751,"text":1561},{"id":1853,"depth":2751,"text":1854},{"id":1866,"depth":2751,"text":1867},{"id":2264,"depth":2751,"text":2265},{"id":2809,"depth":2751,"text":2810},{"id":2828,"depth":2751,"text":2828},{"id":2908,"depth":2751,"text":2908},{"id":2942,"depth":2751,"text":2942},[3007],"学习笔记","\u002Fimages\u002Fposts\u002Fcs336-rmsnorm-swiglu\u002Fcs336-rmsnorm-swiglu-cover.webp","特征尺度整理和双分支门控的 AI 辅助概念插画","2026-10-16","用一个 token 向量理解均方根归一化，拆开 dim=-1 与 keepdim=True 的作用，再沿着两条分支读懂 SiLU、逐元素乘法和 SwiGLU。",false,"md",{"codeUrl":2914,"attachmentUrl":3015},"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-02-rmsnorm-swiglu-20261016.zip",true,"\u002Fposts\u002Fcs336-rmsnorm-swiglu",null,{"title":5,"description":3011},"Stanford CS336","posts\u002Fcs336-rmsnorm-swiglu",[3023,3024,3025,2857,2366],"CS336","PyTorch","Transformer","oVsWQgFrOBMOeCwPcac81EOpzUCAhHIawqsYHaDwLE0",[3028,3044,3060,3075,3089,3103,3118,3133,3147,3160,3175,3188,3202,3214,3218,3231,3248,3263,3274,3286,3301,3314,3328,3340,3354,3366,3381,3395,3409,3420,3433,3450,3465,3478,3490,3502,3515,3528],{"slug":3029,"path":3030,"title":3031,"date":3032,"tags":3033,"description":3037,"cover":3038,"coverAlt":3039,"excerpt":3040,"draft":3012,"hidden":3012,"published":3016,"readingTime":3041,"series":3020,"seriesOrder":3041,"codeUrl":3042,"attachmentUrl":3043},"cs336-cross-entropy-adamw-training-loop","\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop","Stanford CS336 学习记录 08：数值稳定交叉熵、AdamW、余弦调度与完整训练闭环","2026-10-27",[3023,3024,3034,3035,3036],"AdamW","CrossEntropy","训练闭环","用 LogSumExp 消去交叉熵中的指数溢出，手写解耦权重衰减的 AdamW 优化器、全局 L2 梯度裁剪、Warmup 余弦退火调度、memmap 数据采样、Checkpoint 序列化与 Top-p 解码，完成 CS336 Assignment 1 全部 48 项测试。","\u002Fimages\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop\u002Fcs336-cross-entropy-adamw-training-loop-cover.webp","滑动窗口采样、余弦退火曲线、双动量优化轨迹与检查点归档构成的训练闭环 AI 辅助概念插画","作为 Stanford CS336 Assignment 1 系列的收官篇，我们已经拥有了分词器（bpe.py、tokenizer.py）和完整的语言模型架构（transformer.py）。要让模型在语料上真正训练并生成文本，还需",8,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fb5c93ad0cc73178b2b2a62e518a0900da47a0fb7","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-08-adamw-training-loop-20261027.zip",{"slug":3045,"path":3046,"title":3047,"date":3048,"tags":3049,"description":3056,"cover":3057,"coverAlt":3058,"excerpt":3059,"draft":3012,"hidden":3012,"published":3016,"readingTime":3041},"editcache-video-dit-inference-acceleration","\u002Fposts\u002Feditcache-video-dit-inference-acceleration","EditCache：视频 Diffusion Transformer (DiT) 局部编辑的跨步特征复用与区域重算加速","2026-10-25",[3050,3051,3052,3053,3054,3055],"扩散模型","视频生成","AI Infra","推理加速","GPU","论文解读","拆解视频扩散模型（Wan2.1-VACE）局部编辑中的三重算力冗余，通过可重入 Latent、跨步 Block 隐状态缓存复用、带 Halo 边界的局部区域重算与自适应推理起点，在单卡 H100 上实现 2.46× 模型计算加速与 1.49× 严格同步端到端加速。","\u002Fimages\u002Fposts\u002Feditcache-video-dit-inference-acceleration\u002Feditcache-dit-cover-20261025.webp","视频扩散 Transformer 时空三维体素网格中冻结背景缓存与暖橙色局部编辑重算通道的等距技术插画","在基于 Diffusion Transformer（DiT）的视频编辑与局部重绘（Video Inpainting \u002F Local Editing）任务中，用户的典型工作流并不是一次性生成最终成片，而是针对同一段源视频反复调整掩码区",{"slug":3061,"path":3062,"title":3063,"date":3048,"tags":3064,"description":3069,"cover":3070,"coverAlt":3071,"excerpt":3072,"draft":3012,"hidden":3012,"published":3016,"readingTime":3041,"series":3020,"seriesOrder":3073,"codeUrl":3042,"attachmentUrl":3074},"cs336-bpe-tokenizer-training-encoding","\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding","Stanford CS336 学习记录 07：Byte-Level BPE 分词器训练、倒排索引加速与流式编码",[3023,3065,3066,3067,3068],"Tokenizer","BPE","Tiktoken","算法优化","实现 GPT-2 正则预分词、特殊 token 硬边界切分、基于 pair-to-word 倒排索引的增量 BPE 合并算法（\u003C 0.25s 通过速度基准），以及与 tiktoken 完全对齐的 Tokenizer。","\u002Fimages\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding\u002Fcs336-bpe-tokenizer-training-encoding-cover.webp","UTF-8 字节块经过正则切分闸口后沿合并树聚合成子词单元的 AI 辅助概念插画","前面六篇完成了 Transformer 语言模型本身的全部神经网络组件。要让模型真正处理自然语言文本，还需要打通入口处的 ByteLevel BPE（BytePair Encoding）分词器。这一篇记录 cs336basics\u002Fb",7,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-07-bpe-tokenizer-20261025.zip",{"slug":3076,"path":3077,"title":3078,"date":3079,"tags":3080,"description":3085,"cover":3086,"coverAlt":3087,"excerpt":3088,"draft":3012,"hidden":3012,"published":3016,"readingTime":3041},"in-situ-autoguidance-editable-scientific-illustration","\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration","从原位自校正引导（In-situ Autoguidance）到可编辑科研插图生成（AutoFigure-Edit）","2026-10-24",[3050,3081,3082,3083,3084,3055],"多模态","Agent","ICML","ACL","串联两项生成式模型与多模态智能体研究：如何通过推理期局部随机微扰在单模型内构造免训练扩散自引导向量（ICML 2025 Workshop 第一作者，arXiv:2510.17136），以及如何通过五阶段视觉-符号混合流水线生成原生可编辑 SVG 科研插图（ACL 2026 Demo，FigureBench 83.0% 胜率）。","\u002Fimages\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration\u002Fautoguidance-autofigure-cover-20261024.webp","左侧扩散流形自校正轨迹矢量场与右侧五阶段可编辑 SVG 科研插图分层网格相衔接的等距技术插画","生成式视觉模型正在经历从“生成高保真像素（Pixel Synthesis）”向“可控结构化生成（Controllable & Editable Generation）”的演进。在这一进程中，我作为第一作者或核心贡献者参与了两项分别处",{"slug":3090,"path":3091,"title":3092,"date":3093,"tags":3094,"description":3097,"cover":3098,"coverAlt":3099,"excerpt":3100,"draft":3012,"hidden":3012,"published":3016,"readingTime":3041,"series":3020,"seriesOrder":3101,"codeUrl":3042,"attachmentUrl":3102},"cs336-pre-norm-transformer-block-lm","\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm","Stanford CS336 学习记录 06：Pre-Norm Transformer Block 与完整语言模型装配","2026-10-23",[3023,3024,3025,3095,3096],"Pre-Norm","FLOPs","将 RMSNorm、带 RoPE 的因果多头自注意力和 SwiGLU 组装成 Pre-Norm Transformer Block，搭建完整 TransformerLM，并核算 GPT-2 XL 的参数量与前向 FLOPs。","\u002Fimages\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm\u002Fcs336-pre-norm-transformer-block-lm-cover.webp","干净残差流主干穿过多个 Pre-Norm 子层并汇聚到词表投影层的 AI 辅助概念插画","前五篇把 Linear、Embedding、RMSNorm、SwiGLU、RotaryPositionalEmbedding 和因果 MultiHeadSelfAttention 全部单独打通了。这一篇进入 Assignment 1",6,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-06-transformer-block-lm-20261023.zip",{"slug":3104,"path":3105,"title":3106,"date":3093,"tags":3107,"description":3113,"cover":3114,"coverAlt":3115,"excerpt":3116,"draft":3012,"hidden":3012,"published":3016,"readingTime":3117},"xgrammar-concurrent-cache-generation-fix","\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix","XGrammar 结构化解码缓存剖析：Unlimited 模式并发 Clear 字节记账竞态与代际修复（PR #1011）",[3052,3108,3109,3110,3111,3112],"C++","XGrammar","结构化解码","并发编程","源码分析","拆解 XGrammar GrammarCompiler 的双层缓存架构与 ThreadSafeLRUCache 锁分离实现，定位 Unlimited 模式下 Clear() 字节残留与在途编译（In-flight Computation）跨代污染根因，给出代际号 + 读写锁原子修复与零 sleep 确定性并发 GoogleTest 验证。","\u002Fimages\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix\u002Fxgrammar-cache-fix-cover-20261022.webp","双层磨砂玻璃流道经暖橙色代际闸门过滤后汇入晶格缓存矩阵的等距技术插画","在大语言模型服务（如 SGLang、vLLM）的结构化输出场景中，JSON Schema 或 EBNF 语法需要先编译为词表级掩码缓存（CompiledGrammar），才能在每个 Decode 步以微秒级开销生成合法 token ",14,{"slug":3119,"path":3120,"title":3121,"date":3122,"tags":3123,"description":3129,"cover":3130,"coverAlt":3131,"excerpt":3132,"draft":3012,"hidden":3012,"published":3016,"readingTime":3117},"multi-constraint-agent-routing-dag-eval","\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval","多约束生产级 Agent 系统实践：两阶段工具路由、DAG 局部重规划与轨迹级评测闭环","2026-10-22",[3082,3124,3125,3126,3127,3128],"LLM","工具调用","上下文工程","DPO","强化学习","围绕跨境酒旅等强耦合多约束业务场景，拆解向量 Top-K 召回 + LLM 精排 + 跨语言实体链接与 Schema 门控（96%+ 调用合法率）、基于状态机的 DAG 局部子图重规划、Observation 冗余裁剪（节省 40% Token），以及从线上失败恢复轨迹构造 SFT\u002FDPO 偏好对的评测与训练闭环。","\u002Fimages\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval\u002Fagent-dag-routing-cover-20261020.webp","候选模块经漏斗滤网进入双分支有向无环图轨道并在受阻节点触发暖橙色局部旁路重连的等距技术插画","在玩具级 Demo 里，几个工具加上一段 ReAct 循环就能跑通大多数单步问答。但当 Agent 进入跨境酒旅、复杂交易或研发沙箱等真实业务生产环境时，单靠线性 ReAct 循环很快会撞上四堵墙： 1. 工具空间与参数耦合爆炸：候",{"slug":3134,"path":3135,"title":3136,"date":3137,"tags":3138,"description":3142,"cover":3143,"coverAlt":3144,"excerpt":3145,"draft":3012,"hidden":3012,"published":3016,"readingTime":3146},"llm-mrd-distillation-auditable-review-agents","\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents","多视角推理蒸馏（LLM-MRD）与可审计生成式评审系统（DeepReviewer 2.0）深度拆解","2026-10-21",[3139,3081,3140,3082,3141,3055],"知识蒸馏","Qwen2.5-VL","Post-training","对比大模型落地的两条互补路径：如何用 Qwen2.5-VL 离线多视角分析结合 MLP 残差校准与温度缩放 KL 蒸馏，将推理能力压入 358M 学生模型实现 17.2 ms 独立推理（LLM-MRD，DASFAA 2026 Oral）；以及如何通过主张-证据-风险台账、PASA 可比性门控与导出预算校验构建可审计的同行评审 Agent（DeepReviewer 2.0）。","\u002Fimages\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents\u002Fmrd-review-distill-cover-20261016.webp","三块半透明几何棱镜发出平行光束穿过中央校准环汇聚至右侧高密度晶体模块的等距技术插画","把大语言模型或视觉语言模型（VLM）的深度推理能力用到实际系统中，通常会面临一个两难选择： 如果场景受限于毫秒级在线延迟（如高吞吐多模态内容审核与虚假信息检测）：直接在线调用 7B～72B 的多模态大模型做长链条思维链（CoT）生成",18,{"slug":3148,"path":3149,"title":3150,"date":3137,"tags":3151,"description":3154,"cover":3155,"coverAlt":3156,"excerpt":3157,"draft":3012,"hidden":3012,"published":3016,"readingTime":3073,"series":3020,"seriesOrder":3158,"codeUrl":3042,"attachmentUrl":3159},"cs336-rope-rotary-position-embedding","\u002Fposts\u002Fcs336-rope-rotary-position-embedding","Stanford CS336 学习记录 05：RoPE 旋转位置编码与带位置的多头注意力",[3023,3024,3025,3152,3153],"RoPE","Attention","从二维平面正交旋转推导 RoPE 的奇偶特征配对公式，理解为什么内积只依赖相对位置差 (i - j)，并用 register_buffer 预计算 cos\u002Fsin 缓存接入因果多头自注意力。","\u002Fimages\u002Fposts\u002Fcs336-rope-rotary-position-embedding\u002Fcs336-rope-rotary-position-embedding-cover.webp","二维特征平面中的旋转向量对与预计算余弦正弦网格的 AI 辅助概念插画","上一篇完成了不含位置编码的因果多头自注意力。如果只看那一套矩阵乘法，把输入序列前两个 token 交换顺序（并在因果 mask 允许的范围内看同一个后缀 query 对它们的点积），点积本身并不知道哪个 key 离当前 query ",5,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-05-rope-mha-20261021.zip",{"slug":3161,"path":3162,"title":3163,"date":3164,"tags":3165,"description":3171,"cover":3172,"coverAlt":3173,"excerpt":3174,"draft":3012,"hidden":3012,"published":3016,"readingTime":3041},"llm-gateway-semantic-cache-dynamic-batching","\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching","高并发 LLM 推理网关架构实践：两级精确\u002F语义缓存、50ms 有界动态批处理与尾部延迟治理","2026-10-19",[3052,3166,3167,3168,3169,3170],"LLM 推理","分布式系统","Redis","FastAPI","后端","深度拆解生产级大模型推理网关 LLM-Gateway 的核心系统设计：基于 SHA-256 与 Redis 向量余弦相似度的两级精确\u002F语义缓存、50ms 有界等待动态批处理调度器（Dynamic Batching）、令牌桶多租户限流与 Prometheus 全链路可观测性，将命中请求 P99 延迟从 5,307 ms 压降至 17 ms。","\u002Fimages\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching\u002Fllm-gateway-cover-20261019.webp","高并发请求流依次穿过精确哈希滤网、向量球面语义缓存层与 50ms 动态批处理时间窗拼装器的等距系统架构插画","在将大语言模型接入高并发业务系统时，如果让上游微服务直接裸连底层推理引擎（vLLM \u002F SGLang 或外部模型 API），系统很快会暴露出三个典型的生产痛点： 1. 高频重复与近义查询消耗巨量算力：大量用户请求在语义上高度等价（例",{"slug":3176,"path":3177,"title":3178,"date":3164,"tags":3179,"description":3181,"cover":3182,"coverAlt":3183,"excerpt":3184,"draft":3012,"hidden":3012,"published":3016,"readingTime":3117,"series":3020,"seriesOrder":3185,"codeUrl":3186,"attachmentUrl":3187},"cs336-multihead-attention-shapes","\u002Fposts\u002Fcs336-multihead-attention-shapes","Stanford CS336 学习记录 04：多头注意力怎样拆头、计算和合头",[3023,3024,3025,3153,3180],"张量形状","沿着 (2, 5, 64) 的输入追踪 QKV 投影、reshape、transpose、共享因果 mask 和合头，理解每个头仍处理完整序列，以及最后为什么还需要输出 Linear。","\u002Fimages\u002Fposts\u002Fcs336-multihead-attention-shapes\u002Fcs336-multihead-attention-shapes-cover.webp","宽向量束分成多条并行特征轨道后重新合并的 AI 辅助概念插画","上一篇完成了缩放点积注意力。那个函数接收已经准备好的 Q、K、V，计算匹配分数，应用 mask，再用权重汇总 V。这一篇继续把它放进一个带参数的多头自注意力模块。 这次最需要看清的是“拆头”。代码里只是几行 reshape 和 tr",4,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002F31363223498fafd1528c550a51efbe0aed231736","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-04-multihead-attention-20261019.zip",{"slug":3189,"path":3190,"title":3191,"date":3192,"tags":3193,"description":3197,"cover":3198,"coverAlt":3199,"excerpt":3200,"draft":3012,"hidden":3012,"published":3016,"readingTime":3201},"qwen25-sft-dpo-alignment-tax-ablation","\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation","大模型后训练偏好对齐实录：Qwen2.5 SFT 多维消融、动态参考 DPO 与对齐税（Alignment Tax）量化分析","2026-10-18",[3141,3127,3194,3195,3128,3196],"SFT","LoRA","Qwen2.5","基于 Qwen2.5 完整复现 Instruct SFT → Bradley-Terry DPO 两阶段后训练管线：通过 3 随机种子消融 LoRA Rank、学习率与数据规模（GSM8K +12.73pp），实现免双倍权重的动态 Adapter 切换 DPO 训练，并定量揭示偏好对齐对确定性数学推理的对齐税效应。","\u002Fimages\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation\u002Fqwen25-sft-dpo-cover-20261018.webp","左侧监督微调轨迹流经中央 Bradley-Terry 偏好分流天平分离出正负样本通道并连接右侧对齐税监控仪表盘的等距技术插画","在大语言模型后训练（PostTraining）管线中，监督微调（SFT）负责教会模型任务格式与领域推理模式，而直接偏好优化（DPO, Direct Preference Optimization）则利用成对偏好数据 $(x, yw,",9,{"slug":3203,"path":3204,"title":3205,"date":3192,"tags":3206,"description":3208,"cover":3209,"coverAlt":3210,"excerpt":3211,"draft":3012,"hidden":3012,"published":3016,"readingTime":3212,"series":3020,"seriesOrder":2994,"codeUrl":2914,"attachmentUrl":3213},"cs336-softmax-attention-mask","\u002Fposts\u002Fcs336-softmax-attention-mask","Stanford CS336 学习记录 03：Softmax 的数值稳定与 Attention Mask",[3023,3024,3025,3207,3153],"Softmax","从减去最大值为何不改变 Softmax 开始，理清函数与有参数层的区别，再沿着 Q、K、V 的形状理解缩放点积注意力，以及 mask 为什么必须填负无穷。","\u002Fimages\u002Fposts\u002Fcs336-softmax-attention-mask\u002Fcs336-softmax-attention-mask-cover.webp","注意力分数、权重与因果遮罩的 AI 辅助概念插画","上一篇写到 SwiGLU，接下来进入 Attention。真正开始相乘之前，先做了 Softmax：它把一组分数变成总和为 1 的权重，后面用这些权重汇总不同 token 提供的信息。 这段学习里，有两个很具体的问题值得记下来。一个",11,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-03-softmax-sdpa-20261018.zip",{"slug":3215,"path":3017,"title":5,"date":3010,"tags":3216,"description":3011,"cover":3008,"coverAlt":3009,"excerpt":3217,"draft":3012,"hidden":3012,"published":3016,"readingTime":3201,"series":3020,"seriesOrder":2751,"codeUrl":2914,"attachmentUrl":3015},"cs336-rmsnorm-swiglu",[3023,3024,3025,2857,2366],"上一篇把 Linear 和 Embedding 的参数形状理顺了。继续往下，最先遇到的是 RMSNorm：公式不算长，但把它落到一批 token 上时，dim=1、keepdim=True 和广播必须一起理解。 然后是 SiLU 和",{"slug":3219,"path":3220,"title":3221,"date":3222,"tags":3223,"description":3225,"cover":3226,"coverAlt":3227,"excerpt":3228,"draft":3012,"hidden":3012,"published":3016,"readingTime":3229,"series":3020,"seriesOrder":1584,"codeUrl":2914,"attachmentUrl":3230},"cs336-linear-embedding-shapes","\u002Fposts\u002Fcs336-linear-embedding-shapes","Stanford CS336 学习记录 01：从 Linear、Embedding 搞清张量形状","2026-10-14",[3023,3024,3025,2867,3224],"Embedding","从 PyTorch 类、参数与 forward 的关系开始，理解 Linear 为什么保存两行三列的权重、计算时为什么要转置，再区分 Embedding 查表与线性变换。","\u002Fimages\u002Fposts\u002Fcs336-linear-embedding-shapes\u002Fcs336-linear-embedding-shapes-cover.webp","token 方块、向量表与矩阵变换的 AI 辅助概念插画","开始沿着 Stanford CS336 的 Assignment 1 一步步实现。BPE 之前已经接触过，这轮先跳过分词器，从 Transformer 的基础层开始。这里记录的是当时写代码时真正卡住的问题，以及对应测试能确认到的范围",10,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-01-linear-embedding-20261014.zip",{"slug":3232,"path":3233,"title":3234,"date":3235,"tags":3236,"description":3241,"cover":3242,"coverAlt":3243,"excerpt":3244,"draft":3012,"hidden":3012,"published":3016,"readingTime":3245,"series":3237,"seriesOrder":3117,"codeUrl":3246,"attachmentUrl":3247},"cmu-15-779-kv-cache-granularity-study","\u002Fposts\u002Fcmu-15-779-kv-cache-granularity-study","CMU 15-779 学习记录 14：KV 页大小、共享边界与 CPU 实现成本","2026-10-12",[3237,3238,3239,3240],"CMU 15-779","KV cache","PagedAttention","系统实验","用 210 组 CPU 配置比较连续与分页 KV cache，观察 31\u002F32-token 共享前缀的页边界，分别记录活跃槽位、池容量、复制计数和完整调用耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling\u002Fcmu-serving-cover-20261019.webp","浅色手绘分页存储抽屉与共享前缀卡片，旁边连接不同长度的请求队列","个人课程项目中的 CPU 实验记录，延续记录 11 的分页 KV 实现。已完成 210 组配置与 1470 次计时，输出均通过连续存储参考对照；本篇不代表正式作业提交、整门课程或项目全部交付完成。 同样是共享 31 个 token，",12,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F200faf569eccf94fb125e65110bb5d14094d454a","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fkv-cache-granularity-20261019.zip",{"slug":3249,"path":3250,"title":3251,"date":3252,"tags":3253,"description":3256,"cover":3257,"coverAlt":3258,"excerpt":3259,"draft":3012,"hidden":3012,"published":3016,"readingTime":3158,"series":3237,"seriesOrder":3260,"codeUrl":3261,"attachmentUrl":3262},"cmu-15-779-moe-routing-capacity","\u002Fposts\u002Fcmu-15-779-moe-routing-capacity","CMU 15-779 学习记录 13：MoE 路由、稀疏分发与容量溢出","2026-10-09",[3237,3254,3255],"MoE","Routing","用 CPU top-2 MoE 检查稀疏分发的输出与梯度，统计 expert-token 行数，并保留容量丢弃会改变输出的反例。","\u002Fimages\u002Fposts\u002Fcmu-15-779-moe-routing-capacity\u002Fcmu-moe-cover-20261019.webp","浅色手绘分拣器将小珠分配到四个专家工作屋，队列呈现负载不均","第 18 讲的个人配套实验。36 号脚本串行运行 CPU 专家，检查 topk 路由后的分发与合并；没有训练 router，也没有执行 GPU grouped GEMM 或跨设备 alltoall。 上一节记录低秩适配，这次转向每个",13,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fc4851e0cedadf13af7c421867e83c8ff57244fa0","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-parallel-serving-lora-moe-20261019.zip",{"slug":3264,"path":3265,"title":3266,"date":3267,"tags":3268,"description":3270,"cover":3271,"coverAlt":3272,"excerpt":3273,"draft":3012,"hidden":3012,"published":3016,"readingTime":3158,"series":3237,"seriesOrder":3245,"codeUrl":3261,"attachmentUrl":3262},"cmu-15-779-lora-uniform-quantization","\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization","CMU 15-779 学习记录 12：LoRA 首步梯度、权重合并与均匀 4-bit 打包","2026-10-06",[3237,3195,3269],"Quantization","用一次 CPU SGD 更新检查 LoRA 初始化、冻结基座与合并等价，再分别记录均匀 4-bit 存储字节数和量化误差。","\u002Fimages\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization\u002Fcmu-lora-cover-20261019.webp","浅色手绘的大计算板、两块细长适配板和成对打包的小方块","第 17 讲的个人配套实验，35 号脚本实际执行一次 CPU LoRA 更新，并实现均匀 4bit 打包。实验检查机制与数值误差，不评估微调任务效果，也没有实现完整 QLoRA。 这次把两个问题分开记录：低秩分支是否能合并进原权重，",{"slug":3275,"path":3276,"title":3277,"date":3278,"tags":3279,"description":3283,"cover":3242,"coverAlt":3284,"excerpt":3285,"draft":3012,"hidden":3012,"published":3016,"readingTime":3073,"series":3237,"seriesOrder":3212,"codeUrl":3261,"attachmentUrl":3262},"cmu-15-779-kv-serving-speculative-sampling","\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling","CMU 15-779 学习记录 11：分页 KV、连续批处理与推测采样","2026-10-03",[3237,3280,3281,3282],"LLM Serving","KV Cache","Speculative Decoding","在 CPU 上验证 KV 页共享和写时复制，用单位成本模型检查连续批处理，再用条件概率小模型核对推测采样的接受与补采样。","浅色手绘的共享卡片抽屉与不断补入小珠子的请求托盘","第 15、16 讲的个人配套实验。33 号实现 CPU 分页 KV 存储，37 号模拟请求准入，34 号运行小概率模型的推测采样。它们分别验证存储、调度和分布问题，没有测量真实 LLM 服务吞吐或 GPU 延迟。 记录 08已经对照",{"slug":3287,"path":3288,"title":3289,"date":3290,"tags":3291,"description":3296,"cover":3297,"coverAlt":3298,"excerpt":3299,"draft":3012,"hidden":3012,"published":3016,"readingTime":3300},"qwen3-mid-post-train-mlx","\u002Fposts\u002Fqwen3-mid-post-train-mlx","在 M2 Pro 上完成 Qwen3 两阶段训练：Mid Train、SFT 与可复核评估","2026-09-30 11:19:10",[3124,3292,3293,3195,3294,3194,3295],"Qwen3","MLX","持续预训练","Apple Silicon","用 Qwen3 0.6B Base 和 MLX 完成领域持续训练与监督微调，记录数据划分、聊天模板、损失掩码、数值稳定性修复和独立测试结果。","\u002Fimages\u002Fposts\u002Fqwen3-mid-post-train-mlx\u002Fqwen3-mlx-cover-20261018.webp","Apple Silicon 统一内存芯片底座支撑两阶段持续预训练与带精确 Loss 掩码滤网的监督微调桥接插画","在一台 M2 Pro、16 GB 内存的 Mac 上，能否把一个 Base 语言模型接续训练成输出 JSON 的运维事件分级助手？这次项目完成了领域持续训练、监督微调、检查点选择、重新加载和独立测试，并保留了权重、日志和数据哈希。 ",16,{"slug":3302,"path":3303,"title":3304,"date":3305,"tags":3306,"description":3310,"cover":3311,"coverAlt":3312,"excerpt":3313,"draft":3012,"hidden":3012,"published":3016,"readingTime":3101,"series":3237,"seriesOrder":3229,"codeUrl":3261,"attachmentUrl":3262},"cmu-15-779-cpu-parallelism-pipeline","\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline","CMU 15-779 学习记录 10：加权梯度、状态分片与同步流水线","2026-09-30",[3237,3307,3308,3309],"Parallel Training","Gloo","Pipeline","用双 CPU 进程 Gloo 检查不等数据分片的梯度，推导持久状态字节数，再用依赖模型比较 GPipe 与同步 1F1B 的激活存活。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline\u002Fcmu-parallel-cover-20261019.webp","浅色手绘的四阶段桌面流水线与按数量汇总的托盘","第 12 至 14 讲的个人配套实验，脚本编号 31、32 不对应官方作业。Gloo 实际运行两个 CPU 进程；张量切分、状态字节数和流水线分别按下文的数学或成本模型解释。真实 GPU 实测按学习安排跳过。 上一节检查单个计算图的",{"slug":3315,"path":3316,"title":3317,"date":3318,"tags":3319,"description":3322,"cover":3323,"coverAlt":3324,"excerpt":3325,"draft":3012,"hidden":3012,"published":3016,"readingTime":3073,"series":3237,"seriesOrder":3201,"codeUrl":3326,"attachmentUrl":3327},"cmu-15-779-cpu-compiler-optimizations","\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations","CMU 15-779 学习记录 09：CPU 编译优化、合法重写与候选搜索","2026-09-27",[3237,3320,3321,3024],"ML Compiler","TorchInductor","用 CPU 实验验证 BatchNorm 折叠、RMSNorm 重排与非线性反例，测量分块候选，并运行 TorchInductor，分别记录首调用与预热后的时间。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations\u002Fcmu-cpu-compiler-cover-20261019.webp","淡紫计算积木经过薄荷绿折叠模具变成浅蓝积木，前方摆放四种候选托盘的浅色手绘插画","编译章节的个人配套实验，编号 30 不对应官方作业。图重写、分块候选和 TorchInductor 均在 CPU 上运行；真实 GPU 实验按学习安排跳过。 上一节记录了分块 Attention 和 KV cache。这次转到编译优","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F9261bea6aedf31cdccbbab2541ca693fc97b3743","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-compiler-optimizations-20261019.zip",{"slug":3329,"path":3330,"title":3331,"date":3332,"tags":3333,"description":3335,"cover":3336,"coverAlt":3337,"excerpt":3338,"draft":3012,"hidden":3012,"published":3016,"readingTime":3245,"series":3237,"seriesOrder":3041,"codeUrl":3246,"attachmentUrl":3339},"cmu-15-779-attention-online-softmax","\u002Fposts\u002Fcmu-15-779-attention-online-softmax","CMU 15-779 学习记录 08：分块 Attention、在线 Softmax 与 KV cache","2026-09-24",[3237,3153,3207,3334,3024],"FlashAttention","在 CPU 上对照完整 Attention、在线 Softmax 分块实现与 PyTorch SDPA，检查因果掩码、浮点误差和分数块大小，理解 FlashAttention 的一部分算法基础。","\u002Fimages\u002Fposts\u002Fcmu-15-779-attention-online-softmax\u002Fcmu-online-attention-cover-20261016.webp","浅紫色卡片分批进入绿色计算托盘，旁边三个状态容器保存累积量的浅色手绘插画","第 5 讲 Transformers & Attention 的个人配套实验。28\u002F29 号是 CPU 单头前向与 KV cache；后续补入 38 号 CUDA 多头合成张量的显式 SDPA MATH\u002FFLASH 对照。没有手写 ","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcuda-triton-sdpa-20261019.zip",{"slug":3341,"path":3342,"title":3343,"date":3344,"tags":3345,"description":3350,"cover":3351,"coverAlt":3352,"excerpt":3353,"draft":3012,"hidden":3012,"published":3016,"readingTime":3229,"series":3237,"seriesOrder":3073,"codeUrl":3246,"attachmentUrl":3339},"cmu-15-779-matmul-tiling-tree-reduction","\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction","CMU 15-779 学习记录 07：矩阵分块、数据复用与树形归约","2026-09-21",[3237,3346,3347,3348,3349],"CUDA","矩阵乘法","数据复用","归约","用 CPU 模型统计矩阵分块的输入读取次数，再比较串行求和和树形归约的依赖深度，记录补零、同步与浮点加法顺序的边界。","\u002Fimages\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction\u002Fcmu-matmul-reduction-cover-20261016.webp","矩阵积木在共享托盘中复用，旁边的小珠沿树形管道逐层合并的浅色手绘插画","GPU Architecture & CUDA Programming 的个人配套实验。首次记录用 CPU 模型检查数据复用与依赖深度；后续补入单卡 GPU 原生 CUDA 矩阵验证，以及 Triton 两阶段归约和矩阵乘法实测。 ",{"slug":3355,"path":3356,"title":3357,"date":3358,"tags":3359,"description":3362,"cover":3363,"coverAlt":3364,"excerpt":3365,"draft":3012,"hidden":3012,"published":3016,"readingTime":3229,"series":3237,"seriesOrder":3101,"codeUrl":3246,"attachmentUrl":3339},"cmu-15-779-cuda-thread-indexing","\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing","CMU 15-779 学习记录 06：CUDA 线程索引与边界检查","2026-09-18",[3237,3346,3054,3360,3361],"线程索引","边界检查","用 CPU 演示 CUDA 的 block、thread 与全局下标：10 个元素如何分给 3 个线程块，为什么要跳过越界线程，以及这些验证能说明什么。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing\u002Fcmu-cuda-indexing-cover-20261015.webp","分组的浅紫色计算方块与末端边界挡板组成线程分配教具的浅色插画","GPU Architecture & CUDA Programming 的个人配套练习。首次记录为 CPU 索引演示；后续在单卡 GPU 环境补入原生 CUDA 编译、运行与 memcheck，以及 Triton 向量加法实测。 上",{"slug":3367,"path":3368,"title":3369,"date":3370,"tags":3371,"description":3375,"cover":3376,"coverAlt":3377,"excerpt":3378,"draft":3012,"hidden":3012,"published":3016,"readingTime":3229,"series":3237,"seriesOrder":3158,"codeUrl":3379,"attachmentUrl":3380},"cmu-15-779-profiler-operator-batching","\u002Fposts\u002Fcmu-15-779-profiler-operator-batching","CMU 15-779 学习记录 05：Profiler、算子调用与批量推理","2026-09-15",[3237,3024,3372,3373,3374],"Profiler","批量推理","性能分析","对照 ML Systems 基础，用 Profiler 拆开两层网络的执行过程，再固定 1000 个样本扫描 batch size，观察算子调用次数、尾批和 CPU 耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-profiler-operator-batching\u002Fcmu-pytorch-profiling-cover-20261015.webp","放大镜观察线性层与 ReLU 模块，旁边面板呈现执行片段的浅色技术教具插画","本篇对照课程官网的 Introduction to ML Systems 整理，是个人配套 CPU 实验，不是课程官方作业。脚本编号是练习顺序，不对应课程讲次。 前几篇一直在补 PyTorch 基础。到了这一步，我在学习聊天里问：现","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fcecf5110d8a95161bb5763eadfca94cd7a0cc50f","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-profiler-batching-20261015.zip",{"slug":3382,"path":3383,"title":3384,"date":3385,"tags":3386,"description":3389,"cover":3390,"coverAlt":3391,"excerpt":3392,"draft":3012,"hidden":3012,"published":3016,"readingTime":3201,"series":3237,"seriesOrder":3185,"codeUrl":3393,"attachmentUrl":3394},"cmu-15-779-pytorch-linear-relu-absolute-value","\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value","CMU 15-779 学习记录 04：多输出线性层、ReLU 与绝对值网络","2026-09-12",[3237,3024,3347,3387,3388],"ReLU","神经网络","把 nn.Linear 对应到矩阵乘法，弄清多个输出怎样计算，再逐层拆开一个绝对值网络，比较手动设置参数与训练得到的结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value\u002Fcmu-pytorch-absolute-cover-20261015.webp","双分支经过折角模块后汇入紫色 V 形曲线面板的浅色技术教具插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的批量推理与矩阵乘法，我继续把 model(x) 拆开看：一层里有多个输出时，每个数是怎么算出来的？两","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F4cc63904a3a583da4a2122c246a14b43aac38528","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-linear-relu-abs-20261015.zip",{"slug":3396,"path":3397,"title":3398,"date":3399,"tags":3400,"description":3403,"cover":3404,"coverAlt":3405,"excerpt":3406,"draft":3012,"hidden":3012,"published":3016,"readingTime":3201,"series":3237,"seriesOrder":2994,"codeUrl":3407,"attachmentUrl":3408},"cmu-15-779-pytorch-dataloader-checkpoints-batching","\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching","CMU 15-779 学习记录 03：DataLoader、模型保存与批量推理","2026-09-09",[3237,3024,3401,3373,3402],"DataLoader","模型保存","用 DataLoader 分批训练平方函数，保存并加载模型参数，再比较 1000 个样本与 1 个样本的推理耗时，最后把批量计算对应到矩阵乘法。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching\u002Fcmu-pytorch-batching-cover-20261015.webp","样本卡片按批次进入计算模块，旁边配有参数存储盒和输出矩阵面板的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的平方函数实验，这一节继续练习分批训练、保存参数和独立预测。最后做了一次计时：同一个小网络，逐个处理 ","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-batching-20261015.zip",{"slug":3410,"path":3411,"title":3412,"date":3413,"tags":3414,"description":3415,"cover":3416,"coverAlt":3417,"excerpt":3418,"draft":3012,"hidden":3012,"published":3016,"readingTime":3201,"series":3237,"seriesOrder":2751,"codeUrl":3407,"attachmentUrl":3419},"cmu-15-779-pytorch-linear-module-relu","\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu","CMU 15-779 学习记录 02：nn.Linear、张量形状与 ReLU","2026-09-06",[3237,3024,3388,3387],"从带偏置的线性回归开始，弄清 nn.Linear 和 nn.Module 的分工，再用平方函数观察线性模型的限制、ReLU 的作用和测试点的选择。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu\u002Fcmu-pytorch-relu-cover-20261015.webp","输入经过两层神经网络和 ReLU 折角模块，输出曲线拟合结果的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 上一篇里，我用一个参数拟合了 $y=2x$。这一篇继续顺着往下写，先加上偏置，再把手写参数换成 nn.Linea","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-models-20261015.zip",{"slug":3421,"path":3422,"title":3423,"date":3424,"tags":3425,"description":3428,"cover":3429,"coverAlt":3430,"excerpt":3431,"draft":3012,"hidden":3012,"published":3016,"readingTime":3229,"series":3237,"seriesOrder":1584,"codeUrl":3407,"attachmentUrl":3432},"cmu-15-779-pytorch-autograd-linear-regression","\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression","CMU 15-779 学习记录 01：PyTorch 自动求导与线性回归","2026-09-03",[3237,3024,3426,3427],"自动求导","线性回归","从自动求导和手动梯度下降开始，比较学习率，再用 SGD 训练线性回归。记录 w * 4 写错后停在 1.25 的原因和修正结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression\u002Fcmu-pytorch-autograd-cover-20261015.webp","梯度曲面上的小球沿路径下降，前侧面板展示散点与拟合直线的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 开始学习 CMU 15779 时，我会 Python，但还没用过 PyTorch。这次先在 Mac 上用 CPU","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-basics-20261013.zip",{"slug":3434,"path":3435,"title":3436,"date":3437,"tags":3438,"description":3444,"cover":3445,"coverAlt":3446,"excerpt":3447,"draft":3012,"hidden":3012,"published":3016,"readingTime":3260,"codeUrl":3448,"attachmentUrl":3449},"embodied-pickplace-lab-robustness","\u002Fposts\u002Fembodied-pickplace-lab-robustness","Pickplace Lab 项目复盘：抓取成功率之外的时延与失败分析","2026-08-29",[3439,3440,3441,3024,3442,3443],"具身智能","模仿学习","MuJoCo","机器人","鲁棒性","从 MuJoCo 抓取搬运闭环出发，记录 22,100 个评测回合。动作平滑在自建任务的 50 ms 延迟下有效，换到 Panda Lift 却没有支持同样的收益。","\u002Fimages\u002Fposts\u002Fembodied-pickplace-lab-robustness\u002Fpickplace-grasp-cover-20261016.webp","MuJoCo 实际渲染的笛卡尔夹爪抓取方块场景，桌面上标出放置目标","Pickplace Lab 从一个简单任务开始：让夹爪抓起方块，搬到指定位置，放下，再退开。先把仿真、示范采集、训练和闭环评测接起来，随后再问：干净条件下成功的策略，换一种执行条件，还能不能完成任务？ 最值得讨论的结果出现在指令延迟","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Ftree\u002Fv0.1.0","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Freleases\u002Fdownload\u002Fv0.1.0\u002Fpickplace-lab-research-v1.zip",{"slug":3451,"path":3452,"title":3453,"date":3454,"tags":3455,"description":3460,"cover":3461,"coverAlt":3462,"excerpt":3463,"draft":3012,"hidden":3012,"published":3016,"readingTime":3464},"qwen25-vl-sglang-verl-single-gpu-lab","\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab","从 Mac 到单卡 GPU：Qwen2.5-VL 部署、GRPO 与计数 LoRA 实践","2026-08-19",[3140,3081,3456,3457,3458,3459,3293],"SGLang","verl","GRPO","推理优化","在 M2 Pro 和云端单卡上部署 Qwen2.5-VL-3B，实测连续批处理、前缀缓存与 SGLang 参数调优，再用 verl 完成多模态 GRPO 更新和同条件前后评测，记录有效优化、兼容性问题与未观察到提升的结果。","\u002Fimages\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab\u002Fqvl-lab-cover-20261010.webp","笔记本与单卡计算模块通过图像和 token 光轨连接的实验台插画","这次实验从一个很具体的问题开始：M2 Pro、16 GB 统一内存，能不能在本机跑 Qwen2.5VL？跑起来之后，我又想把图文问答、推理性能测量和强化学习训练接成一条完整流程。 本机的 MLX 4bit 服务和云端的 SGLang",29,{"slug":3466,"path":3467,"title":3468,"date":3469,"tags":3470,"description":3474,"cover":3475,"coverAlt":3476,"excerpt":3477,"draft":3012,"hidden":3012,"published":3016,"readingTime":3260},"cycleresearcher-post-training","\u002Fposts\u002Fcycleresearcher-post-training","从 DeepReviewer 2.0 到 CycleResearcher：科研 Agent 如何真正做 Post-training","2026-08-07 17:33:05",[3124,3141,3082,3471,3472,3473],"SimPO","Reward Model","ICLR","区分推理时 Agent 编排与模型后训练，并拆解 CycleResearcher 的领域 SFT、生成式 Reward Model、Iterative SimPO 和评测风险。","\u002Fimages\u002Fposts\u002Fcycleresearcher-post-training\u002Fcycleresearcher-cover-20261018.webp","策略生成模块产出多份候选文稿经多维评审棱镜排序后通过偏好梯度反馈弧闭环优化的等距技术插画","看到一个系统会检索、规划、反思和调用工具，不能据此判断它做过 Posttraining。Agent 描述的是模型在运行时如何完成任务；Posttraining 描述的是预训练之后，是否继续用监督数据、偏好数据或奖励信号更新模型参数。",{"slug":3479,"path":3480,"title":3481,"date":3482,"tags":3483,"description":3485,"cover":3486,"coverAlt":3487,"excerpt":3488,"draft":3012,"hidden":3012,"published":3016,"readingTime":3117,"series":3489,"seriesOrder":1584},"cuda-fused-rmsnorm-kernel","\u002Fposts\u002Fcuda-fused-rmsnorm-kernel","从 PyTorch 到 CUDA Kernel：融合 RMSNorm 在 RTX 4090D 上加速 4.27 倍","2026-07-22 18:16:50",[3346,3052,3054,3024,3166,3484],"Kernel 优化","从零实现 PyTorch CUDA 扩展，拆解 RMSNorm 的访存瓶颈、warp 归约、向量化加载、算子融合与冷热缓存 benchmark。","\u002Fimages\u002Fposts\u002Fcuda-fused-rmsnorm-kernel\u002Fcuda-rmsnorm-cover-20261018.webp","融合残差加法与向量化 float4 访存经 Warp Shuffle 蝴蝶树归约输出归一化张量的等距硬件架构插画","为了准备 AI Infra 岗位，我做了一个尽量小而完整的 CUDA 项目：不追求堆很多算子，而是选一个真实的 LLM 推理路径，把实现、正确性、benchmark 和性能解释全部做扎实。 项目实现了两个 PyTorch CUDA ","AI Infra 实战",{"slug":3491,"path":3492,"title":3493,"date":3494,"tags":3495,"description":3497,"cover":3498,"coverAlt":3499,"excerpt":3500,"draft":3012,"hidden":3012,"published":3016,"readingTime":3041,"series":3501,"seriesOrder":1584},"agent-memory","\u002Fposts\u002Fagent-memory","Agent 上下文工程与分层记忆架构：从 Observation 剪枝、滚动摘要到向量召回闭环","2026-04-24",[3124,3082,3496,3052],"RAG","结合生产级多约束 Agent 与超 100K Token 科研智能体实践，系统拆解长程会话中的上下文膨胀与注意力稀释问题：涵盖 Observation 结构化剪枝（降低约 40% Token 开销）、滚动工作摘要、向量检索长期记忆及 Prefix Cache 友好的上下文组装。","\u002Fimages\u002Fposts\u002Fagent-memory\u002Fagent-memory-cover-20261018.webp","底层高噪数据经剪枝滤网进入中期滑动窗口与金色状态台账并由顶层向量星座按需召回的三层记忆架构插画","在构建多步工具调用（ToolUse）与长周期自主规划（Autonomous Planning）的 Agent 系统时，上下文管理（Context Engineering）往往比单纯改写 System Prompt 更决定线上稳定性。","Agent 上下文工程",{"slug":3503,"path":3504,"title":3505,"date":3506,"tags":3507,"description":3510,"cover":3511,"coverAlt":3512,"excerpt":3513,"draft":3012,"hidden":3012,"published":3016,"readingTime":3158,"series":3514,"seriesOrder":2751},"backend-stack-deep-dive","\u002Fposts\u002Fbackend-stack-deep-dive","高并发 AI 平台后端基础设施实践：FastAPI 异步网关、SQLAlchemy 2.0 Async、PostgreSQL 与 Docker 沙箱编排","2026-04-07",[3170,3167,3169,3508,3509,3052],"PostgreSQL","Docker","结合 DeepScientist 生产级 AI 科研平台实践，拆解 FastAPI ASGI 异步事件循环、SQLAlchemy 2.0 + AsyncPG 连接池调优、PostgreSQL + MinIO 存储分层与 Docker SDK 动态沙箱容器编排的职责边界与核心设计。","\u002Fimages\u002Fposts\u002Fbackend-stack-deep-dive\u002Fbackend-stack-cover-20261018.webp","ASGI 异步事件循环涡轮将非阻塞协程流分发至 PostgreSQL 连接池、MinIO 存储矩阵与动态 Docker 沙箱的等距架构插画","结合 DeepScientist 生产级 AI 科研平台（服务 7,000+ 用户）的实际工程落地，本文系统拆解支撑高并发流式推理与容器沙箱调度的后端基础设施栈——FastAPI（ASGI 异步网关 + SSE 流式推送）、SQLA","DeepScientist 工程复盘",{"slug":3516,"path":3517,"title":3518,"date":3519,"tags":3520,"description":3523,"cover":3524,"coverAlt":3525,"excerpt":3526,"draft":3012,"hidden":3012,"published":3016,"readingTime":3212,"series":3527,"seriesOrder":1584},"minicode-source-analysis","\u002Fposts\u002Fminicode-source-analysis","MiniCode 架构与工具评测深度拆解：开源终端 Agentic Coding 引擎（1k+ Stars）的 ReAct、MCP 与 30 项基准验证","2026-04-06",[3082,3521,3522,3112,3052],"TypeScript","MCP","深度剖析开源终端 AI 编程智能体 MiniCode（1k+ GitHub Stars）的核心架构：从零重型依赖的 ReAct 规划循环、声明式工具注册表与权限拦截层、MCP 协议帧自动协商，到 30 项可执行 Tool-Use 评测基准（86.7% 完成率、100% 错误恢复率）的失败分布归因与闭环优化。","\u002Fimages\u002Fposts\u002Fminicode-source-analysis\u002Fminicode-cover-20261018.webp","终端编程智能体控制台被 ReAct 状态环、Unified Diff 补丁预览门与 MCP 协议插槽环绕的等距技术插画","MiniCode 是我作为核心开发者参与构建的开源终端 Agentic Coding 引擎（目前已在 GitHub 获得 1,000+ Stars）。项目的核心设计目标是：在不引入庞大运行时框架的前提下，用约 5,000 行高内聚的","AI 项目源码导读",{"slug":3529,"path":3530,"title":3531,"date":3519,"tags":3532,"description":3533,"cover":3534,"coverAlt":3535,"excerpt":3536,"draft":3012,"hidden":3012,"published":3016,"readingTime":3229,"series":3514,"seriesOrder":1584},"deepscientist-tech-stack","\u002Fposts\u002Fdeepscientist-tech-stack","DeepScientist 架构与运行时全解析：超 100K 上下文科研智能体、MCP 沙箱与高并发流式系统",[3082,3052,3522,3509,3169,3167],"深度拆解服务 7,000+ 用户的科研自主智能体平台 DeepScientist：从一题一仓库（Quest）持久化状态机、MCP 工具栈（memory \u002F artifact \u002F bash_exec）、Docker SDK 动态容器编排，到 SSE + WebSocket 双流通信与 PostgreSQL \u002F Redis \u002F Prometheus 生产级基础设施。","\u002Fimages\u002Fposts\u002Fdeepscientist-tech-stack\u002Fdeepscientist-cover-20261018.webp","科研智能体编排核心通过 memory、artifact 与隔离容器沙箱三座协议桥连接双流工作台的等距架构插画","DeepScientist（deepscientist.cc）是我们在西湖大学自然语言处理实验室研发的自主科学发现与科研管理平台，目前累计服务 7,000+ 科研用户。与单轮对话或纯文本文献问答工具不同，科研智能体需要在数小时乃至数",1795088946029]