[{"data":1,"prerenderedAt":4445},["ShallowReactive",2],{"post-\u002Fposts\u002Fcs336-softmax-attention-mask":3,"archive-public-posts":3936},{"id":4,"title":5,"body":6,"categories":3914,"cover":3916,"coverAlt":3917,"date":3918,"description":3919,"draft":3920,"extension":3921,"haloPublished":3920,"haloSync":3920,"hidden":3920,"legacy":3920,"meta":3922,"navigation":3924,"path":3925,"published":3920,"readingTime":3926,"seo":3927,"series":3928,"seriesOrder":67,"stem":3929,"tags":3930,"__hash__":3935},"posts\u002Fposts\u002Fcs336-softmax-attention-mask.md","Stanford CS336 学习记录 03：Softmax 的数值稳定与 Attention Mask",{"type":7,"value":8,"toc":3901},"minimark",[9,18,21,25,192,618,621,624,627,630,737,893,1887,1960,1971,2059,2066,2147,2165,2168,2172,2175,2178,2181,2257,2264,2268,2271,2647,2650,2653,2725,2971,2974,2977,2984,2990,2993,2996,3057,3068,3071,3146,3407,3410,3414,3428,3481,3484,3487,3585,3588,3591,3630,3639,3723,3727,3738,3757,3761,3782,3789,3792,3800,3803,3822,3834,3837,3897],[10,11,12,17],"p",{},[13,14,16],"a",{"href":15},"\u002Fposts\u002Fcs336-rmsnorm-swiglu\u002F","上一篇","写到 SwiGLU，接下来进入 Attention。真正开始相乘之前，先做了 Softmax：它把一组分数变成总和为 1 的权重，后面用这些权重汇总不同 token 提供的信息。",[10,19,20],{},"这段学习里，有两个很具体的问题值得记下来。一个是“前面都要创建 layer，为什么 Softmax 直接传输入就行”；另一个是 mask 中少写了一个负号。前者关系到代码组织，后者会直接改变被关注的位置。",[22,23,24],"h2",{"id":24},"从一组分数到一组权重",[10,26,27,28,122,123,191],{},"对分数 ",[29,30,33,76],"span",{"className":31},[32],"katex",[29,34,37],{"className":35},[36],"katex-mathml",[38,39,41],"math",{"xmlns":40},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[42,43,44,71],"semantics",{},[45,46,47,52,56,60,63,65,68],"mrow",{},[48,49,51],"mo",{"stretchy":50},"false","[",[53,54,55],"mn",{},"1",[48,57,59],{"separator":58},"true",",",[53,61,62],{},"2",[48,64,59],{"separator":58},[53,66,67],{},"3",[48,69,70],{"stretchy":50},"]",[72,73,75],"annotation",{"encoding":74},"application\u002Fx-tex","[1,2,3]",[29,77,80],{"className":78,"ariaHidden":58},[79],"katex-html",[29,81,84,89,93,97,101,106,109,112,115,118],{"className":82},[83],"base",[29,85],{"className":86,"style":88},[87],"strut","height:1em;vertical-align:-0.25em;",[29,90,51],{"className":91},[92],"mopen",[29,94,55],{"className":95},[96],"mord",[29,98,59],{"className":99},[100],"mpunct",[29,102],{"className":103,"style":105},[104],"mspace","margin-right:0.1667em;",[29,107,62],{"className":108},[96],[29,110,59],{"className":111},[100],[29,113],{"className":114,"style":105},[104],[29,116,67],{"className":117},[96],[29,119,70],{"className":120},[121],"mclose","，Softmax 的结果大约是 ",[29,124,126,155],{"className":125},[32],[29,127,129],{"className":128},[36],[38,130,131],{"xmlns":40},[42,132,133,152],{},[45,134,135,137,140,142,145,147,150],{},[48,136,51],{"stretchy":50},[53,138,139],{},"0.090",[48,141,59],{"separator":58},[53,143,144],{},"0.245",[48,146,59],{"separator":58},[53,148,149],{},"0.665",[48,151,70],{"stretchy":50},[72,153,154],{"encoding":74},"[0.090,0.245,0.665]",[29,156,158],{"className":157,"ariaHidden":58},[79],[29,159,161,164,167,170,173,176,179,182,185,188],{"className":160},[83],[29,162],{"className":163,"style":88},[87],[29,165,51],{"className":166},[92],[29,168,139],{"className":169},[96],[29,171,59],{"className":172},[100],[29,174],{"className":175,"style":105},[104],[29,177,144],{"className":178},[96],[29,180,59],{"className":181},[100],[29,183],{"className":184,"style":105},[104],[29,186,149],{"className":187},[96],[29,189,70],{"className":190},[121],"。",[29,193,196],{"className":194},[195],"katex-display",[29,197,199,265],{"className":198},[32],[29,200,202],{"className":201},[36],[38,203,205],{"xmlns":40,"display":204},"block",[42,206,207,262],{},[45,208,209,218,221,258],{},[210,211,212,215],"msub",{},[213,214,10],"mi",{},[213,216,217],{},"i",[48,219,220],{},"=",[222,223,224,237],"mfrac",{},[225,226,227,230],"msup",{},[213,228,229],{},"e",[210,231,232,235],{},[213,233,234],{},"x",[213,236,217],{},[45,238,239,248],{},[240,241,242,245],"munder",{},[48,243,244],{},"∑",[213,246,247],{},"j",[225,249,250,252],{},[213,251,229],{},[210,253,254,256],{},[213,255,234],{},[213,257,247],{},[213,259,261],{"mathvariant":260},"normal",".",[72,263,264],{"encoding":74},"p_i=\\frac{e^{x_i}}{\\sum_j e^{x_j}}.",[29,266,268,343],{"className":267,"ariaHidden":58},[79],[29,269,271,275,332,336,340],{"className":270},[83],[29,272],{"className":273,"style":274},[87],"height:0.625em;vertical-align:-0.1944em;",[29,276,278,282],{"className":277},[96],[29,279,10],{"className":280},[96,281],"mathnormal",[29,283,286],{"className":284},[285],"msupsub",[29,287,291,323],{"className":288},[289,290],"vlist-t","vlist-t2",[29,292,295,318],{"className":293},[294],"vlist-r",[29,296,300],{"className":297,"style":299},[298],"vlist","height:0.3117em;",[29,301,303,308],{"style":302},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[29,304],{"className":305,"style":307},[306],"pstrut","height:2.7em;",[29,309,315],{"className":310},[311,312,313,314],"sizing","reset-size6","size3","mtight",[29,316,217],{"className":317},[96,281,314],[29,319,322],{"className":320},[321],"vlist-s","​",[29,324,326],{"className":325},[294],[29,327,330],{"className":328,"style":329},[298],"height:0.15em;",[29,331],{},[29,333],{"className":334,"style":335},[104],"margin-right:0.2778em;",[29,337,220],{"className":338},[339],"mrel",[29,341],{"className":342,"style":335},[104],[29,344,346,350,615],{"className":345},[83],[29,347],{"className":348,"style":349},[87],"height:2.4632em;vertical-align:-1.1218em;",[29,351,353,357,612],{"className":352},[96],[29,354],{"className":355},[92,356],"nulldelimiter",[29,358,360],{"className":359},[222],[29,361,363,603],{"className":362},[289,290],[29,364,366,600],{"className":365},[294],[29,367,370,508,519],{"className":368,"style":369},[298],"height:1.3414em;",[29,371,373,377],{"style":372},"top:-2.314em;",[29,374],{"className":375,"style":376},[306],"height:3em;",[29,378,380,428,431],{"className":379},[96],[29,381,384,390],{"className":382},[383],"mop",[29,385,244],{"className":386,"style":389},[383,387,388],"op-symbol","small-op","position:relative;top:0em;",[29,391,393],{"className":392},[285],[29,394,396,419],{"className":395},[289,290],[29,397,399,416],{"className":398},[294],[29,400,403],{"className":401,"style":402},[298],"height:0.162em;",[29,404,406,409],{"style":405},"top:-2.4003em;margin-left:0em;margin-right:0.05em;",[29,407],{"className":408,"style":307},[306],[29,410,412],{"className":411},[311,312,313,314],[29,413,247],{"className":414,"style":415},[96,281,314],"margin-right:0.0572em;",[29,417,322],{"className":418},[321],[29,420,422],{"className":421},[294],[29,423,426],{"className":424,"style":425},[298],"height:0.4358em;",[29,427],{},[29,429],{"className":430,"style":105},[104],[29,432,434,437],{"className":433},[96],[29,435,229],{"className":436},[96,281],[29,438,440],{"className":439},[285],[29,441,443],{"className":442},[289],[29,444,446],{"className":445},[294],[29,447,450],{"className":448,"style":449},[298],"height:0.6065em;",[29,451,453,456],{"style":452},"top:-3.0051em;margin-right:0.05em;",[29,454],{"className":455,"style":307},[306],[29,457,459],{"className":458},[311,312,313,314],[29,460,462],{"className":461},[96,314],[29,463,465,468],{"className":464},[96,314],[29,466,234],{"className":467},[96,281,314],[29,469,471],{"className":470},[285],[29,472,474,499],{"className":473},[289,290],[29,475,477,496],{"className":476},[294],[29,478,481],{"className":479,"style":480},[298],"height:0.3281em;",[29,482,484,488],{"style":483},"top:-2.357em;margin-left:0em;margin-right:0.0714em;",[29,485],{"className":486,"style":487},[306],"height:2.5em;",[29,489,493],{"className":490},[311,491,492,314],"reset-size3","size1",[29,494,247],{"className":495,"style":415},[96,281,314],[29,497,322],{"className":498},[321],[29,500,502],{"className":501},[294],[29,503,506],{"className":504,"style":505},[298],"height:0.2819em;",[29,507],{},[29,509,511,514],{"style":510},"top:-3.23em;",[29,512],{"className":513,"style":376},[306],[29,515],{"className":516,"style":518},[517],"frac-line","border-bottom-width:0.04em;",[29,520,522,525],{"style":521},"top:-3.677em;",[29,523],{"className":524,"style":376},[306],[29,526,528],{"className":527},[96],[29,529,531,534],{"className":530},[96],[29,532,229],{"className":533},[96,281],[29,535,537],{"className":536},[285],[29,538,540],{"className":539},[289],[29,541,543],{"className":542},[294],[29,544,547],{"className":545,"style":546},[298],"height:0.6644em;",[29,548,550,553],{"style":549},"top:-3.063em;margin-right:0.05em;",[29,551],{"className":552,"style":307},[306],[29,554,556],{"className":555},[311,312,313,314],[29,557,559],{"className":558},[96,314],[29,560,562,565],{"className":561},[96,314],[29,563,234],{"className":564},[96,281,314],[29,566,568],{"className":567},[285],[29,569,571,591],{"className":570},[289,290],[29,572,574,588],{"className":573},[294],[29,575,577],{"className":576,"style":480},[298],[29,578,579,582],{"style":483},[29,580],{"className":581,"style":487},[306],[29,583,585],{"className":584},[311,491,492,314],[29,586,217],{"className":587},[96,281,314],[29,589,322],{"className":590},[321],[29,592,594],{"className":593},[294],[29,595,598],{"className":596,"style":597},[298],"height:0.143em;",[29,599],{},[29,601,322],{"className":602},[321],[29,604,606],{"className":605},[294],[29,607,610],{"className":608,"style":609},[298],"height:1.1218em;",[29,611],{},[29,613],{"className":614},[121,356],[29,616,261],{"className":617},[96],[10,619,620],{},"先把每个分数取指数，再除以这些指数的总和。对于有限输入，得到的权重为正，总和为 1；分数越高，分到的权重越大。",[10,622,623],{},"“除以总和”之前为什么要取指数？这里先记住 Softmax 本身的定义：它不是把原始分数直接除以原始分数之和。分数可以是负数，而指数把它们映射为正值；共同平移所有分数，也不会改变最终分配。",[10,625,626],{},"最后这个性质，正好用于改善数值稳定性。",[22,628,629],{"id":629},"减去最大值为什么不改变结果",[10,631,632,633,736],{},"直接计算很大的指数容易溢出。例如分数已经上千，",[29,634,636,658],{"className":635},[32],[29,637,639],{"className":638},[36],[38,640,641],{"xmlns":40},[42,642,643,655],{},[45,644,645],{},[225,646,647,649],{},[213,648,229],{},[210,650,651,653],{},[213,652,234],{},[213,654,217],{},[72,656,657],{"encoding":74},"e^{x_i}",[29,659,661],{"className":660,"ariaHidden":58},[79],[29,662,664,667],{"className":663},[83],[29,665],{"className":666,"style":546},[87],[29,668,670,673],{"className":669},[96],[29,671,229],{"className":672},[96,281],[29,674,676],{"className":675},[285],[29,677,679],{"className":678},[289],[29,680,682],{"className":681},[294],[29,683,685],{"className":684,"style":546},[298],[29,686,687,690],{"style":549},[29,688],{"className":689,"style":307},[306],[29,691,693],{"className":692},[311,312,313,314],[29,694,696],{"className":695},[96,314],[29,697,699,702],{"className":698},[96,314],[29,700,234],{"className":701},[96,281,314],[29,703,705],{"className":704},[285],[29,706,708,728],{"className":707},[289,290],[29,709,711,725],{"className":710},[294],[29,712,714],{"className":713,"style":480},[298],[29,715,716,719],{"style":483},[29,717],{"className":718,"style":487},[306],[29,720,722],{"className":721},[311,491,492,314],[29,723,217],{"className":724},[96,281,314],[29,726,322],{"className":727},[321],[29,729,731],{"className":730},[294],[29,732,734],{"className":733,"style":597},[298],[29,735],{}," 远超常见浮点类型的范围。即使分数之间的差别很普通，中间计算仍可能先变成无穷大。",[10,738,739,740,892],{},"令 ",[29,741,743,778],{"className":742},[32],[29,744,746],{"className":745},[36],[38,747,748],{"xmlns":40},[42,749,750,775],{},[45,751,752,755,757,769],{},[213,753,754],{},"m",[48,756,220],{},[210,758,759,767],{},[45,760,761,764],{},[213,762,763],{},"max",[48,765,766],{},"⁡",[213,768,247],{},[210,770,771,773],{},[213,772,234],{},[213,774,247],{},[72,776,777],{"encoding":74},"m=\\max_j x_j",[29,779,781,800],{"className":780,"ariaHidden":58},[79],[29,782,784,788,791,794,797],{"className":783},[83],[29,785],{"className":786,"style":787},[87],"height:0.4306em;",[29,789,754],{"className":790},[96,281],[29,792],{"className":793,"style":335},[104],[29,795,220],{"className":796},[339],[29,798],{"className":799,"style":335},[104],[29,801,803,807,849,852],{"className":802},[83],[29,804],{"className":805,"style":806},[87],"height:0.7167em;vertical-align:-0.2861em;",[29,808,810,813],{"className":809},[383],[29,811,763],{"className":812},[383],[29,814,816],{"className":815},[285],[29,817,819,840],{"className":818},[289,290],[29,820,822,837],{"className":821},[294],[29,823,825],{"className":824,"style":299},[298],[29,826,828,831],{"style":827},"top:-2.55em;margin-right:0.05em;",[29,829],{"className":830,"style":307},[306],[29,832,834],{"className":833},[311,312,313,314],[29,835,247],{"className":836,"style":415},[96,281,314],[29,838,322],{"className":839},[321],[29,841,843],{"className":842},[294],[29,844,847],{"className":845,"style":846},[298],"height:0.2861em;",[29,848],{},[29,850],{"className":851,"style":105},[104],[29,853,855,858],{"className":854},[96],[29,856,234],{"className":857},[96,281],[29,859,861],{"className":860},[285],[29,862,864,884],{"className":863},[289,290],[29,865,867,881],{"className":866},[294],[29,868,870],{"className":869,"style":299},[298],[29,871,872,875],{"style":302},[29,873],{"className":874,"style":307},[306],[29,876,878],{"className":877},[311,312,313,314],[29,879,247],{"className":880,"style":415},[96,281,314],[29,882,322],{"className":883},[321],[29,885,887],{"className":886},[294],[29,888,890],{"className":889,"style":846},[298],[29,891],{},"，则：",[29,894,896],{"className":895},[195],[29,897,899,1042],{"className":898},[32],[29,900,902],{"className":901},[36],[38,903,904],{"xmlns":40,"display":204},[42,905,906,1039],{},[45,907,908,951,953,1005,1007,1037],{},[222,909,910,927],{},[225,911,912,914],{},[213,913,229],{},[45,915,916,922,925],{},[210,917,918,920],{},[213,919,234],{},[213,921,217],{},[48,923,924],{},"−",[213,926,754],{},[45,928,929,935],{},[240,930,931,933],{},[48,932,244],{},[213,934,247],{},[225,936,937,939],{},[213,938,229],{},[45,940,941,947,949],{},[210,942,943,945],{},[213,944,234],{},[213,946,247],{},[48,948,924],{},[213,950,754],{},[48,952,220],{},[222,954,955,977],{},[45,956,957,967],{},[225,958,959,961],{},[213,960,229],{},[45,962,963,965],{},[48,964,924],{},[213,966,754],{},[225,968,969,971],{},[213,970,229],{},[210,972,973,975],{},[213,974,234],{},[213,976,217],{},[45,978,979,989,995],{},[225,980,981,983],{},[213,982,229],{},[45,984,985,987],{},[48,986,924],{},[213,988,754],{},[240,990,991,993],{},[48,992,244],{},[213,994,247],{},[225,996,997,999],{},[213,998,229],{},[210,1000,1001,1003],{},[213,1002,234],{},[213,1004,247],{},[48,1006,220],{},[222,1008,1009,1019],{},[225,1010,1011,1013],{},[213,1012,229],{},[210,1014,1015,1017],{},[213,1016,234],{},[213,1018,217],{},[45,1020,1021,1027],{},[240,1022,1023,1025],{},[48,1024,244],{},[213,1026,247],{},[225,1028,1029,1031],{},[213,1030,229],{},[210,1032,1033,1035],{},[213,1034,234],{},[213,1036,247],{},[213,1038,261],{"mathvariant":260},[72,1040,1041],{"encoding":74},"\\frac{e^{x_i-m}}{\\sum_j e^{x_j-m}}\n=\\frac{e^{-m}e^{x_i}}{e^{-m}\\sum_j e^{x_j}}\n=\\frac{e^{x_i}}{\\sum_j e^{x_j}}.",[29,1043,1045,1314,1641],{"className":1044,"ariaHidden":58},[79],[29,1046,1048,1052,1305,1308,1311],{"className":1047},[83],[29,1049],{"className":1050,"style":1051},[87],"height:2.5701em;vertical-align:-1.1218em;",[29,1053,1055,1058,1302],{"className":1054},[96],[29,1056],{"className":1057},[92,356],[29,1059,1061],{"className":1060},[222],[29,1062,1064,1294],{"className":1063},[289,290],[29,1065,1067,1291],{"className":1066},[294],[29,1068,1071,1199,1207],{"className":1069,"style":1070},[298],"height:1.4483em;",[29,1072,1073,1076],{"style":372},[29,1074],{"className":1075,"style":376},[306],[29,1077,1079,1119,1122],{"className":1078},[96],[29,1080,1082,1085],{"className":1081},[383],[29,1083,244],{"className":1084,"style":389},[383,387,388],[29,1086,1088],{"className":1087},[285],[29,1089,1091,1111],{"className":1090},[289,290],[29,1092,1094,1108],{"className":1093},[294],[29,1095,1097],{"className":1096,"style":402},[298],[29,1098,1099,1102],{"style":405},[29,1100],{"className":1101,"style":307},[306],[29,1103,1105],{"className":1104},[311,312,313,314],[29,1106,247],{"className":1107,"style":415},[96,281,314],[29,1109,322],{"className":1110},[321],[29,1112,1114],{"className":1113},[294],[29,1115,1117],{"className":1116,"style":425},[298],[29,1118],{},[29,1120],{"className":1121,"style":105},[104],[29,1123,1125,1128],{"className":1124},[96],[29,1126,229],{"className":1127},[96,281],[29,1129,1131],{"className":1130},[285],[29,1132,1134],{"className":1133},[289],[29,1135,1137],{"className":1136},[294],[29,1138,1141],{"className":1139,"style":1140},[298],"height:0.7134em;",[29,1142,1143,1146],{"style":452},[29,1144],{"className":1145,"style":307},[306],[29,1147,1149],{"className":1148},[311,312,313,314],[29,1150,1152,1192,1196],{"className":1151},[96,314],[29,1153,1155,1158],{"className":1154},[96,314],[29,1156,234],{"className":1157},[96,281,314],[29,1159,1161],{"className":1160},[285],[29,1162,1164,1184],{"className":1163},[289,290],[29,1165,1167,1181],{"className":1166},[294],[29,1168,1170],{"className":1169,"style":480},[298],[29,1171,1172,1175],{"style":483},[29,1173],{"className":1174,"style":487},[306],[29,1176,1178],{"className":1177},[311,491,492,314],[29,1179,247],{"className":1180,"style":415},[96,281,314],[29,1182,322],{"className":1183},[321],[29,1185,1187],{"className":1186},[294],[29,1188,1190],{"className":1189,"style":505},[298],[29,1191],{},[29,1193,924],{"className":1194},[1195,314],"mbin",[29,1197,754],{"className":1198},[96,281,314],[29,1200,1201,1204],{"style":510},[29,1202],{"className":1203,"style":376},[306],[29,1205],{"className":1206,"style":518},[517],[29,1208,1209,1212],{"style":521},[29,1210],{"className":1211,"style":376},[306],[29,1213,1215],{"className":1214},[96],[29,1216,1218,1221],{"className":1217},[96],[29,1219,229],{"className":1220},[96,281],[29,1222,1224],{"className":1223},[285],[29,1225,1227],{"className":1226},[289],[29,1228,1230],{"className":1229},[294],[29,1231,1234],{"className":1232,"style":1233},[298],"height:0.7713em;",[29,1235,1236,1239],{"style":549},[29,1237],{"className":1238,"style":307},[306],[29,1240,1242],{"className":1241},[311,312,313,314],[29,1243,1245,1285,1288],{"className":1244},[96,314],[29,1246,1248,1251],{"className":1247},[96,314],[29,1249,234],{"className":1250},[96,281,314],[29,1252,1254],{"className":1253},[285],[29,1255,1257,1277],{"className":1256},[289,290],[29,1258,1260,1274],{"className":1259},[294],[29,1261,1263],{"className":1262,"style":480},[298],[29,1264,1265,1268],{"style":483},[29,1266],{"className":1267,"style":487},[306],[29,1269,1271],{"className":1270},[311,491,492,314],[29,1272,217],{"className":1273},[96,281,314],[29,1275,322],{"className":1276},[321],[29,1278,1280],{"className":1279},[294],[29,1281,1283],{"className":1282,"style":597},[298],[29,1284],{},[29,1286,924],{"className":1287},[1195,314],[29,1289,754],{"className":1290},[96,281,314],[29,1292,322],{"className":1293},[321],[29,1295,1297],{"className":1296},[294],[29,1298,1300],{"className":1299,"style":609},[298],[29,1301],{},[29,1303],{"className":1304},[121,356],[29,1306],{"className":1307,"style":335},[104],[29,1309,220],{"className":1310},[339],[29,1312],{"className":1313,"style":335},[104],[29,1315,1317,1320,1632,1635,1638],{"className":1316},[83],[29,1318],{"className":1319,"style":1051},[87],[29,1321,1323,1326,1629],{"className":1322},[96],[29,1324],{"className":1325},[92,356],[29,1327,1329],{"className":1328},[222],[29,1330,1332,1621],{"className":1331},[289,290],[29,1333,1335,1618],{"className":1334},[294],[29,1336,1338,1498,1506],{"className":1337,"style":1070},[298],[29,1339,1340,1343],{"style":372},[29,1341],{"className":1342,"style":376},[306],[29,1344,1346,1383,1386,1426,1429],{"className":1345},[96],[29,1347,1349,1352],{"className":1348},[96],[29,1350,229],{"className":1351},[96,281],[29,1353,1355],{"className":1354},[285],[29,1356,1358],{"className":1357},[289],[29,1359,1361],{"className":1360},[294],[29,1362,1365],{"className":1363,"style":1364},[298],"height:0.6973em;",[29,1366,1368,1371],{"style":1367},"top:-2.989em;margin-right:0.05em;",[29,1369],{"className":1370,"style":307},[306],[29,1372,1374],{"className":1373},[311,312,313,314],[29,1375,1377,1380],{"className":1376},[96,314],[29,1378,924],{"className":1379},[96,314],[29,1381,754],{"className":1382},[96,281,314],[29,1384],{"className":1385,"style":105},[104],[29,1387,1389,1392],{"className":1388},[383],[29,1390,244],{"className":1391,"style":389},[383,387,388],[29,1393,1395],{"className":1394},[285],[29,1396,1398,1418],{"className":1397},[289,290],[29,1399,1401,1415],{"className":1400},[294],[29,1402,1404],{"className":1403,"style":402},[298],[29,1405,1406,1409],{"style":405},[29,1407],{"className":1408,"style":307},[306],[29,1410,1412],{"className":1411},[311,312,313,314],[29,1413,247],{"className":1414,"style":415},[96,281,314],[29,1416,322],{"className":1417},[321],[29,1419,1421],{"className":1420},[294],[29,1422,1424],{"className":1423,"style":425},[298],[29,1425],{},[29,1427],{"className":1428,"style":105},[104],[29,1430,1432,1435],{"className":1431},[96],[29,1433,229],{"className":1434},[96,281],[29,1436,1438],{"className":1437},[285],[29,1439,1441],{"className":1440},[289],[29,1442,1444],{"className":1443},[294],[29,1445,1447],{"className":1446,"style":449},[298],[29,1448,1449,1452],{"style":452},[29,1450],{"className":1451,"style":307},[306],[29,1453,1455],{"className":1454},[311,312,313,314],[29,1456,1458],{"className":1457},[96,314],[29,1459,1461,1464],{"className":1460},[96,314],[29,1462,234],{"className":1463},[96,281,314],[29,1465,1467],{"className":1466},[285],[29,1468,1470,1490],{"className":1469},[289,290],[29,1471,1473,1487],{"className":1472},[294],[29,1474,1476],{"className":1475,"style":480},[298],[29,1477,1478,1481],{"style":483},[29,1479],{"className":1480,"style":487},[306],[29,1482,1484],{"className":1483},[311,491,492,314],[29,1485,247],{"className":1486,"style":415},[96,281,314],[29,1488,322],{"className":1489},[321],[29,1491,1493],{"className":1492},[294],[29,1494,1496],{"className":1495,"style":505},[298],[29,1497],{},[29,1499,1500,1503],{"style":510},[29,1501],{"className":1502,"style":376},[306],[29,1504],{"className":1505,"style":518},[517],[29,1507,1508,1511],{"style":521},[29,1509],{"className":1510,"style":376},[306],[29,1512,1514,1549],{"className":1513},[96],[29,1515,1517,1520],{"className":1516},[96],[29,1518,229],{"className":1519},[96,281],[29,1521,1523],{"className":1522},[285],[29,1524,1526],{"className":1525},[289],[29,1527,1529],{"className":1528},[294],[29,1530,1532],{"className":1531,"style":1233},[298],[29,1533,1534,1537],{"style":549},[29,1535],{"className":1536,"style":307},[306],[29,1538,1540],{"className":1539},[311,312,313,314],[29,1541,1543,1546],{"className":1542},[96,314],[29,1544,924],{"className":1545},[96,314],[29,1547,754],{"className":1548},[96,281,314],[29,1550,1552,1555],{"className":1551},[96],[29,1553,229],{"className":1554},[96,281],[29,1556,1558],{"className":1557},[285],[29,1559,1561],{"className":1560},[289],[29,1562,1564],{"className":1563},[294],[29,1565,1567],{"className":1566,"style":546},[298],[29,1568,1569,1572],{"style":549},[29,1570],{"className":1571,"style":307},[306],[29,1573,1575],{"className":1574},[311,312,313,314],[29,1576,1578],{"className":1577},[96,314],[29,1579,1581,1584],{"className":1580},[96,314],[29,1582,234],{"className":1583},[96,281,314],[29,1585,1587],{"className":1586},[285],[29,1588,1590,1610],{"className":1589},[289,290],[29,1591,1593,1607],{"className":1592},[294],[29,1594,1596],{"className":1595,"style":480},[298],[29,1597,1598,1601],{"style":483},[29,1599],{"className":1600,"style":487},[306],[29,1602,1604],{"className":1603},[311,491,492,314],[29,1605,217],{"className":1606},[96,281,314],[29,1608,322],{"className":1609},[321],[29,1611,1613],{"className":1612},[294],[29,1614,1616],{"className":1615,"style":597},[298],[29,1617],{},[29,1619,322],{"className":1620},[321],[29,1622,1624],{"className":1623},[294],[29,1625,1627],{"className":1626,"style":609},[298],[29,1628],{},[29,1630],{"className":1631},[121,356],[29,1633],{"className":1634,"style":335},[104],[29,1636,220],{"className":1637},[339],[29,1639],{"className":1640,"style":335},[104],[29,1642,1644,1647,1884],{"className":1643},[83],[29,1645],{"className":1646,"style":349},[87],[29,1648,1650,1653,1881],{"className":1649},[96],[29,1651],{"className":1652},[92,356],[29,1654,1656],{"className":1655},[222],[29,1657,1659,1873],{"className":1658},[289,290],[29,1660,1662,1870],{"className":1661},[294],[29,1663,1665,1785,1793],{"className":1664,"style":369},[298],[29,1666,1667,1670],{"style":372},[29,1668],{"className":1669,"style":376},[306],[29,1671,1673,1713,1716],{"className":1672},[96],[29,1674,1676,1679],{"className":1675},[383],[29,1677,244],{"className":1678,"style":389},[383,387,388],[29,1680,1682],{"className":1681},[285],[29,1683,1685,1705],{"className":1684},[289,290],[29,1686,1688,1702],{"className":1687},[294],[29,1689,1691],{"className":1690,"style":402},[298],[29,1692,1693,1696],{"style":405},[29,1694],{"className":1695,"style":307},[306],[29,1697,1699],{"className":1698},[311,312,313,314],[29,1700,247],{"className":1701,"style":415},[96,281,314],[29,1703,322],{"className":1704},[321],[29,1706,1708],{"className":1707},[294],[29,1709,1711],{"className":1710,"style":425},[298],[29,1712],{},[29,1714],{"className":1715,"style":105},[104],[29,1717,1719,1722],{"className":1718},[96],[29,1720,229],{"className":1721},[96,281],[29,1723,1725],{"className":1724},[285],[29,1726,1728],{"className":1727},[289],[29,1729,1731],{"className":1730},[294],[29,1732,1734],{"className":1733,"style":449},[298],[29,1735,1736,1739],{"style":452},[29,1737],{"className":1738,"style":307},[306],[29,1740,1742],{"className":1741},[311,312,313,314],[29,1743,1745],{"className":1744},[96,314],[29,1746,1748,1751],{"className":1747},[96,314],[29,1749,234],{"className":1750},[96,281,314],[29,1752,1754],{"className":1753},[285],[29,1755,1757,1777],{"className":1756},[289,290],[29,1758,1760,1774],{"className":1759},[294],[29,1761,1763],{"className":1762,"style":480},[298],[29,1764,1765,1768],{"style":483},[29,1766],{"className":1767,"style":487},[306],[29,1769,1771],{"className":1770},[311,491,492,314],[29,1772,247],{"className":1773,"style":415},[96,281,314],[29,1775,322],{"className":1776},[321],[29,1778,1780],{"className":1779},[294],[29,1781,1783],{"className":1782,"style":505},[298],[29,1784],{},[29,1786,1787,1790],{"style":510},[29,1788],{"className":1789,"style":376},[306],[29,1791],{"className":1792,"style":518},[517],[29,1794,1795,1798],{"style":521},[29,1796],{"className":1797,"style":376},[306],[29,1799,1801],{"className":1800},[96],[29,1802,1804,1807],{"className":1803},[96],[29,1805,229],{"className":1806},[96,281],[29,1808,1810],{"className":1809},[285],[29,1811,1813],{"className":1812},[289],[29,1814,1816],{"className":1815},[294],[29,1817,1819],{"className":1818,"style":546},[298],[29,1820,1821,1824],{"style":549},[29,1822],{"className":1823,"style":307},[306],[29,1825,1827],{"className":1826},[311,312,313,314],[29,1828,1830],{"className":1829},[96,314],[29,1831,1833,1836],{"className":1832},[96,314],[29,1834,234],{"className":1835},[96,281,314],[29,1837,1839],{"className":1838},[285],[29,1840,1842,1862],{"className":1841},[289,290],[29,1843,1845,1859],{"className":1844},[294],[29,1846,1848],{"className":1847,"style":480},[298],[29,1849,1850,1853],{"style":483},[29,1851],{"className":1852,"style":487},[306],[29,1854,1856],{"className":1855},[311,491,492,314],[29,1857,217],{"className":1858},[96,281,314],[29,1860,322],{"className":1861},[321],[29,1863,1865],{"className":1864},[294],[29,1866,1868],{"className":1867,"style":597},[298],[29,1869],{},[29,1871,322],{"className":1872},[321],[29,1874,1876],{"className":1875},[294],[29,1877,1879],{"className":1878,"style":609},[298],[29,1880],{},[29,1882],{"className":1883},[121,356],[29,1885,261],{"className":1886},[96],[10,1888,1889,1890,1959],{},"分子与分母同时乘上 ",[29,1891,1893,1915],{"className":1892},[32],[29,1894,1896],{"className":1895},[36],[38,1897,1898],{"xmlns":40},[42,1899,1900,1912],{},[45,1901,1902],{},[225,1903,1904,1906],{},[213,1905,229],{},[45,1907,1908,1910],{},[48,1909,924],{},[213,1911,754],{},[72,1913,1914],{"encoding":74},"e^{-m}",[29,1916,1918],{"className":1917,"ariaHidden":58},[79],[29,1919,1921,1924],{"className":1920},[83],[29,1922],{"className":1923,"style":1233},[87],[29,1925,1927,1930],{"className":1926},[96],[29,1928,229],{"className":1929},[96,281],[29,1931,1933],{"className":1932},[285],[29,1934,1936],{"className":1935},[289],[29,1937,1939],{"className":1938},[294],[29,1940,1942],{"className":1941,"style":1233},[298],[29,1943,1944,1947],{"style":549},[29,1945],{"className":1946,"style":307},[306],[29,1948,1950],{"className":1949},[311,312,313,314],[29,1951,1953,1956],{"className":1952},[96,314],[29,1954,924],{"className":1955},[96,314],[29,1957,754],{"className":1958},[96,281,314],"，比值不变。对最初那组分数：",[1961,1962,1968],"pre",{"className":1963,"code":1965,"language":1966,"meta":1967},[1964],"language-text","原始分数： [1, 2, 3]\n减最大值： [-2, -1, 0]\n取指数：   [0.135, 0.368, 1]\n归一化：   [0.090, 0.245, 0.665]\n","text","",[1969,1970,1965],"code",{"__ignoreMap":1967},[10,1972,1973,1974,2058],{},"减去最大值后，最大的指数项是 ",[29,1975,1977,2000],{"className":1976},[32],[29,1978,1980],{"className":1979},[36],[38,1981,1982],{"xmlns":40},[42,1983,1984,1997],{},[45,1985,1986,1993,1995],{},[225,1987,1988,1990],{},[213,1989,229],{},[53,1991,1992],{},"0",[48,1994,220],{},[53,1996,55],{},[72,1998,1999],{"encoding":74},"e^0=1",[29,2001,2003,2048],{"className":2002,"ariaHidden":58},[79],[29,2004,2006,2010,2039,2042,2045],{"className":2005},[83],[29,2007],{"className":2008,"style":2009},[87],"height:0.8141em;",[29,2011,2013,2016],{"className":2012},[96],[29,2014,229],{"className":2015},[96,281],[29,2017,2019],{"className":2018},[285],[29,2020,2022],{"className":2021},[289],[29,2023,2025],{"className":2024},[294],[29,2026,2028],{"className":2027,"style":2009},[298],[29,2029,2030,2033],{"style":549},[29,2031],{"className":2032,"style":307},[306],[29,2034,2036],{"className":2035},[311,312,313,314],[29,2037,1992],{"className":2038},[96,314],[29,2040],{"className":2041,"style":335},[104],[29,2043,220],{"className":2044},[339],[29,2046],{"className":2047,"style":335},[104],[29,2049,2051,2055],{"className":2050},[83],[29,2052],{"className":2053,"style":2054},[87],"height:0.6444em;",[29,2056,55],{"className":2057},[96],"，其他项都不大于 1。这样避开了由很大的正指数带来的溢出。",[10,2060,2061,2062,2065],{},"已有 ",[1969,2063,2064],{},"activations.py"," 中，对应的关键步骤是：",[1961,2067,2071],{"className":2068,"code":2069,"language":2070,"meta":1967,"style":1967},"language-python shiki shiki-themes github-dark-dimmed github-light houston","max_value=torch.amax(x,dim=dim,keepdim=True)\nshifted=x-max_value\nexp_x=torch.exp(shifted)\n","python",[1969,2072,2073,2115,2131],{"__ignoreMap":1967},[29,2074,2077,2081,2084,2087,2091,2094,2098,2100,2103,2106,2108,2112],{"class":2075,"line":2076},"line",1,[29,2078,2080],{"class":2079},"sWURj","max_value",[29,2082,220],{"class":2083},"sf2cM",[29,2085,2086],{"class":2079},"torch.",[29,2088,2090],{"class":2089},"sq3c-","amax",[29,2092,2093],{"class":2079},"(x,",[29,2095,2097],{"class":2096},"sA9aF","dim",[29,2099,220],{"class":2083},[29,2101,2102],{"class":2079},"dim,",[29,2104,2105],{"class":2096},"keepdim",[29,2107,220],{"class":2083},[29,2109,2111],{"class":2110},"sgENW","True",[29,2113,2114],{"class":2079},")\n",[29,2116,2118,2121,2123,2125,2128],{"class":2075,"line":2117},2,[29,2119,2120],{"class":2079},"shifted",[29,2122,220],{"class":2083},[29,2124,234],{"class":2079},[29,2126,2127],{"class":2083},"-",[29,2129,2130],{"class":2079},"max_value\n",[29,2132,2134,2137,2139,2141,2144],{"class":2075,"line":2133},3,[29,2135,2136],{"class":2079},"exp_x",[29,2138,220],{"class":2083},[29,2140,2086],{"class":2079},[29,2142,2143],{"class":2089},"exp",[29,2145,2146],{"class":2079},"(shifted)\n",[10,2148,2149,2150,2152,2153,2156,2157,2160,2161,2164],{},"再沿相同的 ",[1969,2151,2097],{}," 求指数之和并相除。这里又出现了 ",[1969,2154,2155],{},"keepdim=True","：例如分数形状为 ",[1969,2158,2159],{},"(batch, queries, keys)","，沿最后一维取最大值后得到 ",[1969,2162,2163],{},"(batch, queries, 1)","，每个 query 都减去自己这一行的最大值。",[10,2166,2167],{},"这个技巧改善的是有限输入下的指数计算。它不能自动解决所有特殊值问题；例如一整行全为负无穷时，减最大值会涉及负无穷减负无穷。当前实现没有特殊处理全屏蔽行，这种情况下会产生 NaN；本次官方用例没有覆盖这个边界。",[22,2169,2171],{"id":2170},"为什么-softmax-不需要先创建-layer","为什么 Softmax 不需要先创建 layer",[10,2173,2174],{},"前面 Linear、Embedding、RMSNorm、SwiGLU 都先创建对象，再把输入交给它。到了 Softmax，适配器直接调用函数。我当时问，这两种写法为什么不一样。",[10,2176,2177],{},"在当前实现里，前面几个组件用类来保存参数。Linear 有权重矩阵，RMSNorm 有可训练缩放参数，SwiGLU 内部还保存三个 Linear 子层。对象创建后，这些参数会被反复使用。",[10,2179,2180],{},"Softmax 只依赖这次传入的张量和计算维度，没有需要保存的可训练权重，所以写成普通函数就够了。SiLU 也是同样的情况。",[2182,2183,2184,2200],"table",{},[2185,2186,2187],"thead",{},[2188,2189,2190,2194,2197],"tr",{},[2191,2192,2193],"th",{},"当前组件",[2191,2195,2196],{},"当前实现形式",[2191,2198,2199],{},"需要保存的可训练参数",[2201,2202,2203,2215,2225,2236,2247],"tbody",{},[2188,2204,2205,2209,2212],{},[2206,2207,2208],"td",{},"Linear、Embedding",[2206,2210,2211],{},"类",[2206,2213,2214],{},"权重矩阵或向量表",[2188,2216,2217,2220,2222],{},[2206,2218,2219],{},"RMSNorm",[2206,2221,2211],{},[2206,2223,2224],{},"每个特征的缩放权重",[2188,2226,2227,2230,2233],{},[2206,2228,2229],{},"SwiGLU",[2206,2231,2232],{},"类，内部有子层",[2206,2234,2235],{},"三组线性层权重",[2188,2237,2238,2241,2244],{},[2206,2239,2240],{},"SiLU、Softmax",[2206,2242,2243],{},"普通函数",[2206,2245,2246],{},"无",[2188,2248,2249,2252,2254],{},[2206,2250,2251],{},"缩放点积 Attention",[2206,2253,2243],{},[2206,2255,2256],{},"当前函数没有额外权重，接收外部 Q、K、V",[10,2258,2259,2260,2263],{},"这是这次代码的组织选择，不是“没有参数就不许写类”的规则。PyTorch 同时提供 ",[1969,2261,2262],{},"nn.Softmax"," 与函数接口。一个无参数计算也可以用模块形式封装，取决于组合方式和接口需要。",[22,2265,2267],{"id":2266},"qkv-的三个角色","Q、K、V 的三个角色",[10,2269,2270],{},"Softmax 之后，开始看缩放点积注意力：",[29,2272,2274],{"className":2273},[195],[29,2275,2277,2354],{"className":2276},[32],[29,2278,2280],{"className":2279},[36],[38,2281,2282],{"xmlns":40,"display":204},[42,2283,2284,2351],{},[45,2285,2286,2289,2291,2294,2297,2299,2302,2304,2307,2310,2312,2315,2317,2347,2349],{},[213,2287,2288],{"mathvariant":260},"Attention",[48,2290,766],{},[48,2292,2293],{"stretchy":50},"(",[213,2295,2296],{},"Q",[48,2298,59],{"separator":58},[213,2300,2301],{},"K",[48,2303,59],{"separator":58},[213,2305,2306],{},"V",[48,2308,2309],{"stretchy":50},")",[48,2311,220],{},[213,2313,2314],{"mathvariant":260},"softmax",[48,2316,766],{},[45,2318,2319,2321,2345],{},[48,2320,2293],{"fence":58},[222,2322,2323,2334],{},[45,2324,2325,2327],{},[213,2326,2296],{},[225,2328,2329,2331],{},[213,2330,2301],{},[213,2332,2333],{"mathvariant":260},"⊤",[2335,2336,2337],"msqrt",{},[210,2338,2339,2342],{},[213,2340,2341],{},"d",[213,2343,2344],{},"k",[48,2346,2309],{"fence":58},[213,2348,2306],{},[213,2350,261],{"mathvariant":260},[72,2352,2353],{"encoding":74},"\\operatorname{Attention}(Q,K,V)\n=\\operatorname{softmax}\\left(\\frac{QK^\\top}{\\sqrt{d_k}}\\right)V.",[29,2355,2357,2408],{"className":2356,"ariaHidden":58},[79],[29,2358,2360,2363,2370,2373,2376,2379,2382,2386,2389,2392,2396,2399,2402,2405],{"className":2359},[83],[29,2361],{"className":2362,"style":88},[87],[29,2364,2366],{"className":2365},[383],[29,2367,2288],{"className":2368},[96,2369],"mathrm",[29,2371,2293],{"className":2372},[92],[29,2374,2296],{"className":2375},[96,281],[29,2377,59],{"className":2378},[100],[29,2380],{"className":2381,"style":105},[104],[29,2383,2301],{"className":2384,"style":2385},[96,281],"margin-right:0.0715em;",[29,2387,59],{"className":2388},[100],[29,2390],{"className":2391,"style":105},[104],[29,2393,2306],{"className":2394,"style":2395},[96,281],"margin-right:0.2222em;",[29,2397,2309],{"className":2398},[121],[29,2400],{"className":2401,"style":335},[104],[29,2403,220],{"className":2404},[339],[29,2406],{"className":2407,"style":335},[104],[29,2409,2411,2415,2421,2424,2638,2641,2644],{"className":2410},[83],[29,2412],{"className":2413,"style":2414},[87],"height:2.4761em;vertical-align:-0.95em;",[29,2416,2418],{"className":2417},[383],[29,2419,2314],{"className":2420},[96,2369],[29,2422],{"className":2423,"style":105},[104],[29,2425,2428,2437,2632],{"className":2426},[2427],"minner",[29,2429,2433],{"className":2430,"style":2432},[92,2431],"delimcenter","top:0em;",[29,2434,2293],{"className":2435},[2436,313],"delimsizing",[29,2438,2440,2443,2629],{"className":2439},[96],[29,2441],{"className":2442},[92,356],[29,2444,2446],{"className":2445},[222],[29,2447,2449,2620],{"className":2448},[289,290],[29,2450,2452,2617],{"className":2451},[294],[29,2453,2456,2568,2576],{"className":2454,"style":2455},[298],"height:1.5261em;",[29,2457,2459,2462],{"style":2458},"top:-2.2528em;",[29,2460],{"className":2461,"style":376},[306],[29,2463,2465],{"className":2464},[96],[29,2466,2469],{"className":2467},[96,2468],"sqrt",[29,2470,2472,2559],{"className":2471},[289,290],[29,2473,2475,2556],{"className":2474},[294],[29,2476,2479,2533],{"className":2477,"style":2478},[298],"height:0.8572em;",[29,2480,2484,2487],{"className":2481,"style":2483},[2482],"svg-align","top:-3em;",[29,2485],{"className":2486,"style":376},[306],[29,2488,2491],{"className":2489,"style":2490},[96],"padding-left:0.833em;",[29,2492,2494,2497],{"className":2493},[96],[29,2495,2341],{"className":2496},[96,281],[29,2498,2500],{"className":2499},[285],[29,2501,2503,2525],{"className":2502},[289,290],[29,2504,2506,2522],{"className":2505},[294],[29,2507,2510],{"className":2508,"style":2509},[298],"height:0.3361em;",[29,2511,2512,2515],{"style":302},[29,2513],{"className":2514,"style":307},[306],[29,2516,2518],{"className":2517},[311,312,313,314],[29,2519,2344],{"className":2520,"style":2521},[96,281,314],"margin-right:0.0315em;",[29,2523,322],{"className":2524},[321],[29,2526,2528],{"className":2527},[294],[29,2529,2531],{"className":2530,"style":329},[298],[29,2532],{},[29,2534,2536,2539],{"style":2535},"top:-2.8172em;",[29,2537],{"className":2538,"style":376},[306],[29,2540,2544],{"className":2541,"style":2543},[2542],"hide-tail","min-width:0.853em;height:1.08em;",[2545,2546,2552],"svg",{"xmlns":2547,"width":2548,"height":2549,"viewBox":2550,"preserveAspectRatio":2551},"http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","400em","1.08em","0 0 400000 1080","xMinYMin slice",[2553,2554],"path",{"d":2555},"M95,702\nc-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14\nc0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54\nc44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10\ns173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429\nc69,-144,104.5,-217.7,106.5,-221\nl0 -0\nc5.3,-9.3,12,-14,20,-14\nH400000v40H845.2724\ns-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7\nc-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z\nM834 80h400000v40h-400000z",[29,2557,322],{"className":2558},[321],[29,2560,2562],{"className":2561},[294],[29,2563,2566],{"className":2564,"style":2565},[298],"height:0.1828em;",[29,2567],{},[29,2569,2570,2573],{"style":510},[29,2571],{"className":2572,"style":376},[306],[29,2574],{"className":2575,"style":518},[517],[29,2577,2578,2581],{"style":521},[29,2579],{"className":2580,"style":376},[306],[29,2582,2584,2587],{"className":2583},[96],[29,2585,2296],{"className":2586},[96,281],[29,2588,2590,2593],{"className":2589},[96],[29,2591,2301],{"className":2592,"style":2385},[96,281],[29,2594,2596],{"className":2595},[285],[29,2597,2599],{"className":2598},[289],[29,2600,2602],{"className":2601},[294],[29,2603,2606],{"className":2604,"style":2605},[298],"height:0.8491em;",[29,2607,2608,2611],{"style":549},[29,2609],{"className":2610,"style":307},[306],[29,2612,2614],{"className":2613},[311,312,313,314],[29,2615,2333],{"className":2616},[96,314],[29,2618,322],{"className":2619},[321],[29,2621,2623],{"className":2622},[294],[29,2624,2627],{"className":2625,"style":2626},[298],"height:0.93em;",[29,2628],{},[29,2630],{"className":2631},[121,356],[29,2633,2635],{"className":2634,"style":2432},[121,2431],[29,2636,2309],{"className":2637},[2436,313],[29,2639],{"className":2640,"style":105},[104],[29,2642,2306],{"className":2643,"style":2395},[96,281],[29,2645,261],{"className":2646},[96],[10,2648,2649],{},"三个输入承担不同角色：Q 是 query，表示当前位置用什么向量查询；K 是 key，参与计算匹配程度；V 是 value，是实际被加权汇总的内容。",[10,2651,2652],{},"“我想找什么”“我能怎样被找到”“我能提供什么信息”有助于初次理解，但计算里它们都是向量。匹配分数来自 Q 与 K 的点积，汇总对象是 V，不能把 K 与 V 的用途混在一起。",[10,2654,2655,2656,2724],{},"例如某个 query 最后得到的权重为 ",[29,2657,2659,2688],{"className":2658},[32],[29,2660,2662],{"className":2661},[36],[38,2663,2664],{"xmlns":40},[42,2665,2666,2685],{},[45,2667,2668,2670,2673,2675,2678,2680,2683],{},[48,2669,51],{"stretchy":50},[53,2671,2672],{},"0.2",[48,2674,59],{"separator":58},[53,2676,2677],{},"0.3",[48,2679,59],{"separator":58},[53,2681,2682],{},"0.5",[48,2684,70],{"stretchy":50},[72,2686,2687],{"encoding":74},"[0.2,0.3,0.5]",[29,2689,2691],{"className":2690,"ariaHidden":58},[79],[29,2692,2694,2697,2700,2703,2706,2709,2712,2715,2718,2721],{"className":2693},[83],[29,2695],{"className":2696,"style":88},[87],[29,2698,51],{"className":2699},[92],[29,2701,2672],{"className":2702},[96],[29,2704,59],{"className":2705},[100],[29,2707],{"className":2708,"style":105},[104],[29,2710,2677],{"className":2711},[96],[29,2713,59],{"className":2714},[100],[29,2716],{"className":2717,"style":105},[104],[29,2719,2682],{"className":2720},[96],[29,2722,70],{"className":2723},[121],"，它的输出就是：",[29,2726,2728],{"className":2727},[195],[29,2729,2731,2779],{"className":2730},[32],[29,2732,2734],{"className":2733},[36],[38,2735,2736],{"xmlns":40,"display":204},[42,2737,2738,2776],{},[45,2739,2740,2743,2745,2747,2753,2756,2758,2764,2766,2768,2774],{},[213,2741,2742],{},"o",[48,2744,220],{},[53,2746,2672],{},[210,2748,2749,2751],{},[213,2750,2306],{},[53,2752,55],{},[48,2754,2755],{},"+",[53,2757,2677],{},[210,2759,2760,2762],{},[213,2761,2306],{},[53,2763,62],{},[48,2765,2755],{},[53,2767,2682],{},[210,2769,2770,2772],{},[213,2771,2306],{},[53,2773,67],{},[213,2775,261],{"mathvariant":260},[72,2777,2778],{"encoding":74},"o=0.2V_1+0.3V_2+0.5V_3.",[29,2780,2782,2800,2861,2919],{"className":2781,"ariaHidden":58},[79],[29,2783,2785,2788,2791,2794,2797],{"className":2784},[83],[29,2786],{"className":2787,"style":787},[87],[29,2789,2742],{"className":2790},[96,281],[29,2792],{"className":2793,"style":335},[104],[29,2795,220],{"className":2796},[339],[29,2798],{"className":2799,"style":335},[104],[29,2801,2803,2807,2810,2852,2855,2858],{"className":2802},[83],[29,2804],{"className":2805,"style":2806},[87],"height:0.8333em;vertical-align:-0.15em;",[29,2808,2672],{"className":2809},[96],[29,2811,2813,2816],{"className":2812},[96],[29,2814,2306],{"className":2815,"style":2395},[96,281],[29,2817,2819],{"className":2818},[285],[29,2820,2822,2844],{"className":2821},[289,290],[29,2823,2825,2841],{"className":2824},[294],[29,2826,2829],{"className":2827,"style":2828},[298],"height:0.3011em;",[29,2830,2832,2835],{"style":2831},"top:-2.55em;margin-left:-0.2222em;margin-right:0.05em;",[29,2833],{"className":2834,"style":307},[306],[29,2836,2838],{"className":2837},[311,312,313,314],[29,2839,55],{"className":2840},[96,314],[29,2842,322],{"className":2843},[321],[29,2845,2847],{"className":2846},[294],[29,2848,2850],{"className":2849,"style":329},[298],[29,2851],{},[29,2853],{"className":2854,"style":2395},[104],[29,2856,2755],{"className":2857},[1195],[29,2859],{"className":2860,"style":2395},[104],[29,2862,2864,2867,2870,2910,2913,2916],{"className":2863},[83],[29,2865],{"className":2866,"style":2806},[87],[29,2868,2677],{"className":2869},[96],[29,2871,2873,2876],{"className":2872},[96],[29,2874,2306],{"className":2875,"style":2395},[96,281],[29,2877,2879],{"className":2878},[285],[29,2880,2882,2902],{"className":2881},[289,290],[29,2883,2885,2899],{"className":2884},[294],[29,2886,2888],{"className":2887,"style":2828},[298],[29,2889,2890,2893],{"style":2831},[29,2891],{"className":2892,"style":307},[306],[29,2894,2896],{"className":2895},[311,312,313,314],[29,2897,62],{"className":2898},[96,314],[29,2900,322],{"className":2901},[321],[29,2903,2905],{"className":2904},[294],[29,2906,2908],{"className":2907,"style":329},[298],[29,2909],{},[29,2911],{"className":2912,"style":2395},[104],[29,2914,2755],{"className":2915},[1195],[29,2917],{"className":2918,"style":2395},[104],[29,2920,2922,2925,2928,2968],{"className":2921},[83],[29,2923],{"className":2924,"style":2806},[87],[29,2926,2682],{"className":2927},[96],[29,2929,2931,2934],{"className":2930},[96],[29,2932,2306],{"className":2933,"style":2395},[96,281],[29,2935,2937],{"className":2936},[285],[29,2938,2940,2960],{"className":2939},[289,290],[29,2941,2943,2957],{"className":2942},[294],[29,2944,2946],{"className":2945,"style":2828},[298],[29,2947,2948,2951],{"style":2831},[29,2949],{"className":2950,"style":307},[306],[29,2952,2954],{"className":2953},[311,312,313,314],[29,2955,67],{"className":2956},[96,314],[29,2958,322],{"className":2959},[321],[29,2961,2963],{"className":2962},[294],[29,2964,2966],{"className":2965,"style":329},[298],[29,2967],{},[29,2969,261],{"className":2970},[96],[10,2972,2973],{},"Linear 和 SwiGLU 都在一个 token 内变换特征；这里才明确出现了按照权重，从多个位置汇总信息的计算。",[22,2975,2976],{"id":2976},"先把两次矩阵乘法的形状对上",[10,2978,2979,2980,2983],{},"可以先忽略 batch，只看三个位置、每个向量两个特征的例子。Q、K、V 都是 ",[1969,2981,2982],{},"(3, 2)","：",[1961,2985,2988],{"className":2986,"code":2987,"language":1966,"meta":1967},[1964],"匹配分数： (3, 2) @ (2, 3) → (3, 3)\n注意力权重：                  (3, 3)\n加权汇总： (3, 3) @ (3, 2) → (3, 2)\n",[1969,2989,2987],{"__ignoreMap":1967},[10,2991,2992],{},"分数矩阵的一行对应一个 query，一列对应一个 key。Softmax 沿最后一维计算，意味着对每个 query，在所有 key 之间分配权重。随后用这一行权重汇总相应的 V。",[10,2994,2995],{},"query 数量和 key 数量也不必相同。更一般地：",[2182,2997,2998,3008],{},[2185,2999,3000],{},[2188,3001,3002,3005],{},[2191,3003,3004],{},"张量",[2191,3006,3007],{},"最后两维",[2201,3009,3010,3019,3028,3037,3047],{},[2188,3011,3012,3014],{},[2206,3013,2296],{},[2206,3015,3016],{},[1969,3017,3018],{},"(queries, d_k)",[2188,3020,3021,3023],{},[2206,3022,2301],{},[2206,3024,3025],{},[1969,3026,3027],{},"(keys, d_k)",[2188,3029,3030,3032],{},[2206,3031,2306],{},[2206,3033,3034],{},[1969,3035,3036],{},"(keys, d_v)",[2188,3038,3039,3042],{},[2206,3040,3041],{},"分数与权重",[2206,3043,3044],{},[1969,3045,3046],{},"(queries, keys)",[2188,3048,3049,3052],{},[2206,3050,3051],{},"输出",[2206,3053,3054],{},[1969,3055,3056],{},"(queries, d_v)",[10,3058,3059,3060,3063,3064,3067],{},"代码使用 ",[1969,3061,3062],{},"K.transpose(-2, -1)","，只交换最后两维，前面的批量维度可以保留。这与对一个已知二维权重使用 ",[1969,3065,3066],{},".T"," 的场景不同；进入带 batch 或 head 维度的张量时，需要明确到底交换哪两根轴。",[22,3069,3070],{"id":3070},"分数为什么除以平方根",[10,3072,3073,3074,3145],{},"这里的 ",[29,3075,3077,3095],{"className":3076},[32],[29,3078,3080],{"className":3079},[36],[38,3081,3082],{"xmlns":40},[42,3083,3084,3092],{},[45,3085,3086],{},[210,3087,3088,3090],{},[213,3089,2341],{},[213,3091,2344],{},[72,3093,3094],{"encoding":74},"d_k",[29,3096,3098],{"className":3097,"ariaHidden":58},[79],[29,3099,3101,3105],{"className":3100},[83],[29,3102],{"className":3103,"style":3104},[87],"height:0.8444em;vertical-align:-0.15em;",[29,3106,3108,3111],{"className":3107},[96],[29,3109,2341],{"className":3110},[96,281],[29,3112,3114],{"className":3113},[285],[29,3115,3117,3137],{"className":3116},[289,290],[29,3118,3120,3134],{"className":3119},[294],[29,3121,3123],{"className":3122,"style":2509},[298],[29,3124,3125,3128],{"style":302},[29,3126],{"className":3127,"style":307},[306],[29,3129,3131],{"className":3130},[311,312,313,314],[29,3132,2344],{"className":3133,"style":2521},[96,281,314],[29,3135,322],{"className":3136},[321],[29,3138,3140],{"className":3139},[294],[29,3141,3143],{"className":3142,"style":329},[298],[29,3144],{}," 是 Q 与 K 的最后一维长度。随着参与点积的特征数增加，分数的波动规模也容易增加，直接送进 Softmax 时可能让权重变得很集中。",[10,3147,3148,3149,3218,3219,3288,3289,3406],{},"在各分量近似独立、均值为 0、方差为 1 的简化假设下，",[29,3150,3152,3169],{"className":3151},[32],[29,3153,3155],{"className":3154},[36],[38,3156,3157],{"xmlns":40},[42,3158,3159,3167],{},[45,3160,3161],{},[210,3162,3163,3165],{},[213,3164,2341],{},[213,3166,2344],{},[72,3168,3094],{"encoding":74},[29,3170,3172],{"className":3171,"ariaHidden":58},[79],[29,3173,3175,3178],{"className":3174},[83],[29,3176],{"className":3177,"style":3104},[87],[29,3179,3181,3184],{"className":3180},[96],[29,3182,2341],{"className":3183},[96,281],[29,3185,3187],{"className":3186},[285],[29,3188,3190,3210],{"className":3189},[289,290],[29,3191,3193,3207],{"className":3192},[294],[29,3194,3196],{"className":3195,"style":2509},[298],[29,3197,3198,3201],{"style":302},[29,3199],{"className":3200,"style":307},[306],[29,3202,3204],{"className":3203},[311,312,313,314],[29,3205,2344],{"className":3206,"style":2521},[96,281,314],[29,3208,322],{"className":3209},[321],[29,3211,3213],{"className":3212},[294],[29,3214,3216],{"className":3215,"style":329},[298],[29,3217],{}," 项乘积之和的方差约为 ",[29,3220,3222,3239],{"className":3221},[32],[29,3223,3225],{"className":3224},[36],[38,3226,3227],{"xmlns":40},[42,3228,3229,3237],{},[45,3230,3231],{},[210,3232,3233,3235],{},[213,3234,2341],{},[213,3236,2344],{},[72,3238,3094],{"encoding":74},[29,3240,3242],{"className":3241,"ariaHidden":58},[79],[29,3243,3245,3248],{"className":3244},[83],[29,3246],{"className":3247,"style":3104},[87],[29,3249,3251,3254],{"className":3250},[96],[29,3252,2341],{"className":3253},[96,281],[29,3255,3257],{"className":3256},[285],[29,3258,3260,3280],{"className":3259},[289,290],[29,3261,3263,3277],{"className":3262},[294],[29,3264,3266],{"className":3265,"style":2509},[298],[29,3267,3268,3271],{"style":302},[29,3269],{"className":3270,"style":307},[306],[29,3272,3274],{"className":3273},[311,312,313,314],[29,3275,2344],{"className":3276,"style":2521},[96,281,314],[29,3278,322],{"className":3279},[321],[29,3281,3283],{"className":3282},[294],[29,3284,3286],{"className":3285,"style":329},[298],[29,3287],{},"。除以 ",[29,3290,3292,3312],{"className":3291},[32],[29,3293,3295],{"className":3294},[36],[38,3296,3297],{"xmlns":40},[42,3298,3299,3309],{},[45,3300,3301],{},[2335,3302,3303],{},[210,3304,3305,3307],{},[213,3306,2341],{},[213,3308,2344],{},[72,3310,3311],{"encoding":74},"\\sqrt{d_k}",[29,3313,3315],{"className":3314,"ariaHidden":58},[79],[29,3316,3318,3322],{"className":3317},[83],[29,3319],{"className":3320,"style":3321},[87],"height:1.04em;vertical-align:-0.1828em;",[29,3323,3325],{"className":3324},[96,2468],[29,3326,3328,3398],{"className":3327},[289,290],[29,3329,3331,3395],{"className":3330},[294],[29,3332,3334,3383],{"className":3333,"style":2478},[298],[29,3335,3337,3340],{"className":3336,"style":2483},[2482],[29,3338],{"className":3339,"style":376},[306],[29,3341,3343],{"className":3342,"style":2490},[96],[29,3344,3346,3349],{"className":3345},[96],[29,3347,2341],{"className":3348},[96,281],[29,3350,3352],{"className":3351},[285],[29,3353,3355,3375],{"className":3354},[289,290],[29,3356,3358,3372],{"className":3357},[294],[29,3359,3361],{"className":3360,"style":2509},[298],[29,3362,3363,3366],{"style":302},[29,3364],{"className":3365,"style":307},[306],[29,3367,3369],{"className":3368},[311,312,313,314],[29,3370,2344],{"className":3371,"style":2521},[96,281,314],[29,3373,322],{"className":3374},[321],[29,3376,3378],{"className":3377},[294],[29,3379,3381],{"className":3380,"style":329},[298],[29,3382],{},[29,3384,3385,3388],{"style":2535},[29,3386],{"className":3387,"style":376},[306],[29,3389,3391],{"className":3390,"style":2543},[2542],[2545,3392,3393],{"xmlns":2547,"width":2548,"height":2549,"viewBox":2550,"preserveAspectRatio":2551},[2553,3394],{"d":2555},[29,3396,322],{"className":3397},[321],[29,3399,3401],{"className":3400},[294],[29,3402,3404],{"className":3403,"style":2565},[298],[29,3405],{}," 后，方差回到约 1 的量级。这说明了缩放因子的动机，实际向量分布当然不必严格满足这些假设。",[10,3408,3409],{},"因此，缩放针对的是点积分数的规模。它与 Softmax 的“减最大值”解决不同问题，两步不能相互替代。",[22,3411,3413],{"id":3412},"masktrue-允许false-屏蔽","Mask：True 允许，False 屏蔽",[10,3415,3416,3417,3419,3420,3423,3424,3427],{},"本次接口接收布尔 mask，约定是 ",[1969,3418,2111],{}," 允许关注，",[1969,3421,3422],{},"False"," 屏蔽。",[1969,3425,3426],{},"mask=None"," 时不会自动启用因果约束，需要调用者显式传入 mask。以三个位置的因果 mask 为例：",[2182,3429,3430,3446],{},[2185,3431,3432],{},[2188,3433,3434,3437,3440,3443],{},[2191,3435,3436],{},"当前 query",[2191,3438,3439],{},"看位置 1",[2191,3441,3442],{},"看位置 2",[2191,3444,3445],{},"看位置 3",[2201,3447,3448,3459,3470],{},[2188,3449,3450,3453,3455,3457],{},[2206,3451,3452],{},"位置 1",[2206,3454,2111],{},[2206,3456,3422],{},[2206,3458,3422],{},[2188,3460,3461,3464,3466,3468],{},[2206,3462,3463],{},"位置 2",[2206,3465,2111],{},[2206,3467,2111],{},[2206,3469,3422],{},[2188,3471,3472,3475,3477,3479],{},[2206,3473,3474],{},"位置 3",[2206,3476,2111],{},[2206,3478,2111],{},[2206,3480,2111],{},[10,3482,3483],{},"每个位置只能看自己和更早的位置，不能利用后面的 token。这张表控制的是 query-key 配对，不是在删除某个 token 的特征维度。",[10,3485,3486],{},"屏蔽发生在 Softmax 之前：将不允许的位置填为负无穷，因为",[29,3488,3490],{"className":3489},[195],[29,3491,3493,3521],{"className":3492},[32],[29,3494,3496],{"className":3495},[36],[38,3497,3498],{"xmlns":40,"display":204},[42,3499,3500,3518],{},[45,3501,3502,3513,3515],{},[225,3503,3504,3506],{},[213,3505,229],{},[45,3507,3508,3510],{},[48,3509,924],{},[213,3511,3512],{"mathvariant":260},"∞",[48,3514,220],{},[53,3516,3517],{},"0.",[72,3519,3520],{"encoding":74},"e^{-\\infty}=0.",[29,3522,3524,3576],{"className":3523,"ariaHidden":58},[79],[29,3525,3527,3531,3567,3570,3573],{"className":3526},[83],[29,3528],{"className":3529,"style":3530},[87],"height:0.8213em;",[29,3532,3534,3537],{"className":3533},[96],[29,3535,229],{"className":3536},[96,281],[29,3538,3540],{"className":3539},[285],[29,3541,3543],{"className":3542},[289],[29,3544,3546],{"className":3545},[294],[29,3547,3549],{"className":3548,"style":3530},[298],[29,3550,3552,3555],{"style":3551},"top:-3.113em;margin-right:0.05em;",[29,3553],{"className":3554,"style":307},[306],[29,3556,3558],{"className":3557},[311,312,313,314],[29,3559,3561,3564],{"className":3560},[96,314],[29,3562,924],{"className":3563},[96,314],[29,3565,3512],{"className":3566},[96,314],[29,3568],{"className":3569,"style":335},[104],[29,3571,220],{"className":3572},[339],[29,3574],{"className":3575,"style":335},[104],[29,3577,3579,3582],{"className":3578},[83],[29,3580],{"className":3581,"style":2054},[87],[29,3583,3517],{"className":3584},[96],[10,3586,3587],{},"在这一行仍有有效 key 的情况下，被屏蔽位置就会获得 0 权重，其余允许的位置重新归一化。",[10,3589,3590],{},"已有实现里，关键的一行是：",[1961,3592,3594],{"className":2068,"code":3593,"language":2070,"meta":1967,"style":1967},"scores=scores.masked_fill(~mask,float(\"-inf\"))\n",[1969,3595,3596],{"__ignoreMap":1967},[29,3597,3598,3601,3603,3606,3609,3611,3614,3617,3621,3623,3627],{"class":2075,"line":2076},[29,3599,3600],{"class":2079},"scores",[29,3602,220],{"class":2083},[29,3604,3605],{"class":2079},"scores.",[29,3607,3608],{"class":2089},"masked_fill",[29,3610,2293],{"class":2079},[29,3612,3613],{"class":2083},"~",[29,3615,3616],{"class":2079},"mask,",[29,3618,3620],{"class":3619},"svkvy","float",[29,3622,2293],{"class":2079},[29,3624,3626],{"class":3625},"sWqY2","\"-inf\"",[29,3628,3629],{"class":2079},"))\n",[10,3631,3632,3635,3636,3638],{},[1969,3633,3634],{},"~mask"," 将 True、False 取反，从而选中需要填充的位置。",[1969,3637,3608],{}," 填的是分数，不是最终输出。",[10,3640,3641,3642,3722],{},"如果把屏蔽分数写成 0，Softmax 中对应项仍是 ",[29,3643,3645,3666],{"className":3644},[32],[29,3646,3648],{"className":3647},[36],[38,3649,3650],{"xmlns":40},[42,3651,3652,3664],{},[45,3653,3654,3660,3662],{},[225,3655,3656,3658],{},[213,3657,229],{},[53,3659,1992],{},[48,3661,220],{},[53,3663,55],{},[72,3665,1999],{"encoding":74},[29,3667,3669,3713],{"className":3668,"ariaHidden":58},[79],[29,3670,3672,3675,3704,3707,3710],{"className":3671},[83],[29,3673],{"className":3674,"style":2009},[87],[29,3676,3678,3681],{"className":3677},[96],[29,3679,229],{"className":3680},[96,281],[29,3682,3684],{"className":3683},[285],[29,3685,3687],{"className":3686},[289],[29,3688,3690],{"className":3689},[294],[29,3691,3693],{"className":3692,"style":2009},[298],[29,3694,3695,3698],{"style":549},[29,3696],{"className":3697,"style":307},[306],[29,3699,3701],{"className":3700},[311,312,313,314],[29,3702,1992],{"className":3703},[96,314],[29,3705],{"className":3706,"style":335},[104],[29,3708,220],{"className":3709},[339],[29,3711],{"className":3712,"style":335},[104],[29,3714,3716,3719],{"className":3715},[83],[29,3717],{"className":3718,"style":2054},[87],[29,3720,55],{"className":3721},[96],"，这个位置还会获得权重。只有放在正确的计算位置、使用符合约定的数值，mask 才能起到屏蔽作用。",[22,3724,3726],{"id":3725},"少一个负号会把方向整个写反","少一个负号，会把方向整个写反",[10,3728,3729,3730,3733,3734,3737],{},"这次确实把 ",[1969,3731,3732],{},"float(\"-inf\")"," 写成过 ",[1969,3735,3736],{},"float(\"inf\")","。公式里只差一个字符，含义却完全相反。正无穷不能让概率归零，还可能在稳定 Softmax 的平移过程中产生无效值。",[10,3739,3740,3741,3744,3745,3748,3749,3752,3753,3756],{},"Softmax 里也出现过 ",[1969,3742,3743],{},"torch,exp","，应该表达的却是对 ",[1969,3746,3747],{},"torch.exp"," 的调用；Attention 的导入语句还混淆过 ",[1969,3750,3751],{},"import"," 与 ",[1969,3754,3755],{},"from ... import ...","。这些都是可以定位到具体行的错误，不必一看到 Attention 输出异常就先怀疑整套数学公式。",[22,3758,3760],{"id":3759},"两项测试通过还差一个多头注意力层","两项测试通过，还差一个多头注意力层",[10,3762,3763,3764,3767,3768,3771,3772,3752,3775,3778,3779,191],{},"历史运行记录里，",[1969,3765,3766],{},"test_softmax_matches_pytorch"," 得到 ",[1969,3769,3770],{},"1 passed","。缩放点积注意力运行了两项官方测试：",[1969,3773,3774],{},"test_scaled_dot_product_attention",[1969,3776,3777],{},"test_4d_scaled_dot_product_attention","，结果是 ",[1969,3780,3781],{},"2 passed",[10,3783,3784,3785,3788],{},"第二项把输入整理成 ",[1969,3786,3787],{},"(batch, heads, seq_len, d_k)","，确认当前注意力函数能在这类四维输入下给出参考结果。它没有验证完整的多头自注意力层：Q、K、V 的投影、拆头、合并和输出投影，都是下一步才要继续学习的内容。",[10,3790,3791],{},"到这篇为止，已经验证的是若干基础组件和缩放点积注意力。多头注意力刚进入形状理解，RoPE、完整 Transformer、优化器及训练流程还不在本轮完成范围内。",[10,3793,3794,3795,3799],{},"同日后续完成的多头自注意力另记在",[13,3796,3798],{"href":3797},"\u002Fposts\u002Fcs336-multihead-attention-shapes\u002F","学习记录 04：拆头、合头与因果 Mask","，保留这里对应的阶段性代码和验证结果。",[22,3801,3802],{"id":3802},"发布时的复核",[10,3804,3805,3806,3812,3813,3816,3817,191],{},"发布前对",[13,3807,3811],{"href":3808,"rel":3809},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fa39eb684710817f7f389626423fcb26ca0f6d629",[3810],"nofollow","独立代码快照","重新运行了 8 项官方组件测试，结果为 ",[1969,3814,3815],{},"8 passed","。完整清单和运行环境见",[13,3818,3821],{"href":3819,"rel":3820},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Fblob\u002Fa39eb684710817f7f389626423fcb26ca0f6d629\u002Fdocs\u002Fprogress.md",[3810],"进度记录",[10,3823,3824],{},[3825,3826,3827,3828,3833],"em",{},"封面使用 ",[13,3829,3832],{"href":3830,"rel":3831},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Faigc-image-generator",[3810],"aigc-image-generator"," 辅助生成，属于概念插画；准确的形状、公式与运算关系以正文为准。",[22,3835,3836],{"id":3836},"参考资料",[3838,3839,3840,3848,3855,3862,3869,3876,3883,3890],"ul",{},[3841,3842,3843],"li",{},[13,3844,3847],{"href":3845,"rel":3846},"https:\u002F\u002Fcs336.stanford.edu\u002F",[3810],"Stanford CS336 官方课程页面",[3841,3849,3850],{},[13,3851,3854],{"href":3852,"rel":3853},"https:\u002F\u002Fgithub.com\u002Fstanford-cs336\u002Fassignment1-basics",[3810],"Assignment 1 官方仓库与讲义",[3841,3856,3857],{},[13,3858,3861],{"href":3859,"rel":3860},"https:\u002F\u002Fgithub.com\u002Fstanford-cs336\u002Fassignment1-basics\u002Fblob\u002Fmain\u002Ftests\u002Ftest_model.py",[3810],"Assignment 1 官方模型测试",[3841,3863,3864],{},[13,3865,3868],{"href":3866,"rel":3867},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fgenerated\u002Ftorch.nn.Softmax.html",[3810],"PyTorch：nn.Softmax",[3841,3870,3871],{},[13,3872,3875],{"href":3873,"rel":3874},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fgenerated\u002Ftorch.amax.html",[3810],"PyTorch：torch.amax",[3841,3877,3878],{},[13,3879,3882],{"href":3880,"rel":3881},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fgenerated\u002Ftorch.transpose.html",[3810],"PyTorch：torch.transpose",[3841,3884,3885],{},[13,3886,3889],{"href":3887,"rel":3888},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fgenerated\u002Ftorch.Tensor.masked_fill.html",[3810],"PyTorch：Tensor.masked_fill",[3841,3891,3892],{},[13,3893,3896],{"href":3894,"rel":3895},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fgenerated\u002Ftorch.nn.functional.scaled_dot_product_attention.html",[3810],"PyTorch：scaled_dot_product_attention",[3898,3899,3900],"style",{},"html pre.shiki code .sWURj, html code.shiki .sWURj{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#EEF0F9}html pre.shiki code .sf2cM, html code.shiki .sf2cM{--shiki-default:#F47067;--shiki-light:#D73A49;--shiki-cyber:#EEF0F9}html pre.shiki code .sq3c-, html code.shiki .sq3c-{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#00DAEF}html pre.shiki code .sA9aF, html code.shiki .sA9aF{--shiki-default:#F69D50;--shiki-default-font-style:inherit;--shiki-light:#E36209;--shiki-light-font-style:inherit;--shiki-cyber:#4BF3C8;--shiki-cyber-font-style:italic}html pre.shiki code .sgENW, html code.shiki .sgENW{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#FFD493}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html.cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html pre.shiki code .svkvy, html code.shiki .svkvy{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#EEF0F9}html pre.shiki code .sWqY2, html code.shiki .sWqY2{--shiki-default:#96D0FF;--shiki-light:#032F62;--shiki-cyber:#FFD493}",{"title":1967,"searchDepth":2117,"depth":2133,"links":3902},[3903,3904,3905,3906,3907,3908,3909,3910,3911,3912,3913],{"id":24,"depth":2117,"text":24},{"id":629,"depth":2117,"text":629},{"id":2170,"depth":2117,"text":2171},{"id":2266,"depth":2117,"text":2267},{"id":2976,"depth":2117,"text":2976},{"id":3070,"depth":2117,"text":3070},{"id":3412,"depth":2117,"text":3413},{"id":3725,"depth":2117,"text":3726},{"id":3759,"depth":2117,"text":3760},{"id":3802,"depth":2117,"text":3802},{"id":3836,"depth":2117,"text":3836},[3915],"学习笔记","\u002Fimages\u002Fposts\u002Fcs336-softmax-attention-mask\u002Fcs336-softmax-attention-mask-cover.webp","注意力分数、权重与因果遮罩的 AI 辅助概念插画","2026-10-18","从减去最大值为何不改变 Softmax 开始，理清函数与有参数层的区别，再沿着 Q、K、V 的形状理解缩放点积注意力，以及 mask 为什么必须填负无穷。",false,"md",{"codeUrl":3808,"attachmentUrl":3923},"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-03-softmax-sdpa-20261018.zip",true,"\u002Fposts\u002Fcs336-softmax-attention-mask",null,{"title":5,"description":3919},"Stanford CS336","posts\u002Fcs336-softmax-attention-mask",[3931,3932,3933,3934,2288],"CS336","PyTorch","Transformer","Softmax","bHI7vrC3mqAMFVWoiWok2WMHIDAja9Yc40OXba4gaiQ",[3937,3953,3969,3984,3998,4012,4027,4042,4056,4068,4083,4096,4109,4114,4125,4139,4156,4171,4182,4194,4209,4222,4236,4248,4262,4274,4289,4303,4317,4328,4341,4358,4373,4386,4398,4410,4423,4436],{"slug":3938,"path":3939,"title":3940,"date":3941,"tags":3942,"description":3946,"cover":3947,"coverAlt":3948,"excerpt":3949,"draft":3920,"hidden":3920,"published":3924,"readingTime":3950,"series":3928,"seriesOrder":3950,"codeUrl":3951,"attachmentUrl":3952},"cs336-cross-entropy-adamw-training-loop","\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop","Stanford CS336 学习记录 08：数值稳定交叉熵、AdamW、余弦调度与完整训练闭环","2026-10-27",[3931,3932,3943,3944,3945],"AdamW","CrossEntropy","训练闭环","用 LogSumExp 消去交叉熵中的指数溢出，手写解耦权重衰减的 AdamW 优化器、全局 L2 梯度裁剪、Warmup 余弦退火调度、memmap 数据采样、Checkpoint 序列化与 Top-p 解码，完成 CS336 Assignment 1 全部 48 项测试。","\u002Fimages\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop\u002Fcs336-cross-entropy-adamw-training-loop-cover.webp","滑动窗口采样、余弦退火曲线、双动量优化轨迹与检查点归档构成的训练闭环 AI 辅助概念插画","作为 Stanford CS336 Assignment 1 系列的收官篇，我们已经拥有了分词器（bpe.py、tokenizer.py）和完整的语言模型架构（transformer.py）。要让模型在语料上真正训练并生成文本，还需",8,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fb5c93ad0cc73178b2b2a62e518a0900da47a0fb7","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-08-adamw-training-loop-20261027.zip",{"slug":3954,"path":3955,"title":3956,"date":3957,"tags":3958,"description":3965,"cover":3966,"coverAlt":3967,"excerpt":3968,"draft":3920,"hidden":3920,"published":3924,"readingTime":3950},"editcache-video-dit-inference-acceleration","\u002Fposts\u002Feditcache-video-dit-inference-acceleration","EditCache：视频 Diffusion Transformer (DiT) 局部编辑的跨步特征复用与区域重算加速","2026-10-25",[3959,3960,3961,3962,3963,3964],"扩散模型","视频生成","AI Infra","推理加速","GPU","论文解读","拆解视频扩散模型（Wan2.1-VACE）局部编辑中的三重算力冗余，通过可重入 Latent、跨步 Block 隐状态缓存复用、带 Halo 边界的局部区域重算与自适应推理起点，在单卡 H100 上实现 2.46× 模型计算加速与 1.49× 严格同步端到端加速。","\u002Fimages\u002Fposts\u002Feditcache-video-dit-inference-acceleration\u002Feditcache-dit-cover-20261025.webp","视频扩散 Transformer 时空三维体素网格中冻结背景缓存与暖橙色局部编辑重算通道的等距技术插画","在基于 Diffusion Transformer（DiT）的视频编辑与局部重绘（Video Inpainting \u002F Local Editing）任务中，用户的典型工作流并不是一次性生成最终成片，而是针对同一段源视频反复调整掩码区",{"slug":3970,"path":3971,"title":3972,"date":3957,"tags":3973,"description":3978,"cover":3979,"coverAlt":3980,"excerpt":3981,"draft":3920,"hidden":3920,"published":3924,"readingTime":3950,"series":3928,"seriesOrder":3982,"codeUrl":3951,"attachmentUrl":3983},"cs336-bpe-tokenizer-training-encoding","\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding","Stanford CS336 学习记录 07：Byte-Level BPE 分词器训练、倒排索引加速与流式编码",[3931,3974,3975,3976,3977],"Tokenizer","BPE","Tiktoken","算法优化","实现 GPT-2 正则预分词、特殊 token 硬边界切分、基于 pair-to-word 倒排索引的增量 BPE 合并算法（\u003C 0.25s 通过速度基准），以及与 tiktoken 完全对齐的 Tokenizer。","\u002Fimages\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding\u002Fcs336-bpe-tokenizer-training-encoding-cover.webp","UTF-8 字节块经过正则切分闸口后沿合并树聚合成子词单元的 AI 辅助概念插画","前面六篇完成了 Transformer 语言模型本身的全部神经网络组件。要让模型真正处理自然语言文本，还需要打通入口处的 ByteLevel BPE（BytePair Encoding）分词器。这一篇记录 cs336basics\u002Fb",7,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-07-bpe-tokenizer-20261025.zip",{"slug":3985,"path":3986,"title":3987,"date":3988,"tags":3989,"description":3994,"cover":3995,"coverAlt":3996,"excerpt":3997,"draft":3920,"hidden":3920,"published":3924,"readingTime":3950},"in-situ-autoguidance-editable-scientific-illustration","\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration","从原位自校正引导（In-situ Autoguidance）到可编辑科研插图生成（AutoFigure-Edit）","2026-10-24",[3959,3990,3991,3992,3993,3964],"多模态","Agent","ICML","ACL","串联两项生成式模型与多模态智能体研究：如何通过推理期局部随机微扰在单模型内构造免训练扩散自引导向量（ICML 2025 Workshop 第一作者，arXiv:2510.17136），以及如何通过五阶段视觉-符号混合流水线生成原生可编辑 SVG 科研插图（ACL 2026 Demo，FigureBench 83.0% 胜率）。","\u002Fimages\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration\u002Fautoguidance-autofigure-cover-20261024.webp","左侧扩散流形自校正轨迹矢量场与右侧五阶段可编辑 SVG 科研插图分层网格相衔接的等距技术插画","生成式视觉模型正在经历从“生成高保真像素（Pixel Synthesis）”向“可控结构化生成（Controllable & Editable Generation）”的演进。在这一进程中，我作为第一作者或核心贡献者参与了两项分别处",{"slug":3999,"path":4000,"title":4001,"date":4002,"tags":4003,"description":4006,"cover":4007,"coverAlt":4008,"excerpt":4009,"draft":3920,"hidden":3920,"published":3924,"readingTime":3950,"series":3928,"seriesOrder":4010,"codeUrl":3951,"attachmentUrl":4011},"cs336-pre-norm-transformer-block-lm","\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm","Stanford CS336 学习记录 06：Pre-Norm Transformer Block 与完整语言模型装配","2026-10-23",[3931,3932,3933,4004,4005],"Pre-Norm","FLOPs","将 RMSNorm、带 RoPE 的因果多头自注意力和 SwiGLU 组装成 Pre-Norm Transformer Block，搭建完整 TransformerLM，并核算 GPT-2 XL 的参数量与前向 FLOPs。","\u002Fimages\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm\u002Fcs336-pre-norm-transformer-block-lm-cover.webp","干净残差流主干穿过多个 Pre-Norm 子层并汇聚到词表投影层的 AI 辅助概念插画","前五篇把 Linear、Embedding、RMSNorm、SwiGLU、RotaryPositionalEmbedding 和因果 MultiHeadSelfAttention 全部单独打通了。这一篇进入 Assignment 1",6,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-06-transformer-block-lm-20261023.zip",{"slug":4013,"path":4014,"title":4015,"date":4002,"tags":4016,"description":4022,"cover":4023,"coverAlt":4024,"excerpt":4025,"draft":3920,"hidden":3920,"published":3924,"readingTime":4026},"xgrammar-concurrent-cache-generation-fix","\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix","XGrammar 结构化解码缓存剖析：Unlimited 模式并发 Clear 字节记账竞态与代际修复（PR #1011）",[3961,4017,4018,4019,4020,4021],"C++","XGrammar","结构化解码","并发编程","源码分析","拆解 XGrammar GrammarCompiler 的双层缓存架构与 ThreadSafeLRUCache 锁分离实现，定位 Unlimited 模式下 Clear() 字节残留与在途编译（In-flight Computation）跨代污染根因，给出代际号 + 读写锁原子修复与零 sleep 确定性并发 GoogleTest 验证。","\u002Fimages\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix\u002Fxgrammar-cache-fix-cover-20261022.webp","双层磨砂玻璃流道经暖橙色代际闸门过滤后汇入晶格缓存矩阵的等距技术插画","在大语言模型服务（如 SGLang、vLLM）的结构化输出场景中，JSON Schema 或 EBNF 语法需要先编译为词表级掩码缓存（CompiledGrammar），才能在每个 Decode 步以微秒级开销生成合法 token ",14,{"slug":4028,"path":4029,"title":4030,"date":4031,"tags":4032,"description":4038,"cover":4039,"coverAlt":4040,"excerpt":4041,"draft":3920,"hidden":3920,"published":3924,"readingTime":4026},"multi-constraint-agent-routing-dag-eval","\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval","多约束生产级 Agent 系统实践：两阶段工具路由、DAG 局部重规划与轨迹级评测闭环","2026-10-22",[3991,4033,4034,4035,4036,4037],"LLM","工具调用","上下文工程","DPO","强化学习","围绕跨境酒旅等强耦合多约束业务场景，拆解向量 Top-K 召回 + LLM 精排 + 跨语言实体链接与 Schema 门控（96%+ 调用合法率）、基于状态机的 DAG 局部子图重规划、Observation 冗余裁剪（节省 40% Token），以及从线上失败恢复轨迹构造 SFT\u002FDPO 偏好对的评测与训练闭环。","\u002Fimages\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval\u002Fagent-dag-routing-cover-20261020.webp","候选模块经漏斗滤网进入双分支有向无环图轨道并在受阻节点触发暖橙色局部旁路重连的等距技术插画","在玩具级 Demo 里，几个工具加上一段 ReAct 循环就能跑通大多数单步问答。但当 Agent 进入跨境酒旅、复杂交易或研发沙箱等真实业务生产环境时，单靠线性 ReAct 循环很快会撞上四堵墙： 1. 工具空间与参数耦合爆炸：候",{"slug":4043,"path":4044,"title":4045,"date":4046,"tags":4047,"description":4051,"cover":4052,"coverAlt":4053,"excerpt":4054,"draft":3920,"hidden":3920,"published":3924,"readingTime":4055},"llm-mrd-distillation-auditable-review-agents","\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents","多视角推理蒸馏（LLM-MRD）与可审计生成式评审系统（DeepReviewer 2.0）深度拆解","2026-10-21",[4048,3990,4049,3991,4050,3964],"知识蒸馏","Qwen2.5-VL","Post-training","对比大模型落地的两条互补路径：如何用 Qwen2.5-VL 离线多视角分析结合 MLP 残差校准与温度缩放 KL 蒸馏，将推理能力压入 358M 学生模型实现 17.2 ms 独立推理（LLM-MRD，DASFAA 2026 Oral）；以及如何通过主张-证据-风险台账、PASA 可比性门控与导出预算校验构建可审计的同行评审 Agent（DeepReviewer 2.0）。","\u002Fimages\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents\u002Fmrd-review-distill-cover-20261016.webp","三块半透明几何棱镜发出平行光束穿过中央校准环汇聚至右侧高密度晶体模块的等距技术插画","把大语言模型或视觉语言模型（VLM）的深度推理能力用到实际系统中，通常会面临一个两难选择： 如果场景受限于毫秒级在线延迟（如高吞吐多模态内容审核与虚假信息检测）：直接在线调用 7B～72B 的多模态大模型做长链条思维链（CoT）生成",18,{"slug":4057,"path":4058,"title":4059,"date":4046,"tags":4060,"description":4062,"cover":4063,"coverAlt":4064,"excerpt":4065,"draft":3920,"hidden":3920,"published":3924,"readingTime":3982,"series":3928,"seriesOrder":4066,"codeUrl":3951,"attachmentUrl":4067},"cs336-rope-rotary-position-embedding","\u002Fposts\u002Fcs336-rope-rotary-position-embedding","Stanford CS336 学习记录 05：RoPE 旋转位置编码与带位置的多头注意力",[3931,3932,3933,4061,2288],"RoPE","从二维平面正交旋转推导 RoPE 的奇偶特征配对公式，理解为什么内积只依赖相对位置差 (i - j)，并用 register_buffer 预计算 cos\u002Fsin 缓存接入因果多头自注意力。","\u002Fimages\u002Fposts\u002Fcs336-rope-rotary-position-embedding\u002Fcs336-rope-rotary-position-embedding-cover.webp","二维特征平面中的旋转向量对与预计算余弦正弦网格的 AI 辅助概念插画","上一篇完成了不含位置编码的因果多头自注意力。如果只看那一套矩阵乘法，把输入序列前两个 token 交换顺序（并在因果 mask 允许的范围内看同一个后缀 query 对它们的点积），点积本身并不知道哪个 key 离当前 query ",5,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-05-rope-mha-20261021.zip",{"slug":4069,"path":4070,"title":4071,"date":4072,"tags":4073,"description":4079,"cover":4080,"coverAlt":4081,"excerpt":4082,"draft":3920,"hidden":3920,"published":3924,"readingTime":3950},"llm-gateway-semantic-cache-dynamic-batching","\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching","高并发 LLM 推理网关架构实践：两级精确\u002F语义缓存、50ms 有界动态批处理与尾部延迟治理","2026-10-19",[3961,4074,4075,4076,4077,4078],"LLM 推理","分布式系统","Redis","FastAPI","后端","深度拆解生产级大模型推理网关 LLM-Gateway 的核心系统设计：基于 SHA-256 与 Redis 向量余弦相似度的两级精确\u002F语义缓存、50ms 有界等待动态批处理调度器（Dynamic Batching）、令牌桶多租户限流与 Prometheus 全链路可观测性，将命中请求 P99 延迟从 5,307 ms 压降至 17 ms。","\u002Fimages\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching\u002Fllm-gateway-cover-20261019.webp","高并发请求流依次穿过精确哈希滤网、向量球面语义缓存层与 50ms 动态批处理时间窗拼装器的等距系统架构插画","在将大语言模型接入高并发业务系统时，如果让上游微服务直接裸连底层推理引擎（vLLM \u002F SGLang 或外部模型 API），系统很快会暴露出三个典型的生产痛点： 1. 高频重复与近义查询消耗巨量算力：大量用户请求在语义上高度等价（例",{"slug":4084,"path":4085,"title":4086,"date":4072,"tags":4087,"description":4089,"cover":4090,"coverAlt":4091,"excerpt":4092,"draft":3920,"hidden":3920,"published":3924,"readingTime":4026,"series":3928,"seriesOrder":4093,"codeUrl":4094,"attachmentUrl":4095},"cs336-multihead-attention-shapes","\u002Fposts\u002Fcs336-multihead-attention-shapes","Stanford CS336 学习记录 04：多头注意力怎样拆头、计算和合头",[3931,3932,3933,2288,4088],"张量形状","沿着 (2, 5, 64) 的输入追踪 QKV 投影、reshape、transpose、共享因果 mask 和合头，理解每个头仍处理完整序列，以及最后为什么还需要输出 Linear。","\u002Fimages\u002Fposts\u002Fcs336-multihead-attention-shapes\u002Fcs336-multihead-attention-shapes-cover.webp","宽向量束分成多条并行特征轨道后重新合并的 AI 辅助概念插画","上一篇完成了缩放点积注意力。那个函数接收已经准备好的 Q、K、V，计算匹配分数，应用 mask，再用权重汇总 V。这一篇继续把它放进一个带参数的多头自注意力模块。 这次最需要看清的是“拆头”。代码里只是几行 reshape 和 tr",4,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002F31363223498fafd1528c550a51efbe0aed231736","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-04-multihead-attention-20261019.zip",{"slug":4097,"path":4098,"title":4099,"date":3918,"tags":4100,"description":4104,"cover":4105,"coverAlt":4106,"excerpt":4107,"draft":3920,"hidden":3920,"published":3924,"readingTime":4108},"qwen25-sft-dpo-alignment-tax-ablation","\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation","大模型后训练偏好对齐实录：Qwen2.5 SFT 多维消融、动态参考 DPO 与对齐税（Alignment Tax）量化分析",[4050,4036,4101,4102,4037,4103],"SFT","LoRA","Qwen2.5","基于 Qwen2.5 完整复现 Instruct SFT → Bradley-Terry DPO 两阶段后训练管线：通过 3 随机种子消融 LoRA Rank、学习率与数据规模（GSM8K +12.73pp），实现免双倍权重的动态 Adapter 切换 DPO 训练，并定量揭示偏好对齐对确定性数学推理的对齐税效应。","\u002Fimages\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation\u002Fqwen25-sft-dpo-cover-20261018.webp","左侧监督微调轨迹流经中央 Bradley-Terry 偏好分流天平分离出正负样本通道并连接右侧对齐税监控仪表盘的等距技术插画","在大语言模型后训练（PostTraining）管线中，监督微调（SFT）负责教会模型任务格式与领域推理模式，而直接偏好优化（DPO, Direct Preference Optimization）则利用成对偏好数据 $(x, yw,",9,{"slug":4110,"path":3925,"title":5,"date":3918,"tags":4111,"description":3919,"cover":3916,"coverAlt":3917,"excerpt":4112,"draft":3920,"hidden":3920,"published":3924,"readingTime":4113,"series":3928,"seriesOrder":2133,"codeUrl":3808,"attachmentUrl":3923},"cs336-softmax-attention-mask",[3931,3932,3933,3934,2288],"上一篇写到 SwiGLU，接下来进入 Attention。真正开始相乘之前，先做了 Softmax：它把一组分数变成总和为 1 的权重，后面用这些权重汇总不同 token 提供的信息。 这段学习里，有两个很具体的问题值得记下来。一个",11,{"slug":4115,"path":4116,"title":4117,"date":4118,"tags":4119,"description":4120,"cover":4121,"coverAlt":4122,"excerpt":4123,"draft":3920,"hidden":3920,"published":3924,"readingTime":4108,"series":3928,"seriesOrder":2117,"codeUrl":3808,"attachmentUrl":4124},"cs336-rmsnorm-swiglu","\u002Fposts\u002Fcs336-rmsnorm-swiglu","Stanford CS336 学习记录 02：RMSNorm 的广播与 SwiGLU 的门控","2026-10-16",[3931,3932,3933,2219,2229],"用一个 token 向量理解均方根归一化，拆开 dim=-1 与 keepdim=True 的作用，再沿着两条分支读懂 SiLU、逐元素乘法和 SwiGLU。","\u002Fimages\u002Fposts\u002Fcs336-rmsnorm-swiglu\u002Fcs336-rmsnorm-swiglu-cover.webp","特征尺度整理和双分支门控的 AI 辅助概念插画","上一篇把 Linear 和 Embedding 的参数形状理顺了。继续往下，最先遇到的是 RMSNorm：公式不算长，但把它落到一批 token 上时，dim=1、keepdim=True 和广播必须一起理解。 然后是 SiLU 和","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-02-rmsnorm-swiglu-20261016.zip",{"slug":4126,"path":4127,"title":4128,"date":4129,"tags":4130,"description":4133,"cover":4134,"coverAlt":4135,"excerpt":4136,"draft":3920,"hidden":3920,"published":3924,"readingTime":4137,"series":3928,"seriesOrder":2076,"codeUrl":3808,"attachmentUrl":4138},"cs336-linear-embedding-shapes","\u002Fposts\u002Fcs336-linear-embedding-shapes","Stanford CS336 学习记录 01：从 Linear、Embedding 搞清张量形状","2026-10-14",[3931,3932,3933,4131,4132],"Linear","Embedding","从 PyTorch 类、参数与 forward 的关系开始，理解 Linear 为什么保存两行三列的权重、计算时为什么要转置，再区分 Embedding 查表与线性变换。","\u002Fimages\u002Fposts\u002Fcs336-linear-embedding-shapes\u002Fcs336-linear-embedding-shapes-cover.webp","token 方块、向量表与矩阵变换的 AI 辅助概念插画","开始沿着 Stanford CS336 的 Assignment 1 一步步实现。BPE 之前已经接触过，这轮先跳过分词器，从 Transformer 的基础层开始。这里记录的是当时写代码时真正卡住的问题，以及对应测试能确认到的范围",10,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-01-linear-embedding-20261014.zip",{"slug":4140,"path":4141,"title":4142,"date":4143,"tags":4144,"description":4149,"cover":4150,"coverAlt":4151,"excerpt":4152,"draft":3920,"hidden":3920,"published":3924,"readingTime":4153,"series":4145,"seriesOrder":4026,"codeUrl":4154,"attachmentUrl":4155},"cmu-15-779-kv-cache-granularity-study","\u002Fposts\u002Fcmu-15-779-kv-cache-granularity-study","CMU 15-779 学习记录 14：KV 页大小、共享边界与 CPU 实现成本","2026-10-12",[4145,4146,4147,4148],"CMU 15-779","KV cache","PagedAttention","系统实验","用 210 组 CPU 配置比较连续与分页 KV cache，观察 31\u002F32-token 共享前缀的页边界，分别记录活跃槽位、池容量、复制计数和完整调用耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling\u002Fcmu-serving-cover-20261019.webp","浅色手绘分页存储抽屉与共享前缀卡片，旁边连接不同长度的请求队列","个人课程项目中的 CPU 实验记录，延续记录 11 的分页 KV 实现。已完成 210 组配置与 1470 次计时，输出均通过连续存储参考对照；本篇不代表正式作业提交、整门课程或项目全部交付完成。 同样是共享 31 个 token，",12,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F200faf569eccf94fb125e65110bb5d14094d454a","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fkv-cache-granularity-20261019.zip",{"slug":4157,"path":4158,"title":4159,"date":4160,"tags":4161,"description":4164,"cover":4165,"coverAlt":4166,"excerpt":4167,"draft":3920,"hidden":3920,"published":3924,"readingTime":4066,"series":4145,"seriesOrder":4168,"codeUrl":4169,"attachmentUrl":4170},"cmu-15-779-moe-routing-capacity","\u002Fposts\u002Fcmu-15-779-moe-routing-capacity","CMU 15-779 学习记录 13：MoE 路由、稀疏分发与容量溢出","2026-10-09",[4145,4162,4163],"MoE","Routing","用 CPU top-2 MoE 检查稀疏分发的输出与梯度，统计 expert-token 行数，并保留容量丢弃会改变输出的反例。","\u002Fimages\u002Fposts\u002Fcmu-15-779-moe-routing-capacity\u002Fcmu-moe-cover-20261019.webp","浅色手绘分拣器将小珠分配到四个专家工作屋，队列呈现负载不均","第 18 讲的个人配套实验。36 号脚本串行运行 CPU 专家，检查 topk 路由后的分发与合并；没有训练 router，也没有执行 GPU grouped GEMM 或跨设备 alltoall。 上一节记录低秩适配，这次转向每个",13,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fc4851e0cedadf13af7c421867e83c8ff57244fa0","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-parallel-serving-lora-moe-20261019.zip",{"slug":4172,"path":4173,"title":4174,"date":4175,"tags":4176,"description":4178,"cover":4179,"coverAlt":4180,"excerpt":4181,"draft":3920,"hidden":3920,"published":3924,"readingTime":4066,"series":4145,"seriesOrder":4153,"codeUrl":4169,"attachmentUrl":4170},"cmu-15-779-lora-uniform-quantization","\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization","CMU 15-779 学习记录 12：LoRA 首步梯度、权重合并与均匀 4-bit 打包","2026-10-06",[4145,4102,4177],"Quantization","用一次 CPU SGD 更新检查 LoRA 初始化、冻结基座与合并等价，再分别记录均匀 4-bit 存储字节数和量化误差。","\u002Fimages\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization\u002Fcmu-lora-cover-20261019.webp","浅色手绘的大计算板、两块细长适配板和成对打包的小方块","第 17 讲的个人配套实验，35 号脚本实际执行一次 CPU LoRA 更新，并实现均匀 4bit 打包。实验检查机制与数值误差，不评估微调任务效果，也没有实现完整 QLoRA。 这次把两个问题分开记录：低秩分支是否能合并进原权重，",{"slug":4183,"path":4184,"title":4185,"date":4186,"tags":4187,"description":4191,"cover":4150,"coverAlt":4192,"excerpt":4193,"draft":3920,"hidden":3920,"published":3924,"readingTime":3982,"series":4145,"seriesOrder":4113,"codeUrl":4169,"attachmentUrl":4170},"cmu-15-779-kv-serving-speculative-sampling","\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling","CMU 15-779 学习记录 11：分页 KV、连续批处理与推测采样","2026-10-03",[4145,4188,4189,4190],"LLM Serving","KV Cache","Speculative Decoding","在 CPU 上验证 KV 页共享和写时复制，用单位成本模型检查连续批处理，再用条件概率小模型核对推测采样的接受与补采样。","浅色手绘的共享卡片抽屉与不断补入小珠子的请求托盘","第 15、16 讲的个人配套实验。33 号实现 CPU 分页 KV 存储，37 号模拟请求准入，34 号运行小概率模型的推测采样。它们分别验证存储、调度和分布问题，没有测量真实 LLM 服务吞吐或 GPU 延迟。 记录 08已经对照",{"slug":4195,"path":4196,"title":4197,"date":4198,"tags":4199,"description":4204,"cover":4205,"coverAlt":4206,"excerpt":4207,"draft":3920,"hidden":3920,"published":3924,"readingTime":4208},"qwen3-mid-post-train-mlx","\u002Fposts\u002Fqwen3-mid-post-train-mlx","在 M2 Pro 上完成 Qwen3 两阶段训练：Mid Train、SFT 与可复核评估","2026-09-30 11:19:10",[4033,4200,4201,4102,4202,4101,4203],"Qwen3","MLX","持续预训练","Apple Silicon","用 Qwen3 0.6B Base 和 MLX 完成领域持续训练与监督微调，记录数据划分、聊天模板、损失掩码、数值稳定性修复和独立测试结果。","\u002Fimages\u002Fposts\u002Fqwen3-mid-post-train-mlx\u002Fqwen3-mlx-cover-20261018.webp","Apple Silicon 统一内存芯片底座支撑两阶段持续预训练与带精确 Loss 掩码滤网的监督微调桥接插画","在一台 M2 Pro、16 GB 内存的 Mac 上，能否把一个 Base 语言模型接续训练成输出 JSON 的运维事件分级助手？这次项目完成了领域持续训练、监督微调、检查点选择、重新加载和独立测试，并保留了权重、日志和数据哈希。 ",16,{"slug":4210,"path":4211,"title":4212,"date":4213,"tags":4214,"description":4218,"cover":4219,"coverAlt":4220,"excerpt":4221,"draft":3920,"hidden":3920,"published":3924,"readingTime":4010,"series":4145,"seriesOrder":4137,"codeUrl":4169,"attachmentUrl":4170},"cmu-15-779-cpu-parallelism-pipeline","\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline","CMU 15-779 学习记录 10：加权梯度、状态分片与同步流水线","2026-09-30",[4145,4215,4216,4217],"Parallel Training","Gloo","Pipeline","用双 CPU 进程 Gloo 检查不等数据分片的梯度，推导持久状态字节数，再用依赖模型比较 GPipe 与同步 1F1B 的激活存活。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline\u002Fcmu-parallel-cover-20261019.webp","浅色手绘的四阶段桌面流水线与按数量汇总的托盘","第 12 至 14 讲的个人配套实验，脚本编号 31、32 不对应官方作业。Gloo 实际运行两个 CPU 进程；张量切分、状态字节数和流水线分别按下文的数学或成本模型解释。真实 GPU 实测按学习安排跳过。 上一节检查单个计算图的",{"slug":4223,"path":4224,"title":4225,"date":4226,"tags":4227,"description":4230,"cover":4231,"coverAlt":4232,"excerpt":4233,"draft":3920,"hidden":3920,"published":3924,"readingTime":3982,"series":4145,"seriesOrder":4108,"codeUrl":4234,"attachmentUrl":4235},"cmu-15-779-cpu-compiler-optimizations","\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations","CMU 15-779 学习记录 09：CPU 编译优化、合法重写与候选搜索","2026-09-27",[4145,4228,4229,3932],"ML Compiler","TorchInductor","用 CPU 实验验证 BatchNorm 折叠、RMSNorm 重排与非线性反例，测量分块候选，并运行 TorchInductor，分别记录首调用与预热后的时间。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations\u002Fcmu-cpu-compiler-cover-20261019.webp","淡紫计算积木经过薄荷绿折叠模具变成浅蓝积木，前方摆放四种候选托盘的浅色手绘插画","编译章节的个人配套实验，编号 30 不对应官方作业。图重写、分块候选和 TorchInductor 均在 CPU 上运行；真实 GPU 实验按学习安排跳过。 上一节记录了分块 Attention 和 KV cache。这次转到编译优","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F9261bea6aedf31cdccbbab2541ca693fc97b3743","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-compiler-optimizations-20261019.zip",{"slug":4237,"path":4238,"title":4239,"date":4240,"tags":4241,"description":4243,"cover":4244,"coverAlt":4245,"excerpt":4246,"draft":3920,"hidden":3920,"published":3924,"readingTime":4153,"series":4145,"seriesOrder":3950,"codeUrl":4154,"attachmentUrl":4247},"cmu-15-779-attention-online-softmax","\u002Fposts\u002Fcmu-15-779-attention-online-softmax","CMU 15-779 学习记录 08：分块 Attention、在线 Softmax 与 KV cache","2026-09-24",[4145,2288,3934,4242,3932],"FlashAttention","在 CPU 上对照完整 Attention、在线 Softmax 分块实现与 PyTorch SDPA，检查因果掩码、浮点误差和分数块大小，理解 FlashAttention 的一部分算法基础。","\u002Fimages\u002Fposts\u002Fcmu-15-779-attention-online-softmax\u002Fcmu-online-attention-cover-20261016.webp","浅紫色卡片分批进入绿色计算托盘，旁边三个状态容器保存累积量的浅色手绘插画","第 5 讲 Transformers & Attention 的个人配套实验。28\u002F29 号是 CPU 单头前向与 KV cache；后续补入 38 号 CUDA 多头合成张量的显式 SDPA MATH\u002FFLASH 对照。没有手写 ","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcuda-triton-sdpa-20261019.zip",{"slug":4249,"path":4250,"title":4251,"date":4252,"tags":4253,"description":4258,"cover":4259,"coverAlt":4260,"excerpt":4261,"draft":3920,"hidden":3920,"published":3924,"readingTime":4137,"series":4145,"seriesOrder":3982,"codeUrl":4154,"attachmentUrl":4247},"cmu-15-779-matmul-tiling-tree-reduction","\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction","CMU 15-779 学习记录 07：矩阵分块、数据复用与树形归约","2026-09-21",[4145,4254,4255,4256,4257],"CUDA","矩阵乘法","数据复用","归约","用 CPU 模型统计矩阵分块的输入读取次数，再比较串行求和和树形归约的依赖深度，记录补零、同步与浮点加法顺序的边界。","\u002Fimages\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction\u002Fcmu-matmul-reduction-cover-20261016.webp","矩阵积木在共享托盘中复用，旁边的小珠沿树形管道逐层合并的浅色手绘插画","GPU Architecture & CUDA Programming 的个人配套实验。首次记录用 CPU 模型检查数据复用与依赖深度；后续补入单卡 GPU 原生 CUDA 矩阵验证，以及 Triton 两阶段归约和矩阵乘法实测。 ",{"slug":4263,"path":4264,"title":4265,"date":4266,"tags":4267,"description":4270,"cover":4271,"coverAlt":4272,"excerpt":4273,"draft":3920,"hidden":3920,"published":3924,"readingTime":4137,"series":4145,"seriesOrder":4010,"codeUrl":4154,"attachmentUrl":4247},"cmu-15-779-cuda-thread-indexing","\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing","CMU 15-779 学习记录 06：CUDA 线程索引与边界检查","2026-09-18",[4145,4254,3963,4268,4269],"线程索引","边界检查","用 CPU 演示 CUDA 的 block、thread 与全局下标：10 个元素如何分给 3 个线程块，为什么要跳过越界线程，以及这些验证能说明什么。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing\u002Fcmu-cuda-indexing-cover-20261015.webp","分组的浅紫色计算方块与末端边界挡板组成线程分配教具的浅色插画","GPU Architecture & CUDA Programming 的个人配套练习。首次记录为 CPU 索引演示；后续在单卡 GPU 环境补入原生 CUDA 编译、运行与 memcheck，以及 Triton 向量加法实测。 上",{"slug":4275,"path":4276,"title":4277,"date":4278,"tags":4279,"description":4283,"cover":4284,"coverAlt":4285,"excerpt":4286,"draft":3920,"hidden":3920,"published":3924,"readingTime":4137,"series":4145,"seriesOrder":4066,"codeUrl":4287,"attachmentUrl":4288},"cmu-15-779-profiler-operator-batching","\u002Fposts\u002Fcmu-15-779-profiler-operator-batching","CMU 15-779 学习记录 05：Profiler、算子调用与批量推理","2026-09-15",[4145,3932,4280,4281,4282],"Profiler","批量推理","性能分析","对照 ML Systems 基础，用 Profiler 拆开两层网络的执行过程，再固定 1000 个样本扫描 batch size，观察算子调用次数、尾批和 CPU 耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-profiler-operator-batching\u002Fcmu-pytorch-profiling-cover-20261015.webp","放大镜观察线性层与 ReLU 模块，旁边面板呈现执行片段的浅色技术教具插画","本篇对照课程官网的 Introduction to ML Systems 整理，是个人配套 CPU 实验，不是课程官方作业。脚本编号是练习顺序，不对应课程讲次。 前几篇一直在补 PyTorch 基础。到了这一步，我在学习聊天里问：现","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fcecf5110d8a95161bb5763eadfca94cd7a0cc50f","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-profiler-batching-20261015.zip",{"slug":4290,"path":4291,"title":4292,"date":4293,"tags":4294,"description":4297,"cover":4298,"coverAlt":4299,"excerpt":4300,"draft":3920,"hidden":3920,"published":3924,"readingTime":4108,"series":4145,"seriesOrder":4093,"codeUrl":4301,"attachmentUrl":4302},"cmu-15-779-pytorch-linear-relu-absolute-value","\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value","CMU 15-779 学习记录 04：多输出线性层、ReLU 与绝对值网络","2026-09-12",[4145,3932,4255,4295,4296],"ReLU","神经网络","把 nn.Linear 对应到矩阵乘法，弄清多个输出怎样计算，再逐层拆开一个绝对值网络，比较手动设置参数与训练得到的结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value\u002Fcmu-pytorch-absolute-cover-20261015.webp","双分支经过折角模块后汇入紫色 V 形曲线面板的浅色技术教具插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的批量推理与矩阵乘法，我继续把 model(x) 拆开看：一层里有多个输出时，每个数是怎么算出来的？两","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F4cc63904a3a583da4a2122c246a14b43aac38528","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-linear-relu-abs-20261015.zip",{"slug":4304,"path":4305,"title":4306,"date":4307,"tags":4308,"description":4311,"cover":4312,"coverAlt":4313,"excerpt":4314,"draft":3920,"hidden":3920,"published":3924,"readingTime":4108,"series":4145,"seriesOrder":2133,"codeUrl":4315,"attachmentUrl":4316},"cmu-15-779-pytorch-dataloader-checkpoints-batching","\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching","CMU 15-779 学习记录 03：DataLoader、模型保存与批量推理","2026-09-09",[4145,3932,4309,4281,4310],"DataLoader","模型保存","用 DataLoader 分批训练平方函数，保存并加载模型参数，再比较 1000 个样本与 1 个样本的推理耗时，最后把批量计算对应到矩阵乘法。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching\u002Fcmu-pytorch-batching-cover-20261015.webp","样本卡片按批次进入计算模块，旁边配有参数存储盒和输出矩阵面板的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的平方函数实验，这一节继续练习分批训练、保存参数和独立预测。最后做了一次计时：同一个小网络，逐个处理 ","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-batching-20261015.zip",{"slug":4318,"path":4319,"title":4320,"date":4321,"tags":4322,"description":4323,"cover":4324,"coverAlt":4325,"excerpt":4326,"draft":3920,"hidden":3920,"published":3924,"readingTime":4108,"series":4145,"seriesOrder":2117,"codeUrl":4315,"attachmentUrl":4327},"cmu-15-779-pytorch-linear-module-relu","\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu","CMU 15-779 学习记录 02：nn.Linear、张量形状与 ReLU","2026-09-06",[4145,3932,4296,4295],"从带偏置的线性回归开始，弄清 nn.Linear 和 nn.Module 的分工，再用平方函数观察线性模型的限制、ReLU 的作用和测试点的选择。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu\u002Fcmu-pytorch-relu-cover-20261015.webp","输入经过两层神经网络和 ReLU 折角模块，输出曲线拟合结果的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 上一篇里，我用一个参数拟合了 $y=2x$。这一篇继续顺着往下写，先加上偏置，再把手写参数换成 nn.Linea","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-models-20261015.zip",{"slug":4329,"path":4330,"title":4331,"date":4332,"tags":4333,"description":4336,"cover":4337,"coverAlt":4338,"excerpt":4339,"draft":3920,"hidden":3920,"published":3924,"readingTime":4137,"series":4145,"seriesOrder":2076,"codeUrl":4315,"attachmentUrl":4340},"cmu-15-779-pytorch-autograd-linear-regression","\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression","CMU 15-779 学习记录 01：PyTorch 自动求导与线性回归","2026-09-03",[4145,3932,4334,4335],"自动求导","线性回归","从自动求导和手动梯度下降开始，比较学习率，再用 SGD 训练线性回归。记录 w * 4 写错后停在 1.25 的原因和修正结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression\u002Fcmu-pytorch-autograd-cover-20261015.webp","梯度曲面上的小球沿路径下降，前侧面板展示散点与拟合直线的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 开始学习 CMU 15779 时，我会 Python，但还没用过 PyTorch。这次先在 Mac 上用 CPU","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-basics-20261013.zip",{"slug":4342,"path":4343,"title":4344,"date":4345,"tags":4346,"description":4352,"cover":4353,"coverAlt":4354,"excerpt":4355,"draft":3920,"hidden":3920,"published":3924,"readingTime":4168,"codeUrl":4356,"attachmentUrl":4357},"embodied-pickplace-lab-robustness","\u002Fposts\u002Fembodied-pickplace-lab-robustness","Pickplace Lab 项目复盘：抓取成功率之外的时延与失败分析","2026-08-29",[4347,4348,4349,3932,4350,4351],"具身智能","模仿学习","MuJoCo","机器人","鲁棒性","从 MuJoCo 抓取搬运闭环出发，记录 22,100 个评测回合。动作平滑在自建任务的 50 ms 延迟下有效，换到 Panda Lift 却没有支持同样的收益。","\u002Fimages\u002Fposts\u002Fembodied-pickplace-lab-robustness\u002Fpickplace-grasp-cover-20261016.webp","MuJoCo 实际渲染的笛卡尔夹爪抓取方块场景，桌面上标出放置目标","Pickplace Lab 从一个简单任务开始：让夹爪抓起方块，搬到指定位置，放下，再退开。先把仿真、示范采集、训练和闭环评测接起来，随后再问：干净条件下成功的策略，换一种执行条件，还能不能完成任务？ 最值得讨论的结果出现在指令延迟","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Ftree\u002Fv0.1.0","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Freleases\u002Fdownload\u002Fv0.1.0\u002Fpickplace-lab-research-v1.zip",{"slug":4359,"path":4360,"title":4361,"date":4362,"tags":4363,"description":4368,"cover":4369,"coverAlt":4370,"excerpt":4371,"draft":3920,"hidden":3920,"published":3924,"readingTime":4372},"qwen25-vl-sglang-verl-single-gpu-lab","\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab","从 Mac 到单卡 GPU：Qwen2.5-VL 部署、GRPO 与计数 LoRA 实践","2026-08-19",[4049,3990,4364,4365,4366,4367,4201],"SGLang","verl","GRPO","推理优化","在 M2 Pro 和云端单卡上部署 Qwen2.5-VL-3B，实测连续批处理、前缀缓存与 SGLang 参数调优，再用 verl 完成多模态 GRPO 更新和同条件前后评测，记录有效优化、兼容性问题与未观察到提升的结果。","\u002Fimages\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab\u002Fqvl-lab-cover-20261010.webp","笔记本与单卡计算模块通过图像和 token 光轨连接的实验台插画","这次实验从一个很具体的问题开始：M2 Pro、16 GB 统一内存，能不能在本机跑 Qwen2.5VL？跑起来之后，我又想把图文问答、推理性能测量和强化学习训练接成一条完整流程。 本机的 MLX 4bit 服务和云端的 SGLang",29,{"slug":4374,"path":4375,"title":4376,"date":4377,"tags":4378,"description":4382,"cover":4383,"coverAlt":4384,"excerpt":4385,"draft":3920,"hidden":3920,"published":3924,"readingTime":4168},"cycleresearcher-post-training","\u002Fposts\u002Fcycleresearcher-post-training","从 DeepReviewer 2.0 到 CycleResearcher：科研 Agent 如何真正做 Post-training","2026-08-07 17:33:05",[4033,4050,3991,4379,4380,4381],"SimPO","Reward Model","ICLR","区分推理时 Agent 编排与模型后训练，并拆解 CycleResearcher 的领域 SFT、生成式 Reward Model、Iterative SimPO 和评测风险。","\u002Fimages\u002Fposts\u002Fcycleresearcher-post-training\u002Fcycleresearcher-cover-20261018.webp","策略生成模块产出多份候选文稿经多维评审棱镜排序后通过偏好梯度反馈弧闭环优化的等距技术插画","看到一个系统会检索、规划、反思和调用工具，不能据此判断它做过 Posttraining。Agent 描述的是模型在运行时如何完成任务；Posttraining 描述的是预训练之后，是否继续用监督数据、偏好数据或奖励信号更新模型参数。",{"slug":4387,"path":4388,"title":4389,"date":4390,"tags":4391,"description":4393,"cover":4394,"coverAlt":4395,"excerpt":4396,"draft":3920,"hidden":3920,"published":3924,"readingTime":4026,"series":4397,"seriesOrder":2076},"cuda-fused-rmsnorm-kernel","\u002Fposts\u002Fcuda-fused-rmsnorm-kernel","从 PyTorch 到 CUDA Kernel：融合 RMSNorm 在 RTX 4090D 上加速 4.27 倍","2026-07-22 18:16:50",[4254,3961,3963,3932,4074,4392],"Kernel 优化","从零实现 PyTorch CUDA 扩展，拆解 RMSNorm 的访存瓶颈、warp 归约、向量化加载、算子融合与冷热缓存 benchmark。","\u002Fimages\u002Fposts\u002Fcuda-fused-rmsnorm-kernel\u002Fcuda-rmsnorm-cover-20261018.webp","融合残差加法与向量化 float4 访存经 Warp Shuffle 蝴蝶树归约输出归一化张量的等距硬件架构插画","为了准备 AI Infra 岗位，我做了一个尽量小而完整的 CUDA 项目：不追求堆很多算子，而是选一个真实的 LLM 推理路径，把实现、正确性、benchmark 和性能解释全部做扎实。 项目实现了两个 PyTorch CUDA ","AI Infra 实战",{"slug":4399,"path":4400,"title":4401,"date":4402,"tags":4403,"description":4405,"cover":4406,"coverAlt":4407,"excerpt":4408,"draft":3920,"hidden":3920,"published":3924,"readingTime":3950,"series":4409,"seriesOrder":2076},"agent-memory","\u002Fposts\u002Fagent-memory","Agent 上下文工程与分层记忆架构：从 Observation 剪枝、滚动摘要到向量召回闭环","2026-04-24",[4033,3991,4404,3961],"RAG","结合生产级多约束 Agent 与超 100K Token 科研智能体实践，系统拆解长程会话中的上下文膨胀与注意力稀释问题：涵盖 Observation 结构化剪枝（降低约 40% Token 开销）、滚动工作摘要、向量检索长期记忆及 Prefix Cache 友好的上下文组装。","\u002Fimages\u002Fposts\u002Fagent-memory\u002Fagent-memory-cover-20261018.webp","底层高噪数据经剪枝滤网进入中期滑动窗口与金色状态台账并由顶层向量星座按需召回的三层记忆架构插画","在构建多步工具调用（ToolUse）与长周期自主规划（Autonomous Planning）的 Agent 系统时，上下文管理（Context Engineering）往往比单纯改写 System Prompt 更决定线上稳定性。","Agent 上下文工程",{"slug":4411,"path":4412,"title":4413,"date":4414,"tags":4415,"description":4418,"cover":4419,"coverAlt":4420,"excerpt":4421,"draft":3920,"hidden":3920,"published":3924,"readingTime":4066,"series":4422,"seriesOrder":2117},"backend-stack-deep-dive","\u002Fposts\u002Fbackend-stack-deep-dive","高并发 AI 平台后端基础设施实践：FastAPI 异步网关、SQLAlchemy 2.0 Async、PostgreSQL 与 Docker 沙箱编排","2026-04-07",[4078,4075,4077,4416,4417,3961],"PostgreSQL","Docker","结合 DeepScientist 生产级 AI 科研平台实践，拆解 FastAPI ASGI 异步事件循环、SQLAlchemy 2.0 + AsyncPG 连接池调优、PostgreSQL + MinIO 存储分层与 Docker SDK 动态沙箱容器编排的职责边界与核心设计。","\u002Fimages\u002Fposts\u002Fbackend-stack-deep-dive\u002Fbackend-stack-cover-20261018.webp","ASGI 异步事件循环涡轮将非阻塞协程流分发至 PostgreSQL 连接池、MinIO 存储矩阵与动态 Docker 沙箱的等距架构插画","结合 DeepScientist 生产级 AI 科研平台（服务 7,000+ 用户）的实际工程落地，本文系统拆解支撑高并发流式推理与容器沙箱调度的后端基础设施栈——FastAPI（ASGI 异步网关 + SSE 流式推送）、SQLA","DeepScientist 工程复盘",{"slug":4424,"path":4425,"title":4426,"date":4427,"tags":4428,"description":4431,"cover":4432,"coverAlt":4433,"excerpt":4434,"draft":3920,"hidden":3920,"published":3924,"readingTime":4113,"series":4435,"seriesOrder":2076},"minicode-source-analysis","\u002Fposts\u002Fminicode-source-analysis","MiniCode 架构与工具评测深度拆解：开源终端 Agentic Coding 引擎（1k+ Stars）的 ReAct、MCP 与 30 项基准验证","2026-04-06",[3991,4429,4430,4021,3961],"TypeScript","MCP","深度剖析开源终端 AI 编程智能体 MiniCode（1k+ GitHub Stars）的核心架构：从零重型依赖的 ReAct 规划循环、声明式工具注册表与权限拦截层、MCP 协议帧自动协商，到 30 项可执行 Tool-Use 评测基准（86.7% 完成率、100% 错误恢复率）的失败分布归因与闭环优化。","\u002Fimages\u002Fposts\u002Fminicode-source-analysis\u002Fminicode-cover-20261018.webp","终端编程智能体控制台被 ReAct 状态环、Unified Diff 补丁预览门与 MCP 协议插槽环绕的等距技术插画","MiniCode 是我作为核心开发者参与构建的开源终端 Agentic Coding 引擎（目前已在 GitHub 获得 1,000+ Stars）。项目的核心设计目标是：在不引入庞大运行时框架的前提下，用约 5,000 行高内聚的","AI 项目源码导读",{"slug":4437,"path":4438,"title":4439,"date":4427,"tags":4440,"description":4441,"cover":4442,"coverAlt":4443,"excerpt":4444,"draft":3920,"hidden":3920,"published":3924,"readingTime":4137,"series":4422,"seriesOrder":2076},"deepscientist-tech-stack","\u002Fposts\u002Fdeepscientist-tech-stack","DeepScientist 架构与运行时全解析：超 100K 上下文科研智能体、MCP 沙箱与高并发流式系统",[3991,3961,4430,4417,4077,4075],"深度拆解服务 7,000+ 用户的科研自主智能体平台 DeepScientist：从一题一仓库（Quest）持久化状态机、MCP 工具栈（memory \u002F artifact \u002F bash_exec）、Docker SDK 动态容器编排，到 SSE + WebSocket 双流通信与 PostgreSQL \u002F Redis \u002F Prometheus 生产级基础设施。","\u002Fimages\u002Fposts\u002Fdeepscientist-tech-stack\u002Fdeepscientist-cover-20261018.webp","科研智能体编排核心通过 memory、artifact 与隔离容器沙箱三座协议桥连接双流工作台的等距架构插画","DeepScientist（deepscientist.cc）是我们在西湖大学自然语言处理实验室研发的自主科学发现与科研管理平台，目前累计服务 7,000+ 科研用户。与单轮对话或纯文本文献问答工具不同，科研智能体需要在数小时乃至数",1795088946049]