[{"data":1,"prerenderedAt":5217},["ShallowReactive",2],{"post-\u002Fposts\u002Fcs336-rope-rotary-position-embedding":3,"archive-public-posts":4714},{"id":4,"title":5,"body":6,"categories":4692,"cover":4694,"coverAlt":4695,"date":4696,"description":4697,"draft":4698,"extension":4699,"haloPublished":4698,"haloSync":4698,"hidden":4698,"legacy":4698,"meta":4700,"navigation":3441,"path":4701,"published":4698,"readingTime":4702,"seo":4703,"series":4704,"seriesOrder":4705,"stem":4706,"tags":4707,"__hash__":4713},"posts\u002Fposts\u002Fcs336-rope-rotary-position-embedding.md","Stanford CS336 学习记录 05：RoPE 旋转位置编码与带位置的多头注意力",{"type":7,"value":8,"toc":4680},"minimark",[9,18,35,48,245,504,1402,1539,1965,2145,2621,2968,3029,3044,3283,3302,3309,3507,3510,3778,3786,3811,3949,4141,4156,4159,4219,4227,4233,4339,4365,4372,4410,4427,4571,4604,4607,4621,4627,4638,4650,4653,4676],[10,11,12,17],"p",{},[13,14,16],"a",{"href":15},"\u002Fposts\u002Fcs336-multihead-attention-shapes\u002F","上一篇","完成了不含位置编码的因果多头自注意力。如果只看那一套矩阵乘法，把输入序列前两个 token 交换顺序（并在因果 mask 允许的范围内看同一个后缀 query 对它们的点积），点积本身并不知道哪个 key 离当前 query 更近、哪个更远。",[10,19,20,21,25,26,30,31,34],{},"现代语言模型（LLaMA、Qwen 等）普遍采用 ",[22,23,24],"strong",{},"RoPE（Rotary Position Embedding，旋转位置编码）"," 来注入相对位置信息。这一篇记录 ",[27,28,29],"code",{},"cs336_basics\u002Frope.py"," 与带 RoPE 的 ",[27,32,33],{},"MultiHeadSelfAttention"," 的完整推导、形状对齐与单元测试验证。",[10,36,37,43,44],{},[13,38,42],{"href":39,"rel":40},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fb5c93ad0cc73178b2b2a62e518a0900da47a0fb7",[41],"nofollow","查看代码仓库"," · ",[13,45,47],{"href":46},"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-05-rope-mha-20261021.zip","下载本节代码与测试 ZIP",[49,50,52,53,160,161,244],"h2",{"id":51},"为什么把-dkd_kdk-维向量拆成-dk2d_k2dk2-个二维平面","为什么把 ",[54,55,58,87],"span",{"className":56},[57],"katex",[54,59,62],{"className":60},[61],"katex-mathml",[63,64,66],"math",{"xmlns":65},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[67,68,69,82],"semantics",{},[70,71,72],"mrow",{},[73,74,75,79],"msub",{},[76,77,78],"mi",{},"d",[76,80,81],{},"k",[83,84,86],"annotation",{"encoding":85},"application\u002Fx-tex","d_k",[54,88,92],{"className":89,"ariaHidden":91},[90],"katex-html","true",[54,93,96,101],{"className":94},[95],"base",[54,97],{"className":98,"style":100},[99],"strut","height:0.8444em;vertical-align:-0.15em;",[54,102,105,109],{"className":103},[104],"mord",[54,106,78],{"className":107},[104,108],"mathnormal",[54,110,113],{"className":111},[112],"msupsub",[54,114,118,151],{"className":115},[116,117],"vlist-t","vlist-t2",[54,119,122,146],{"className":120},[121],"vlist-r",[54,123,127],{"className":124,"style":126},[125],"vlist","height:0.3361em;",[54,128,130,135],{"style":129},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[54,131],{"className":132,"style":134},[133],"pstrut","height:2.7em;",[54,136,142],{"className":137},[138,139,140,141],"sizing","reset-size6","size3","mtight",[54,143,81],{"className":144,"style":145},[104,108,141],"margin-right:0.0315em;",[54,147,150],{"className":148},[149],"vlist-s","​",[54,152,154],{"className":153},[121],[54,155,158],{"className":156,"style":157},[125],"height:0.15em;",[54,159],{}," 维向量拆成 ",[54,162,164,190],{"className":163},[57],[54,165,167],{"className":166},[61],[63,168,169],{"xmlns":65},[67,170,171,187],{},[70,172,173,179,183],{},[73,174,175,177],{},[76,176,78],{},[76,178,81],{},[76,180,182],{"mathvariant":181},"normal","\u002F",[184,185,186],"mn",{},"2",[83,188,189],{"encoding":85},"d_k\u002F2",[54,191,193],{"className":192,"ariaHidden":91},[90],[54,194,196,200,240],{"className":195},[95],[54,197],{"className":198,"style":199},[99],"height:1em;vertical-align:-0.25em;",[54,201,203,206],{"className":202},[104],[54,204,78],{"className":205},[104,108],[54,207,209],{"className":208},[112],[54,210,212,232],{"className":211},[116,117],[54,213,215,229],{"className":214},[121],[54,216,218],{"className":217,"style":126},[125],[54,219,220,223],{"style":129},[54,221],{"className":222,"style":134},[133],[54,224,226],{"className":225},[138,139,140,141],[54,227,81],{"className":228,"style":145},[104,108,141],[54,230,150],{"className":231},[149],[54,233,235],{"className":234},[121],[54,236,238],{"className":237,"style":157},[125],[54,239],{},[54,241,243],{"className":242},[104],"\u002F2"," 个二维平面",[10,246,247,248,470,471,503],{},"在二维平面上，把一个向量 ",[54,249,251,307],{"className":250},[57],[54,252,254],{"className":253},[61],[63,255,256],{"xmlns":65},[67,257,258,304],{},[70,259,260,264,301],{},[261,262,263],"mo",{"fence":91},"[",[265,266,270,288],"mtable",{"rowspacing":267,"columnalign":268,"columnspacing":269},"0.16em","center","1em",[271,272,273],"mtr",{},[274,275,276],"mtd",{},[277,278,281],"mstyle",{"scriptlevel":279,"displaystyle":280},"0","false",[73,282,283,286],{},[76,284,285],{},"x",[184,287,279],{},[271,289,290],{},[274,291,292],{},[277,293,294],{"scriptlevel":279,"displaystyle":280},[73,295,296,298],{},[76,297,285],{},[184,299,300],{},"1",[261,302,303],{"fence":91},"]",[83,305,306],{"encoding":85},"\\begin{bmatrix}x_0 \\\\ x_1\\end{bmatrix}",[54,308,310],{"className":309,"ariaHidden":91},[90],[54,311,313,317],{"className":312},[95],[54,314],{"className":315,"style":316},[99],"height:2.4em;vertical-align:-0.95em;",[54,318,321,331,463],{"className":319},[320],"minner",[54,322,327],{"className":323,"style":326},[324,325],"mopen","delimcenter","top:0em;",[54,328,263],{"className":329},[330,140],"delimsizing",[54,332,334],{"className":333},[104],[54,335,337],{"className":336},[265],[54,338,341],{"className":339},[340],"col-align-c",[54,342,344,454],{"className":343},[116,117],[54,345,347,451],{"className":346},[121],[54,348,351,402],{"className":349,"style":350},[125],"height:1.45em;",[54,352,354,358],{"style":353},"top:-3.61em;",[54,355],{"className":356,"style":357},[133],"height:3em;",[54,359,361],{"className":360},[104],[54,362,364,367],{"className":363},[104],[54,365,285],{"className":366},[104,108],[54,368,370],{"className":369},[112],[54,371,373,394],{"className":372},[116,117],[54,374,376,391],{"className":375},[121],[54,377,380],{"className":378,"style":379},[125],"height:0.3011em;",[54,381,382,385],{"style":129},[54,383],{"className":384,"style":134},[133],[54,386,388],{"className":387},[138,139,140,141],[54,389,279],{"className":390},[104,141],[54,392,150],{"className":393},[149],[54,395,397],{"className":396},[121],[54,398,400],{"className":399,"style":157},[125],[54,401],{},[54,403,405,408],{"style":404},"top:-2.41em;",[54,406],{"className":407,"style":357},[133],[54,409,411],{"className":410},[104],[54,412,414,417],{"className":413},[104],[54,415,285],{"className":416},[104,108],[54,418,420],{"className":419},[112],[54,421,423,443],{"className":422},[116,117],[54,424,426,440],{"className":425},[121],[54,427,429],{"className":428,"style":379},[125],[54,430,431,434],{"style":129},[54,432],{"className":433,"style":134},[133],[54,435,437],{"className":436},[138,139,140,141],[54,438,300],{"className":439},[104,141],[54,441,150],{"className":442},[149],[54,444,446],{"className":445},[121],[54,447,449],{"className":448,"style":157},[125],[54,450],{},[54,452,150],{"className":453},[149],[54,455,457],{"className":456},[121],[54,458,461],{"className":459,"style":460},[125],"height:0.95em;",[54,462],{},[54,464,467],{"className":465,"style":326},[466,325],"mclose",[54,468,303],{"className":469},[330,140]," 逆时针旋转角度 ",[54,472,474,489],{"className":473},[57],[54,475,477],{"className":476},[61],[63,478,479],{"xmlns":65},[67,480,481,486],{},[70,482,483],{},[76,484,485],{},"θ",[83,487,488],{"encoding":85},"\\theta",[54,490,492],{"className":491,"ariaHidden":91},[90],[54,493,495,499],{"className":494},[95],[54,496],{"className":497,"style":498},[99],"height:0.6944em;",[54,500,485],{"className":501,"style":502},[104,108],"margin-right:0.0278em;","，对应的正交变换矩阵是：",[54,505,508],{"className":506},[507],"katex-display",[54,509,511,734],{"className":510},[57],[54,512,514],{"className":513},[61],[63,515,517],{"xmlns":65,"display":516},"block",[67,518,519,731],{},[70,520,521,524,527,529,532,535,602,605,609,611,613,615,617,649,651,728],{},[76,522,523],{},"R",[261,525,526],{"stretchy":280},"(",[76,528,485],{},[261,530,531],{"stretchy":280},")",[261,533,534],{},"=",[70,536,537,539,600],{},[261,538,263],{"fence":91},[265,540,542,574],{"rowspacing":267,"columnalign":541,"columnspacing":269},"center center",[271,543,544,558],{},[274,545,546],{},[277,547,548],{"scriptlevel":279,"displaystyle":280},[70,549,550,553,556],{},[76,551,552],{},"cos",[261,554,555],{},"⁡",[76,557,485],{},[274,559,560],{},[277,561,562],{"scriptlevel":279,"displaystyle":280},[70,563,564,567,570,572],{},[261,565,566],{},"−",[76,568,569],{},"sin",[261,571,555],{},[76,573,485],{},[271,575,576,588],{},[274,577,578],{},[277,579,580],{"scriptlevel":279,"displaystyle":280},[70,581,582,584,586],{},[76,583,569],{},[261,585,555],{},[76,587,485],{},[274,589,590],{},[277,591,592],{"scriptlevel":279,"displaystyle":280},[70,593,594,596,598],{},[76,595,552],{},[261,597,555],{},[76,599,485],{},[261,601,303],{"fence":91},[261,603,604],{"separator":91},",",[606,607],"mspace",{"width":608},"2em",[76,610,523],{},[261,612,526],{"stretchy":280},[76,614,485],{},[261,616,531],{"stretchy":280},[70,618,619,621,647],{},[261,620,263],{"fence":91},[265,622,623,635],{"rowspacing":267,"columnalign":268,"columnspacing":269},[271,624,625],{},[274,626,627],{},[277,628,629],{"scriptlevel":279,"displaystyle":280},[73,630,631,633],{},[76,632,285],{},[184,634,279],{},[271,636,637],{},[274,638,639],{},[277,640,641],{"scriptlevel":279,"displaystyle":280},[73,642,643,645],{},[76,644,285],{},[184,646,300],{},[261,648,303],{"fence":91},[261,650,534],{},[70,652,653,655,726],{},[261,654,263],{"fence":91},[265,656,657,691],{"rowspacing":267,"columnalign":268,"columnspacing":269},[271,658,659],{},[274,660,661],{},[277,662,663],{"scriptlevel":279,"displaystyle":280},[70,664,665,671,673,675,677,679,685,687,689],{},[73,666,667,669],{},[76,668,285],{},[184,670,279],{},[76,672,552],{},[261,674,555],{},[76,676,485],{},[261,678,566],{},[73,680,681,683],{},[76,682,285],{},[184,684,300],{},[76,686,569],{},[261,688,555],{},[76,690,485],{},[271,692,693],{},[274,694,695],{},[277,696,697],{"scriptlevel":279,"displaystyle":280},[70,698,699,705,707,709,711,714,720,722,724],{},[73,700,701,703],{},[76,702,285],{},[184,704,279],{},[76,706,569],{},[261,708,555],{},[76,710,485],{},[261,712,713],{},"+",[73,715,716,718],{},[76,717,285],{},[184,719,300],{},[76,721,552],{},[261,723,555],{},[76,725,485],{},[261,727,303],{"fence":91},[76,729,730],{"mathvariant":181},".",[83,732,733],{"encoding":85},"R(\\theta) =\n\\begin{bmatrix}\n\\cos\\theta & -\\sin\\theta \\\\\n\\sin\\theta & \\cos\\theta\n\\end{bmatrix},\n\\qquad\nR(\\theta)\n\\begin{bmatrix}\nx_0 \\\\\nx_1\n\\end{bmatrix}\n=\n\\begin{bmatrix}\nx_0\\cos\\theta - x_1\\sin\\theta \\\\\nx_0\\sin\\theta + x_1\\cos\\theta\n\\end{bmatrix}.",[54,735,737,767,1102],{"className":736,"ariaHidden":91},[90],[54,738,740,743,747,750,753,756,760,764],{"className":739},[95],[54,741],{"className":742,"style":199},[99],[54,744,523],{"className":745,"style":746},[104,108],"margin-right:0.0077em;",[54,748,526],{"className":749},[324],[54,751,485],{"className":752,"style":502},[104,108],[54,754,531],{"className":755},[466],[54,757],{"className":758,"style":759},[606],"margin-right:0.2778em;",[54,761,534],{"className":762},[763],"mrel",[54,765],{"className":766,"style":759},[606],[54,768,770,773,924,927,931,935,938,941,944,947,950,953,1093,1096,1099],{"className":769},[95],[54,771],{"className":772,"style":316},[99],[54,774,776,782,918],{"className":775},[320],[54,777,779],{"className":778,"style":326},[324,325],[54,780,263],{"className":781},[330,140],[54,783,785],{"className":784},[104],[54,786,788,847,852,855],{"className":787},[265],[54,789,791],{"className":790},[340],[54,792,794,839],{"className":793},[116,117],[54,795,797,836],{"className":796},[121],[54,798,800,819],{"className":799,"style":350},[125],[54,801,802,805],{"style":353},[54,803],{"className":804,"style":357},[133],[54,806,808,812,816],{"className":807},[104],[54,809,552],{"className":810},[811],"mop",[54,813],{"className":814,"style":815},[606],"margin-right:0.1667em;",[54,817,485],{"className":818,"style":502},[104,108],[54,820,821,824],{"style":404},[54,822],{"className":823,"style":357},[133],[54,825,827,830,833],{"className":826},[104],[54,828,569],{"className":829},[811],[54,831],{"className":832,"style":815},[606],[54,834,485],{"className":835,"style":502},[104,108],[54,837,150],{"className":838},[149],[54,840,842],{"className":841},[121],[54,843,845],{"className":844,"style":460},[125],[54,846],{},[54,848],{"className":849,"style":851},[850],"arraycolsep","width:0.5em;",[54,853],{"className":854,"style":851},[850],[54,856,858],{"className":857},[340],[54,859,861,910],{"className":860},[116,117],[54,862,864,907],{"className":863},[121],[54,865,867,890],{"className":866,"style":350},[125],[54,868,869,872],{"style":353},[54,870],{"className":871,"style":357},[133],[54,873,875,878,881,884,887],{"className":874},[104],[54,876,566],{"className":877},[104],[54,879],{"className":880,"style":815},[606],[54,882,569],{"className":883},[811],[54,885],{"className":886,"style":815},[606],[54,888,485],{"className":889,"style":502},[104,108],[54,891,892,895],{"style":404},[54,893],{"className":894,"style":357},[133],[54,896,898,901,904],{"className":897},[104],[54,899,552],{"className":900},[811],[54,902],{"className":903,"style":815},[606],[54,905,485],{"className":906,"style":502},[104,108],[54,908,150],{"className":909},[149],[54,911,913],{"className":912},[121],[54,914,916],{"className":915,"style":460},[125],[54,917],{},[54,919,921],{"className":920,"style":326},[466,325],[54,922,303],{"className":923},[330,140],[54,925],{"className":926,"style":815},[606],[54,928,604],{"className":929},[930],"mpunct",[54,932],{"className":933,"style":934},[606],"margin-right:2em;",[54,936],{"className":937,"style":815},[606],[54,939,523],{"className":940,"style":746},[104,108],[54,942,526],{"className":943},[324],[54,945,485],{"className":946,"style":502},[104,108],[54,948,531],{"className":949},[466],[54,951],{"className":952,"style":815},[606],[54,954,956,962,1087],{"className":955},[320],[54,957,959],{"className":958,"style":326},[324,325],[54,960,263],{"className":961},[330,140],[54,963,965],{"className":964},[104],[54,966,968],{"className":967},[265],[54,969,971],{"className":970},[340],[54,972,974,1079],{"className":973},[116,117],[54,975,977,1076],{"className":976},[121],[54,978,980,1028],{"className":979,"style":350},[125],[54,981,982,985],{"style":353},[54,983],{"className":984,"style":357},[133],[54,986,988],{"className":987},[104],[54,989,991,994],{"className":990},[104],[54,992,285],{"className":993},[104,108],[54,995,997],{"className":996},[112],[54,998,1000,1020],{"className":999},[116,117],[54,1001,1003,1017],{"className":1002},[121],[54,1004,1006],{"className":1005,"style":379},[125],[54,1007,1008,1011],{"style":129},[54,1009],{"className":1010,"style":134},[133],[54,1012,1014],{"className":1013},[138,139,140,141],[54,1015,279],{"className":1016},[104,141],[54,1018,150],{"className":1019},[149],[54,1021,1023],{"className":1022},[121],[54,1024,1026],{"className":1025,"style":157},[125],[54,1027],{},[54,1029,1030,1033],{"style":404},[54,1031],{"className":1032,"style":357},[133],[54,1034,1036],{"className":1035},[104],[54,1037,1039,1042],{"className":1038},[104],[54,1040,285],{"className":1041},[104,108],[54,1043,1045],{"className":1044},[112],[54,1046,1048,1068],{"className":1047},[116,117],[54,1049,1051,1065],{"className":1050},[121],[54,1052,1054],{"className":1053,"style":379},[125],[54,1055,1056,1059],{"style":129},[54,1057],{"className":1058,"style":134},[133],[54,1060,1062],{"className":1061},[138,139,140,141],[54,1063,300],{"className":1064},[104,141],[54,1066,150],{"className":1067},[149],[54,1069,1071],{"className":1070},[121],[54,1072,1074],{"className":1073,"style":157},[125],[54,1075],{},[54,1077,150],{"className":1078},[149],[54,1080,1082],{"className":1081},[121],[54,1083,1085],{"className":1084,"style":460},[125],[54,1086],{},[54,1088,1090],{"className":1089,"style":326},[466,325],[54,1091,303],{"className":1092},[330,140],[54,1094],{"className":1095,"style":759},[606],[54,1097,534],{"className":1098},[763],[54,1100],{"className":1101,"style":759},[606],[54,1103,1105,1108,1396,1399],{"className":1104},[95],[54,1106],{"className":1107,"style":316},[99],[54,1109,1111,1117,1390],{"className":1110},[320],[54,1112,1114],{"className":1113,"style":326},[324,325],[54,1115,263],{"className":1116},[330,140],[54,1118,1120],{"className":1119},[104],[54,1121,1123],{"className":1122},[265],[54,1124,1126],{"className":1125},[340],[54,1127,1129,1382],{"className":1128},[116,117],[54,1130,1132,1379],{"className":1131},[121],[54,1133,1135,1258],{"className":1134,"style":350},[125],[54,1136,1137,1140],{"style":353},[54,1138],{"className":1139,"style":357},[133],[54,1141,1143,1183,1186,1189,1192,1195,1199,1203,1206,1246,1249,1252,1255],{"className":1142},[104],[54,1144,1146,1149],{"className":1145},[104],[54,1147,285],{"className":1148},[104,108],[54,1150,1152],{"className":1151},[112],[54,1153,1155,1175],{"className":1154},[116,117],[54,1156,1158,1172],{"className":1157},[121],[54,1159,1161],{"className":1160,"style":379},[125],[54,1162,1163,1166],{"style":129},[54,1164],{"className":1165,"style":134},[133],[54,1167,1169],{"className":1168},[138,139,140,141],[54,1170,279],{"className":1171},[104,141],[54,1173,150],{"className":1174},[149],[54,1176,1178],{"className":1177},[121],[54,1179,1181],{"className":1180,"style":157},[125],[54,1182],{},[54,1184],{"className":1185,"style":815},[606],[54,1187,552],{"className":1188},[811],[54,1190],{"className":1191,"style":815},[606],[54,1193,485],{"className":1194,"style":502},[104,108],[54,1196],{"className":1197,"style":1198},[606],"margin-right:0.2222em;",[54,1200,566],{"className":1201},[1202],"mbin",[54,1204],{"className":1205,"style":1198},[606],[54,1207,1209,1212],{"className":1208},[104],[54,1210,285],{"className":1211},[104,108],[54,1213,1215],{"className":1214},[112],[54,1216,1218,1238],{"className":1217},[116,117],[54,1219,1221,1235],{"className":1220},[121],[54,1222,1224],{"className":1223,"style":379},[125],[54,1225,1226,1229],{"style":129},[54,1227],{"className":1228,"style":134},[133],[54,1230,1232],{"className":1231},[138,139,140,141],[54,1233,300],{"className":1234},[104,141],[54,1236,150],{"className":1237},[149],[54,1239,1241],{"className":1240},[121],[54,1242,1244],{"className":1243,"style":157},[125],[54,1245],{},[54,1247],{"className":1248,"style":815},[606],[54,1250,569],{"className":1251},[811],[54,1253],{"className":1254,"style":815},[606],[54,1256,485],{"className":1257,"style":502},[104,108],[54,1259,1260,1263],{"style":404},[54,1261],{"className":1262,"style":357},[133],[54,1264,1266,1306,1309,1312,1315,1318,1321,1324,1327,1367,1370,1373,1376],{"className":1265},[104],[54,1267,1269,1272],{"className":1268},[104],[54,1270,285],{"className":1271},[104,108],[54,1273,1275],{"className":1274},[112],[54,1276,1278,1298],{"className":1277},[116,117],[54,1279,1281,1295],{"className":1280},[121],[54,1282,1284],{"className":1283,"style":379},[125],[54,1285,1286,1289],{"style":129},[54,1287],{"className":1288,"style":134},[133],[54,1290,1292],{"className":1291},[138,139,140,141],[54,1293,279],{"className":1294},[104,141],[54,1296,150],{"className":1297},[149],[54,1299,1301],{"className":1300},[121],[54,1302,1304],{"className":1303,"style":157},[125],[54,1305],{},[54,1307],{"className":1308,"style":815},[606],[54,1310,569],{"className":1311},[811],[54,1313],{"className":1314,"style":815},[606],[54,1316,485],{"className":1317,"style":502},[104,108],[54,1319],{"className":1320,"style":1198},[606],[54,1322,713],{"className":1323},[1202],[54,1325],{"className":1326,"style":1198},[606],[54,1328,1330,1333],{"className":1329},[104],[54,1331,285],{"className":1332},[104,108],[54,1334,1336],{"className":1335},[112],[54,1337,1339,1359],{"className":1338},[116,117],[54,1340,1342,1356],{"className":1341},[121],[54,1343,1345],{"className":1344,"style":379},[125],[54,1346,1347,1350],{"style":129},[54,1348],{"className":1349,"style":134},[133],[54,1351,1353],{"className":1352},[138,139,140,141],[54,1354,300],{"className":1355},[104,141],[54,1357,150],{"className":1358},[149],[54,1360,1362],{"className":1361},[121],[54,1363,1365],{"className":1364,"style":157},[125],[54,1366],{},[54,1368],{"className":1369,"style":815},[606],[54,1371,552],{"className":1372},[811],[54,1374],{"className":1375,"style":815},[606],[54,1377,485],{"className":1378,"style":502},[104,108],[54,1380,150],{"className":1381},[149],[54,1383,1385],{"className":1384},[121],[54,1386,1388],{"className":1387,"style":460},[125],[54,1389],{},[54,1391,1393],{"className":1392,"style":326},[466,325],[54,1394,303],{"className":1395},[330,140],[54,1397],{"className":1398,"style":815},[606],[54,1400,730],{"className":1401},[104],[10,1403,1404,1405,1435,1436,1470,1471,1502,1503,1538],{},"如果位置 ",[54,1406,1408,1422],{"className":1407},[57],[54,1409,1411],{"className":1410},[61],[63,1412,1413],{"xmlns":65},[67,1414,1415,1420],{},[70,1416,1417],{},[76,1418,1419],{},"i",[83,1421,1419],{"encoding":85},[54,1423,1425],{"className":1424,"ariaHidden":91},[90],[54,1426,1428,1432],{"className":1427},[95],[54,1429],{"className":1430,"style":1431},[99],"height:0.6595em;",[54,1433,1419],{"className":1434},[104,108]," 的 query 二维向量旋转 ",[54,1437,1439,1455],{"className":1438},[57],[54,1440,1442],{"className":1441},[61],[63,1443,1444],{"xmlns":65},[67,1445,1446,1452],{},[70,1447,1448,1450],{},[76,1449,1419],{},[76,1451,485],{},[83,1453,1454],{"encoding":85},"i\\theta",[54,1456,1458],{"className":1457,"ariaHidden":91},[90],[54,1459,1461,1464,1467],{"className":1460},[95],[54,1462],{"className":1463,"style":498},[99],[54,1465,1419],{"className":1466},[104,108],[54,1468,485],{"className":1469,"style":502},[104,108],"，位置 ",[54,1472,1474,1488],{"className":1473},[57],[54,1475,1477],{"className":1476},[61],[63,1478,1479],{"xmlns":65},[67,1480,1481,1486],{},[70,1482,1483],{},[76,1484,1485],{},"j",[83,1487,1485],{"encoding":85},[54,1489,1491],{"className":1490,"ariaHidden":91},[90],[54,1492,1494,1498],{"className":1493},[95],[54,1495],{"className":1496,"style":1497},[99],"height:0.854em;vertical-align:-0.1944em;",[54,1499,1485],{"className":1500,"style":1501},[104,108],"margin-right:0.0572em;"," 的 key 二维向量旋转 ",[54,1504,1506,1522],{"className":1505},[57],[54,1507,1509],{"className":1508},[61],[63,1510,1511],{"xmlns":65},[67,1512,1513,1519],{},[70,1514,1515,1517],{},[76,1516,1485],{},[76,1518,485],{},[83,1520,1521],{"encoding":85},"j\\theta",[54,1523,1525],{"className":1524,"ariaHidden":91},[90],[54,1526,1528,1532,1535],{"className":1527},[95],[54,1529],{"className":1530,"style":1531},[99],"height:0.8889em;vertical-align:-0.1944em;",[54,1533,1485],{"className":1534,"style":1501},[104,108],[54,1536,485],{"className":1537,"style":502},[104,108],"，那么它们旋转后的点积满足正交旋转群的加性性质：",[54,1540,1542],{"className":1541},[507],[54,1543,1545,1665],{"className":1544},[57],[54,1546,1548],{"className":1547},[61],[63,1549,1550],{"xmlns":65,"display":516},[67,1551,1552,1662],{},[70,1553,1554,1579,1597,1599,1605,1607,1609,1611,1613,1619,1621,1623,1625,1627,1629,1631,1633,1639,1641,1644,1646,1648,1650,1652,1654,1656,1658,1660],{},[1555,1556,1557,1576],"msup",{},[70,1558,1559,1561,1563,1565,1567,1569,1571,1574],{},[261,1560,526],{"fence":91},[76,1562,523],{},[261,1564,526],{"stretchy":280},[76,1566,1419],{},[76,1568,485],{},[261,1570,531],{"stretchy":280},[76,1572,1573],{},"q",[261,1575,531],{"fence":91},[76,1577,1578],{"mathvariant":181},"⊤",[70,1580,1581,1583,1585,1587,1589,1591,1593,1595],{},[261,1582,526],{"fence":91},[76,1584,523],{},[261,1586,526],{"stretchy":280},[76,1588,1485],{},[76,1590,485],{},[261,1592,531],{"stretchy":280},[76,1594,81],{},[261,1596,531],{"fence":91},[261,1598,534],{},[1555,1600,1601,1603],{},[76,1602,1573],{},[76,1604,1578],{"mathvariant":181},[76,1606,523],{},[261,1608,526],{"stretchy":280},[76,1610,1419],{},[76,1612,485],{},[1555,1614,1615,1617],{},[261,1616,531],{"stretchy":280},[76,1618,1578],{"mathvariant":181},[76,1620,523],{},[261,1622,526],{"stretchy":280},[76,1624,1485],{},[76,1626,485],{},[261,1628,531],{"stretchy":280},[76,1630,81],{},[261,1632,534],{},[1555,1634,1635,1637],{},[76,1636,1573],{},[76,1638,1578],{"mathvariant":181},[76,1640,523],{},[261,1642,526],{"fence":280,"stretchy":91,"minsize":1643,"maxsize":1643},"1.2em",[261,1645,526],{"stretchy":280},[76,1647,1485],{},[261,1649,566],{},[76,1651,1419],{},[261,1653,531],{"stretchy":280},[76,1655,485],{},[261,1657,531],{"fence":280,"stretchy":91,"minsize":1643,"maxsize":1643},[76,1659,81],{},[76,1661,730],{"mathvariant":181},[83,1663,1664],{"encoding":85},"\\left(R(i\\theta)q\\right)^\\top \\left(R(j\\theta)k\\right)\n= q^\\top R(i\\theta)^\\top R(j\\theta) k\n= q^\\top R\\big((j-i)\\theta\\big) k.",[54,1666,1668,1770,1876,1937],{"className":1667,"ariaHidden":91},[90],[54,1669,1671,1675,1731,1734,1761,1764,1767],{"className":1670},[95],[54,1672],{"className":1673,"style":1674},[99],"height:1.239em;vertical-align:-0.25em;",[54,1676,1678,1706],{"className":1677},[320],[54,1679,1681,1684,1687,1690,1693,1696,1699,1703],{"className":1680},[320],[54,1682,526],{"className":1683,"style":326},[324,325],[54,1685,523],{"className":1686,"style":746},[104,108],[54,1688,526],{"className":1689},[324],[54,1691,1419],{"className":1692},[104,108],[54,1694,485],{"className":1695,"style":502},[104,108],[54,1697,531],{"className":1698},[466],[54,1700,1573],{"className":1701,"style":1702},[104,108],"margin-right:0.0359em;",[54,1704,531],{"className":1705,"style":326},[466,325],[54,1707,1709],{"className":1708},[112],[54,1710,1712],{"className":1711},[116],[54,1713,1715],{"className":1714},[121],[54,1716,1719],{"className":1717,"style":1718},[125],"height:0.989em;",[54,1720,1722,1725],{"style":1721},"top:-3.2029em;margin-right:0.05em;",[54,1723],{"className":1724,"style":134},[133],[54,1726,1728],{"className":1727},[138,139,140,141],[54,1729,1578],{"className":1730},[104,141],[54,1732],{"className":1733,"style":815},[606],[54,1735,1737,1740,1743,1746,1749,1752,1755,1758],{"className":1736},[320],[54,1738,526],{"className":1739,"style":326},[324,325],[54,1741,523],{"className":1742,"style":746},[104,108],[54,1744,526],{"className":1745},[324],[54,1747,1485],{"className":1748,"style":1501},[104,108],[54,1750,485],{"className":1751,"style":502},[104,108],[54,1753,531],{"className":1754},[466],[54,1756,81],{"className":1757,"style":145},[104,108],[54,1759,531],{"className":1760,"style":326},[466,325],[54,1762],{"className":1763,"style":759},[606],[54,1765,534],{"className":1766},[763],[54,1768],{"className":1769,"style":759},[606],[54,1771,1773,1777,1808,1811,1814,1817,1820,1849,1852,1855,1858,1861,1864,1867,1870,1873],{"className":1772},[95],[54,1774],{"className":1775,"style":1776},[99],"height:1.1491em;vertical-align:-0.25em;",[54,1778,1780,1783],{"className":1779},[104],[54,1781,1573],{"className":1782,"style":1702},[104,108],[54,1784,1786],{"className":1785},[112],[54,1787,1789],{"className":1788},[116],[54,1790,1792],{"className":1791},[121],[54,1793,1796],{"className":1794,"style":1795},[125],"height:0.8991em;",[54,1797,1799,1802],{"style":1798},"top:-3.113em;margin-right:0.05em;",[54,1800],{"className":1801,"style":134},[133],[54,1803,1805],{"className":1804},[138,139,140,141],[54,1806,1578],{"className":1807},[104,141],[54,1809,523],{"className":1810,"style":746},[104,108],[54,1812,526],{"className":1813},[324],[54,1815,1419],{"className":1816},[104,108],[54,1818,485],{"className":1819,"style":502},[104,108],[54,1821,1823,1826],{"className":1822},[466],[54,1824,531],{"className":1825},[466],[54,1827,1829],{"className":1828},[112],[54,1830,1832],{"className":1831},[116],[54,1833,1835],{"className":1834},[121],[54,1836,1838],{"className":1837,"style":1795},[125],[54,1839,1840,1843],{"style":1798},[54,1841],{"className":1842,"style":134},[133],[54,1844,1846],{"className":1845},[138,139,140,141],[54,1847,1578],{"className":1848},[104,141],[54,1850,523],{"className":1851,"style":746},[104,108],[54,1853,526],{"className":1854},[324],[54,1856,1485],{"className":1857,"style":1501},[104,108],[54,1859,485],{"className":1860,"style":502},[104,108],[54,1862,531],{"className":1863},[466],[54,1865,81],{"className":1866,"style":145},[104,108],[54,1868],{"className":1869,"style":759},[606],[54,1871,534],{"className":1872},[763],[54,1874],{"className":1875,"style":759},[606],[54,1877,1879,1883,1912,1915,1922,1925,1928,1931,1934],{"className":1878},[95],[54,1880],{"className":1881,"style":1882},[99],"height:1.2491em;vertical-align:-0.35em;",[54,1884,1886,1889],{"className":1885},[104],[54,1887,1573],{"className":1888,"style":1702},[104,108],[54,1890,1892],{"className":1891},[112],[54,1893,1895],{"className":1894},[116],[54,1896,1898],{"className":1897},[121],[54,1899,1901],{"className":1900,"style":1795},[125],[54,1902,1903,1906],{"style":1798},[54,1904],{"className":1905,"style":134},[133],[54,1907,1909],{"className":1908},[138,139,140,141],[54,1910,1578],{"className":1911},[104,141],[54,1913,523],{"className":1914,"style":746},[104,108],[54,1916,1918],{"className":1917},[104],[54,1919,526],{"className":1920},[330,1921],"size1",[54,1923,526],{"className":1924},[324],[54,1926,1485],{"className":1927,"style":1501},[104,108],[54,1929],{"className":1930,"style":1198},[606],[54,1932,566],{"className":1933},[1202],[54,1935],{"className":1936,"style":1198},[606],[54,1938,1940,1944,1947,1950,1953,1959,1962],{"className":1939},[95],[54,1941],{"className":1942,"style":1943},[99],"height:1.2em;vertical-align:-0.35em;",[54,1945,1419],{"className":1946},[104,108],[54,1948,531],{"className":1949},[466],[54,1951,485],{"className":1952,"style":502},[104,108],[54,1954,1956],{"className":1955},[104],[54,1957,531],{"className":1958},[330,1921],[54,1960,81],{"className":1961,"style":145},[104,108],[54,1963,730],{"className":1964},[104],[10,1966,1967,1968,1996,1997,2025,2026,1996,2056,2087,2088,2091,2092,2144],{},"也就是说，虽然旋转操作是按每个 token 自己的绝对位置 ",[54,1969,1971,1984],{"className":1970},[57],[54,1972,1974],{"className":1973},[61],[63,1975,1976],{"xmlns":65},[67,1977,1978,1982],{},[70,1979,1980],{},[76,1981,1419],{},[83,1983,1419],{"encoding":85},[54,1985,1987],{"className":1986,"ariaHidden":91},[90],[54,1988,1990,1993],{"className":1989},[95],[54,1991],{"className":1992,"style":1431},[99],[54,1994,1419],{"className":1995},[104,108]," 和 ",[54,1998,2000,2013],{"className":1999},[57],[54,2001,2003],{"className":2002},[61],[63,2004,2005],{"xmlns":65},[67,2006,2007,2011],{},[70,2008,2009],{},[76,2010,1485],{},[83,2012,1485],{"encoding":85},[54,2014,2016],{"className":2015,"ariaHidden":91},[90],[54,2017,2019,2022],{"className":2018},[95],[54,2020],{"className":2021,"style":1497},[99],[54,2023,1485],{"className":2024,"style":1501},[104,108]," 独立施加在 ",[54,2027,2029,2043],{"className":2028},[57],[54,2030,2032],{"className":2031},[61],[63,2033,2034],{"xmlns":65},[67,2035,2036,2041],{},[70,2037,2038],{},[76,2039,2040],{},"Q",[83,2042,2040],{"encoding":85},[54,2044,2046],{"className":2045,"ariaHidden":91},[90],[54,2047,2049,2053],{"className":2048},[95],[54,2050],{"className":2051,"style":2052},[99],"height:0.8778em;vertical-align:-0.1944em;",[54,2054,2040],{"className":2055},[104,108],[54,2057,2059,2073],{"className":2058},[57],[54,2060,2062],{"className":2061},[61],[63,2063,2064],{"xmlns":65},[67,2065,2066,2071],{},[70,2067,2068],{},[76,2069,2070],{},"K",[83,2072,2070],{"encoding":85},[54,2074,2076],{"className":2075,"ariaHidden":91},[90],[54,2077,2079,2083],{"className":2078},[95],[54,2080],{"className":2081,"style":2082},[99],"height:0.6833em;",[54,2084,2070],{"className":2085,"style":2086},[104,108],"margin-right:0.0715em;"," 上的，但点积注意力分数只依赖它们的",[22,2089,2090],{},"相对位置差"," ",[54,2093,2095,2113],{"className":2094},[57],[54,2096,2098],{"className":2097},[61],[63,2099,2100],{"xmlns":65},[67,2101,2102,2110],{},[70,2103,2104,2106,2108],{},[76,2105,1419],{},[261,2107,566],{},[76,2109,1485],{},[83,2111,2112],{"encoding":85},"i-j",[54,2114,2116,2135],{"className":2115,"ariaHidden":91},[90],[54,2117,2119,2123,2126,2129,2132],{"className":2118},[95],[54,2120],{"className":2121,"style":2122},[99],"height:0.7429em;vertical-align:-0.0833em;",[54,2124,1419],{"className":2125},[104,108],[54,2127],{"className":2128,"style":1198},[606],[54,2130,566],{"className":2131},[1202],[54,2133],{"className":2134,"style":1198},[606],[54,2136,2138,2141],{"className":2137},[95],[54,2139],{"className":2140,"style":1497},[99],[54,2142,1485],{"className":2143,"style":1501},[104,108],"。",[10,2146,2147,2148,2217,2218,2287,2288,2444,2445,2620],{},"推广到每头维度 ",[54,2149,2151,2168],{"className":2150},[57],[54,2152,2154],{"className":2153},[61],[63,2155,2156],{"xmlns":65},[67,2157,2158,2166],{},[70,2159,2160],{},[73,2161,2162,2164],{},[76,2163,78],{},[76,2165,81],{},[83,2167,86],{"encoding":85},[54,2169,2171],{"className":2170,"ariaHidden":91},[90],[54,2172,2174,2177],{"className":2173},[95],[54,2175],{"className":2176,"style":100},[99],[54,2178,2180,2183],{"className":2179},[104],[54,2181,78],{"className":2182},[104,108],[54,2184,2186],{"className":2185},[112],[54,2187,2189,2209],{"className":2188},[116,117],[54,2190,2192,2206],{"className":2191},[121],[54,2193,2195],{"className":2194,"style":126},[125],[54,2196,2197,2200],{"style":129},[54,2198],{"className":2199,"style":134},[133],[54,2201,2203],{"className":2202},[138,139,140,141],[54,2204,81],{"className":2205,"style":145},[104,108,141],[54,2207,150],{"className":2208},[149],[54,2210,2212],{"className":2211},[121],[54,2213,2215],{"className":2214,"style":157},[125],[54,2216],{},"（要求 ",[54,2219,2221,2238],{"className":2220},[57],[54,2222,2224],{"className":2223},[61],[63,2225,2226],{"xmlns":65},[67,2227,2228,2236],{},[70,2229,2230],{},[73,2231,2232,2234],{},[76,2233,78],{},[76,2235,81],{},[83,2237,86],{"encoding":85},[54,2239,2241],{"className":2240,"ariaHidden":91},[90],[54,2242,2244,2247],{"className":2243},[95],[54,2245],{"className":2246,"style":100},[99],[54,2248,2250,2253],{"className":2249},[104],[54,2251,78],{"className":2252},[104,108],[54,2254,2256],{"className":2255},[112],[54,2257,2259,2279],{"className":2258},[116,117],[54,2260,2262,2276],{"className":2261},[121],[54,2263,2265],{"className":2264,"style":126},[125],[54,2266,2267,2270],{"style":129},[54,2268],{"className":2269,"style":134},[133],[54,2271,2273],{"className":2272},[138,139,140,141],[54,2274,81],{"className":2275,"style":145},[104,108,141],[54,2277,150],{"className":2278},[149],[54,2280,2282],{"className":2281},[121],[54,2283,2285],{"className":2284,"style":157},[125],[54,2286],{}," 为偶数），RoPE 将第 ",[54,2289,2291,2337],{"className":2290},[57],[54,2292,2294],{"className":2293},[61],[63,2295,2296],{"xmlns":65},[67,2297,2298,2334],{},[70,2299,2300,2302,2305,2308,2310,2312,2314,2316,2319,2321,2327,2329,2331],{},[76,2301,81],{},[261,2303,2304],{},"∈",[261,2306,2307],{"stretchy":280},"{",[184,2309,300],{},[261,2311,604],{"separator":91},[184,2313,186],{},[261,2315,604],{"separator":91},[261,2317,2318],{},"…",[261,2320,604],{"separator":91},[73,2322,2323,2325],{},[76,2324,78],{},[76,2326,81],{},[76,2328,182],{"mathvariant":181},[184,2330,186],{},[261,2332,2333],{"stretchy":280},"}",[83,2335,2336],{"encoding":85},"k \\in \\{1, 2, \\dots, d_k\u002F2\\}",[54,2338,2340,2359],{"className":2339,"ariaHidden":91},[90],[54,2341,2343,2347,2350,2353,2356],{"className":2342},[95],[54,2344],{"className":2345,"style":2346},[99],"height:0.7335em;vertical-align:-0.0391em;",[54,2348,81],{"className":2349,"style":145},[104,108],[54,2351],{"className":2352,"style":759},[606],[54,2354,2304],{"className":2355},[763],[54,2357],{"className":2358,"style":759},[606],[54,2360,2362,2365,2368,2371,2374,2377,2380,2383,2386,2389,2392,2395,2398,2438,2441],{"className":2361},[95],[54,2363],{"className":2364,"style":199},[99],[54,2366,2307],{"className":2367},[324],[54,2369,300],{"className":2370},[104],[54,2372,604],{"className":2373},[930],[54,2375],{"className":2376,"style":815},[606],[54,2378,186],{"className":2379},[104],[54,2381,604],{"className":2382},[930],[54,2384],{"className":2385,"style":815},[606],[54,2387,2318],{"className":2388},[320],[54,2390],{"className":2391,"style":815},[606],[54,2393,604],{"className":2394},[930],[54,2396],{"className":2397,"style":815},[606],[54,2399,2401,2404],{"className":2400},[104],[54,2402,78],{"className":2403},[104,108],[54,2405,2407],{"className":2406},[112],[54,2408,2410,2430],{"className":2409},[116,117],[54,2411,2413,2427],{"className":2412},[121],[54,2414,2416],{"className":2415,"style":126},[125],[54,2417,2418,2421],{"style":129},[54,2419],{"className":2420,"style":134},[133],[54,2422,2424],{"className":2423},[138,139,140,141],[54,2425,81],{"className":2426,"style":145},[104,108,141],[54,2428,150],{"className":2429},[149],[54,2431,2433],{"className":2432},[121],[54,2434,2436],{"className":2435,"style":157},[125],[54,2437],{},[54,2439,243],{"className":2440},[104],[54,2442,2333],{"className":2443},[466]," 对相邻分量 ",[54,2446,2448,2494],{"className":2447},[57],[54,2449,2451],{"className":2450},[61],[63,2452,2453],{"xmlns":65},[67,2454,2455,2491],{},[70,2456,2457,2459,2473,2475,2489],{},[261,2458,526],{"stretchy":280},[73,2460,2461,2463],{},[76,2462,285],{},[70,2464,2465,2467,2469,2471],{},[184,2466,186],{},[76,2468,81],{},[261,2470,566],{},[184,2472,186],{},[261,2474,604],{"separator":91},[73,2476,2477,2479],{},[76,2478,285],{},[70,2480,2481,2483,2485,2487],{},[184,2482,186],{},[76,2484,81],{},[261,2486,566],{},[184,2488,300],{},[261,2490,531],{"stretchy":280},[83,2492,2493],{"encoding":85},"(x_{2k-2}, x_{2k-1})",[54,2495,2497],{"className":2496,"ariaHidden":91},[90],[54,2498,2500,2503,2506,2559,2562,2565,2617],{"className":2499},[95],[54,2501],{"className":2502,"style":199},[99],[54,2504,526],{"className":2505},[324],[54,2507,2509,2512],{"className":2508},[104],[54,2510,285],{"className":2511},[104,108],[54,2513,2515],{"className":2514},[112],[54,2516,2518,2550],{"className":2517},[116,117],[54,2519,2521,2547],{"className":2520},[121],[54,2522,2524],{"className":2523,"style":126},[125],[54,2525,2526,2529],{"style":129},[54,2527],{"className":2528,"style":134},[133],[54,2530,2532],{"className":2531},[138,139,140,141],[54,2533,2535,2538,2541,2544],{"className":2534},[104,141],[54,2536,186],{"className":2537},[104,141],[54,2539,81],{"className":2540,"style":145},[104,108,141],[54,2542,566],{"className":2543},[1202,141],[54,2545,186],{"className":2546},[104,141],[54,2548,150],{"className":2549},[149],[54,2551,2553],{"className":2552},[121],[54,2554,2557],{"className":2555,"style":2556},[125],"height:0.2083em;",[54,2558],{},[54,2560,604],{"className":2561},[930],[54,2563],{"className":2564,"style":815},[606],[54,2566,2568,2571],{"className":2567},[104],[54,2569,285],{"className":2570},[104,108],[54,2572,2574],{"className":2573},[112],[54,2575,2577,2609],{"className":2576},[116,117],[54,2578,2580,2606],{"className":2579},[121],[54,2581,2583],{"className":2582,"style":126},[125],[54,2584,2585,2588],{"style":129},[54,2586],{"className":2587,"style":134},[133],[54,2589,2591],{"className":2590},[138,139,140,141],[54,2592,2594,2597,2600,2603],{"className":2593},[104,141],[54,2595,186],{"className":2596},[104,141],[54,2598,81],{"className":2599,"style":145},[104,108,141],[54,2601,566],{"className":2602},[1202,141],[54,2604,300],{"className":2605},[104,141],[54,2607,150],{"className":2608},[149],[54,2610,2612],{"className":2611},[121],[54,2613,2615],{"className":2614,"style":2556},[125],[54,2616],{},[54,2618,531],{"className":2619},[466]," 视为一个独立的二维子空间，赋予按几何级数递减的角频率：",[54,2622,2624],{"className":2623},[507],[54,2625,2627,2696],{"className":2626},[57],[54,2628,2630],{"className":2629},[61],[63,2631,2632],{"xmlns":65,"display":516},[67,2633,2634,2693],{},[70,2635,2636,2648,2650,2682,2684,2686,2688,2690],{},[73,2637,2638,2640],{},[76,2639,485],{},[70,2641,2642,2644,2646],{},[76,2643,1419],{},[261,2645,604],{"separator":91},[76,2647,81],{},[261,2649,534],{},[2651,2652,2653,2655],"mfrac",{},[76,2654,1419],{},[1555,2656,2657,2660],{},[76,2658,2659],{"mathvariant":181},"Θ",[70,2661,2662,2664,2666,2668,2670,2672,2674,2676],{},[261,2663,526],{"stretchy":280},[184,2665,186],{},[76,2667,81],{},[261,2669,566],{},[184,2671,186],{},[261,2673,531],{"stretchy":280},[76,2675,182],{"mathvariant":181},[73,2677,2678,2680],{},[76,2679,78],{},[76,2681,81],{},[261,2683,604],{"separator":91},[606,2685],{"width":608},[76,2687,2659],{"mathvariant":181},[261,2689,534],{},[184,2691,2692],{},"10000.0.",[83,2694,2695],{"encoding":85},"\\theta_{i, k} = \\frac{i}{\\Theta^{(2k-2)\u002Fd_k}},\n\\qquad\n\\Theta = 10000.0.",[54,2697,2699,2766,2958],{"className":2698,"ariaHidden":91},[90],[54,2700,2702,2706,2757,2760,2763],{"className":2701},[95],[54,2703],{"className":2704,"style":2705},[99],"height:0.9805em;vertical-align:-0.2861em;",[54,2707,2709,2712],{"className":2708},[104],[54,2710,485],{"className":2711,"style":502},[104,108],[54,2713,2715],{"className":2714},[112],[54,2716,2718,2748],{"className":2717},[116,117],[54,2719,2721,2745],{"className":2720},[121],[54,2722,2724],{"className":2723,"style":126},[125],[54,2725,2727,2730],{"style":2726},"top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;",[54,2728],{"className":2729,"style":134},[133],[54,2731,2733],{"className":2732},[138,139,140,141],[54,2734,2736,2739,2742],{"className":2735},[104,141],[54,2737,1419],{"className":2738},[104,108,141],[54,2740,604],{"className":2741},[930,141],[54,2743,81],{"className":2744,"style":145},[104,108,141],[54,2746,150],{"className":2747},[149],[54,2749,2751],{"className":2750},[121],[54,2752,2755],{"className":2753,"style":2754},[125],"height:0.2861em;",[54,2756],{},[54,2758],{"className":2759,"style":759},[606],[54,2761,534],{"className":2762},[763],[54,2764],{"className":2765,"style":759},[606],[54,2767,2769,2773,2937,2940,2943,2946,2949,2952,2955],{"className":2768},[95],[54,2770],{"className":2771,"style":2772},[99],"height:2.0405em;vertical-align:-0.704em;",[54,2774,2776,2780,2934],{"className":2775},[104],[54,2777],{"className":2778},[324,2779],"nulldelimiter",[54,2781,2783],{"className":2782},[2651],[54,2784,2786,2925],{"className":2785},[116,117],[54,2787,2789,2922],{"className":2788},[121],[54,2790,2793,2899,2910],{"className":2791,"style":2792},[125],"height:1.3365em;",[54,2794,2796,2799],{"style":2795},"top:-2.296em;",[54,2797],{"className":2798,"style":357},[133],[54,2800,2802],{"className":2801},[104],[54,2803,2805,2808],{"className":2804},[104],[54,2806,2659],{"className":2807},[104],[54,2809,2811],{"className":2810},[112],[54,2812,2814],{"className":2813},[116],[54,2815,2817],{"className":2816},[121],[54,2818,2821],{"className":2819,"style":2820},[125],"height:0.814em;",[54,2822,2824,2827],{"style":2823},"top:-2.989em;margin-right:0.05em;",[54,2825],{"className":2826,"style":134},[133],[54,2828,2830],{"className":2829},[138,139,140,141],[54,2831,2833,2836,2839,2842,2845,2848,2851,2854],{"className":2832},[104,141],[54,2834,526],{"className":2835},[324,141],[54,2837,186],{"className":2838},[104,141],[54,2840,81],{"className":2841,"style":145},[104,108,141],[54,2843,566],{"className":2844},[1202,141],[54,2846,186],{"className":2847},[104,141],[54,2849,531],{"className":2850},[466,141],[54,2852,182],{"className":2853},[104,141],[54,2855,2857,2860],{"className":2856},[104,141],[54,2858,78],{"className":2859},[104,108,141],[54,2861,2863],{"className":2862},[112],[54,2864,2866,2890],{"className":2865},[116,117],[54,2867,2869,2887],{"className":2868},[121],[54,2870,2873],{"className":2871,"style":2872},[125],"height:0.3448em;",[54,2874,2876,2880],{"style":2875},"top:-2.3488em;margin-left:0em;margin-right:0.0714em;",[54,2877],{"className":2878,"style":2879},[133],"height:2.5em;",[54,2881,2884],{"className":2882},[138,2883,1921,141],"reset-size3",[54,2885,81],{"className":2886,"style":145},[104,108,141],[54,2888,150],{"className":2889},[149],[54,2891,2893],{"className":2892},[121],[54,2894,2897],{"className":2895,"style":2896},[125],"height:0.1512em;",[54,2898],{},[54,2900,2902,2905],{"style":2901},"top:-3.23em;",[54,2903],{"className":2904,"style":357},[133],[54,2906],{"className":2907,"style":2909},[2908],"frac-line","border-bottom-width:0.04em;",[54,2911,2913,2916],{"style":2912},"top:-3.677em;",[54,2914],{"className":2915,"style":357},[133],[54,2917,2919],{"className":2918},[104],[54,2920,1419],{"className":2921},[104,108],[54,2923,150],{"className":2924},[149],[54,2926,2928],{"className":2927},[121],[54,2929,2932],{"className":2930,"style":2931},[125],"height:0.704em;",[54,2933],{},[54,2935],{"className":2936},[466,2779],[54,2938,604],{"className":2939},[930],[54,2941],{"className":2942,"style":934},[606],[54,2944],{"className":2945,"style":815},[606],[54,2947,2659],{"className":2948},[104],[54,2950],{"className":2951,"style":759},[606],[54,2953,534],{"className":2954},[763],[54,2956],{"className":2957,"style":759},[606],[54,2959,2961,2965],{"className":2960},[95],[54,2962],{"className":2963,"style":2964},[99],"height:0.6444em;",[54,2966,2692],{"className":2967},[104],[10,2969,2970,2971,2999,3000,3028],{},"低维对（",[54,2972,2974,2987],{"className":2973},[57],[54,2975,2977],{"className":2976},[61],[63,2978,2979],{"xmlns":65},[67,2980,2981,2985],{},[70,2982,2983],{},[76,2984,81],{},[83,2986,81],{"encoding":85},[54,2988,2990],{"className":2989,"ariaHidden":91},[90],[54,2991,2993,2996],{"className":2992},[95],[54,2994],{"className":2995,"style":498},[99],[54,2997,81],{"className":2998,"style":145},[104,108]," 小）波长短、旋转快，对局部相邻 token 顺序敏感；高维对（",[54,3001,3003,3016],{"className":3002},[57],[54,3004,3006],{"className":3005},[61],[63,3007,3008],{"xmlns":65},[67,3009,3010,3014],{},[70,3011,3012],{},[76,3013,81],{},[83,3015,81],{"encoding":85},[54,3017,3019],{"className":3018,"ariaHidden":91},[90],[54,3020,3022,3025],{"className":3021},[95],[54,3023],{"className":3024,"style":498},[99],[54,3026,81],{"className":3027,"style":145},[104,108]," 大）波长长、旋转慢，负责承载长距离相对位置关系。",[49,3030,3032,3033,3036,3037,3040,3041],{"id":3031},"预计算-cos_cached-与-sin_cached为什么用-register_buffer","预计算 ",[27,3034,3035],{},"cos_cached"," 与 ",[27,3038,3039],{},"sin_cached","：为什么用 ",[27,3042,3043],{},"register_buffer",[10,3045,3046,3047,3132,3133,3215,3216,3244,3245,3274,3275,3278,3279,3282],{},"因为 ",[54,3048,3050,3074],{"className":3049},[57],[54,3051,3053],{"className":3052},[61],[63,3054,3055],{"xmlns":65},[67,3056,3057,3071],{},[70,3058,3059],{},[73,3060,3061,3063],{},[76,3062,485],{},[70,3064,3065,3067,3069],{},[76,3066,1419],{},[261,3068,604],{"separator":91},[76,3070,81],{},[83,3072,3073],{"encoding":85},"\\theta_{i, k}",[54,3075,3077],{"className":3076,"ariaHidden":91},[90],[54,3078,3080,3083],{"className":3079},[95],[54,3081],{"className":3082,"style":2705},[99],[54,3084,3086,3089],{"className":3085},[104],[54,3087,485],{"className":3088,"style":502},[104,108],[54,3090,3092],{"className":3091},[112],[54,3093,3095,3124],{"className":3094},[116,117],[54,3096,3098,3121],{"className":3097},[121],[54,3099,3101],{"className":3100,"style":126},[125],[54,3102,3103,3106],{"style":2726},[54,3104],{"className":3105,"style":134},[133],[54,3107,3109],{"className":3108},[138,139,140,141],[54,3110,3112,3115,3118],{"className":3111},[104,141],[54,3113,1419],{"className":3114},[104,108,141],[54,3116,604],{"className":3117},[930,141],[54,3119,81],{"className":3120,"style":145},[104,108,141],[54,3122,150],{"className":3123},[149],[54,3125,3127],{"className":3126},[121],[54,3128,3130],{"className":3129,"style":2754},[125],[54,3131],{}," 只依赖位置编号 ",[54,3134,3136,3164],{"className":3135},[57],[54,3137,3139],{"className":3138},[61],[63,3140,3141],{"xmlns":65},[67,3142,3143,3161],{},[70,3144,3145,3147,3149,3151,3153,3155,3159],{},[76,3146,1419],{},[261,3148,2304],{},[261,3150,263],{"stretchy":280},[184,3152,279],{},[261,3154,604],{"separator":91},[3156,3157,3158],"mtext",{},"max_seq_len",[261,3160,531],{"stretchy":280},[83,3162,3163],{"encoding":85},"i \\in [0, \\text{max\\_seq\\_len})",[54,3165,3167,3186],{"className":3166,"ariaHidden":91},[90],[54,3168,3170,3174,3177,3180,3183],{"className":3169},[95],[54,3171],{"className":3172,"style":3173},[99],"height:0.6986em;vertical-align:-0.0391em;",[54,3175,1419],{"className":3176},[104,108],[54,3178],{"className":3179,"style":759},[606],[54,3181,2304],{"className":3182},[763],[54,3184],{"className":3185,"style":759},[606],[54,3187,3189,3193,3196,3199,3202,3205,3212],{"className":3188},[95],[54,3190],{"className":3191,"style":3192},[99],"height:1.06em;vertical-align:-0.31em;",[54,3194,263],{"className":3195},[324],[54,3197,279],{"className":3198},[104],[54,3200,604],{"className":3201},[930],[54,3203],{"className":3204,"style":815},[606],[54,3206,3209],{"className":3207},[104,3208],"text",[54,3210,3158],{"className":3211},[104],[54,3213,531],{"className":3214},[466],"、特征对下标 ",[54,3217,3219,3232],{"className":3218},[57],[54,3220,3222],{"className":3221},[61],[63,3223,3224],{"xmlns":65},[67,3225,3226,3230],{},[70,3227,3228],{},[76,3229,81],{},[83,3231,81],{"encoding":85},[54,3233,3235],{"className":3234,"ariaHidden":91},[90],[54,3236,3238,3241],{"className":3237},[95],[54,3239],{"className":3240,"style":498},[99],[54,3242,81],{"className":3243,"style":145},[104,108]," 和超参数 ",[54,3246,3248,3262],{"className":3247},[57],[54,3249,3251],{"className":3250},[61],[63,3252,3253],{"xmlns":65},[67,3254,3255,3259],{},[70,3256,3257],{},[76,3258,2659],{"mathvariant":181},[83,3260,3261],{"encoding":85},"\\Theta",[54,3263,3265],{"className":3264,"ariaHidden":91},[90],[54,3266,3268,3271],{"className":3267},[95],[54,3269],{"className":3270,"style":2082},[99],[54,3272,2659],{"className":3273},[104],"，完全不包含可训练参数，如果在每次 ",[27,3276,3277],{},"forward"," 时重新调用 ",[27,3280,3281],{},"torch.outer"," 和三角函数，会产生不必要的重复开销。",[10,3284,3285,3286,3289,3290,3293,3294,3297,3298,3301],{},"但也不能直接把预计算好的张量挂在普通属性 ",[27,3287,3288],{},"self.cos_cached"," 上——那样调用 ",[27,3291,3292],{},"model.to(device)"," 时，普通属性不会跟着模型迁移设备；而如果包进 ",[27,3295,3296],{},"nn.Parameter","，优化器又会误以为它是可训练权重，甚至写进 ",[27,3299,3300],{},"state_dict"," 破坏官方权重加载检查。",[10,3303,3304,3305,3308],{},"PyTorch 的标准解法是 ",[27,3306,3307],{},"self.register_buffer(..., persistent=False)","：",[3310,3311,3316],"pre",{"className":3312,"code":3313,"language":3314,"meta":3315,"style":3315},"language-python shiki shiki-themes github-dark-dimmed github-light houston","pair_dims = torch.arange(0, d_k, 2, device=device, dtype=torch.float32)\ninv_freq = theta ** (-pair_dims \u002F d_k)\npositions = torch.arange(max_seq_len, device=device, dtype=torch.float32)\nangles = torch.outer(positions, inv_freq)\n\nself.register_buffer(\"cos_cached\", torch.cos(angles), persistent=False)\nself.register_buffer(\"sin_cached\", torch.sin(angles), persistent=False)\n","python","",[27,3317,3318,3366,3393,3420,3436,3443,3479],{"__ignoreMap":3315},[54,3319,3322,3326,3329,3332,3336,3338,3341,3344,3346,3349,3353,3355,3358,3361,3363],{"class":3320,"line":3321},"line",1,[54,3323,3325],{"class":3324},"sWURj","pair_dims ",[54,3327,534],{"class":3328},"sf2cM",[54,3330,3331],{"class":3324}," torch.",[54,3333,3335],{"class":3334},"sq3c-","arange",[54,3337,526],{"class":3324},[54,3339,279],{"class":3340},"sgENW",[54,3342,3343],{"class":3324},", d_k, ",[54,3345,186],{"class":3340},[54,3347,3348],{"class":3324},", ",[54,3350,3352],{"class":3351},"sA9aF","device",[54,3354,534],{"class":3328},[54,3356,3357],{"class":3324},"device, ",[54,3359,3360],{"class":3351},"dtype",[54,3362,534],{"class":3328},[54,3364,3365],{"class":3324},"torch.float32)\n",[54,3367,3369,3372,3374,3377,3380,3383,3386,3388,3390],{"class":3320,"line":3368},2,[54,3370,3371],{"class":3324},"inv_freq ",[54,3373,534],{"class":3328},[54,3375,3376],{"class":3324}," theta ",[54,3378,3379],{"class":3328},"**",[54,3381,3382],{"class":3324}," (",[54,3384,3385],{"class":3328},"-",[54,3387,3325],{"class":3324},[54,3389,182],{"class":3328},[54,3391,3392],{"class":3324}," d_k)\n",[54,3394,3396,3399,3401,3403,3405,3408,3410,3412,3414,3416,3418],{"class":3320,"line":3395},3,[54,3397,3398],{"class":3324},"positions ",[54,3400,534],{"class":3328},[54,3402,3331],{"class":3324},[54,3404,3335],{"class":3334},[54,3406,3407],{"class":3324},"(max_seq_len, ",[54,3409,3352],{"class":3351},[54,3411,534],{"class":3328},[54,3413,3357],{"class":3324},[54,3415,3360],{"class":3351},[54,3417,534],{"class":3328},[54,3419,3365],{"class":3324},[54,3421,3423,3426,3428,3430,3433],{"class":3320,"line":3422},4,[54,3424,3425],{"class":3324},"angles ",[54,3427,534],{"class":3328},[54,3429,3331],{"class":3324},[54,3431,3432],{"class":3334},"outer",[54,3434,3435],{"class":3324},"(positions, inv_freq)\n",[54,3437,3439],{"class":3320,"line":3438},5,[54,3440,3442],{"emptyLinePlaceholder":3441},true,"\n",[54,3444,3446,3450,3452,3454,3456,3460,3463,3465,3468,3471,3473,3476],{"class":3320,"line":3445},6,[54,3447,3449],{"class":3448},"sRGqv","self",[54,3451,730],{"class":3324},[54,3453,3043],{"class":3334},[54,3455,526],{"class":3324},[54,3457,3459],{"class":3458},"sWqY2","\"cos_cached\"",[54,3461,3462],{"class":3324},", torch.",[54,3464,552],{"class":3334},[54,3466,3467],{"class":3324},"(angles), ",[54,3469,3470],{"class":3351},"persistent",[54,3472,534],{"class":3328},[54,3474,3475],{"class":3340},"False",[54,3477,3478],{"class":3324},")\n",[54,3480,3482,3484,3486,3488,3490,3493,3495,3497,3499,3501,3503,3505],{"class":3320,"line":3481},7,[54,3483,3449],{"class":3448},[54,3485,730],{"class":3324},[54,3487,3043],{"class":3334},[54,3489,526],{"class":3324},[54,3491,3492],{"class":3458},"\"sin_cached\"",[54,3494,3462],{"class":3324},[54,3496,569],{"class":3334},[54,3498,3467],{"class":3324},[54,3500,3470],{"class":3351},[54,3502,534],{"class":3328},[54,3504,3475],{"class":3340},[54,3506,3478],{"class":3324},[10,3508,3509],{},"这里有三个关键细节：",[3511,3512,3513,3526],"table",{},[3514,3515,3516],"thead",{},[3517,3518,3519,3523],"tr",{},[3520,3521,3522],"th",{},"设计点",[3520,3524,3525],{},"原因",[3527,3528,3529,3746,3760],"tbody",{},[3517,3530,3531,3537],{},[3532,3533,3534],"td",{},[27,3535,3536],{},"torch.arange(0, d_k, 2)",[3532,3538,3539,3540,3688,3689],{},"直接生成偶教学下标 ",[54,3541,3543,3582],{"className":3542},[57],[54,3544,3546],{"className":3545},[61],[63,3547,3548],{"xmlns":65},[67,3549,3550,3579],{},[70,3551,3552,3554,3556,3558,3560,3563,3565,3567,3569,3575,3577],{},[184,3553,279],{},[261,3555,604],{"separator":91},[184,3557,186],{},[261,3559,604],{"separator":91},[184,3561,3562],{},"4",[261,3564,604],{"separator":91},[261,3566,2318],{},[261,3568,604],{"separator":91},[73,3570,3571,3573],{},[76,3572,78],{},[76,3574,81],{},[261,3576,566],{},[184,3578,186],{},[83,3580,3581],{"encoding":85},"0, 2, 4, \\dots, d_k-2",[54,3583,3585,3679],{"className":3584,"ariaHidden":91},[90],[54,3586,3588,3591,3594,3597,3600,3603,3606,3609,3612,3615,3618,3621,3624,3627,3630,3670,3673,3676],{"className":3587},[95],[54,3589],{"className":3590,"style":1531},[99],[54,3592,279],{"className":3593},[104],[54,3595,604],{"className":3596},[930],[54,3598],{"className":3599,"style":815},[606],[54,3601,186],{"className":3602},[104],[54,3604,604],{"className":3605},[930],[54,3607],{"className":3608,"style":815},[606],[54,3610,3562],{"className":3611},[104],[54,3613,604],{"className":3614},[930],[54,3616],{"className":3617,"style":815},[606],[54,3619,2318],{"className":3620},[320],[54,3622],{"className":3623,"style":815},[606],[54,3625,604],{"className":3626},[930],[54,3628],{"className":3629,"style":815},[606],[54,3631,3633,3636],{"className":3632},[104],[54,3634,78],{"className":3635},[104,108],[54,3637,3639],{"className":3638},[112],[54,3640,3642,3662],{"className":3641},[116,117],[54,3643,3645,3659],{"className":3644},[121],[54,3646,3648],{"className":3647,"style":126},[125],[54,3649,3650,3653],{"style":129},[54,3651],{"className":3652,"style":134},[133],[54,3654,3656],{"className":3655},[138,139,140,141],[54,3657,81],{"className":3658,"style":145},[104,108,141],[54,3660,150],{"className":3661},[149],[54,3663,3665],{"className":3664},[121],[54,3666,3668],{"className":3667,"style":157},[125],[54,3669],{},[54,3671],{"className":3672,"style":1198},[606],[54,3674,566],{"className":3675},[1202],[54,3677],{"className":3678,"style":1198},[606],[54,3680,3682,3685],{"className":3681},[95],[54,3683],{"className":3684,"style":2964},[99],[54,3686,186],{"className":3687},[104],"，恰好对应公式中的 ",[54,3690,3692,3712],{"className":3691},[57],[54,3693,3695],{"className":3694},[61],[63,3696,3697],{"xmlns":65},[67,3698,3699,3709],{},[70,3700,3701,3703,3705,3707],{},[184,3702,186],{},[76,3704,81],{},[261,3706,566],{},[184,3708,186],{},[83,3710,3711],{"encoding":85},"2k-2",[54,3713,3715,3737],{"className":3714,"ariaHidden":91},[90],[54,3716,3718,3722,3725,3728,3731,3734],{"className":3717},[95],[54,3719],{"className":3720,"style":3721},[99],"height:0.7778em;vertical-align:-0.0833em;",[54,3723,186],{"className":3724},[104],[54,3726,81],{"className":3727,"style":145},[104,108],[54,3729],{"className":3730,"style":1198},[606],[54,3732,566],{"className":3733},[1202],[54,3735],{"className":3736,"style":1198},[606],[54,3738,3740,3743],{"className":3739},[95],[54,3741],{"className":3742,"style":2964},[99],[54,3744,186],{"className":3745},[104],[3517,3747,3748,3753],{},[3532,3749,3750],{},[27,3751,3752],{},"torch.outer(positions, inv_freq)",[3532,3754,3755,3756,3759],{},"一次性生成形状为 ",[27,3757,3758],{},"(max_seq_len, d_k \u002F\u002F 2)"," 的角度表",[3517,3761,3762,3767],{},[3532,3763,3764],{},[27,3765,3766],{},"persistent=False",[3532,3768,3769,3770,3773,3774,3777],{},"跟随模块迁移 device，但不写入 ",[27,3771,3772],{},"state_dict()","，保证 ",[27,3775,3776],{},"load_state_dict"," 与官方测试快照严格对齐",[49,3779,3781,3782,3785],{"id":3780},"奇偶切片与-torchstack-交错还原","奇偶切片与 ",[27,3783,3784],{},"torch.stack"," 交错还原",[10,3787,3788,3789,3792,3793,3795,3796,3799,3800,3803,3804,3807,3808,2144],{},"在 ",[27,3790,3791],{},"RotaryPositionalEmbedding.forward(self, x, token_positions)"," 中，输入 ",[27,3794,285],{}," 的形状是 ",[27,3797,3798],{},"(..., seq_len, d_k)","，其中前导维度 ",[27,3801,3802],{},"..."," 可以是 ",[27,3805,3806],{},"(batch,)","，也可以是拆头后的 ",[27,3809,3810],{},"(batch, num_heads)",[10,3812,3813,3814,3948],{},"我们无需显式拼出稀疏的 ",[54,3815,3817,3844],{"className":3816},[57],[54,3818,3820],{"className":3819},[61],[63,3821,3822],{"xmlns":65},[67,3823,3824,3841],{},[70,3825,3826,3832,3835],{},[73,3827,3828,3830],{},[76,3829,78],{},[76,3831,81],{},[261,3833,3834],{},"×",[73,3836,3837,3839],{},[76,3838,78],{},[76,3840,81],{},[83,3842,3843],{"encoding":85},"d_k \\times d_k",[54,3845,3847,3902],{"className":3846,"ariaHidden":91},[90],[54,3848,3850,3853,3893,3896,3899],{"className":3849},[95],[54,3851],{"className":3852,"style":100},[99],[54,3854,3856,3859],{"className":3855},[104],[54,3857,78],{"className":3858},[104,108],[54,3860,3862],{"className":3861},[112],[54,3863,3865,3885],{"className":3864},[116,117],[54,3866,3868,3882],{"className":3867},[121],[54,3869,3871],{"className":3870,"style":126},[125],[54,3872,3873,3876],{"style":129},[54,3874],{"className":3875,"style":134},[133],[54,3877,3879],{"className":3878},[138,139,140,141],[54,3880,81],{"className":3881,"style":145},[104,108,141],[54,3883,150],{"className":3884},[149],[54,3886,3888],{"className":3887},[121],[54,3889,3891],{"className":3890,"style":157},[125],[54,3892],{},[54,3894],{"className":3895,"style":1198},[606],[54,3897,3834],{"className":3898},[1202],[54,3900],{"className":3901,"style":1198},[606],[54,3903,3905,3908],{"className":3904},[95],[54,3906],{"className":3907,"style":100},[99],[54,3909,3911,3914],{"className":3910},[104],[54,3912,78],{"className":3913},[104,108],[54,3915,3917],{"className":3916},[112],[54,3918,3920,3940],{"className":3919},[116,117],[54,3921,3923,3937],{"className":3922},[121],[54,3924,3926],{"className":3925,"style":126},[125],[54,3927,3928,3931],{"style":129},[54,3929],{"className":3930,"style":134},[133],[54,3932,3934],{"className":3933},[138,139,140,141],[54,3935,81],{"className":3936,"style":145},[104,108,141],[54,3938,150],{"className":3939},[149],[54,3941,3943],{"className":3942},[121],[54,3944,3946],{"className":3945,"style":157},[125],[54,3947],{}," 分块对角矩阵，只需沿最后一维按步长 2 切出偶数位和奇数位：",[3310,3950,3952],{"className":3312,"code":3951,"language":3314,"meta":3315,"style":3315},"cos = self.cos_cached[token_positions]\nsin = self.sin_cached[token_positions]\n\nx_even = x[..., 0::2]\nx_odd = x[..., 1::2]\n\nrotated_even = x_even * cos - x_odd * sin\nrotated_odd = x_even * sin + x_odd * cos\n\nrotate = torch.stack((rotated_even, rotated_odd), dim=-1)\nreturn rotate.reshape(x.shape).to(dtype=x.dtype)\n",[27,3953,3954,3967,3979,3983,4007,4028,4032,4058,4082,4087,4113],{"__ignoreMap":3315},[54,3955,3956,3959,3961,3964],{"class":3320,"line":3321},[54,3957,3958],{"class":3324},"cos ",[54,3960,534],{"class":3328},[54,3962,3963],{"class":3448}," self",[54,3965,3966],{"class":3324},".cos_cached[token_positions]\n",[54,3968,3969,3972,3974,3976],{"class":3320,"line":3368},[54,3970,3971],{"class":3324},"sin ",[54,3973,534],{"class":3328},[54,3975,3963],{"class":3448},[54,3977,3978],{"class":3324},".sin_cached[token_positions]\n",[54,3980,3981],{"class":3320,"line":3395},[54,3982,3442],{"emptyLinePlaceholder":3441},[54,3984,3985,3988,3990,3993,3995,3997,3999,4002,4004],{"class":3320,"line":3422},[54,3986,3987],{"class":3324},"x_even ",[54,3989,534],{"class":3328},[54,3991,3992],{"class":3324}," x[",[54,3994,3802],{"class":3340},[54,3996,3348],{"class":3324},[54,3998,279],{"class":3340},[54,4000,4001],{"class":3324},"::",[54,4003,186],{"class":3340},[54,4005,4006],{"class":3324},"]\n",[54,4008,4009,4012,4014,4016,4018,4020,4022,4024,4026],{"class":3320,"line":3438},[54,4010,4011],{"class":3324},"x_odd ",[54,4013,534],{"class":3328},[54,4015,3992],{"class":3324},[54,4017,3802],{"class":3340},[54,4019,3348],{"class":3324},[54,4021,300],{"class":3340},[54,4023,4001],{"class":3324},[54,4025,186],{"class":3340},[54,4027,4006],{"class":3324},[54,4029,4030],{"class":3320,"line":3445},[54,4031,3442],{"emptyLinePlaceholder":3441},[54,4033,4034,4037,4039,4042,4045,4048,4050,4053,4055],{"class":3320,"line":3481},[54,4035,4036],{"class":3324},"rotated_even ",[54,4038,534],{"class":3328},[54,4040,4041],{"class":3324}," x_even ",[54,4043,4044],{"class":3328},"*",[54,4046,4047],{"class":3324}," cos ",[54,4049,3385],{"class":3328},[54,4051,4052],{"class":3324}," x_odd ",[54,4054,4044],{"class":3328},[54,4056,4057],{"class":3324}," sin\n",[54,4059,4061,4064,4066,4068,4070,4073,4075,4077,4079],{"class":3320,"line":4060},8,[54,4062,4063],{"class":3324},"rotated_odd ",[54,4065,534],{"class":3328},[54,4067,4041],{"class":3324},[54,4069,4044],{"class":3328},[54,4071,4072],{"class":3324}," sin ",[54,4074,713],{"class":3328},[54,4076,4052],{"class":3324},[54,4078,4044],{"class":3328},[54,4080,4081],{"class":3324}," cos\n",[54,4083,4085],{"class":3320,"line":4084},9,[54,4086,3442],{"emptyLinePlaceholder":3441},[54,4088,4090,4093,4095,4097,4100,4103,4106,4109,4111],{"class":3320,"line":4089},10,[54,4091,4092],{"class":3324},"rotate ",[54,4094,534],{"class":3328},[54,4096,3331],{"class":3324},[54,4098,4099],{"class":3334},"stack",[54,4101,4102],{"class":3324},"((rotated_even, rotated_odd), ",[54,4104,4105],{"class":3351},"dim",[54,4107,4108],{"class":3328},"=-",[54,4110,300],{"class":3340},[54,4112,3478],{"class":3324},[54,4114,4116,4120,4123,4126,4129,4132,4134,4136,4138],{"class":3320,"line":4115},11,[54,4117,4119],{"class":4118},"slEfO","return",[54,4121,4122],{"class":3324}," rotate.",[54,4124,4125],{"class":3334},"reshape",[54,4127,4128],{"class":3324},"(x.shape).",[54,4130,4131],{"class":3334},"to",[54,4133,526],{"class":3324},[54,4135,3360],{"class":3351},[54,4137,534],{"class":3328},[54,4139,4140],{"class":3324},"x.dtype)\n",[10,4142,4143,4144,4147,4148,4151,4152,4155],{},"为什么最后要用 ",[27,4145,4146],{},"torch.stack(..., dim=-1)"," 再 ",[27,4149,4150],{},".reshape(x.shape)","，而不是直接 ",[27,4153,4154],{},"torch.cat((rotated_even, rotated_odd), dim=-1)","？",[10,4157,4158],{},"用一组 4 维向量对照就能一眼看清：",[4160,4161,4162,4177,4187,4198],"ul",{},[4163,4164,4165,4168,4169,4172,4173,4176],"li",{},[27,4166,4167],{},"rotated_even"," 包含新向量的第 ",[27,4170,4171],{},"[0, 2]"," 号分量，形状为 ",[27,4174,4175],{},"(..., seq_len, 2)","；",[4163,4178,4179,4168,4182,4172,4185,4176],{},[27,4180,4181],{},"rotated_odd",[27,4183,4184],{},"[1, 3]",[27,4186,4175],{},[4163,4188,4189,4190,4193,4194,4197],{},"如果用 ",[27,4191,4192],{},"torch.cat(..., dim=-1)","，排出来的顺序会变成 ",[27,4195,4196],{},"[0, 2, 1, 3]","，把原本成对的相邻坐标拆成了前后两半；",[4163,4199,4200,4201,4203,4204,4207,4208,4211,4212,4214,4215,4218],{},"用 ",[27,4202,4146],{}," 先得到 ",[27,4205,4206],{},"(..., seq_len, 2, 2)","，其中最后两维分别是 ",[27,4209,4210],{},"[[第0分量, 第1分量], [第2分量, 第3分量]]","，紧接着 ",[27,4213,4150],{}," 按行展平，就精确还原了 ",[27,4216,4217],{},"[0, 1, 2, 3]"," 的交错顺序。",[49,4220,4222,4223,4226],{"id":4221},"接入多头自注意力只旋转-qk以及-token_positions-的广播轴","接入多头自注意力：只旋转 Q、K，以及 ",[27,4224,4225],{},"token_positions"," 的广播轴",[10,4228,4229,4230,4232],{},"把 RoPE 放进 ",[27,4231,33],{}," 时，有两个极易踩坑的位置：",[10,4234,4235,4236,4326,4327,4330,4331,4334,4335,4338],{},"第一，",[22,4237,4238,4239,1996,4267,4295,4296,4325],{},"RoPE 只能作用在拆头后的 ",[54,4240,4242,4255],{"className":4241},[57],[54,4243,4245],{"className":4244},[61],[63,4246,4247],{"xmlns":65},[67,4248,4249,4253],{},[70,4250,4251],{},[76,4252,2040],{},[83,4254,2040],{"encoding":85},[54,4256,4258],{"className":4257,"ariaHidden":91},[90],[54,4259,4261,4264],{"className":4260},[95],[54,4262],{"className":4263,"style":2052},[99],[54,4265,2040],{"className":4266},[104,108],[54,4268,4270,4283],{"className":4269},[57],[54,4271,4273],{"className":4272},[61],[63,4274,4275],{"xmlns":65},[67,4276,4277,4281],{},[70,4278,4279],{},[76,4280,2070],{},[83,4282,2070],{"encoding":85},[54,4284,4286],{"className":4285,"ariaHidden":91},[90],[54,4287,4289,4292],{"className":4288},[95],[54,4290],{"className":4291,"style":2082},[99],[54,4293,2070],{"className":4294,"style":2086},[104,108]," 上，绝对不能作用在 ",[54,4297,4299,4313],{"className":4298},[57],[54,4300,4302],{"className":4301},[61],[63,4303,4304],{"xmlns":65},[67,4305,4306,4311],{},[70,4307,4308],{},[76,4309,4310],{},"V",[83,4312,4310],{"encoding":85},[54,4314,4316],{"className":4315,"ariaHidden":91},[90],[54,4317,4319,4322],{"className":4318},[95],[54,4320],{"className":4321,"style":2082},[99],[54,4323,4310],{"className":4324,"style":1198},[104,108]," 上","，而且每个头的旋转维度必须是 ",[27,4328,4329],{},"d_head = d_model \u002F\u002F num_heads","，而不是 ",[27,4332,4333],{},"d_model","。因为每个注意力头各自在一个 ",[27,4336,4337],{},"d_head"," 维空间内计算点积，所有头共享同一组位置编码规则。",[10,4340,4341,4342,4344,4345,4348,4349,4352,4353,4355,4356,4359,4360,4355,4362,4364],{},"第二，",[27,4343,4225],{}," 与四维张量 ",[27,4346,4347],{},"(batch, num_heads, seq_len, d_head)"," 的广播对齐。在官方测试 ",[27,4350,4351],{},"test_multihead_self_attention_with_rope"," 中，传入的 ",[27,4354,4225],{}," 形状是 ",[27,4357,4358],{},"(1, seq_len)","（二维），而拆头后的 ",[27,4361,1573],{},[27,4363,4347],{},"（四维）：",[3310,4366,4370],{"className":4367,"code":4369,"language":3208,"meta":3315},[4368],"language-text","q_even 的形状：                      (batch, num_heads, seq_len, d_head \u002F\u002F 2)\n直接索引 cos_cached[token_positions]：(1, seq_len, d_head \u002F\u002F 2)\n",[27,4371,4369],{"__ignoreMap":3315},[10,4373,4374,4375,4377,4378,4381,4382,4355,4384,4387,4388,4391,4392,4394,4395,4398,4399,4402,4403,4406,4407,4409],{},"如果直接相乘，PyTorch 从右往左对齐维度时，会把 ",[27,4376,300],{}," 对齐到 ",[27,4379,4380],{},"num_heads","（4），完全没问题；但如果调用者传入的 ",[27,4383,4225],{},[27,4385,4386],{},"(batch, seq_len)","（即 ",[27,4389,4390],{},"batch > 1"," 且没有 ",[27,4393,4380],{}," 这一维），那么 ",[27,4396,4397],{},"(batch, seq_len, d_head \u002F\u002F 2)"," 的第 0 维 ",[27,4400,4401],{},"batch"," 就会错误地和 ",[27,4404,4405],{},"q_even"," 的第 1 维 ",[27,4408,4380],{}," 对齐！",[10,4411,4412,4413,4416,4417,4419,4420,4422,4423,4426],{},"为了同时兼容一维 ",[27,4414,4415],{},"(seq_len,)","、二维 ",[27,4418,4386],{}," 以及带 head 维度的 ",[27,4421,4225],{},"，我们在 ",[27,4424,4425],{},"MultiHeadSelfAttention.forward"," 中加入显式维度对齐：",[3310,4428,4430],{"className":3312,"code":4429,"language":3314,"meta":3315,"style":3315},"if self.rope is not None:\n    if token_positions is None:\n        token_positions = torch.arange(seq_len, device=x.device, dtype=torch.long)\n    elif token_positions.ndim == x.ndim - 1:\n        token_positions = token_positions.unsqueeze(-2)\n    q = self.rope(q, token_positions)\n    k = self.rope(k, token_positions)\n",[27,4431,4432,4455,4469,4497,4518,4538,4555],{"__ignoreMap":3315},[54,4433,4434,4437,4439,4442,4446,4449,4452],{"class":3320,"line":3321},[54,4435,4436],{"class":4118},"if",[54,4438,3963],{"class":3448},[54,4440,4441],{"class":3324},".rope ",[54,4443,4445],{"class":4444},"sST-V","is",[54,4447,4448],{"class":4444}," not",[54,4450,4451],{"class":3340}," None",[54,4453,4454],{"class":3324},":\n",[54,4456,4457,4460,4463,4465,4467],{"class":3320,"line":3368},[54,4458,4459],{"class":4118},"    if",[54,4461,4462],{"class":3324}," token_positions ",[54,4464,4445],{"class":4444},[54,4466,4451],{"class":3340},[54,4468,4454],{"class":3324},[54,4470,4471,4474,4476,4478,4480,4483,4485,4487,4490,4492,4494],{"class":3320,"line":3395},[54,4472,4473],{"class":3324},"        token_positions ",[54,4475,534],{"class":3328},[54,4477,3331],{"class":3324},[54,4479,3335],{"class":3334},[54,4481,4482],{"class":3324},"(seq_len, ",[54,4484,3352],{"class":3351},[54,4486,534],{"class":3328},[54,4488,4489],{"class":3324},"x.device, ",[54,4491,3360],{"class":3351},[54,4493,534],{"class":3328},[54,4495,4496],{"class":3324},"torch.long)\n",[54,4498,4499,4502,4505,4508,4511,4513,4516],{"class":3320,"line":3422},[54,4500,4501],{"class":4118},"    elif",[54,4503,4504],{"class":3324}," token_positions.ndim ",[54,4506,4507],{"class":3328},"==",[54,4509,4510],{"class":3324}," x.ndim ",[54,4512,3385],{"class":3328},[54,4514,4515],{"class":3340}," 1",[54,4517,4454],{"class":3324},[54,4519,4520,4522,4524,4527,4530,4532,4534,4536],{"class":3320,"line":3438},[54,4521,4473],{"class":3324},[54,4523,534],{"class":3328},[54,4525,4526],{"class":3324}," token_positions.",[54,4528,4529],{"class":3334},"unsqueeze",[54,4531,526],{"class":3324},[54,4533,3385],{"class":3328},[54,4535,186],{"class":3340},[54,4537,3478],{"class":3324},[54,4539,4540,4543,4545,4547,4549,4552],{"class":3320,"line":3445},[54,4541,4542],{"class":3324},"    q ",[54,4544,534],{"class":3328},[54,4546,3963],{"class":3448},[54,4548,730],{"class":3324},[54,4550,4551],{"class":3334},"rope",[54,4553,4554],{"class":3324},"(q, token_positions)\n",[54,4556,4557,4560,4562,4564,4566,4568],{"class":3320,"line":3481},[54,4558,4559],{"class":3324},"    k ",[54,4561,534],{"class":3328},[54,4563,3963],{"class":3448},[54,4565,730],{"class":3324},[54,4567,4551],{"class":3334},[54,4569,4570],{"class":3324},"(k, token_positions)\n",[10,4572,4573,4574,4576,4577,4580,4581,4576,4583,4585,4586,4589,4590,4593,4594,4596,4597,4600,4601,4603],{},"当输入 ",[27,4575,285],{}," 是 ",[27,4578,4579],{},"(batch, seq_len, d_model)","（3 维）而 ",[27,4582,4225],{},[27,4584,4386],{},"（2 维）时，",[27,4587,4588],{},"unsqueeze(-2)"," 把它变成 ",[27,4591,4592],{},"(batch, 1, seq_len)","。这样索引 ",[27,4595,3035],{}," 后得到的 ",[27,4598,4599],{},"(batch, 1, seq_len, d_head \u002F\u002F 2)"," 就能在 ",[27,4602,4380],{}," 轴上安全广播给所有头。",[49,4605,4606],{"id":4606},"官方测试验证结果",[10,4608,4609,4610,4613,4614,3036,4617,4620],{},"对接 ",[27,4611,4612],{},"adapters.py"," 中的 ",[27,4615,4616],{},"run_rope",[27,4618,4619],{},"run_multihead_self_attention_with_rope"," 后，运行官方测试：",[3310,4622,4625],{"className":4623,"code":4624,"language":3208,"meta":3315},[4368],"tests\u002Ftest_model.py::test_rope PASSED\ntests\u002Ftest_model.py::test_multihead_self_attention_with_rope PASSED\n",[27,4626,4624],{"__ignoreMap":3315},[10,4628,4629,4630,4633,4634,2144],{},"两项测试均在 ",[27,4631,4632],{},"atol=1e-5"," 容差下与官方快照完全吻合。下一篇继续把带 RoPE 的多头自注意力、RMSNorm 和 SwiGLU 组装成 ",[13,4635,4637],{"href":4636},"\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm\u002F","Pre-Norm Transformer Block 与完整 Transformer LM",[10,4639,4640],{},[4641,4642,4643,4644,4649],"em",{},"封面使用 ",[13,4645,4648],{"href":4646,"rel":4647},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Faigc-image-generator",[41],"aigc-image-generator"," 辅助生成，属于概念插画；准确的公式、形状与代码逻辑以正文为准。",[49,4651,4652],{"id":4652},"参考资料",[4160,4654,4655,4662,4669],{},[4163,4656,4657],{},[13,4658,4661],{"href":4659,"rel":4660},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2104.09864",[41],"Su et al., 2021: RoFormer: Enhanced Transformer with Rotary Position Embedding",[4163,4663,4664],{},[13,4665,4668],{"href":4666,"rel":4667},"https:\u002F\u002Fgithub.com\u002Fstanford-cs336\u002Fassignment1-basics",[41],"Stanford CS336 Assignment 1 官方仓库",[4163,4670,4671],{},[13,4672,4675],{"href":4673,"rel":4674},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fgenerated\u002Ftorch.nn.Module.html#torch.nn.Module.register_buffer",[41],"PyTorch：Module.register_buffer",[4677,4678,4679],"style",{},"html pre.shiki code .sWURj, html code.shiki .sWURj{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#EEF0F9}html pre.shiki code .sf2cM, html code.shiki .sf2cM{--shiki-default:#F47067;--shiki-light:#D73A49;--shiki-cyber:#EEF0F9}html pre.shiki code .sq3c-, html code.shiki .sq3c-{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#00DAEF}html pre.shiki code .sgENW, html code.shiki .sgENW{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#FFD493}html pre.shiki code .sA9aF, html code.shiki .sA9aF{--shiki-default:#F69D50;--shiki-default-font-style:inherit;--shiki-light:#E36209;--shiki-light-font-style:inherit;--shiki-cyber:#4BF3C8;--shiki-cyber-font-style:italic}html pre.shiki code .sRGqv, html code.shiki .sRGqv{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#ACAFFF}html pre.shiki code .sWqY2, html code.shiki .sWqY2{--shiki-default:#96D0FF;--shiki-light:#032F62;--shiki-cyber:#FFD493}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html.cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html pre.shiki code .slEfO, html code.shiki .slEfO{--shiki-default:#F47067;--shiki-default-font-style:inherit;--shiki-light:#D73A49;--shiki-light-font-style:inherit;--shiki-cyber:#54B9FF;--shiki-cyber-font-style:italic}html pre.shiki code .sST-V, html code.shiki .sST-V{--shiki-default:#F47067;--shiki-light:#D73A49;--shiki-cyber:#54B9FF}",{"title":3315,"searchDepth":3368,"depth":3395,"links":4681},[4682,4684,4686,4688,4690,4691],{"id":51,"depth":3368,"text":4683},"为什么把 dkd_kdk​ 维向量拆成 dk\u002F2d_k\u002F2dk​\u002F2 个二维平面",{"id":3031,"depth":3368,"text":4685},"预计算 cos_cached 与 sin_cached：为什么用 register_buffer",{"id":3780,"depth":3368,"text":4687},"奇偶切片与 torch.stack 交错还原",{"id":4221,"depth":3368,"text":4689},"接入多头自注意力：只旋转 Q、K，以及 token_positions 的广播轴",{"id":4606,"depth":3368,"text":4606},{"id":4652,"depth":3368,"text":4652},[4693],"学习笔记","\u002Fimages\u002Fposts\u002Fcs336-rope-rotary-position-embedding\u002Fcs336-rope-rotary-position-embedding-cover.webp","二维特征平面中的旋转向量对与预计算余弦正弦网格的 AI 辅助概念插画","2026-10-21","从二维平面正交旋转推导 RoPE 的奇偶特征配对公式，理解为什么内积只依赖相对位置差 (i - j)，并用 register_buffer 预计算 cos\u002Fsin 缓存接入因果多头自注意力。",false,"md",{"codeUrl":39,"attachmentUrl":46},"\u002Fposts\u002Fcs336-rope-rotary-position-embedding",null,{"title":5,"description":4697},"Stanford CS336","5","posts\u002Fcs336-rope-rotary-position-embedding",[4708,4709,4710,4711,4712],"CS336","PyTorch","Transformer","RoPE","Attention","HZbahgMm4vHLOC4bt8Ri1UKRea6tdJ-XpNEk25fGO2Q",[4715,4729,4745,4759,4773,4786,4801,4816,4829,4833,4848,4860,4873,4885,4898,4911,4928,4943,4954,4966,4981,4994,5008,5020,5034,5046,5061,5075,5089,5100,5113,5130,5145,5158,5170,5182,5195,5208],{"slug":4716,"path":4717,"title":4718,"date":4719,"tags":4720,"description":4724,"cover":4725,"coverAlt":4726,"excerpt":4727,"draft":4698,"hidden":4698,"published":3441,"readingTime":4060,"series":4704,"seriesOrder":4060,"codeUrl":39,"attachmentUrl":4728},"cs336-cross-entropy-adamw-training-loop","\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop","Stanford CS336 学习记录 08：数值稳定交叉熵、AdamW、余弦调度与完整训练闭环","2026-10-27",[4708,4709,4721,4722,4723],"AdamW","CrossEntropy","训练闭环","用 LogSumExp 消去交叉熵中的指数溢出，手写解耦权重衰减的 AdamW 优化器、全局 L2 梯度裁剪、Warmup 余弦退火调度、memmap 数据采样、Checkpoint 序列化与 Top-p 解码，完成 CS336 Assignment 1 全部 48 项测试。","\u002Fimages\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop\u002Fcs336-cross-entropy-adamw-training-loop-cover.webp","滑动窗口采样、余弦退火曲线、双动量优化轨迹与检查点归档构成的训练闭环 AI 辅助概念插画","作为 Stanford CS336 Assignment 1 系列的收官篇，我们已经拥有了分词器（bpe.py、tokenizer.py）和完整的语言模型架构（transformer.py）。要让模型在语料上真正训练并生成文本，还需","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-08-adamw-training-loop-20261027.zip",{"slug":4730,"path":4731,"title":4732,"date":4733,"tags":4734,"description":4741,"cover":4742,"coverAlt":4743,"excerpt":4744,"draft":4698,"hidden":4698,"published":3441,"readingTime":4060},"editcache-video-dit-inference-acceleration","\u002Fposts\u002Feditcache-video-dit-inference-acceleration","EditCache：视频 Diffusion Transformer (DiT) 局部编辑的跨步特征复用与区域重算加速","2026-10-25",[4735,4736,4737,4738,4739,4740],"扩散模型","视频生成","AI Infra","推理加速","GPU","论文解读","拆解视频扩散模型（Wan2.1-VACE）局部编辑中的三重算力冗余，通过可重入 Latent、跨步 Block 隐状态缓存复用、带 Halo 边界的局部区域重算与自适应推理起点，在单卡 H100 上实现 2.46× 模型计算加速与 1.49× 严格同步端到端加速。","\u002Fimages\u002Fposts\u002Feditcache-video-dit-inference-acceleration\u002Feditcache-dit-cover-20261025.webp","视频扩散 Transformer 时空三维体素网格中冻结背景缓存与暖橙色局部编辑重算通道的等距技术插画","在基于 Diffusion Transformer（DiT）的视频编辑与局部重绘（Video Inpainting \u002F Local Editing）任务中，用户的典型工作流并不是一次性生成最终成片，而是针对同一段源视频反复调整掩码区",{"slug":4746,"path":4747,"title":4748,"date":4733,"tags":4749,"description":4754,"cover":4755,"coverAlt":4756,"excerpt":4757,"draft":4698,"hidden":4698,"published":3441,"readingTime":4060,"series":4704,"seriesOrder":3481,"codeUrl":39,"attachmentUrl":4758},"cs336-bpe-tokenizer-training-encoding","\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding","Stanford CS336 学习记录 07：Byte-Level BPE 分词器训练、倒排索引加速与流式编码",[4708,4750,4751,4752,4753],"Tokenizer","BPE","Tiktoken","算法优化","实现 GPT-2 正则预分词、特殊 token 硬边界切分、基于 pair-to-word 倒排索引的增量 BPE 合并算法（\u003C 0.25s 通过速度基准），以及与 tiktoken 完全对齐的 Tokenizer。","\u002Fimages\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding\u002Fcs336-bpe-tokenizer-training-encoding-cover.webp","UTF-8 字节块经过正则切分闸口后沿合并树聚合成子词单元的 AI 辅助概念插画","前面六篇完成了 Transformer 语言模型本身的全部神经网络组件。要让模型真正处理自然语言文本，还需要打通入口处的 ByteLevel BPE（BytePair Encoding）分词器。这一篇记录 cs336basics\u002Fb","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-07-bpe-tokenizer-20261025.zip",{"slug":4760,"path":4761,"title":4762,"date":4763,"tags":4764,"description":4769,"cover":4770,"coverAlt":4771,"excerpt":4772,"draft":4698,"hidden":4698,"published":3441,"readingTime":4060},"in-situ-autoguidance-editable-scientific-illustration","\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration","从原位自校正引导（In-situ Autoguidance）到可编辑科研插图生成（AutoFigure-Edit）","2026-10-24",[4735,4765,4766,4767,4768,4740],"多模态","Agent","ICML","ACL","串联两项生成式模型与多模态智能体研究：如何通过推理期局部随机微扰在单模型内构造免训练扩散自引导向量（ICML 2025 Workshop 第一作者，arXiv:2510.17136），以及如何通过五阶段视觉-符号混合流水线生成原生可编辑 SVG 科研插图（ACL 2026 Demo，FigureBench 83.0% 胜率）。","\u002Fimages\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration\u002Fautoguidance-autofigure-cover-20261024.webp","左侧扩散流形自校正轨迹矢量场与右侧五阶段可编辑 SVG 科研插图分层网格相衔接的等距技术插画","生成式视觉模型正在经历从“生成高保真像素（Pixel Synthesis）”向“可控结构化生成（Controllable & Editable Generation）”的演进。在这一进程中，我作为第一作者或核心贡献者参与了两项分别处",{"slug":4774,"path":4775,"title":4776,"date":4777,"tags":4778,"description":4781,"cover":4782,"coverAlt":4783,"excerpt":4784,"draft":4698,"hidden":4698,"published":3441,"readingTime":4060,"series":4704,"seriesOrder":3445,"codeUrl":39,"attachmentUrl":4785},"cs336-pre-norm-transformer-block-lm","\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm","Stanford CS336 学习记录 06：Pre-Norm Transformer Block 与完整语言模型装配","2026-10-23",[4708,4709,4710,4779,4780],"Pre-Norm","FLOPs","将 RMSNorm、带 RoPE 的因果多头自注意力和 SwiGLU 组装成 Pre-Norm Transformer Block，搭建完整 TransformerLM，并核算 GPT-2 XL 的参数量与前向 FLOPs。","\u002Fimages\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm\u002Fcs336-pre-norm-transformer-block-lm-cover.webp","干净残差流主干穿过多个 Pre-Norm 子层并汇聚到词表投影层的 AI 辅助概念插画","前五篇把 Linear、Embedding、RMSNorm、SwiGLU、RotaryPositionalEmbedding 和因果 MultiHeadSelfAttention 全部单独打通了。这一篇进入 Assignment 1","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-06-transformer-block-lm-20261023.zip",{"slug":4787,"path":4788,"title":4789,"date":4777,"tags":4790,"description":4796,"cover":4797,"coverAlt":4798,"excerpt":4799,"draft":4698,"hidden":4698,"published":3441,"readingTime":4800},"xgrammar-concurrent-cache-generation-fix","\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix","XGrammar 结构化解码缓存剖析：Unlimited 模式并发 Clear 字节记账竞态与代际修复（PR #1011）",[4737,4791,4792,4793,4794,4795],"C++","XGrammar","结构化解码","并发编程","源码分析","拆解 XGrammar GrammarCompiler 的双层缓存架构与 ThreadSafeLRUCache 锁分离实现，定位 Unlimited 模式下 Clear() 字节残留与在途编译（In-flight Computation）跨代污染根因，给出代际号 + 读写锁原子修复与零 sleep 确定性并发 GoogleTest 验证。","\u002Fimages\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix\u002Fxgrammar-cache-fix-cover-20261022.webp","双层磨砂玻璃流道经暖橙色代际闸门过滤后汇入晶格缓存矩阵的等距技术插画","在大语言模型服务（如 SGLang、vLLM）的结构化输出场景中，JSON Schema 或 EBNF 语法需要先编译为词表级掩码缓存（CompiledGrammar），才能在每个 Decode 步以微秒级开销生成合法 token ",14,{"slug":4802,"path":4803,"title":4804,"date":4805,"tags":4806,"description":4812,"cover":4813,"coverAlt":4814,"excerpt":4815,"draft":4698,"hidden":4698,"published":3441,"readingTime":4800},"multi-constraint-agent-routing-dag-eval","\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval","多约束生产级 Agent 系统实践：两阶段工具路由、DAG 局部重规划与轨迹级评测闭环","2026-10-22",[4766,4807,4808,4809,4810,4811],"LLM","工具调用","上下文工程","DPO","强化学习","围绕跨境酒旅等强耦合多约束业务场景，拆解向量 Top-K 召回 + LLM 精排 + 跨语言实体链接与 Schema 门控（96%+ 调用合法率）、基于状态机的 DAG 局部子图重规划、Observation 冗余裁剪（节省 40% Token），以及从线上失败恢复轨迹构造 SFT\u002FDPO 偏好对的评测与训练闭环。","\u002Fimages\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval\u002Fagent-dag-routing-cover-20261020.webp","候选模块经漏斗滤网进入双分支有向无环图轨道并在受阻节点触发暖橙色局部旁路重连的等距技术插画","在玩具级 Demo 里，几个工具加上一段 ReAct 循环就能跑通大多数单步问答。但当 Agent 进入跨境酒旅、复杂交易或研发沙箱等真实业务生产环境时，单靠线性 ReAct 循环很快会撞上四堵墙： 1. 工具空间与参数耦合爆炸：候",{"slug":4817,"path":4818,"title":4819,"date":4696,"tags":4820,"description":4824,"cover":4825,"coverAlt":4826,"excerpt":4827,"draft":4698,"hidden":4698,"published":3441,"readingTime":4828},"llm-mrd-distillation-auditable-review-agents","\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents","多视角推理蒸馏（LLM-MRD）与可审计生成式评审系统（DeepReviewer 2.0）深度拆解",[4821,4765,4822,4766,4823,4740],"知识蒸馏","Qwen2.5-VL","Post-training","对比大模型落地的两条互补路径：如何用 Qwen2.5-VL 离线多视角分析结合 MLP 残差校准与温度缩放 KL 蒸馏，将推理能力压入 358M 学生模型实现 17.2 ms 独立推理（LLM-MRD，DASFAA 2026 Oral）；以及如何通过主张-证据-风险台账、PASA 可比性门控与导出预算校验构建可审计的同行评审 Agent（DeepReviewer 2.0）。","\u002Fimages\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents\u002Fmrd-review-distill-cover-20261016.webp","三块半透明几何棱镜发出平行光束穿过中央校准环汇聚至右侧高密度晶体模块的等距技术插画","把大语言模型或视觉语言模型（VLM）的深度推理能力用到实际系统中，通常会面临一个两难选择： 如果场景受限于毫秒级在线延迟（如高吞吐多模态内容审核与虚假信息检测）：直接在线调用 7B～72B 的多模态大模型做长链条思维链（CoT）生成",18,{"slug":4830,"path":4701,"title":5,"date":4696,"tags":4831,"description":4697,"cover":4694,"coverAlt":4695,"excerpt":4832,"draft":4698,"hidden":4698,"published":3441,"readingTime":3481,"series":4704,"seriesOrder":3438,"codeUrl":39,"attachmentUrl":46},"cs336-rope-rotary-position-embedding",[4708,4709,4710,4711,4712],"上一篇完成了不含位置编码的因果多头自注意力。如果只看那一套矩阵乘法，把输入序列前两个 token 交换顺序（并在因果 mask 允许的范围内看同一个后缀 query 对它们的点积），点积本身并不知道哪个 key 离当前 query ",{"slug":4834,"path":4835,"title":4836,"date":4837,"tags":4838,"description":4844,"cover":4845,"coverAlt":4846,"excerpt":4847,"draft":4698,"hidden":4698,"published":3441,"readingTime":4060},"llm-gateway-semantic-cache-dynamic-batching","\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching","高并发 LLM 推理网关架构实践：两级精确\u002F语义缓存、50ms 有界动态批处理与尾部延迟治理","2026-10-19",[4737,4839,4840,4841,4842,4843],"LLM 推理","分布式系统","Redis","FastAPI","后端","深度拆解生产级大模型推理网关 LLM-Gateway 的核心系统设计：基于 SHA-256 与 Redis 向量余弦相似度的两级精确\u002F语义缓存、50ms 有界等待动态批处理调度器（Dynamic Batching）、令牌桶多租户限流与 Prometheus 全链路可观测性，将命中请求 P99 延迟从 5,307 ms 压降至 17 ms。","\u002Fimages\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching\u002Fllm-gateway-cover-20261019.webp","高并发请求流依次穿过精确哈希滤网、向量球面语义缓存层与 50ms 动态批处理时间窗拼装器的等距系统架构插画","在将大语言模型接入高并发业务系统时，如果让上游微服务直接裸连底层推理引擎（vLLM \u002F SGLang 或外部模型 API），系统很快会暴露出三个典型的生产痛点： 1. 高频重复与近义查询消耗巨量算力：大量用户请求在语义上高度等价（例",{"slug":4849,"path":4850,"title":4851,"date":4837,"tags":4852,"description":4854,"cover":4855,"coverAlt":4856,"excerpt":4857,"draft":4698,"hidden":4698,"published":3441,"readingTime":4800,"series":4704,"seriesOrder":3422,"codeUrl":4858,"attachmentUrl":4859},"cs336-multihead-attention-shapes","\u002Fposts\u002Fcs336-multihead-attention-shapes","Stanford CS336 学习记录 04：多头注意力怎样拆头、计算和合头",[4708,4709,4710,4712,4853],"张量形状","沿着 (2, 5, 64) 的输入追踪 QKV 投影、reshape、transpose、共享因果 mask 和合头，理解每个头仍处理完整序列，以及最后为什么还需要输出 Linear。","\u002Fimages\u002Fposts\u002Fcs336-multihead-attention-shapes\u002Fcs336-multihead-attention-shapes-cover.webp","宽向量束分成多条并行特征轨道后重新合并的 AI 辅助概念插画","上一篇完成了缩放点积注意力。那个函数接收已经准备好的 Q、K、V，计算匹配分数，应用 mask，再用权重汇总 V。这一篇继续把它放进一个带参数的多头自注意力模块。 这次最需要看清的是“拆头”。代码里只是几行 reshape 和 tr","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002F31363223498fafd1528c550a51efbe0aed231736","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-04-multihead-attention-20261019.zip",{"slug":4861,"path":4862,"title":4863,"date":4864,"tags":4865,"description":4869,"cover":4870,"coverAlt":4871,"excerpt":4872,"draft":4698,"hidden":4698,"published":3441,"readingTime":4084},"qwen25-sft-dpo-alignment-tax-ablation","\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation","大模型后训练偏好对齐实录：Qwen2.5 SFT 多维消融、动态参考 DPO 与对齐税（Alignment Tax）量化分析","2026-10-18",[4823,4810,4866,4867,4811,4868],"SFT","LoRA","Qwen2.5","基于 Qwen2.5 完整复现 Instruct SFT → Bradley-Terry DPO 两阶段后训练管线：通过 3 随机种子消融 LoRA Rank、学习率与数据规模（GSM8K +12.73pp），实现免双倍权重的动态 Adapter 切换 DPO 训练，并定量揭示偏好对齐对确定性数学推理的对齐税效应。","\u002Fimages\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation\u002Fqwen25-sft-dpo-cover-20261018.webp","左侧监督微调轨迹流经中央 Bradley-Terry 偏好分流天平分离出正负样本通道并连接右侧对齐税监控仪表盘的等距技术插画","在大语言模型后训练（PostTraining）管线中，监督微调（SFT）负责教会模型任务格式与领域推理模式，而直接偏好优化（DPO, Direct Preference Optimization）则利用成对偏好数据 $(x, yw,",{"slug":4874,"path":4875,"title":4876,"date":4864,"tags":4877,"description":4879,"cover":4880,"coverAlt":4881,"excerpt":4882,"draft":4698,"hidden":4698,"published":3441,"readingTime":4115,"series":4704,"seriesOrder":3395,"codeUrl":4883,"attachmentUrl":4884},"cs336-softmax-attention-mask","\u002Fposts\u002Fcs336-softmax-attention-mask","Stanford CS336 学习记录 03：Softmax 的数值稳定与 Attention Mask",[4708,4709,4710,4878,4712],"Softmax","从减去最大值为何不改变 Softmax 开始，理清函数与有参数层的区别，再沿着 Q、K、V 的形状理解缩放点积注意力，以及 mask 为什么必须填负无穷。","\u002Fimages\u002Fposts\u002Fcs336-softmax-attention-mask\u002Fcs336-softmax-attention-mask-cover.webp","注意力分数、权重与因果遮罩的 AI 辅助概念插画","上一篇写到 SwiGLU，接下来进入 Attention。真正开始相乘之前，先做了 Softmax：它把一组分数变成总和为 1 的权重，后面用这些权重汇总不同 token 提供的信息。 这段学习里，有两个很具体的问题值得记下来。一个","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fa39eb684710817f7f389626423fcb26ca0f6d629","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-03-softmax-sdpa-20261018.zip",{"slug":4886,"path":4887,"title":4888,"date":4889,"tags":4890,"description":4893,"cover":4894,"coverAlt":4895,"excerpt":4896,"draft":4698,"hidden":4698,"published":3441,"readingTime":4084,"series":4704,"seriesOrder":3368,"codeUrl":4883,"attachmentUrl":4897},"cs336-rmsnorm-swiglu","\u002Fposts\u002Fcs336-rmsnorm-swiglu","Stanford CS336 学习记录 02：RMSNorm 的广播与 SwiGLU 的门控","2026-10-16",[4708,4709,4710,4891,4892],"RMSNorm","SwiGLU","用一个 token 向量理解均方根归一化，拆开 dim=-1 与 keepdim=True 的作用，再沿着两条分支读懂 SiLU、逐元素乘法和 SwiGLU。","\u002Fimages\u002Fposts\u002Fcs336-rmsnorm-swiglu\u002Fcs336-rmsnorm-swiglu-cover.webp","特征尺度整理和双分支门控的 AI 辅助概念插画","上一篇把 Linear 和 Embedding 的参数形状理顺了。继续往下，最先遇到的是 RMSNorm：公式不算长，但把它落到一批 token 上时，dim=1、keepdim=True 和广播必须一起理解。 然后是 SiLU 和","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-02-rmsnorm-swiglu-20261016.zip",{"slug":4899,"path":4900,"title":4901,"date":4902,"tags":4903,"description":4906,"cover":4907,"coverAlt":4908,"excerpt":4909,"draft":4698,"hidden":4698,"published":3441,"readingTime":4089,"series":4704,"seriesOrder":3321,"codeUrl":4883,"attachmentUrl":4910},"cs336-linear-embedding-shapes","\u002Fposts\u002Fcs336-linear-embedding-shapes","Stanford CS336 学习记录 01：从 Linear、Embedding 搞清张量形状","2026-10-14",[4708,4709,4710,4904,4905],"Linear","Embedding","从 PyTorch 类、参数与 forward 的关系开始，理解 Linear 为什么保存两行三列的权重、计算时为什么要转置，再区分 Embedding 查表与线性变换。","\u002Fimages\u002Fposts\u002Fcs336-linear-embedding-shapes\u002Fcs336-linear-embedding-shapes-cover.webp","token 方块、向量表与矩阵变换的 AI 辅助概念插画","开始沿着 Stanford CS336 的 Assignment 1 一步步实现。BPE 之前已经接触过，这轮先跳过分词器，从 Transformer 的基础层开始。这里记录的是当时写代码时真正卡住的问题，以及对应测试能确认到的范围","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-01-linear-embedding-20261014.zip",{"slug":4912,"path":4913,"title":4914,"date":4915,"tags":4916,"description":4921,"cover":4922,"coverAlt":4923,"excerpt":4924,"draft":4698,"hidden":4698,"published":3441,"readingTime":4925,"series":4917,"seriesOrder":4800,"codeUrl":4926,"attachmentUrl":4927},"cmu-15-779-kv-cache-granularity-study","\u002Fposts\u002Fcmu-15-779-kv-cache-granularity-study","CMU 15-779 学习记录 14：KV 页大小、共享边界与 CPU 实现成本","2026-10-12",[4917,4918,4919,4920],"CMU 15-779","KV cache","PagedAttention","系统实验","用 210 组 CPU 配置比较连续与分页 KV cache，观察 31\u002F32-token 共享前缀的页边界，分别记录活跃槽位、池容量、复制计数和完整调用耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling\u002Fcmu-serving-cover-20261019.webp","浅色手绘分页存储抽屉与共享前缀卡片，旁边连接不同长度的请求队列","个人课程项目中的 CPU 实验记录，延续记录 11 的分页 KV 实现。已完成 210 组配置与 1470 次计时，输出均通过连续存储参考对照；本篇不代表正式作业提交、整门课程或项目全部交付完成。 同样是共享 31 个 token，",12,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F200faf569eccf94fb125e65110bb5d14094d454a","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fkv-cache-granularity-20261019.zip",{"slug":4929,"path":4930,"title":4931,"date":4932,"tags":4933,"description":4936,"cover":4937,"coverAlt":4938,"excerpt":4939,"draft":4698,"hidden":4698,"published":3441,"readingTime":3438,"series":4917,"seriesOrder":4940,"codeUrl":4941,"attachmentUrl":4942},"cmu-15-779-moe-routing-capacity","\u002Fposts\u002Fcmu-15-779-moe-routing-capacity","CMU 15-779 学习记录 13：MoE 路由、稀疏分发与容量溢出","2026-10-09",[4917,4934,4935],"MoE","Routing","用 CPU top-2 MoE 检查稀疏分发的输出与梯度，统计 expert-token 行数，并保留容量丢弃会改变输出的反例。","\u002Fimages\u002Fposts\u002Fcmu-15-779-moe-routing-capacity\u002Fcmu-moe-cover-20261019.webp","浅色手绘分拣器将小珠分配到四个专家工作屋，队列呈现负载不均","第 18 讲的个人配套实验。36 号脚本串行运行 CPU 专家，检查 topk 路由后的分发与合并；没有训练 router，也没有执行 GPU grouped GEMM 或跨设备 alltoall。 上一节记录低秩适配，这次转向每个",13,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fc4851e0cedadf13af7c421867e83c8ff57244fa0","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-parallel-serving-lora-moe-20261019.zip",{"slug":4944,"path":4945,"title":4946,"date":4947,"tags":4948,"description":4950,"cover":4951,"coverAlt":4952,"excerpt":4953,"draft":4698,"hidden":4698,"published":3441,"readingTime":3438,"series":4917,"seriesOrder":4925,"codeUrl":4941,"attachmentUrl":4942},"cmu-15-779-lora-uniform-quantization","\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization","CMU 15-779 学习记录 12：LoRA 首步梯度、权重合并与均匀 4-bit 打包","2026-10-06",[4917,4867,4949],"Quantization","用一次 CPU SGD 更新检查 LoRA 初始化、冻结基座与合并等价，再分别记录均匀 4-bit 存储字节数和量化误差。","\u002Fimages\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization\u002Fcmu-lora-cover-20261019.webp","浅色手绘的大计算板、两块细长适配板和成对打包的小方块","第 17 讲的个人配套实验，35 号脚本实际执行一次 CPU LoRA 更新，并实现均匀 4bit 打包。实验检查机制与数值误差，不评估微调任务效果，也没有实现完整 QLoRA。 这次把两个问题分开记录：低秩分支是否能合并进原权重，",{"slug":4955,"path":4956,"title":4957,"date":4958,"tags":4959,"description":4963,"cover":4922,"coverAlt":4964,"excerpt":4965,"draft":4698,"hidden":4698,"published":3441,"readingTime":3481,"series":4917,"seriesOrder":4115,"codeUrl":4941,"attachmentUrl":4942},"cmu-15-779-kv-serving-speculative-sampling","\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling","CMU 15-779 学习记录 11：分页 KV、连续批处理与推测采样","2026-10-03",[4917,4960,4961,4962],"LLM Serving","KV Cache","Speculative Decoding","在 CPU 上验证 KV 页共享和写时复制，用单位成本模型检查连续批处理，再用条件概率小模型核对推测采样的接受与补采样。","浅色手绘的共享卡片抽屉与不断补入小珠子的请求托盘","第 15、16 讲的个人配套实验。33 号实现 CPU 分页 KV 存储，37 号模拟请求准入，34 号运行小概率模型的推测采样。它们分别验证存储、调度和分布问题，没有测量真实 LLM 服务吞吐或 GPU 延迟。 记录 08已经对照",{"slug":4967,"path":4968,"title":4969,"date":4970,"tags":4971,"description":4976,"cover":4977,"coverAlt":4978,"excerpt":4979,"draft":4698,"hidden":4698,"published":3441,"readingTime":4980},"qwen3-mid-post-train-mlx","\u002Fposts\u002Fqwen3-mid-post-train-mlx","在 M2 Pro 上完成 Qwen3 两阶段训练：Mid Train、SFT 与可复核评估","2026-09-30 11:19:10",[4807,4972,4973,4867,4974,4866,4975],"Qwen3","MLX","持续预训练","Apple Silicon","用 Qwen3 0.6B Base 和 MLX 完成领域持续训练与监督微调，记录数据划分、聊天模板、损失掩码、数值稳定性修复和独立测试结果。","\u002Fimages\u002Fposts\u002Fqwen3-mid-post-train-mlx\u002Fqwen3-mlx-cover-20261018.webp","Apple Silicon 统一内存芯片底座支撑两阶段持续预训练与带精确 Loss 掩码滤网的监督微调桥接插画","在一台 M2 Pro、16 GB 内存的 Mac 上，能否把一个 Base 语言模型接续训练成输出 JSON 的运维事件分级助手？这次项目完成了领域持续训练、监督微调、检查点选择、重新加载和独立测试，并保留了权重、日志和数据哈希。 ",16,{"slug":4982,"path":4983,"title":4984,"date":4985,"tags":4986,"description":4990,"cover":4991,"coverAlt":4992,"excerpt":4993,"draft":4698,"hidden":4698,"published":3441,"readingTime":3445,"series":4917,"seriesOrder":4089,"codeUrl":4941,"attachmentUrl":4942},"cmu-15-779-cpu-parallelism-pipeline","\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline","CMU 15-779 学习记录 10：加权梯度、状态分片与同步流水线","2026-09-30",[4917,4987,4988,4989],"Parallel Training","Gloo","Pipeline","用双 CPU 进程 Gloo 检查不等数据分片的梯度，推导持久状态字节数，再用依赖模型比较 GPipe 与同步 1F1B 的激活存活。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline\u002Fcmu-parallel-cover-20261019.webp","浅色手绘的四阶段桌面流水线与按数量汇总的托盘","第 12 至 14 讲的个人配套实验，脚本编号 31、32 不对应官方作业。Gloo 实际运行两个 CPU 进程；张量切分、状态字节数和流水线分别按下文的数学或成本模型解释。真实 GPU 实测按学习安排跳过。 上一节检查单个计算图的",{"slug":4995,"path":4996,"title":4997,"date":4998,"tags":4999,"description":5002,"cover":5003,"coverAlt":5004,"excerpt":5005,"draft":4698,"hidden":4698,"published":3441,"readingTime":3481,"series":4917,"seriesOrder":4084,"codeUrl":5006,"attachmentUrl":5007},"cmu-15-779-cpu-compiler-optimizations","\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations","CMU 15-779 学习记录 09：CPU 编译优化、合法重写与候选搜索","2026-09-27",[4917,5000,5001,4709],"ML Compiler","TorchInductor","用 CPU 实验验证 BatchNorm 折叠、RMSNorm 重排与非线性反例，测量分块候选，并运行 TorchInductor，分别记录首调用与预热后的时间。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations\u002Fcmu-cpu-compiler-cover-20261019.webp","淡紫计算积木经过薄荷绿折叠模具变成浅蓝积木，前方摆放四种候选托盘的浅色手绘插画","编译章节的个人配套实验，编号 30 不对应官方作业。图重写、分块候选和 TorchInductor 均在 CPU 上运行；真实 GPU 实验按学习安排跳过。 上一节记录了分块 Attention 和 KV cache。这次转到编译优","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F9261bea6aedf31cdccbbab2541ca693fc97b3743","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-compiler-optimizations-20261019.zip",{"slug":5009,"path":5010,"title":5011,"date":5012,"tags":5013,"description":5015,"cover":5016,"coverAlt":5017,"excerpt":5018,"draft":4698,"hidden":4698,"published":3441,"readingTime":4925,"series":4917,"seriesOrder":4060,"codeUrl":4926,"attachmentUrl":5019},"cmu-15-779-attention-online-softmax","\u002Fposts\u002Fcmu-15-779-attention-online-softmax","CMU 15-779 学习记录 08：分块 Attention、在线 Softmax 与 KV cache","2026-09-24",[4917,4712,4878,5014,4709],"FlashAttention","在 CPU 上对照完整 Attention、在线 Softmax 分块实现与 PyTorch SDPA，检查因果掩码、浮点误差和分数块大小，理解 FlashAttention 的一部分算法基础。","\u002Fimages\u002Fposts\u002Fcmu-15-779-attention-online-softmax\u002Fcmu-online-attention-cover-20261016.webp","浅紫色卡片分批进入绿色计算托盘，旁边三个状态容器保存累积量的浅色手绘插画","第 5 讲 Transformers & Attention 的个人配套实验。28\u002F29 号是 CPU 单头前向与 KV cache；后续补入 38 号 CUDA 多头合成张量的显式 SDPA MATH\u002FFLASH 对照。没有手写 ","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcuda-triton-sdpa-20261019.zip",{"slug":5021,"path":5022,"title":5023,"date":5024,"tags":5025,"description":5030,"cover":5031,"coverAlt":5032,"excerpt":5033,"draft":4698,"hidden":4698,"published":3441,"readingTime":4089,"series":4917,"seriesOrder":3481,"codeUrl":4926,"attachmentUrl":5019},"cmu-15-779-matmul-tiling-tree-reduction","\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction","CMU 15-779 学习记录 07：矩阵分块、数据复用与树形归约","2026-09-21",[4917,5026,5027,5028,5029],"CUDA","矩阵乘法","数据复用","归约","用 CPU 模型统计矩阵分块的输入读取次数，再比较串行求和和树形归约的依赖深度，记录补零、同步与浮点加法顺序的边界。","\u002Fimages\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction\u002Fcmu-matmul-reduction-cover-20261016.webp","矩阵积木在共享托盘中复用，旁边的小珠沿树形管道逐层合并的浅色手绘插画","GPU Architecture & CUDA Programming 的个人配套实验。首次记录用 CPU 模型检查数据复用与依赖深度；后续补入单卡 GPU 原生 CUDA 矩阵验证，以及 Triton 两阶段归约和矩阵乘法实测。 ",{"slug":5035,"path":5036,"title":5037,"date":5038,"tags":5039,"description":5042,"cover":5043,"coverAlt":5044,"excerpt":5045,"draft":4698,"hidden":4698,"published":3441,"readingTime":4089,"series":4917,"seriesOrder":3445,"codeUrl":4926,"attachmentUrl":5019},"cmu-15-779-cuda-thread-indexing","\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing","CMU 15-779 学习记录 06：CUDA 线程索引与边界检查","2026-09-18",[4917,5026,4739,5040,5041],"线程索引","边界检查","用 CPU 演示 CUDA 的 block、thread 与全局下标：10 个元素如何分给 3 个线程块，为什么要跳过越界线程，以及这些验证能说明什么。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing\u002Fcmu-cuda-indexing-cover-20261015.webp","分组的浅紫色计算方块与末端边界挡板组成线程分配教具的浅色插画","GPU Architecture & CUDA Programming 的个人配套练习。首次记录为 CPU 索引演示；后续在单卡 GPU 环境补入原生 CUDA 编译、运行与 memcheck，以及 Triton 向量加法实测。 上",{"slug":5047,"path":5048,"title":5049,"date":5050,"tags":5051,"description":5055,"cover":5056,"coverAlt":5057,"excerpt":5058,"draft":4698,"hidden":4698,"published":3441,"readingTime":4089,"series":4917,"seriesOrder":3438,"codeUrl":5059,"attachmentUrl":5060},"cmu-15-779-profiler-operator-batching","\u002Fposts\u002Fcmu-15-779-profiler-operator-batching","CMU 15-779 学习记录 05：Profiler、算子调用与批量推理","2026-09-15",[4917,4709,5052,5053,5054],"Profiler","批量推理","性能分析","对照 ML Systems 基础，用 Profiler 拆开两层网络的执行过程，再固定 1000 个样本扫描 batch size，观察算子调用次数、尾批和 CPU 耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-profiler-operator-batching\u002Fcmu-pytorch-profiling-cover-20261015.webp","放大镜观察线性层与 ReLU 模块，旁边面板呈现执行片段的浅色技术教具插画","本篇对照课程官网的 Introduction to ML Systems 整理，是个人配套 CPU 实验，不是课程官方作业。脚本编号是练习顺序，不对应课程讲次。 前几篇一直在补 PyTorch 基础。到了这一步，我在学习聊天里问：现","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fcecf5110d8a95161bb5763eadfca94cd7a0cc50f","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-profiler-batching-20261015.zip",{"slug":5062,"path":5063,"title":5064,"date":5065,"tags":5066,"description":5069,"cover":5070,"coverAlt":5071,"excerpt":5072,"draft":4698,"hidden":4698,"published":3441,"readingTime":4084,"series":4917,"seriesOrder":3422,"codeUrl":5073,"attachmentUrl":5074},"cmu-15-779-pytorch-linear-relu-absolute-value","\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value","CMU 15-779 学习记录 04：多输出线性层、ReLU 与绝对值网络","2026-09-12",[4917,4709,5027,5067,5068],"ReLU","神经网络","把 nn.Linear 对应到矩阵乘法，弄清多个输出怎样计算，再逐层拆开一个绝对值网络，比较手动设置参数与训练得到的结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value\u002Fcmu-pytorch-absolute-cover-20261015.webp","双分支经过折角模块后汇入紫色 V 形曲线面板的浅色技术教具插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的批量推理与矩阵乘法，我继续把 model(x) 拆开看：一层里有多个输出时，每个数是怎么算出来的？两","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F4cc63904a3a583da4a2122c246a14b43aac38528","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-linear-relu-abs-20261015.zip",{"slug":5076,"path":5077,"title":5078,"date":5079,"tags":5080,"description":5083,"cover":5084,"coverAlt":5085,"excerpt":5086,"draft":4698,"hidden":4698,"published":3441,"readingTime":4084,"series":4917,"seriesOrder":3395,"codeUrl":5087,"attachmentUrl":5088},"cmu-15-779-pytorch-dataloader-checkpoints-batching","\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching","CMU 15-779 学习记录 03：DataLoader、模型保存与批量推理","2026-09-09",[4917,4709,5081,5053,5082],"DataLoader","模型保存","用 DataLoader 分批训练平方函数，保存并加载模型参数，再比较 1000 个样本与 1 个样本的推理耗时，最后把批量计算对应到矩阵乘法。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching\u002Fcmu-pytorch-batching-cover-20261015.webp","样本卡片按批次进入计算模块，旁边配有参数存储盒和输出矩阵面板的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的平方函数实验，这一节继续练习分批训练、保存参数和独立预测。最后做了一次计时：同一个小网络，逐个处理 ","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-batching-20261015.zip",{"slug":5090,"path":5091,"title":5092,"date":5093,"tags":5094,"description":5095,"cover":5096,"coverAlt":5097,"excerpt":5098,"draft":4698,"hidden":4698,"published":3441,"readingTime":4084,"series":4917,"seriesOrder":3368,"codeUrl":5087,"attachmentUrl":5099},"cmu-15-779-pytorch-linear-module-relu","\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu","CMU 15-779 学习记录 02：nn.Linear、张量形状与 ReLU","2026-09-06",[4917,4709,5068,5067],"从带偏置的线性回归开始，弄清 nn.Linear 和 nn.Module 的分工，再用平方函数观察线性模型的限制、ReLU 的作用和测试点的选择。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu\u002Fcmu-pytorch-relu-cover-20261015.webp","输入经过两层神经网络和 ReLU 折角模块，输出曲线拟合结果的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 上一篇里，我用一个参数拟合了 $y=2x$。这一篇继续顺着往下写，先加上偏置，再把手写参数换成 nn.Linea","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-models-20261015.zip",{"slug":5101,"path":5102,"title":5103,"date":5104,"tags":5105,"description":5108,"cover":5109,"coverAlt":5110,"excerpt":5111,"draft":4698,"hidden":4698,"published":3441,"readingTime":4089,"series":4917,"seriesOrder":3321,"codeUrl":5087,"attachmentUrl":5112},"cmu-15-779-pytorch-autograd-linear-regression","\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression","CMU 15-779 学习记录 01：PyTorch 自动求导与线性回归","2026-09-03",[4917,4709,5106,5107],"自动求导","线性回归","从自动求导和手动梯度下降开始，比较学习率，再用 SGD 训练线性回归。记录 w * 4 写错后停在 1.25 的原因和修正结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression\u002Fcmu-pytorch-autograd-cover-20261015.webp","梯度曲面上的小球沿路径下降，前侧面板展示散点与拟合直线的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 开始学习 CMU 15779 时，我会 Python，但还没用过 PyTorch。这次先在 Mac 上用 CPU","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-basics-20261013.zip",{"slug":5114,"path":5115,"title":5116,"date":5117,"tags":5118,"description":5124,"cover":5125,"coverAlt":5126,"excerpt":5127,"draft":4698,"hidden":4698,"published":3441,"readingTime":4940,"codeUrl":5128,"attachmentUrl":5129},"embodied-pickplace-lab-robustness","\u002Fposts\u002Fembodied-pickplace-lab-robustness","Pickplace Lab 项目复盘：抓取成功率之外的时延与失败分析","2026-08-29",[5119,5120,5121,4709,5122,5123],"具身智能","模仿学习","MuJoCo","机器人","鲁棒性","从 MuJoCo 抓取搬运闭环出发，记录 22,100 个评测回合。动作平滑在自建任务的 50 ms 延迟下有效，换到 Panda Lift 却没有支持同样的收益。","\u002Fimages\u002Fposts\u002Fembodied-pickplace-lab-robustness\u002Fpickplace-grasp-cover-20261016.webp","MuJoCo 实际渲染的笛卡尔夹爪抓取方块场景，桌面上标出放置目标","Pickplace Lab 从一个简单任务开始：让夹爪抓起方块，搬到指定位置，放下，再退开。先把仿真、示范采集、训练和闭环评测接起来，随后再问：干净条件下成功的策略，换一种执行条件，还能不能完成任务？ 最值得讨论的结果出现在指令延迟","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Ftree\u002Fv0.1.0","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Freleases\u002Fdownload\u002Fv0.1.0\u002Fpickplace-lab-research-v1.zip",{"slug":5131,"path":5132,"title":5133,"date":5134,"tags":5135,"description":5140,"cover":5141,"coverAlt":5142,"excerpt":5143,"draft":4698,"hidden":4698,"published":3441,"readingTime":5144},"qwen25-vl-sglang-verl-single-gpu-lab","\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab","从 Mac 到单卡 GPU：Qwen2.5-VL 部署、GRPO 与计数 LoRA 实践","2026-08-19",[4822,4765,5136,5137,5138,5139,4973],"SGLang","verl","GRPO","推理优化","在 M2 Pro 和云端单卡上部署 Qwen2.5-VL-3B，实测连续批处理、前缀缓存与 SGLang 参数调优，再用 verl 完成多模态 GRPO 更新和同条件前后评测，记录有效优化、兼容性问题与未观察到提升的结果。","\u002Fimages\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab\u002Fqvl-lab-cover-20261010.webp","笔记本与单卡计算模块通过图像和 token 光轨连接的实验台插画","这次实验从一个很具体的问题开始：M2 Pro、16 GB 统一内存，能不能在本机跑 Qwen2.5VL？跑起来之后，我又想把图文问答、推理性能测量和强化学习训练接成一条完整流程。 本机的 MLX 4bit 服务和云端的 SGLang",29,{"slug":5146,"path":5147,"title":5148,"date":5149,"tags":5150,"description":5154,"cover":5155,"coverAlt":5156,"excerpt":5157,"draft":4698,"hidden":4698,"published":3441,"readingTime":4940},"cycleresearcher-post-training","\u002Fposts\u002Fcycleresearcher-post-training","从 DeepReviewer 2.0 到 CycleResearcher：科研 Agent 如何真正做 Post-training","2026-08-07 17:33:05",[4807,4823,4766,5151,5152,5153],"SimPO","Reward Model","ICLR","区分推理时 Agent 编排与模型后训练，并拆解 CycleResearcher 的领域 SFT、生成式 Reward Model、Iterative SimPO 和评测风险。","\u002Fimages\u002Fposts\u002Fcycleresearcher-post-training\u002Fcycleresearcher-cover-20261018.webp","策略生成模块产出多份候选文稿经多维评审棱镜排序后通过偏好梯度反馈弧闭环优化的等距技术插画","看到一个系统会检索、规划、反思和调用工具，不能据此判断它做过 Posttraining。Agent 描述的是模型在运行时如何完成任务；Posttraining 描述的是预训练之后，是否继续用监督数据、偏好数据或奖励信号更新模型参数。",{"slug":5159,"path":5160,"title":5161,"date":5162,"tags":5163,"description":5165,"cover":5166,"coverAlt":5167,"excerpt":5168,"draft":4698,"hidden":4698,"published":3441,"readingTime":4800,"series":5169,"seriesOrder":3321},"cuda-fused-rmsnorm-kernel","\u002Fposts\u002Fcuda-fused-rmsnorm-kernel","从 PyTorch 到 CUDA Kernel：融合 RMSNorm 在 RTX 4090D 上加速 4.27 倍","2026-07-22 18:16:50",[5026,4737,4739,4709,4839,5164],"Kernel 优化","从零实现 PyTorch CUDA 扩展，拆解 RMSNorm 的访存瓶颈、warp 归约、向量化加载、算子融合与冷热缓存 benchmark。","\u002Fimages\u002Fposts\u002Fcuda-fused-rmsnorm-kernel\u002Fcuda-rmsnorm-cover-20261018.webp","融合残差加法与向量化 float4 访存经 Warp Shuffle 蝴蝶树归约输出归一化张量的等距硬件架构插画","为了准备 AI Infra 岗位，我做了一个尽量小而完整的 CUDA 项目：不追求堆很多算子，而是选一个真实的 LLM 推理路径，把实现、正确性、benchmark 和性能解释全部做扎实。 项目实现了两个 PyTorch CUDA ","AI Infra 实战",{"slug":5171,"path":5172,"title":5173,"date":5174,"tags":5175,"description":5177,"cover":5178,"coverAlt":5179,"excerpt":5180,"draft":4698,"hidden":4698,"published":3441,"readingTime":4060,"series":5181,"seriesOrder":3321},"agent-memory","\u002Fposts\u002Fagent-memory","Agent 上下文工程与分层记忆架构：从 Observation 剪枝、滚动摘要到向量召回闭环","2026-04-24",[4807,4766,5176,4737],"RAG","结合生产级多约束 Agent 与超 100K Token 科研智能体实践，系统拆解长程会话中的上下文膨胀与注意力稀释问题：涵盖 Observation 结构化剪枝（降低约 40% Token 开销）、滚动工作摘要、向量检索长期记忆及 Prefix Cache 友好的上下文组装。","\u002Fimages\u002Fposts\u002Fagent-memory\u002Fagent-memory-cover-20261018.webp","底层高噪数据经剪枝滤网进入中期滑动窗口与金色状态台账并由顶层向量星座按需召回的三层记忆架构插画","在构建多步工具调用（ToolUse）与长周期自主规划（Autonomous Planning）的 Agent 系统时，上下文管理（Context Engineering）往往比单纯改写 System Prompt 更决定线上稳定性。","Agent 上下文工程",{"slug":5183,"path":5184,"title":5185,"date":5186,"tags":5187,"description":5190,"cover":5191,"coverAlt":5192,"excerpt":5193,"draft":4698,"hidden":4698,"published":3441,"readingTime":3438,"series":5194,"seriesOrder":3368},"backend-stack-deep-dive","\u002Fposts\u002Fbackend-stack-deep-dive","高并发 AI 平台后端基础设施实践：FastAPI 异步网关、SQLAlchemy 2.0 Async、PostgreSQL 与 Docker 沙箱编排","2026-04-07",[4843,4840,4842,5188,5189,4737],"PostgreSQL","Docker","结合 DeepScientist 生产级 AI 科研平台实践，拆解 FastAPI ASGI 异步事件循环、SQLAlchemy 2.0 + AsyncPG 连接池调优、PostgreSQL + MinIO 存储分层与 Docker SDK 动态沙箱容器编排的职责边界与核心设计。","\u002Fimages\u002Fposts\u002Fbackend-stack-deep-dive\u002Fbackend-stack-cover-20261018.webp","ASGI 异步事件循环涡轮将非阻塞协程流分发至 PostgreSQL 连接池、MinIO 存储矩阵与动态 Docker 沙箱的等距架构插画","结合 DeepScientist 生产级 AI 科研平台（服务 7,000+ 用户）的实际工程落地，本文系统拆解支撑高并发流式推理与容器沙箱调度的后端基础设施栈——FastAPI（ASGI 异步网关 + SSE 流式推送）、SQLA","DeepScientist 工程复盘",{"slug":5196,"path":5197,"title":5198,"date":5199,"tags":5200,"description":5203,"cover":5204,"coverAlt":5205,"excerpt":5206,"draft":4698,"hidden":4698,"published":3441,"readingTime":4115,"series":5207,"seriesOrder":3321},"minicode-source-analysis","\u002Fposts\u002Fminicode-source-analysis","MiniCode 架构与工具评测深度拆解：开源终端 Agentic Coding 引擎（1k+ Stars）的 ReAct、MCP 与 30 项基准验证","2026-04-06",[4766,5201,5202,4795,4737],"TypeScript","MCP","深度剖析开源终端 AI 编程智能体 MiniCode（1k+ GitHub Stars）的核心架构：从零重型依赖的 ReAct 规划循环、声明式工具注册表与权限拦截层、MCP 协议帧自动协商，到 30 项可执行 Tool-Use 评测基准（86.7% 完成率、100% 错误恢复率）的失败分布归因与闭环优化。","\u002Fimages\u002Fposts\u002Fminicode-source-analysis\u002Fminicode-cover-20261018.webp","终端编程智能体控制台被 ReAct 状态环、Unified Diff 补丁预览门与 MCP 协议插槽环绕的等距技术插画","MiniCode 是我作为核心开发者参与构建的开源终端 Agentic Coding 引擎（目前已在 GitHub 获得 1,000+ Stars）。项目的核心设计目标是：在不引入庞大运行时框架的前提下，用约 5,000 行高内聚的","AI 项目源码导读",{"slug":5209,"path":5210,"title":5211,"date":5199,"tags":5212,"description":5213,"cover":5214,"coverAlt":5215,"excerpt":5216,"draft":4698,"hidden":4698,"published":3441,"readingTime":4089,"series":5194,"seriesOrder":3321},"deepscientist-tech-stack","\u002Fposts\u002Fdeepscientist-tech-stack","DeepScientist 架构与运行时全解析：超 100K 上下文科研智能体、MCP 沙箱与高并发流式系统",[4766,4737,5202,5189,4842,4840],"深度拆解服务 7,000+ 用户的科研自主智能体平台 DeepScientist：从一题一仓库（Quest）持久化状态机、MCP 工具栈（memory \u002F artifact \u002F bash_exec）、Docker SDK 动态容器编排，到 SSE + WebSocket 双流通信与 PostgreSQL \u002F Redis \u002F Prometheus 生产级基础设施。","\u002Fimages\u002Fposts\u002Fdeepscientist-tech-stack\u002Fdeepscientist-cover-20261018.webp","科研智能体编排核心通过 memory、artifact 与隔离容器沙箱三座协议桥连接双流工作台的等距架构插画","DeepScientist（deepscientist.cc）是我们在西湖大学自然语言处理实验室研发的自主科学发现与科研管理平台，目前累计服务 7,000+ 科研用户。与单轮对话或纯文本文献问答工具不同，科研智能体需要在数小时乃至数",1795088946092]