[{"data":1,"prerenderedAt":2808},["ShallowReactive",2],{"post-\u002Fposts\u002Fcs336-linear-embedding-shapes":3,"archive-public-posts":2297},{"id":4,"title":5,"body":6,"categories":2274,"cover":2276,"coverAlt":2277,"date":2278,"description":2279,"draft":2280,"extension":2281,"haloPublished":2280,"haloSync":2280,"hidden":2280,"legacy":2280,"meta":2282,"navigation":2284,"path":2285,"published":2280,"readingTime":2286,"seo":2287,"series":2288,"seriesOrder":187,"stem":2289,"tags":2290,"__hash__":2296},"posts\u002Fposts\u002Fcs336-linear-embedding-shapes.md","Stanford CS336 学习记录 01：从 Linear、Embedding 搞清张量形状",{"type":7,"value":8,"toc":2260},"minimark",[9,13,28,33,36,45,112,128,134,138,145,148,576,579,825,912,915,1155,1162,1193,1211,1222,1226,1240,1272,1281,1340,1347,1351,1356,1359,1691,1877,1887,1891,1906,1915,1921,1965,1978,1982,1985,1992,1998,2012,2023,2031,2034,2095,2098,2105,2123,2126,2132,2139,2142,2157,2166,2169,2188,2200,2203,2256],[10,11,12],"p",{},"开始沿着 Stanford CS336 的 Assignment 1 一步步实现。BPE 之前已经接触过，这轮先跳过分词器，从 Transformer 的基础层开始。这里记录的是当时写代码时真正卡住的问题，以及对应测试能确认到的范围。",[10,14,15,16,20,21,20,24,27],{},"起步时，我连类的骨架都没完全写顺。",[17,18,19],"code",{},"nn.Module","、",[17,22,23],{},"nn.Parameter",[17,25,26],{},"forward"," 看起来都与“模型”有关，但分别负责什么并不清楚。与其急着拼一个完整网络，不如先把一个输入 3 维、输出 2 维的 Linear 看明白。",[29,30,32],"h2",{"id":31},"类保存什么forward-计算什么","类保存什么，forward 计算什么",[10,34,35],{},"一个带可训练参数的层，需要同时描述两件事：它拥有什么参数，以及输入来了以后怎样使用这些参数。",[10,37,38,41,42,44],{},[17,39,40],{},"__init__"," 在对象创建时准备参数。",[17,43,26],{}," 描述一次前向计算。同一个层可以处理很多批输入，权重保存在对象里，不会因为每来一批数据就重新创建。",[46,47,48,61],"table",{},[49,50,51],"thead",{},[52,53,54,58],"tr",{},[55,56,57],"th",{},"名字",[55,59,60],{},"在这一层里的工作",[62,63,64,74,84,93,103],"tbody",{},[52,65,66,71],{},[67,68,69],"td",{},[17,70,19],{},[67,72,73],{},"提供模块、子模块和参数管理机制",[52,75,76,81],{},[67,77,78],{},[17,79,80],{},"super().__init__()",[67,82,83],{},"初始化父类的模块管理机制",[52,85,86,90],{},[67,87,88],{},[17,89,23],{},[67,91,92],{},"把张量登记为可训练参数",[52,94,95,100],{},[67,96,97],{},[17,98,99],{},"self.weight",[67,101,102],{},"保存这个层自己的权重",[52,104,105,109],{},[67,106,107],{},[17,108,26],{},[67,110,111],{},"定义输入怎样变成输出",[10,113,114,115,118,119,121,122,124,125,127],{},"调用层时，通常使用 ",[17,116,117],{},"layer(x)","。它经过 ",[17,120,19],{}," 的调用机制，执行 ",[17,123,26],{},"，也保留 PyTorch 提供的 hooks 等功能。可以把 ",[17,126,26],{}," 理解为主要的计算内容，但不能把二者的实现机制完全画上等号。",[10,129,130,133],{},[17,131,132],{},"layer"," 也不是特殊关键字，只是给对象起的变量名。前面反复出现“先创建 layer，再把输入传进去”，原因是这个对象需要保存和复用权重。",[29,135,137],{"id":136},"输入-3-维输出-2-维权重到底是几行几列","输入 3 维、输出 2 维，权重到底是几行几列",[10,139,140,141,144],{},"当时被问到权重形状时，我回答了“三行两列”。这个回答对应着一种可以成立的数学存储方向，但本次作业约定权重按 ",[17,142,143],{},"(out_features, in_features)"," 存储，所以应是两行三列。",[10,146,147],{},"每一行生成一个输出，每一列对应一个输入特征。用一组便于手算的数：",[149,150,153],"span",{"className":151},[152],"katex-display",[149,154,157,285],{"className":155},[156],"katex",[149,158,161],{"className":159},[160],"katex-mathml",[162,163,166],"math",{"xmlns":164,"display":165},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML","block",[167,168,169,280],"semantics",{},[170,171,172,176,180,184,188,192,195,197,200,203,205,209,212,214,276],"mrow",{},[173,174,175],"mi",{},"x",[177,178,179],"mo",{},"=",[177,181,183],{"stretchy":182},"false","[",[185,186,187],"mn",{},"1",[177,189,191],{"separator":190},"true",",",[185,193,194],{},"2",[177,196,191],{"separator":190},[185,198,199],{},"3",[177,201,202],{"stretchy":182},"]",[177,204,191],{"separator":190},[206,207],"mspace",{"width":208},"2em",[173,210,211],{},"W",[177,213,179],{},[170,215,216,218,274],{},[177,217,183],{"fence":190},[219,220,224,251],"mtable",{"rowspacing":221,"columnalign":222,"columnspacing":223},"0.16em","center center center","1em",[225,226,227,237,244],"mtr",{},[228,229,230],"mtd",{},[231,232,234],"mstyle",{"scriptlevel":233,"displaystyle":182},"0",[185,235,236],{},"0.1",[228,238,239],{},[231,240,241],{"scriptlevel":233,"displaystyle":182},[185,242,243],{},"0.2",[228,245,246],{},[231,247,248],{"scriptlevel":233,"displaystyle":182},[185,249,250],{},"0.3",[225,252,253,260,267],{},[228,254,255],{},[231,256,257],{"scriptlevel":233,"displaystyle":182},[185,258,259],{},"0.4",[228,261,262],{},[231,263,264],{"scriptlevel":233,"displaystyle":182},[185,265,266],{},"0.5",[228,268,269],{},[231,270,271],{"scriptlevel":233,"displaystyle":182},[185,272,273],{},"0.6",[177,275,202],{"fence":190},[173,277,279],{"mathvariant":278},"normal",".",[281,282,284],"annotation",{"encoding":283},"application\u002Fx-tex","x=[1,2,3],\\qquad\nW=\\begin{bmatrix}\n0.1&0.2&0.3\\\\\n0.4&0.5&0.6\n\\end{bmatrix}.",[149,286,289,314,375],{"className":287,"ariaHidden":190},[288],"katex-html",[149,290,293,298,303,307,311],{"className":291},[292],"base",[149,294],{"className":295,"style":297},[296],"strut","height:0.4306em;",[149,299,175],{"className":300},[301,302],"mord","mathnormal",[149,304],{"className":305,"style":306},[206],"margin-right:0.2778em;",[149,308,179],{"className":309},[310],"mrel",[149,312],{"className":313,"style":306},[206],[149,315,317,321,325,328,332,336,339,342,345,348,352,355,359,362,366,369,372],{"className":316},[292],[149,318],{"className":319,"style":320},[296],"height:1em;vertical-align:-0.25em;",[149,322,183],{"className":323},[324],"mopen",[149,326,187],{"className":327},[301],[149,329,191],{"className":330},[331],"mpunct",[149,333],{"className":334,"style":335},[206],"margin-right:0.1667em;",[149,337,194],{"className":338},[301],[149,340,191],{"className":341},[331],[149,343],{"className":344,"style":335},[206],[149,346,199],{"className":347},[301],[149,349,202],{"className":350},[351],"mclose",[149,353,191],{"className":354},[331],[149,356],{"className":357,"style":358},[206],"margin-right:2em;",[149,360],{"className":361,"style":335},[206],[149,363,211],{"className":364,"style":365},[301,302],"margin-right:0.1389em;",[149,367],{"className":368,"style":306},[206],[149,370,179],{"className":371},[310],[149,373],{"className":374,"style":306},[206],[149,376,378,382,570,573],{"className":377},[292],[149,379],{"className":380,"style":381},[296],"height:2.4em;vertical-align:-0.95em;",[149,383,386,396,564],{"className":384},[385],"minner",[149,387,391],{"className":388,"style":390},[324,389],"delimcenter","top:0em;",[149,392,183],{"className":393},[394,395],"delimsizing","size3",[149,397,399],{"className":398},[301],[149,400,402,460,465,468,513,516,519],{"className":401},[219],[149,403,406],{"className":404},[405],"col-align-c",[149,407,411,451],{"className":408},[409,410],"vlist-t","vlist-t2",[149,412,415,446],{"className":413},[414],"vlist-r",[149,416,420,434],{"className":417,"style":419},[418],"vlist","height:1.45em;",[149,421,423,428],{"style":422},"top:-3.61em;",[149,424],{"className":425,"style":427},[426],"pstrut","height:3em;",[149,429,431],{"className":430},[301],[149,432,236],{"className":433},[301],[149,435,437,440],{"style":436},"top:-2.41em;",[149,438],{"className":439,"style":427},[426],[149,441,443],{"className":442},[301],[149,444,259],{"className":445},[301],[149,447,450],{"className":448},[449],"vlist-s","​",[149,452,454],{"className":453},[414],[149,455,458],{"className":456,"style":457},[418],"height:0.95em;",[149,459],{},[149,461],{"className":462,"style":464},[463],"arraycolsep","width:0.5em;",[149,466],{"className":467,"style":464},[463],[149,469,471],{"className":470},[405],[149,472,474,505],{"className":473},[409,410],[149,475,477,502],{"className":476},[414],[149,478,480,491],{"className":479,"style":419},[418],[149,481,482,485],{"style":422},[149,483],{"className":484,"style":427},[426],[149,486,488],{"className":487},[301],[149,489,243],{"className":490},[301],[149,492,493,496],{"style":436},[149,494],{"className":495,"style":427},[426],[149,497,499],{"className":498},[301],[149,500,266],{"className":501},[301],[149,503,450],{"className":504},[449],[149,506,508],{"className":507},[414],[149,509,511],{"className":510,"style":457},[418],[149,512],{},[149,514],{"className":515,"style":464},[463],[149,517],{"className":518,"style":464},[463],[149,520,522],{"className":521},[405],[149,523,525,556],{"className":524},[409,410],[149,526,528,553],{"className":527},[414],[149,529,531,542],{"className":530,"style":419},[418],[149,532,533,536],{"style":422},[149,534],{"className":535,"style":427},[426],[149,537,539],{"className":538},[301],[149,540,250],{"className":541},[301],[149,543,544,547],{"style":436},[149,545],{"className":546,"style":427},[426],[149,548,550],{"className":549},[301],[149,551,273],{"className":552},[301],[149,554,450],{"className":555},[449],[149,557,559],{"className":558},[414],[149,560,562],{"className":561,"style":457},[418],[149,563],{},[149,565,567],{"className":566,"style":390},[351,389],[149,568,202],{"className":569},[394,395],[149,571],{"className":572,"style":335},[206],[149,574,279],{"className":575},[301],[10,577,578],{},"第一个输出把输入与第一行逐项相乘再求和：",[149,580,582],{"className":581},[152],[149,583,585,636],{"className":584},[156],[149,586,588],{"className":587},[160],[162,589,590],{"xmlns":164,"display":165},[167,591,592,633],{},[170,593,594,602,604,606,609,611,614,616,618,620,622,624,626,628,630],{},[595,596,597,600],"msub",{},[173,598,599],{},"y",[185,601,187],{},[177,603,179],{},[185,605,187],{},[177,607,608],{},"×",[185,610,236],{},[177,612,613],{},"+",[185,615,194],{},[177,617,608],{},[185,619,243],{},[177,621,613],{},[185,623,199],{},[177,625,608],{},[185,627,250],{},[177,629,179],{},[185,631,632],{},"1.4.",[281,634,635],{"encoding":283},"y_1=1\\times0.1+2\\times0.2+3\\times0.3=1.4.",[149,637,639,704,725,743,761,779,797,816],{"className":638,"ariaHidden":190},[288],[149,640,642,646,695,698,701],{"className":641},[292],[149,643],{"className":644,"style":645},[296],"height:0.625em;vertical-align:-0.1944em;",[149,647,649,653],{"className":648},[301],[149,650,599],{"className":651,"style":652},[301,302],"margin-right:0.0359em;",[149,654,657],{"className":655},[656],"msupsub",[149,658,660,686],{"className":659},[409,410],[149,661,663,683],{"className":662},[414],[149,664,667],{"className":665,"style":666},[418],"height:0.3011em;",[149,668,670,674],{"style":669},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[149,671],{"className":672,"style":673},[426],"height:2.7em;",[149,675,680],{"className":676},[677,678,395,679],"sizing","reset-size6","mtight",[149,681,187],{"className":682},[301,679],[149,684,450],{"className":685},[449],[149,687,689],{"className":688},[414],[149,690,693],{"className":691,"style":692},[418],"height:0.15em;",[149,694],{},[149,696],{"className":697,"style":306},[206],[149,699,179],{"className":700},[310],[149,702],{"className":703,"style":306},[206],[149,705,707,711,714,718,722],{"className":706},[292],[149,708],{"className":709,"style":710},[296],"height:0.7278em;vertical-align:-0.0833em;",[149,712,187],{"className":713},[301],[149,715],{"className":716,"style":717},[206],"margin-right:0.2222em;",[149,719,608],{"className":720},[721],"mbin",[149,723],{"className":724,"style":717},[206],[149,726,728,731,734,737,740],{"className":727},[292],[149,729],{"className":730,"style":710},[296],[149,732,236],{"className":733},[301],[149,735],{"className":736,"style":717},[206],[149,738,613],{"className":739},[721],[149,741],{"className":742,"style":717},[206],[149,744,746,749,752,755,758],{"className":745},[292],[149,747],{"className":748,"style":710},[296],[149,750,194],{"className":751},[301],[149,753],{"className":754,"style":717},[206],[149,756,608],{"className":757},[721],[149,759],{"className":760,"style":717},[206],[149,762,764,767,770,773,776],{"className":763},[292],[149,765],{"className":766,"style":710},[296],[149,768,243],{"className":769},[301],[149,771],{"className":772,"style":717},[206],[149,774,613],{"className":775},[721],[149,777],{"className":778,"style":717},[206],[149,780,782,785,788,791,794],{"className":781},[292],[149,783],{"className":784,"style":710},[296],[149,786,199],{"className":787},[301],[149,789],{"className":790,"style":717},[206],[149,792,608],{"className":793},[721],[149,795],{"className":796,"style":717},[206],[149,798,800,804,807,810,813],{"className":799},[292],[149,801],{"className":802,"style":803},[296],"height:0.6444em;",[149,805,250],{"className":806},[301],[149,808],{"className":809,"style":306},[206],[149,811,179],{"className":812},[310],[149,814],{"className":815,"style":306},[206],[149,817,819,822],{"className":818},[292],[149,820],{"className":821,"style":803},[296],[149,823,632],{"className":824},[301],[10,826,827,828,857,858,911],{},"同样，第二个输出是 ",[149,829,831,845],{"className":830},[156],[149,832,834],{"className":833},[160],[162,835,836],{"xmlns":164},[167,837,838,843],{},[170,839,840],{},[185,841,842],{},"3.2",[281,844,842],{"encoding":283},[149,846,848],{"className":847,"ariaHidden":190},[288],[149,849,851,854],{"className":850},[292],[149,852],{"className":853,"style":803},[296],[149,855,842],{"className":856},[301],"。最后得到 ",[149,859,861,884],{"className":860},[156],[149,862,864],{"className":863},[160],[162,865,866],{"xmlns":164},[167,867,868,881],{},[170,869,870,872,875,877,879],{},[177,871,183],{"stretchy":182},[185,873,874],{},"1.4",[177,876,191],{"separator":190},[185,878,842],{},[177,880,202],{"stretchy":182},[281,882,883],{"encoding":283},"[1.4,3.2]",[149,885,887],{"className":886,"ariaHidden":190},[288],[149,888,890,893,896,899,902,905,908],{"className":889},[292],[149,891],{"className":892,"style":320},[296],[149,894,183],{"className":895},[324],[149,897,874],{"className":898},[301],[149,900,191],{"className":901},[331],[149,903],{"className":904,"style":335},[206],[149,906,842],{"className":907},[301],[149,909,202],{"className":910},[351],"，特征数从 3 变成了 2。",[10,913,914],{},"输入按行摆放时，对应的矩阵计算是：",[149,916,918],{"className":917},[152],[149,919,921,1000],{"className":920},[156],[149,922,924],{"className":923},[160],[162,925,926],{"xmlns":164,"display":165},[167,927,928,997],{},[170,929,930,933,935,938,946,948,950,953,956,958,960,963,967,970,972,974,976,978,980,982,985,987,989,991,993,995],{},[173,931,932],{},"Y",[177,934,179],{},[173,936,937],{},"X",[939,940,941,943],"msup",{},[173,942,211],{},[173,944,945],{"mathvariant":278},"⊤",[177,947,191],{"separator":190},[206,949],{"width":208},[177,951,952],{"stretchy":182},"(",[173,954,955],{},"B",[177,957,191],{"separator":190},[185,959,199],{},[177,961,962],{"stretchy":182},")",[964,965,966],"mtext",{}," ",[173,968,969],{"mathvariant":278},"@",[964,971,966],{},[177,973,952],{"stretchy":182},[185,975,199],{},[177,977,191],{"separator":190},[185,979,194],{},[177,981,962],{"stretchy":182},[177,983,984],{},"⟶",[177,986,952],{"stretchy":182},[173,988,955],{},[177,990,191],{"separator":190},[185,992,194],{},[177,994,962],{"stretchy":182},[173,996,279],{"mathvariant":278},[281,998,999],{"encoding":283},"Y=XW^\\top,\n\\qquad\n(B,3)\\ @\\ (3,2)\\longrightarrow(B,2).",[149,1001,1003,1022,1128],{"className":1002,"ariaHidden":190},[288],[149,1004,1006,1010,1013,1016,1019],{"className":1005},[292],[149,1007],{"className":1008,"style":1009},[296],"height:0.6833em;",[149,1011,932],{"className":1012,"style":717},[301,302],[149,1014],{"className":1015,"style":306},[206],[149,1017,179],{"className":1018},[310],[149,1020],{"className":1021,"style":306},[206],[149,1023,1025,1029,1033,1064,1067,1070,1073,1076,1080,1083,1086,1089,1092,1095,1098,1101,1104,1107,1110,1113,1116,1119,1122,1125],{"className":1024},[292],[149,1026],{"className":1027,"style":1028},[296],"height:1.1491em;vertical-align:-0.25em;",[149,1030,937],{"className":1031,"style":1032},[301,302],"margin-right:0.0785em;",[149,1034,1036,1039],{"className":1035},[301],[149,1037,211],{"className":1038,"style":365},[301,302],[149,1040,1042],{"className":1041},[656],[149,1043,1045],{"className":1044},[409],[149,1046,1048],{"className":1047},[414],[149,1049,1052],{"className":1050,"style":1051},[418],"height:0.8991em;",[149,1053,1055,1058],{"style":1054},"top:-3.113em;margin-right:0.05em;",[149,1056],{"className":1057,"style":673},[426],[149,1059,1061],{"className":1060},[677,678,395,679],[149,1062,945],{"className":1063},[301,679],[149,1065,191],{"className":1066},[331],[149,1068],{"className":1069,"style":358},[206],[149,1071],{"className":1072,"style":335},[206],[149,1074,952],{"className":1075},[324],[149,1077,955],{"className":1078,"style":1079},[301,302],"margin-right:0.0502em;",[149,1081,191],{"className":1082},[331],[149,1084],{"className":1085,"style":335},[206],[149,1087,199],{"className":1088},[301],[149,1090,962],{"className":1091},[351],[149,1093,966],{"className":1094},[206],[149,1096,969],{"className":1097},[301],[149,1099,966],{"className":1100},[206],[149,1102,952],{"className":1103},[324],[149,1105,199],{"className":1106},[301],[149,1108,191],{"className":1109},[331],[149,1111],{"className":1112,"style":335},[206],[149,1114,194],{"className":1115},[301],[149,1117,962],{"className":1118},[351],[149,1120],{"className":1121,"style":306},[206],[149,1123,984],{"className":1124},[310],[149,1126],{"className":1127,"style":306},[206],[149,1129,1131,1134,1137,1140,1143,1146,1149,1152],{"className":1130},[292],[149,1132],{"className":1133,"style":320},[296],[149,1135,952],{"className":1136},[324],[149,1138,955],{"className":1139,"style":1079},[301,302],[149,1141,191],{"className":1142},[331],[149,1144],{"className":1145,"style":335},[206],[149,1147,194],{"className":1148},[301],[149,1150,962],{"className":1151},[351],[149,1153,279],{"className":1154},[301],[10,1156,1157,1158,1161],{},"已经写好的 ",[17,1159,1160],{},"linear.py"," 中，核心计算只有一行：",[1163,1164,1169],"pre",{"className":1165,"code":1166,"language":1167,"meta":1168,"style":1168},"language-python shiki shiki-themes github-dark-dimmed github-light houston","return x @ self.weight.T\n","python","",[17,1170,1171],{"__ignoreMap":1168},[149,1172,1175,1179,1183,1186,1190],{"class":1173,"line":1174},"line",1,[149,1176,1178],{"class":1177},"slEfO","return",[149,1180,1182],{"class":1181},"sWURj"," x ",[149,1184,969],{"class":1185},"sf2cM",[149,1187,1189],{"class":1188},"sRGqv"," self",[149,1191,1192],{"class":1181},".weight.T\n",[10,1194,1195,1196,1198,1199,1202,1203,1206,1207,1210],{},"这里的 ",[17,1197,969],{}," 做矩阵乘法，",[17,1200,1201],{},".T"," 把二维权重从 ",[17,1204,1205],{},"(2, 3)"," 转成 ",[17,1208,1209],{},"(3, 2)","。转置服务于所选的存储约定；判断方向时，先写出形状，比死记“Linear 一定要转置”可靠。",[10,1212,1213,1214,1217,1218,1221],{},"批量和序列维度也能保留。例如输入是 ",[17,1215,1216],{},"(2, 5, 3)","，可以理解为两段序列、每段五个 token、每个 token 三个特征。输出是 ",[17,1219,1220],{},"(2, 5, 2)","。这个 Linear 对所有位置应用同一组权重，改变的是最后一维，不会让第一个 token 从第二个 token 那里读取信息。",[29,1223,1225],{"id":1224},"形状device-和-dtype-是三个独立问题","形状、device 和 dtype 是三个独立问题",[10,1227,1228,1229,1232,1233,20,1236,1239],{},"创建参数时，",[17,1230,1231],{},"torch.empty"," 后面既有数字，又有 ",[17,1234,1235],{},"device",[17,1237,1238],{},"dtype","，最初很容易把它们当成一组含义相近的配置。已有代码中的创建部分是：",[1163,1241,1243],{"className":1165,"code":1242,"language":1167,"meta":1168,"style":1168},"torch.empty(out_features, in_features, device=device, dtype=dtype)\n",[17,1244,1245],{"__ignoreMap":1168},[149,1246,1247,1250,1254,1257,1260,1262,1265,1267,1269],{"class":1173,"line":1174},[149,1248,1249],{"class":1181},"torch.",[149,1251,1253],{"class":1252},"sq3c-","empty",[149,1255,1256],{"class":1181},"(out_features, in_features, ",[149,1258,1235],{"class":1259},"sA9aF",[149,1261,179],{"class":1185},[149,1263,1264],{"class":1181},"device, ",[149,1266,1238],{"class":1259},[149,1268,179],{"class":1185},[149,1270,1271],{"class":1181},"dtype)\n",[10,1273,1274,1275,1277,1278,1280],{},"前两个整数决定形状；",[17,1276,1235],{}," 决定张量放在哪里；",[17,1279,1238],{}," 决定每个元素使用什么数值类型。",[46,1282,1283,1296],{},[49,1284,1285],{},[52,1286,1287,1290,1293],{},[55,1288,1289],{},"配置",[55,1291,1292],{},"例子",[55,1294,1295],{},"回答的问题",[62,1297,1298,1310,1328],{},[52,1299,1300,1303,1307],{},[67,1301,1302],{},"shape",[67,1304,1305],{},[17,1306,1205],{},[67,1308,1309],{},"有几行、几列",[52,1311,1312,1314,1325],{},[67,1313,1235],{},[67,1315,1316,20,1319,20,1322],{},[17,1317,1318],{},"cpu",[17,1320,1321],{},"cuda",[17,1323,1324],{},"mps",[67,1326,1327],{},"数据在哪个计算设备上",[52,1329,1330,1332,1337],{},[67,1331,1238],{},[67,1333,1334],{},[17,1335,1336],{},"torch.float32",[67,1338,1339],{},"每个数怎样存储和计算",[10,1341,1342,1343,1346],{},"这次先在 CPU 上验证前向计算。构造函数接受 ",[17,1344,1345],{},"device=None, dtype=None","，表示允许调用者沿用 PyTorch 当前的默认设置。测试适配器则会采用测试权重的设备和类型，避免输入、参数的配置对不上。",[29,1348,1350],{"id":1349},"empty-申请空间初始化决定起点","empty 申请空间，初始化决定起点",[10,1352,1353,1355],{},[17,1354,1253],{}," 并不是“得到一张全零矩阵”。它分配指定形状的空间，其中的值尚未按模型需要初始化，不能直接拿来当可靠的起始参数。",[10,1357,1358],{},"这份作业的 Linear 使用截断正态初始化，标准差按以下规则选取：",[149,1360,1362],{"className":1361},[152],[149,1363,1365,1424],{"className":1364},[156],[149,1366,1368],{"className":1367},[160],[162,1369,1370],{"xmlns":164,"display":165},[167,1371,1372,1421],{},[170,1373,1374,1377,1379,1419],{},[173,1375,1376],{},"σ",[177,1378,179],{},[1380,1381,1382],"msqrt",{},[1383,1384,1385,1387],"mfrac",{},[185,1386,194],{},[170,1388,1389,1402,1404],{},[595,1390,1391,1394],{},[173,1392,1393],{},"d",[170,1395,1396,1399],{},[173,1397,1398],{"mathvariant":278},"i",[173,1400,1401],{"mathvariant":278},"n",[177,1403,613],{},[595,1405,1406,1408],{},[173,1407,1393],{},[170,1409,1410,1413,1416],{},[173,1411,1412],{"mathvariant":278},"o",[173,1414,1415],{"mathvariant":278},"u",[173,1417,1418],{"mathvariant":278},"t",[173,1420,279],{"mathvariant":278},[281,1422,1423],{"encoding":283},"\\sigma=\\sqrt{\\frac{2}{d_{\\mathrm{in}}+d_{\\mathrm{out}}}}.",[149,1425,1427,1445],{"className":1426,"ariaHidden":190},[288],[149,1428,1430,1433,1436,1439,1442],{"className":1429},[292],[149,1431],{"className":1432,"style":297},[296],[149,1434,1376],{"className":1435,"style":652},[301,302],[149,1437],{"className":1438,"style":306},[206],[149,1440,179],{"className":1441},[310],[149,1443],{"className":1444,"style":306},[206],[149,1446,1448,1452,1688],{"className":1447},[292],[149,1449],{"className":1450,"style":1451},[296],"height:2.44em;vertical-align:-0.8634em;",[149,1453,1456],{"className":1454},[301,1455],"sqrt",[149,1457,1459,1679],{"className":1458},[409,410],[149,1460,1462,1676],{"className":1461},[414],[149,1463,1466,1653],{"className":1464,"style":1465},[418],"height:1.5766em;",[149,1467,1471,1475],{"className":1468,"style":1470},[1469],"svg-align","top:-4.4em;",[149,1472],{"className":1473,"style":1474},[426],"height:4.4em;",[149,1476,1479],{"className":1477,"style":1478},[301],"padding-left:1em;",[149,1480,1482,1486,1650],{"className":1481},[301],[149,1483],{"className":1484},[324,1485],"nulldelimiter",[149,1487,1489],{"className":1488},[1383],[149,1490,1492,1641],{"className":1491},[409,410],[149,1493,1495,1638],{"className":1494},[414],[149,1496,1499,1615,1626],{"className":1497,"style":1498},[418],"height:1.3214em;",[149,1500,1502,1505],{"style":1501},"top:-2.314em;",[149,1503],{"className":1504,"style":427},[426],[149,1506,1508,1558,1561,1564,1567],{"className":1507},[301],[149,1509,1511,1514],{"className":1510},[301],[149,1512,1393],{"className":1513},[301,302],[149,1515,1517],{"className":1516},[656],[149,1518,1520,1550],{"className":1519},[409,410],[149,1521,1523,1547],{"className":1522},[414],[149,1524,1527],{"className":1525,"style":1526},[418],"height:0.3175em;",[149,1528,1530,1533],{"style":1529},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[149,1531],{"className":1532,"style":673},[426],[149,1534,1536],{"className":1535},[677,678,395,679],[149,1537,1539],{"className":1538},[301,679],[149,1540,1542],{"className":1541},[301,679],[149,1543,1546],{"className":1544},[301,1545,679],"mathrm","in",[149,1548,450],{"className":1549},[449],[149,1551,1553],{"className":1552},[414],[149,1554,1556],{"className":1555,"style":692},[418],[149,1557],{},[149,1559],{"className":1560,"style":717},[206],[149,1562,613],{"className":1563},[721],[149,1565],{"className":1566,"style":717},[206],[149,1568,1570,1573],{"className":1569},[301],[149,1571,1393],{"className":1572},[301,302],[149,1574,1576],{"className":1575},[656],[149,1577,1579,1607],{"className":1578},[409,410],[149,1580,1582,1604],{"className":1581},[414],[149,1583,1586],{"className":1584,"style":1585},[418],"height:0.2806em;",[149,1587,1588,1591],{"style":1529},[149,1589],{"className":1590,"style":673},[426],[149,1592,1594],{"className":1593},[677,678,395,679],[149,1595,1597],{"className":1596},[301,679],[149,1598,1600],{"className":1599},[301,679],[149,1601,1603],{"className":1602},[301,1545,679],"out",[149,1605,450],{"className":1606},[449],[149,1608,1610],{"className":1609},[414],[149,1611,1613],{"className":1612,"style":692},[418],[149,1614],{},[149,1616,1618,1621],{"style":1617},"top:-3.23em;",[149,1619],{"className":1620,"style":427},[426],[149,1622],{"className":1623,"style":1625},[1624],"frac-line","border-bottom-width:0.04em;",[149,1627,1629,1632],{"style":1628},"top:-3.677em;",[149,1630],{"className":1631,"style":427},[426],[149,1633,1635],{"className":1634},[301],[149,1636,194],{"className":1637},[301],[149,1639,450],{"className":1640},[449],[149,1642,1644],{"className":1643},[414],[149,1645,1648],{"className":1646,"style":1647},[418],"height:0.836em;",[149,1649],{},[149,1651],{"className":1652},[351,1485],[149,1654,1656,1659],{"style":1655},"top:-3.5366em;",[149,1657],{"className":1658,"style":1474},[426],[149,1660,1664],{"className":1661,"style":1663},[1662],"hide-tail","min-width:1.02em;height:2.48em;",[1665,1666,1672],"svg",{"xmlns":1667,"width":1668,"height":1669,"viewBox":1670,"preserveAspectRatio":1671},"http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","400em","2.48em","0 0 400000 2592","xMinYMin slice",[1673,1674],"path",{"d":1675},"M424,2478\nc-1.3,-0.7,-38.5,-172,-111.5,-514c-73,-342,-109.8,-513.3,-110.5,-514\nc0,-2,-10.7,14.3,-32,49c-4.7,7.3,-9.8,15.7,-15.5,25c-5.7,9.3,-9.8,16,-12.5,20\ns-5,7,-5,7c-4,-3.3,-8.3,-7.7,-13,-13s-13,-13,-13,-13s76,-122,76,-122s77,-121,77,-121\ns209,968,209,968c0,-2,84.7,-361.7,254,-1079c169.3,-717.3,254.7,-1077.7,256,-1081\nl0 -0c4,-6.7,10,-10,18,-10 H400000\nv40H1014.6\ns-87.3,378.7,-272.6,1166c-185.3,787.3,-279.3,1182.3,-282,1185\nc-2,6,-10,9,-24,9\nc-8,0,-12,-0.7,-12,-2z M1001 80\nh400000v40h-400000z",[149,1677,450],{"className":1678},[449],[149,1680,1682],{"className":1681},[414],[149,1683,1686],{"className":1684,"style":1685},[418],"height:0.8634em;",[149,1687],{},[149,1689,279],{"className":1690},[301],[10,1692,1693,1694,1762,1763,1817,1818,1876],{},"均值设为 0，截断边界是 ",[149,1695,1697,1726],{"className":1696},[156],[149,1698,1700],{"className":1699},[160],[162,1701,1702],{"xmlns":164},[167,1703,1704,1723],{},[170,1705,1706,1708,1711,1713,1715,1717,1719,1721],{},[177,1707,183],{"stretchy":182},[177,1709,1710],{},"−",[185,1712,199],{},[173,1714,1376],{},[177,1716,191],{"separator":190},[185,1718,199],{},[173,1720,1376],{},[177,1722,202],{"stretchy":182},[281,1724,1725],{"encoding":283},"[-3\\sigma,3\\sigma]",[149,1727,1729],{"className":1728,"ariaHidden":190},[288],[149,1730,1732,1735,1738,1741,1744,1747,1750,1753,1756,1759],{"className":1731},[292],[149,1733],{"className":1734,"style":320},[296],[149,1736,183],{"className":1737},[324],[149,1739,1710],{"className":1740},[301],[149,1742,199],{"className":1743},[301],[149,1745,1376],{"className":1746,"style":652},[301,302],[149,1748,191],{"className":1749},[331],[149,1751],{"className":1752,"style":335},[206],[149,1754,199],{"className":1755},[301],[149,1757,1376],{"className":1758,"style":652},[301,302],[149,1760,202],{"className":1761},[351],"。输入 3 维、输出 2 维时，",[149,1764,1766,1786],{"className":1765},[156],[149,1767,1769],{"className":1768},[160],[162,1770,1771],{"xmlns":164},[167,1772,1773,1783],{},[170,1774,1775,1777,1780],{},[173,1776,1376],{},[177,1778,1779],{},"≈",[185,1781,1782],{},"0.632",[281,1784,1785],{"encoding":283},"\\sigma\\approx0.632",[149,1787,1789,1808],{"className":1788,"ariaHidden":190},[288],[149,1790,1792,1796,1799,1802,1805],{"className":1791},[292],[149,1793],{"className":1794,"style":1795},[296],"height:0.4831em;",[149,1797,1376],{"className":1798,"style":652},[301,302],[149,1800],{"className":1801,"style":306},[206],[149,1803,1779],{"className":1804},[310],[149,1806],{"className":1807,"style":306},[206],[149,1809,1811,1814],{"className":1810},[292],[149,1812],{"className":1813,"style":803},[296],[149,1815,1782],{"className":1816},[301],"，边界大约是 ",[149,1819,1821,1846],{"className":1820},[156],[149,1822,1824],{"className":1823},[160],[162,1825,1826],{"xmlns":164},[167,1827,1828,1843],{},[170,1829,1830,1832,1834,1837,1839,1841],{},[177,1831,183],{"stretchy":182},[177,1833,1710],{},[185,1835,1836],{},"1.897",[177,1838,191],{"separator":190},[185,1840,1836],{},[177,1842,202],{"stretchy":182},[281,1844,1845],{"encoding":283},"[-1.897,1.897]",[149,1847,1849],{"className":1848,"ariaHidden":190},[288],[149,1850,1852,1855,1858,1861,1864,1867,1870,1873],{"className":1851},[292],[149,1853],{"className":1854,"style":320},[296],[149,1856,183],{"className":1857},[324],[149,1859,1710],{"className":1860},[301],[149,1862,1836],{"className":1863},[301],[149,1865,191],{"className":1866},[331],[149,1868],{"className":1869,"style":335},[206],[149,1871,1836],{"className":1872},[301],[149,1874,202],{"className":1875},[351],"。",[10,1878,1879,1880,1883,1884,1886],{},"这里需要分清标准差和方差；公式外面还有一个平方根。",[17,1881,1882],{},"trunc_normal_"," 结尾的下划线则表示操作直接修改已有张量，把抽到的数填入 ",[17,1885,99],{},"。初始化只是训练的起点，之后参数仍然需要通过学习调整。",[29,1888,1890],{"id":1889},"测试里的-weights-与-in_features","测试里的 weights 与 in_features",[10,1892,1893,1894,1897,1898,1901,1902,1905],{},"接上官方 ",[17,1895,1896],{},"tests\u002Fadapters.py"," 时，我又问了一次：",[17,1899,1900],{},"weights"," 是什么，",[17,1903,1904],{},"in_features"," 又是什么？",[10,1907,1908,1909,1911,1912,1914],{},"适配器拿到的是测试提供的具体数据：",[17,1910,1900],{}," 是指定权重张量，",[17,1913,1904],{}," 是实际输入张量。适配器创建自己的层，把指定权重加载到层里，再返回这一层对输入的计算结果。",[10,1916,1917,1918,1920],{},"同一个名字在不同函数里可能表示不同东西，尤其是 ",[17,1919,1904],{},"：",[46,1922,1923,1936],{},[49,1924,1925],{},[52,1926,1927,1930,1933],{},[55,1928,1929],{},"出现位置",[55,1931,1932],{},"本次含义",[55,1934,1935],{},"类型",[62,1937,1938,1951],{},[52,1939,1940,1945,1948],{},[67,1941,1942],{},[17,1943,1944],{},"Linear.__init__",[67,1946,1947],{},"每个输入有几个特征",[67,1949,1950],{},"整数",[52,1952,1953,1959,1962],{},[67,1954,1955,1956],{},"官方 ",[17,1957,1958],{},"run_linear",[67,1960,1961],{},"交给层计算的实际输入",[67,1963,1964],{},"张量",[10,1966,1967,1968,1970,1971,1973,1974,1977],{},"外部的 ",[17,1969,1900],{}," 与内部的 ",[17,1972,99],{}," 也不是同一个变量。",[17,1975,1976],{},"load_state_dict"," 按名称将测试提供的数值加载到层的参数中。因此这项测试主要核对给定输入、给定权重下的前向计算，不能据此推断随机初始化也已经被完整验证。",[29,1979,1981],{"id":1980},"embedding-是取行输出多出一个特征维度","Embedding 是取行，输出多出一个特征维度",[10,1983,1984],{},"Linear 之后写的是 Embedding。它也保存可训练权重，不过输入变成了整数 token ID。",[10,1986,1987,1988,1991],{},"假设词表有 1000 个 token，每个 token 用 64 个数表示，参数表的形状就是 ",[17,1989,1990],{},"(1000, 64)","。ID 为 2，就取编号为 2 的行，也就是第三行。",[10,1993,1157,1994,1997],{},[17,1995,1996],{},"embedding.py"," 用下面这行完成查表：",[1163,1999,2001],{"className":1165,"code":2000,"language":1167,"meta":1168,"style":1168},"return self.weight[token_ids]\n",[17,2002,2003],{"__ignoreMap":1168},[149,2004,2005,2007,2009],{"class":1173,"line":1174},[149,2006,1178],{"class":1177},[149,2008,1189],{"class":1188},[149,2010,2011],{"class":1181},".weight[token_ids]\n",[10,2013,2014,2015,2018,2019,2022],{},"输入 ID ",[17,2016,2017],{},"[2, 0]","，就按这个顺序取出第三行和第一行，输出形状为 ",[17,2020,2021],{},"(2, 64)","。对于一批序列：",[1163,2024,2029],{"className":2025,"code":2027,"language":2028,"meta":1168},[2026],"language-text","整数 ID：  (batch, seq_len)\n权重表：   (vocab_size, embedding_dim)\n输出向量：(batch, seq_len, embedding_dim)\n","text",[17,2030,2027],{"__ignoreMap":1168},[10,2032,2033],{},"输入有多少个位置，输出就有多少个向量。Embedding 在这里没有执行 Linear 那样的特征加权求和。它与 Linear 都有可训练参数，具体的前向运算却不同。",[10,2035,2036,2037,2094],{},"本次 Embedding 的初始化使用标准差 1、范围 ",[149,2038,2040,2064],{"className":2039},[156],[149,2041,2043],{"className":2042},[160],[162,2044,2045],{"xmlns":164},[167,2046,2047,2061],{},[170,2048,2049,2051,2053,2055,2057,2059],{},[177,2050,183],{"stretchy":182},[177,2052,1710],{},[185,2054,199],{},[177,2056,191],{"separator":190},[185,2058,199],{},[177,2060,202],{"stretchy":182},[281,2062,2063],{"encoding":283},"[-3,3]",[149,2065,2067],{"className":2066,"ariaHidden":190},[288],[149,2068,2070,2073,2076,2079,2082,2085,2088,2091],{"className":2069},[292],[149,2071],{"className":2072,"style":320},[296],[149,2074,183],{"className":2075},[324],[149,2077,1710],{"className":2078},[301],[149,2080,199],{"className":2081},[301],[149,2083,191],{"className":2084},[331],[149,2086],{"className":2087,"style":335},[206],[149,2089,199],{"className":2090},[301],[149,2092,202],{"className":2093},[351]," 的截断正态，不能照抄 Linear 计算标准差的规则。",[29,2096,2097],{"id":2097},"这一步真正遇到的错误",[10,2099,2100,2101,2104],{},"除了数学方向，我还在文件位置和 Python 语法上花了不少时间。最初文件放在仓库根目录，检查时找的是 ",[17,2102,2103],{},"cs336_basics\u002Flinear.py","，于是出现了“文件明明在，为什么没读到”的误会。模块导入最终依赖的是包目录下的文件。",[10,2106,2107,2108,2111,2112,2114,2115,2118,2119,2122],{},"占位语句也带来过混淆：",[17,2109,2110],{},"NotImplementedError"," 曾拼错；写完 ",[17,2113,1178],{}," 后，后面又留着不会执行的 ",[17,2116,2117],{},"raise NotImplementedError","。Embedding 里还写过 ",[17,2120,2121],{},"return NotImplementedError","，它会把异常类当成返回结果，并不会报出“尚未实现”的异常。",[10,2124,2125],{},"第一次跑 Embedding 测试时，程序甚至还没有开始比较数值，就在收集测试阶段报了：",[1163,2127,2130],{"className":2128,"code":2129,"language":2028,"meta":1168},[2026],"SyntaxError: positional argument follows keyword argument\n",[17,2131,2129],{"__ignoreMap":1168},[10,2133,2134,2135,2138],{},"原因是适配器中的数据类型参数写成了 ",[17,2136,2137],{},"dtype.dtype","，本意是把测试权重的数据类型传给构造函数。这个错误提醒我，先区分“测试没有运行起来”和“运行后结果不对”，排查方向会清楚很多。",[29,2140,2141],{"id":2141},"已验证到哪里",[10,2143,2144,2145,2148,2149,2152,2153,2156],{},"修正上述问题后，历史运行记录里，官方 ",[17,2146,2147],{},"test_linear"," 与 ",[17,2150,2151],{},"test_embedding"," 分别得到 ",[17,2154,2155],{},"1 passed","。这确认了这两个基础组件在对应官方用例中的结果。",[10,2158,2159,2160,2165],{},"这一阶段还没有训练语言模型，也没有验证完整的 Assignment 1。接下来继续看 ",[2161,2162,2164],"a",{"href":2163},"\u002Fposts\u002Fcs336-rmsnorm-swiglu\u002F","RMSNorm 和 SwiGLU","，重点从矩阵方向转到广播和逐元素门控。",[29,2167,2168],{"id":2168},"发布时的复核",[10,2170,2171,2172,2178,2179,2182,2183,1876],{},"发布前对",[2161,2173,2177],{"href":2174,"rel":2175},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fa39eb684710817f7f389626423fcb26ca0f6d629",[2176],"nofollow","独立代码快照","重新运行了 8 项官方组件测试，结果为 ",[17,2180,2181],{},"8 passed","。完整清单和运行环境见",[2161,2184,2187],{"href":2185,"rel":2186},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Fblob\u002Fa39eb684710817f7f389626423fcb26ca0f6d629\u002Fdocs\u002Fprogress.md",[2176],"进度记录",[10,2189,2190],{},[2191,2192,2193,2194,2199],"em",{},"封面使用 ",[2161,2195,2198],{"href":2196,"rel":2197},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Faigc-image-generator",[2176],"aigc-image-generator"," 辅助生成，属于概念插画；准确的形状、公式与运算关系以正文为准。",[29,2201,2202],{"id":2202},"参考资料",[2204,2205,2206,2214,2221,2228,2235,2242,2249],"ul",{},[2207,2208,2209],"li",{},[2161,2210,2213],{"href":2211,"rel":2212},"https:\u002F\u002Fcs336.stanford.edu\u002F",[2176],"Stanford CS336 官方课程页面",[2207,2215,2216],{},[2161,2217,2220],{"href":2218,"rel":2219},"https:\u002F\u002Fgithub.com\u002Fstanford-cs336\u002Fassignment1-basics",[2176],"Assignment 1 官方仓库与讲义",[2207,2222,2223],{},[2161,2224,2227],{"href":2225,"rel":2226},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fgenerated\u002Ftorch.nn.Module.html",[2176],"PyTorch：nn.Module",[2207,2229,2230],{},[2161,2231,2234],{"href":2232,"rel":2233},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fgenerated\u002Ftorch.nn.parameter.Parameter.html",[2176],"PyTorch：nn.Parameter",[2207,2236,2237],{},[2161,2238,2241],{"href":2239,"rel":2240},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fgenerated\u002Ftorch.empty.html",[2176],"PyTorch：torch.empty",[2207,2243,2244],{},[2161,2245,2248],{"href":2246,"rel":2247},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fgenerated\u002Ftorch.matmul.html",[2176],"PyTorch：torch.matmul",[2207,2250,2251],{},[2161,2252,2255],{"href":2253,"rel":2254},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fgenerated\u002Ftorch.nn.Embedding.html",[2176],"PyTorch：nn.Embedding",[2257,2258,2259],"style",{},"html pre.shiki code .slEfO, html code.shiki .slEfO{--shiki-default:#F47067;--shiki-default-font-style:inherit;--shiki-light:#D73A49;--shiki-light-font-style:inherit;--shiki-cyber:#54B9FF;--shiki-cyber-font-style:italic}html pre.shiki code .sWURj, html code.shiki .sWURj{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#EEF0F9}html pre.shiki code .sf2cM, html code.shiki .sf2cM{--shiki-default:#F47067;--shiki-light:#D73A49;--shiki-cyber:#EEF0F9}html pre.shiki code .sRGqv, html code.shiki .sRGqv{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#ACAFFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html.cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html pre.shiki code .sq3c-, html code.shiki .sq3c-{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#00DAEF}html pre.shiki code .sA9aF, html code.shiki .sA9aF{--shiki-default:#F69D50;--shiki-default-font-style:inherit;--shiki-light:#E36209;--shiki-light-font-style:inherit;--shiki-cyber:#4BF3C8;--shiki-cyber-font-style:italic}",{"title":1168,"searchDepth":2261,"depth":2262,"links":2263},2,3,[2264,2265,2266,2267,2268,2269,2270,2271,2272,2273],{"id":31,"depth":2261,"text":32},{"id":136,"depth":2261,"text":137},{"id":1224,"depth":2261,"text":1225},{"id":1349,"depth":2261,"text":1350},{"id":1889,"depth":2261,"text":1890},{"id":1980,"depth":2261,"text":1981},{"id":2097,"depth":2261,"text":2097},{"id":2141,"depth":2261,"text":2141},{"id":2168,"depth":2261,"text":2168},{"id":2202,"depth":2261,"text":2202},[2275],"学习笔记","\u002Fimages\u002Fposts\u002Fcs336-linear-embedding-shapes\u002Fcs336-linear-embedding-shapes-cover.webp","token 方块、向量表与矩阵变换的 AI 辅助概念插画","2026-10-14","从 PyTorch 类、参数与 forward 的关系开始，理解 Linear 为什么保存两行三列的权重、计算时为什么要转置，再区分 Embedding 查表与线性变换。",false,"md",{"codeUrl":2174,"attachmentUrl":2283},"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-01-linear-embedding-20261014.zip",true,"\u002Fposts\u002Fcs336-linear-embedding-shapes",null,{"title":5,"description":2279},"Stanford CS336","posts\u002Fcs336-linear-embedding-shapes",[2291,2292,2293,2294,2295],"CS336","PyTorch","Transformer","Linear","Embedding","ocnvMxTsawBCzAzVAYpYYmIzntTzOn1kaHQ_0issafw",[2298,2314,2330,2345,2359,2373,2388,2403,2417,2430,2445,2458,2472,2484,2497,2502,2519,2534,2545,2557,2572,2585,2599,2611,2625,2637,2652,2666,2680,2691,2704,2721,2736,2749,2761,2773,2786,2799],{"slug":2299,"path":2300,"title":2301,"date":2302,"tags":2303,"description":2307,"cover":2308,"coverAlt":2309,"excerpt":2310,"draft":2280,"hidden":2280,"published":2284,"readingTime":2311,"series":2288,"seriesOrder":2311,"codeUrl":2312,"attachmentUrl":2313},"cs336-cross-entropy-adamw-training-loop","\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop","Stanford CS336 学习记录 08：数值稳定交叉熵、AdamW、余弦调度与完整训练闭环","2026-10-27",[2291,2292,2304,2305,2306],"AdamW","CrossEntropy","训练闭环","用 LogSumExp 消去交叉熵中的指数溢出，手写解耦权重衰减的 AdamW 优化器、全局 L2 梯度裁剪、Warmup 余弦退火调度、memmap 数据采样、Checkpoint 序列化与 Top-p 解码，完成 CS336 Assignment 1 全部 48 项测试。","\u002Fimages\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop\u002Fcs336-cross-entropy-adamw-training-loop-cover.webp","滑动窗口采样、余弦退火曲线、双动量优化轨迹与检查点归档构成的训练闭环 AI 辅助概念插画","作为 Stanford CS336 Assignment 1 系列的收官篇，我们已经拥有了分词器（bpe.py、tokenizer.py）和完整的语言模型架构（transformer.py）。要让模型在语料上真正训练并生成文本，还需",8,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fb5c93ad0cc73178b2b2a62e518a0900da47a0fb7","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-08-adamw-training-loop-20261027.zip",{"slug":2315,"path":2316,"title":2317,"date":2318,"tags":2319,"description":2326,"cover":2327,"coverAlt":2328,"excerpt":2329,"draft":2280,"hidden":2280,"published":2284,"readingTime":2311},"editcache-video-dit-inference-acceleration","\u002Fposts\u002Feditcache-video-dit-inference-acceleration","EditCache：视频 Diffusion Transformer (DiT) 局部编辑的跨步特征复用与区域重算加速","2026-10-25",[2320,2321,2322,2323,2324,2325],"扩散模型","视频生成","AI Infra","推理加速","GPU","论文解读","拆解视频扩散模型（Wan2.1-VACE）局部编辑中的三重算力冗余，通过可重入 Latent、跨步 Block 隐状态缓存复用、带 Halo 边界的局部区域重算与自适应推理起点，在单卡 H100 上实现 2.46× 模型计算加速与 1.49× 严格同步端到端加速。","\u002Fimages\u002Fposts\u002Feditcache-video-dit-inference-acceleration\u002Feditcache-dit-cover-20261025.webp","视频扩散 Transformer 时空三维体素网格中冻结背景缓存与暖橙色局部编辑重算通道的等距技术插画","在基于 Diffusion Transformer（DiT）的视频编辑与局部重绘（Video Inpainting \u002F Local Editing）任务中，用户的典型工作流并不是一次性生成最终成片，而是针对同一段源视频反复调整掩码区",{"slug":2331,"path":2332,"title":2333,"date":2318,"tags":2334,"description":2339,"cover":2340,"coverAlt":2341,"excerpt":2342,"draft":2280,"hidden":2280,"published":2284,"readingTime":2311,"series":2288,"seriesOrder":2343,"codeUrl":2312,"attachmentUrl":2344},"cs336-bpe-tokenizer-training-encoding","\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding","Stanford CS336 学习记录 07：Byte-Level BPE 分词器训练、倒排索引加速与流式编码",[2291,2335,2336,2337,2338],"Tokenizer","BPE","Tiktoken","算法优化","实现 GPT-2 正则预分词、特殊 token 硬边界切分、基于 pair-to-word 倒排索引的增量 BPE 合并算法（\u003C 0.25s 通过速度基准），以及与 tiktoken 完全对齐的 Tokenizer。","\u002Fimages\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding\u002Fcs336-bpe-tokenizer-training-encoding-cover.webp","UTF-8 字节块经过正则切分闸口后沿合并树聚合成子词单元的 AI 辅助概念插画","前面六篇完成了 Transformer 语言模型本身的全部神经网络组件。要让模型真正处理自然语言文本，还需要打通入口处的 ByteLevel BPE（BytePair Encoding）分词器。这一篇记录 cs336basics\u002Fb",7,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-07-bpe-tokenizer-20261025.zip",{"slug":2346,"path":2347,"title":2348,"date":2349,"tags":2350,"description":2355,"cover":2356,"coverAlt":2357,"excerpt":2358,"draft":2280,"hidden":2280,"published":2284,"readingTime":2311},"in-situ-autoguidance-editable-scientific-illustration","\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration","从原位自校正引导（In-situ Autoguidance）到可编辑科研插图生成（AutoFigure-Edit）","2026-10-24",[2320,2351,2352,2353,2354,2325],"多模态","Agent","ICML","ACL","串联两项生成式模型与多模态智能体研究：如何通过推理期局部随机微扰在单模型内构造免训练扩散自引导向量（ICML 2025 Workshop 第一作者，arXiv:2510.17136），以及如何通过五阶段视觉-符号混合流水线生成原生可编辑 SVG 科研插图（ACL 2026 Demo，FigureBench 83.0% 胜率）。","\u002Fimages\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration\u002Fautoguidance-autofigure-cover-20261024.webp","左侧扩散流形自校正轨迹矢量场与右侧五阶段可编辑 SVG 科研插图分层网格相衔接的等距技术插画","生成式视觉模型正在经历从“生成高保真像素（Pixel Synthesis）”向“可控结构化生成（Controllable & Editable Generation）”的演进。在这一进程中，我作为第一作者或核心贡献者参与了两项分别处",{"slug":2360,"path":2361,"title":2362,"date":2363,"tags":2364,"description":2367,"cover":2368,"coverAlt":2369,"excerpt":2370,"draft":2280,"hidden":2280,"published":2284,"readingTime":2311,"series":2288,"seriesOrder":2371,"codeUrl":2312,"attachmentUrl":2372},"cs336-pre-norm-transformer-block-lm","\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm","Stanford CS336 学习记录 06：Pre-Norm Transformer Block 与完整语言模型装配","2026-10-23",[2291,2292,2293,2365,2366],"Pre-Norm","FLOPs","将 RMSNorm、带 RoPE 的因果多头自注意力和 SwiGLU 组装成 Pre-Norm Transformer Block，搭建完整 TransformerLM，并核算 GPT-2 XL 的参数量与前向 FLOPs。","\u002Fimages\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm\u002Fcs336-pre-norm-transformer-block-lm-cover.webp","干净残差流主干穿过多个 Pre-Norm 子层并汇聚到词表投影层的 AI 辅助概念插画","前五篇把 Linear、Embedding、RMSNorm、SwiGLU、RotaryPositionalEmbedding 和因果 MultiHeadSelfAttention 全部单独打通了。这一篇进入 Assignment 1",6,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-06-transformer-block-lm-20261023.zip",{"slug":2374,"path":2375,"title":2376,"date":2363,"tags":2377,"description":2383,"cover":2384,"coverAlt":2385,"excerpt":2386,"draft":2280,"hidden":2280,"published":2284,"readingTime":2387},"xgrammar-concurrent-cache-generation-fix","\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix","XGrammar 结构化解码缓存剖析：Unlimited 模式并发 Clear 字节记账竞态与代际修复（PR #1011）",[2322,2378,2379,2380,2381,2382],"C++","XGrammar","结构化解码","并发编程","源码分析","拆解 XGrammar GrammarCompiler 的双层缓存架构与 ThreadSafeLRUCache 锁分离实现，定位 Unlimited 模式下 Clear() 字节残留与在途编译（In-flight Computation）跨代污染根因，给出代际号 + 读写锁原子修复与零 sleep 确定性并发 GoogleTest 验证。","\u002Fimages\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix\u002Fxgrammar-cache-fix-cover-20261022.webp","双层磨砂玻璃流道经暖橙色代际闸门过滤后汇入晶格缓存矩阵的等距技术插画","在大语言模型服务（如 SGLang、vLLM）的结构化输出场景中，JSON Schema 或 EBNF 语法需要先编译为词表级掩码缓存（CompiledGrammar），才能在每个 Decode 步以微秒级开销生成合法 token ",14,{"slug":2389,"path":2390,"title":2391,"date":2392,"tags":2393,"description":2399,"cover":2400,"coverAlt":2401,"excerpt":2402,"draft":2280,"hidden":2280,"published":2284,"readingTime":2387},"multi-constraint-agent-routing-dag-eval","\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval","多约束生产级 Agent 系统实践：两阶段工具路由、DAG 局部重规划与轨迹级评测闭环","2026-10-22",[2352,2394,2395,2396,2397,2398],"LLM","工具调用","上下文工程","DPO","强化学习","围绕跨境酒旅等强耦合多约束业务场景，拆解向量 Top-K 召回 + LLM 精排 + 跨语言实体链接与 Schema 门控（96%+ 调用合法率）、基于状态机的 DAG 局部子图重规划、Observation 冗余裁剪（节省 40% Token），以及从线上失败恢复轨迹构造 SFT\u002FDPO 偏好对的评测与训练闭环。","\u002Fimages\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval\u002Fagent-dag-routing-cover-20261020.webp","候选模块经漏斗滤网进入双分支有向无环图轨道并在受阻节点触发暖橙色局部旁路重连的等距技术插画","在玩具级 Demo 里，几个工具加上一段 ReAct 循环就能跑通大多数单步问答。但当 Agent 进入跨境酒旅、复杂交易或研发沙箱等真实业务生产环境时，单靠线性 ReAct 循环很快会撞上四堵墙： 1. 工具空间与参数耦合爆炸：候",{"slug":2404,"path":2405,"title":2406,"date":2407,"tags":2408,"description":2412,"cover":2413,"coverAlt":2414,"excerpt":2415,"draft":2280,"hidden":2280,"published":2284,"readingTime":2416},"llm-mrd-distillation-auditable-review-agents","\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents","多视角推理蒸馏（LLM-MRD）与可审计生成式评审系统（DeepReviewer 2.0）深度拆解","2026-10-21",[2409,2351,2410,2352,2411,2325],"知识蒸馏","Qwen2.5-VL","Post-training","对比大模型落地的两条互补路径：如何用 Qwen2.5-VL 离线多视角分析结合 MLP 残差校准与温度缩放 KL 蒸馏，将推理能力压入 358M 学生模型实现 17.2 ms 独立推理（LLM-MRD，DASFAA 2026 Oral）；以及如何通过主张-证据-风险台账、PASA 可比性门控与导出预算校验构建可审计的同行评审 Agent（DeepReviewer 2.0）。","\u002Fimages\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents\u002Fmrd-review-distill-cover-20261016.webp","三块半透明几何棱镜发出平行光束穿过中央校准环汇聚至右侧高密度晶体模块的等距技术插画","把大语言模型或视觉语言模型（VLM）的深度推理能力用到实际系统中，通常会面临一个两难选择： 如果场景受限于毫秒级在线延迟（如高吞吐多模态内容审核与虚假信息检测）：直接在线调用 7B～72B 的多模态大模型做长链条思维链（CoT）生成",18,{"slug":2418,"path":2419,"title":2420,"date":2407,"tags":2421,"description":2424,"cover":2425,"coverAlt":2426,"excerpt":2427,"draft":2280,"hidden":2280,"published":2284,"readingTime":2343,"series":2288,"seriesOrder":2428,"codeUrl":2312,"attachmentUrl":2429},"cs336-rope-rotary-position-embedding","\u002Fposts\u002Fcs336-rope-rotary-position-embedding","Stanford CS336 学习记录 05：RoPE 旋转位置编码与带位置的多头注意力",[2291,2292,2293,2422,2423],"RoPE","Attention","从二维平面正交旋转推导 RoPE 的奇偶特征配对公式，理解为什么内积只依赖相对位置差 (i - j)，并用 register_buffer 预计算 cos\u002Fsin 缓存接入因果多头自注意力。","\u002Fimages\u002Fposts\u002Fcs336-rope-rotary-position-embedding\u002Fcs336-rope-rotary-position-embedding-cover.webp","二维特征平面中的旋转向量对与预计算余弦正弦网格的 AI 辅助概念插画","上一篇完成了不含位置编码的因果多头自注意力。如果只看那一套矩阵乘法，把输入序列前两个 token 交换顺序（并在因果 mask 允许的范围内看同一个后缀 query 对它们的点积），点积本身并不知道哪个 key 离当前 query ",5,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-05-rope-mha-20261021.zip",{"slug":2431,"path":2432,"title":2433,"date":2434,"tags":2435,"description":2441,"cover":2442,"coverAlt":2443,"excerpt":2444,"draft":2280,"hidden":2280,"published":2284,"readingTime":2311},"llm-gateway-semantic-cache-dynamic-batching","\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching","高并发 LLM 推理网关架构实践：两级精确\u002F语义缓存、50ms 有界动态批处理与尾部延迟治理","2026-10-19",[2322,2436,2437,2438,2439,2440],"LLM 推理","分布式系统","Redis","FastAPI","后端","深度拆解生产级大模型推理网关 LLM-Gateway 的核心系统设计：基于 SHA-256 与 Redis 向量余弦相似度的两级精确\u002F语义缓存、50ms 有界等待动态批处理调度器（Dynamic Batching）、令牌桶多租户限流与 Prometheus 全链路可观测性，将命中请求 P99 延迟从 5,307 ms 压降至 17 ms。","\u002Fimages\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching\u002Fllm-gateway-cover-20261019.webp","高并发请求流依次穿过精确哈希滤网、向量球面语义缓存层与 50ms 动态批处理时间窗拼装器的等距系统架构插画","在将大语言模型接入高并发业务系统时，如果让上游微服务直接裸连底层推理引擎（vLLM \u002F SGLang 或外部模型 API），系统很快会暴露出三个典型的生产痛点： 1. 高频重复与近义查询消耗巨量算力：大量用户请求在语义上高度等价（例",{"slug":2446,"path":2447,"title":2448,"date":2434,"tags":2449,"description":2451,"cover":2452,"coverAlt":2453,"excerpt":2454,"draft":2280,"hidden":2280,"published":2284,"readingTime":2387,"series":2288,"seriesOrder":2455,"codeUrl":2456,"attachmentUrl":2457},"cs336-multihead-attention-shapes","\u002Fposts\u002Fcs336-multihead-attention-shapes","Stanford CS336 学习记录 04：多头注意力怎样拆头、计算和合头",[2291,2292,2293,2423,2450],"张量形状","沿着 (2, 5, 64) 的输入追踪 QKV 投影、reshape、transpose、共享因果 mask 和合头，理解每个头仍处理完整序列，以及最后为什么还需要输出 Linear。","\u002Fimages\u002Fposts\u002Fcs336-multihead-attention-shapes\u002Fcs336-multihead-attention-shapes-cover.webp","宽向量束分成多条并行特征轨道后重新合并的 AI 辅助概念插画","上一篇完成了缩放点积注意力。那个函数接收已经准备好的 Q、K、V，计算匹配分数，应用 mask，再用权重汇总 V。这一篇继续把它放进一个带参数的多头自注意力模块。 这次最需要看清的是“拆头”。代码里只是几行 reshape 和 tr",4,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002F31363223498fafd1528c550a51efbe0aed231736","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-04-multihead-attention-20261019.zip",{"slug":2459,"path":2460,"title":2461,"date":2462,"tags":2463,"description":2467,"cover":2468,"coverAlt":2469,"excerpt":2470,"draft":2280,"hidden":2280,"published":2284,"readingTime":2471},"qwen25-sft-dpo-alignment-tax-ablation","\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation","大模型后训练偏好对齐实录：Qwen2.5 SFT 多维消融、动态参考 DPO 与对齐税（Alignment Tax）量化分析","2026-10-18",[2411,2397,2464,2465,2398,2466],"SFT","LoRA","Qwen2.5","基于 Qwen2.5 完整复现 Instruct SFT → Bradley-Terry DPO 两阶段后训练管线：通过 3 随机种子消融 LoRA Rank、学习率与数据规模（GSM8K +12.73pp），实现免双倍权重的动态 Adapter 切换 DPO 训练，并定量揭示偏好对齐对确定性数学推理的对齐税效应。","\u002Fimages\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation\u002Fqwen25-sft-dpo-cover-20261018.webp","左侧监督微调轨迹流经中央 Bradley-Terry 偏好分流天平分离出正负样本通道并连接右侧对齐税监控仪表盘的等距技术插画","在大语言模型后训练（PostTraining）管线中，监督微调（SFT）负责教会模型任务格式与领域推理模式，而直接偏好优化（DPO, Direct Preference Optimization）则利用成对偏好数据 $(x, yw,",9,{"slug":2473,"path":2474,"title":2475,"date":2462,"tags":2476,"description":2478,"cover":2479,"coverAlt":2480,"excerpt":2481,"draft":2280,"hidden":2280,"published":2284,"readingTime":2482,"series":2288,"seriesOrder":2262,"codeUrl":2174,"attachmentUrl":2483},"cs336-softmax-attention-mask","\u002Fposts\u002Fcs336-softmax-attention-mask","Stanford CS336 学习记录 03：Softmax 的数值稳定与 Attention Mask",[2291,2292,2293,2477,2423],"Softmax","从减去最大值为何不改变 Softmax 开始，理清函数与有参数层的区别，再沿着 Q、K、V 的形状理解缩放点积注意力，以及 mask 为什么必须填负无穷。","\u002Fimages\u002Fposts\u002Fcs336-softmax-attention-mask\u002Fcs336-softmax-attention-mask-cover.webp","注意力分数、权重与因果遮罩的 AI 辅助概念插画","上一篇写到 SwiGLU，接下来进入 Attention。真正开始相乘之前，先做了 Softmax：它把一组分数变成总和为 1 的权重，后面用这些权重汇总不同 token 提供的信息。 这段学习里，有两个很具体的问题值得记下来。一个",11,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-03-softmax-sdpa-20261018.zip",{"slug":2485,"path":2486,"title":2487,"date":2488,"tags":2489,"description":2492,"cover":2493,"coverAlt":2494,"excerpt":2495,"draft":2280,"hidden":2280,"published":2284,"readingTime":2471,"series":2288,"seriesOrder":2261,"codeUrl":2174,"attachmentUrl":2496},"cs336-rmsnorm-swiglu","\u002Fposts\u002Fcs336-rmsnorm-swiglu","Stanford CS336 学习记录 02：RMSNorm 的广播与 SwiGLU 的门控","2026-10-16",[2291,2292,2293,2490,2491],"RMSNorm","SwiGLU","用一个 token 向量理解均方根归一化，拆开 dim=-1 与 keepdim=True 的作用，再沿着两条分支读懂 SiLU、逐元素乘法和 SwiGLU。","\u002Fimages\u002Fposts\u002Fcs336-rmsnorm-swiglu\u002Fcs336-rmsnorm-swiglu-cover.webp","特征尺度整理和双分支门控的 AI 辅助概念插画","上一篇把 Linear 和 Embedding 的参数形状理顺了。继续往下，最先遇到的是 RMSNorm：公式不算长，但把它落到一批 token 上时，dim=1、keepdim=True 和广播必须一起理解。 然后是 SiLU 和","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-02-rmsnorm-swiglu-20261016.zip",{"slug":2498,"path":2285,"title":5,"date":2278,"tags":2499,"description":2279,"cover":2276,"coverAlt":2277,"excerpt":2500,"draft":2280,"hidden":2280,"published":2284,"readingTime":2501,"series":2288,"seriesOrder":1174,"codeUrl":2174,"attachmentUrl":2283},"cs336-linear-embedding-shapes",[2291,2292,2293,2294,2295],"开始沿着 Stanford CS336 的 Assignment 1 一步步实现。BPE 之前已经接触过，这轮先跳过分词器，从 Transformer 的基础层开始。这里记录的是当时写代码时真正卡住的问题，以及对应测试能确认到的范围",10,{"slug":2503,"path":2504,"title":2505,"date":2506,"tags":2507,"description":2512,"cover":2513,"coverAlt":2514,"excerpt":2515,"draft":2280,"hidden":2280,"published":2284,"readingTime":2516,"series":2508,"seriesOrder":2387,"codeUrl":2517,"attachmentUrl":2518},"cmu-15-779-kv-cache-granularity-study","\u002Fposts\u002Fcmu-15-779-kv-cache-granularity-study","CMU 15-779 学习记录 14：KV 页大小、共享边界与 CPU 实现成本","2026-10-12",[2508,2509,2510,2511],"CMU 15-779","KV cache","PagedAttention","系统实验","用 210 组 CPU 配置比较连续与分页 KV cache，观察 31\u002F32-token 共享前缀的页边界，分别记录活跃槽位、池容量、复制计数和完整调用耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling\u002Fcmu-serving-cover-20261019.webp","浅色手绘分页存储抽屉与共享前缀卡片，旁边连接不同长度的请求队列","个人课程项目中的 CPU 实验记录，延续记录 11 的分页 KV 实现。已完成 210 组配置与 1470 次计时，输出均通过连续存储参考对照；本篇不代表正式作业提交、整门课程或项目全部交付完成。 同样是共享 31 个 token，",12,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F200faf569eccf94fb125e65110bb5d14094d454a","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fkv-cache-granularity-20261019.zip",{"slug":2520,"path":2521,"title":2522,"date":2523,"tags":2524,"description":2527,"cover":2528,"coverAlt":2529,"excerpt":2530,"draft":2280,"hidden":2280,"published":2284,"readingTime":2428,"series":2508,"seriesOrder":2531,"codeUrl":2532,"attachmentUrl":2533},"cmu-15-779-moe-routing-capacity","\u002Fposts\u002Fcmu-15-779-moe-routing-capacity","CMU 15-779 学习记录 13：MoE 路由、稀疏分发与容量溢出","2026-10-09",[2508,2525,2526],"MoE","Routing","用 CPU top-2 MoE 检查稀疏分发的输出与梯度，统计 expert-token 行数，并保留容量丢弃会改变输出的反例。","\u002Fimages\u002Fposts\u002Fcmu-15-779-moe-routing-capacity\u002Fcmu-moe-cover-20261019.webp","浅色手绘分拣器将小珠分配到四个专家工作屋，队列呈现负载不均","第 18 讲的个人配套实验。36 号脚本串行运行 CPU 专家，检查 topk 路由后的分发与合并；没有训练 router，也没有执行 GPU grouped GEMM 或跨设备 alltoall。 上一节记录低秩适配，这次转向每个",13,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fc4851e0cedadf13af7c421867e83c8ff57244fa0","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-parallel-serving-lora-moe-20261019.zip",{"slug":2535,"path":2536,"title":2537,"date":2538,"tags":2539,"description":2541,"cover":2542,"coverAlt":2543,"excerpt":2544,"draft":2280,"hidden":2280,"published":2284,"readingTime":2428,"series":2508,"seriesOrder":2516,"codeUrl":2532,"attachmentUrl":2533},"cmu-15-779-lora-uniform-quantization","\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization","CMU 15-779 学习记录 12：LoRA 首步梯度、权重合并与均匀 4-bit 打包","2026-10-06",[2508,2465,2540],"Quantization","用一次 CPU SGD 更新检查 LoRA 初始化、冻结基座与合并等价，再分别记录均匀 4-bit 存储字节数和量化误差。","\u002Fimages\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization\u002Fcmu-lora-cover-20261019.webp","浅色手绘的大计算板、两块细长适配板和成对打包的小方块","第 17 讲的个人配套实验，35 号脚本实际执行一次 CPU LoRA 更新，并实现均匀 4bit 打包。实验检查机制与数值误差，不评估微调任务效果，也没有实现完整 QLoRA。 这次把两个问题分开记录：低秩分支是否能合并进原权重，",{"slug":2546,"path":2547,"title":2548,"date":2549,"tags":2550,"description":2554,"cover":2513,"coverAlt":2555,"excerpt":2556,"draft":2280,"hidden":2280,"published":2284,"readingTime":2343,"series":2508,"seriesOrder":2482,"codeUrl":2532,"attachmentUrl":2533},"cmu-15-779-kv-serving-speculative-sampling","\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling","CMU 15-779 学习记录 11：分页 KV、连续批处理与推测采样","2026-10-03",[2508,2551,2552,2553],"LLM Serving","KV Cache","Speculative Decoding","在 CPU 上验证 KV 页共享和写时复制，用单位成本模型检查连续批处理，再用条件概率小模型核对推测采样的接受与补采样。","浅色手绘的共享卡片抽屉与不断补入小珠子的请求托盘","第 15、16 讲的个人配套实验。33 号实现 CPU 分页 KV 存储，37 号模拟请求准入，34 号运行小概率模型的推测采样。它们分别验证存储、调度和分布问题，没有测量真实 LLM 服务吞吐或 GPU 延迟。 记录 08已经对照",{"slug":2558,"path":2559,"title":2560,"date":2561,"tags":2562,"description":2567,"cover":2568,"coverAlt":2569,"excerpt":2570,"draft":2280,"hidden":2280,"published":2284,"readingTime":2571},"qwen3-mid-post-train-mlx","\u002Fposts\u002Fqwen3-mid-post-train-mlx","在 M2 Pro 上完成 Qwen3 两阶段训练：Mid Train、SFT 与可复核评估","2026-09-30 11:19:10",[2394,2563,2564,2465,2565,2464,2566],"Qwen3","MLX","持续预训练","Apple Silicon","用 Qwen3 0.6B Base 和 MLX 完成领域持续训练与监督微调，记录数据划分、聊天模板、损失掩码、数值稳定性修复和独立测试结果。","\u002Fimages\u002Fposts\u002Fqwen3-mid-post-train-mlx\u002Fqwen3-mlx-cover-20261018.webp","Apple Silicon 统一内存芯片底座支撑两阶段持续预训练与带精确 Loss 掩码滤网的监督微调桥接插画","在一台 M2 Pro、16 GB 内存的 Mac 上，能否把一个 Base 语言模型接续训练成输出 JSON 的运维事件分级助手？这次项目完成了领域持续训练、监督微调、检查点选择、重新加载和独立测试，并保留了权重、日志和数据哈希。 ",16,{"slug":2573,"path":2574,"title":2575,"date":2576,"tags":2577,"description":2581,"cover":2582,"coverAlt":2583,"excerpt":2584,"draft":2280,"hidden":2280,"published":2284,"readingTime":2371,"series":2508,"seriesOrder":2501,"codeUrl":2532,"attachmentUrl":2533},"cmu-15-779-cpu-parallelism-pipeline","\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline","CMU 15-779 学习记录 10：加权梯度、状态分片与同步流水线","2026-09-30",[2508,2578,2579,2580],"Parallel Training","Gloo","Pipeline","用双 CPU 进程 Gloo 检查不等数据分片的梯度，推导持久状态字节数，再用依赖模型比较 GPipe 与同步 1F1B 的激活存活。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline\u002Fcmu-parallel-cover-20261019.webp","浅色手绘的四阶段桌面流水线与按数量汇总的托盘","第 12 至 14 讲的个人配套实验，脚本编号 31、32 不对应官方作业。Gloo 实际运行两个 CPU 进程；张量切分、状态字节数和流水线分别按下文的数学或成本模型解释。真实 GPU 实测按学习安排跳过。 上一节检查单个计算图的",{"slug":2586,"path":2587,"title":2588,"date":2589,"tags":2590,"description":2593,"cover":2594,"coverAlt":2595,"excerpt":2596,"draft":2280,"hidden":2280,"published":2284,"readingTime":2343,"series":2508,"seriesOrder":2471,"codeUrl":2597,"attachmentUrl":2598},"cmu-15-779-cpu-compiler-optimizations","\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations","CMU 15-779 学习记录 09：CPU 编译优化、合法重写与候选搜索","2026-09-27",[2508,2591,2592,2292],"ML Compiler","TorchInductor","用 CPU 实验验证 BatchNorm 折叠、RMSNorm 重排与非线性反例，测量分块候选，并运行 TorchInductor，分别记录首调用与预热后的时间。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations\u002Fcmu-cpu-compiler-cover-20261019.webp","淡紫计算积木经过薄荷绿折叠模具变成浅蓝积木，前方摆放四种候选托盘的浅色手绘插画","编译章节的个人配套实验，编号 30 不对应官方作业。图重写、分块候选和 TorchInductor 均在 CPU 上运行；真实 GPU 实验按学习安排跳过。 上一节记录了分块 Attention 和 KV cache。这次转到编译优","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F9261bea6aedf31cdccbbab2541ca693fc97b3743","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-compiler-optimizations-20261019.zip",{"slug":2600,"path":2601,"title":2602,"date":2603,"tags":2604,"description":2606,"cover":2607,"coverAlt":2608,"excerpt":2609,"draft":2280,"hidden":2280,"published":2284,"readingTime":2516,"series":2508,"seriesOrder":2311,"codeUrl":2517,"attachmentUrl":2610},"cmu-15-779-attention-online-softmax","\u002Fposts\u002Fcmu-15-779-attention-online-softmax","CMU 15-779 学习记录 08：分块 Attention、在线 Softmax 与 KV cache","2026-09-24",[2508,2423,2477,2605,2292],"FlashAttention","在 CPU 上对照完整 Attention、在线 Softmax 分块实现与 PyTorch SDPA，检查因果掩码、浮点误差和分数块大小，理解 FlashAttention 的一部分算法基础。","\u002Fimages\u002Fposts\u002Fcmu-15-779-attention-online-softmax\u002Fcmu-online-attention-cover-20261016.webp","浅紫色卡片分批进入绿色计算托盘，旁边三个状态容器保存累积量的浅色手绘插画","第 5 讲 Transformers & Attention 的个人配套实验。28\u002F29 号是 CPU 单头前向与 KV cache；后续补入 38 号 CUDA 多头合成张量的显式 SDPA MATH\u002FFLASH 对照。没有手写 ","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcuda-triton-sdpa-20261019.zip",{"slug":2612,"path":2613,"title":2614,"date":2615,"tags":2616,"description":2621,"cover":2622,"coverAlt":2623,"excerpt":2624,"draft":2280,"hidden":2280,"published":2284,"readingTime":2501,"series":2508,"seriesOrder":2343,"codeUrl":2517,"attachmentUrl":2610},"cmu-15-779-matmul-tiling-tree-reduction","\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction","CMU 15-779 学习记录 07：矩阵分块、数据复用与树形归约","2026-09-21",[2508,2617,2618,2619,2620],"CUDA","矩阵乘法","数据复用","归约","用 CPU 模型统计矩阵分块的输入读取次数，再比较串行求和和树形归约的依赖深度，记录补零、同步与浮点加法顺序的边界。","\u002Fimages\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction\u002Fcmu-matmul-reduction-cover-20261016.webp","矩阵积木在共享托盘中复用，旁边的小珠沿树形管道逐层合并的浅色手绘插画","GPU Architecture & CUDA Programming 的个人配套实验。首次记录用 CPU 模型检查数据复用与依赖深度；后续补入单卡 GPU 原生 CUDA 矩阵验证，以及 Triton 两阶段归约和矩阵乘法实测。 ",{"slug":2626,"path":2627,"title":2628,"date":2629,"tags":2630,"description":2633,"cover":2634,"coverAlt":2635,"excerpt":2636,"draft":2280,"hidden":2280,"published":2284,"readingTime":2501,"series":2508,"seriesOrder":2371,"codeUrl":2517,"attachmentUrl":2610},"cmu-15-779-cuda-thread-indexing","\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing","CMU 15-779 学习记录 06：CUDA 线程索引与边界检查","2026-09-18",[2508,2617,2324,2631,2632],"线程索引","边界检查","用 CPU 演示 CUDA 的 block、thread 与全局下标：10 个元素如何分给 3 个线程块，为什么要跳过越界线程，以及这些验证能说明什么。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing\u002Fcmu-cuda-indexing-cover-20261015.webp","分组的浅紫色计算方块与末端边界挡板组成线程分配教具的浅色插画","GPU Architecture & CUDA Programming 的个人配套练习。首次记录为 CPU 索引演示；后续在单卡 GPU 环境补入原生 CUDA 编译、运行与 memcheck，以及 Triton 向量加法实测。 上",{"slug":2638,"path":2639,"title":2640,"date":2641,"tags":2642,"description":2646,"cover":2647,"coverAlt":2648,"excerpt":2649,"draft":2280,"hidden":2280,"published":2284,"readingTime":2501,"series":2508,"seriesOrder":2428,"codeUrl":2650,"attachmentUrl":2651},"cmu-15-779-profiler-operator-batching","\u002Fposts\u002Fcmu-15-779-profiler-operator-batching","CMU 15-779 学习记录 05：Profiler、算子调用与批量推理","2026-09-15",[2508,2292,2643,2644,2645],"Profiler","批量推理","性能分析","对照 ML Systems 基础，用 Profiler 拆开两层网络的执行过程，再固定 1000 个样本扫描 batch size，观察算子调用次数、尾批和 CPU 耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-profiler-operator-batching\u002Fcmu-pytorch-profiling-cover-20261015.webp","放大镜观察线性层与 ReLU 模块，旁边面板呈现执行片段的浅色技术教具插画","本篇对照课程官网的 Introduction to ML Systems 整理，是个人配套 CPU 实验，不是课程官方作业。脚本编号是练习顺序，不对应课程讲次。 前几篇一直在补 PyTorch 基础。到了这一步，我在学习聊天里问：现","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fcecf5110d8a95161bb5763eadfca94cd7a0cc50f","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-profiler-batching-20261015.zip",{"slug":2653,"path":2654,"title":2655,"date":2656,"tags":2657,"description":2660,"cover":2661,"coverAlt":2662,"excerpt":2663,"draft":2280,"hidden":2280,"published":2284,"readingTime":2471,"series":2508,"seriesOrder":2455,"codeUrl":2664,"attachmentUrl":2665},"cmu-15-779-pytorch-linear-relu-absolute-value","\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value","CMU 15-779 学习记录 04：多输出线性层、ReLU 与绝对值网络","2026-09-12",[2508,2292,2618,2658,2659],"ReLU","神经网络","把 nn.Linear 对应到矩阵乘法，弄清多个输出怎样计算，再逐层拆开一个绝对值网络，比较手动设置参数与训练得到的结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value\u002Fcmu-pytorch-absolute-cover-20261015.webp","双分支经过折角模块后汇入紫色 V 形曲线面板的浅色技术教具插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的批量推理与矩阵乘法，我继续把 model(x) 拆开看：一层里有多个输出时，每个数是怎么算出来的？两","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F4cc63904a3a583da4a2122c246a14b43aac38528","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-linear-relu-abs-20261015.zip",{"slug":2667,"path":2668,"title":2669,"date":2670,"tags":2671,"description":2674,"cover":2675,"coverAlt":2676,"excerpt":2677,"draft":2280,"hidden":2280,"published":2284,"readingTime":2471,"series":2508,"seriesOrder":2262,"codeUrl":2678,"attachmentUrl":2679},"cmu-15-779-pytorch-dataloader-checkpoints-batching","\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching","CMU 15-779 学习记录 03：DataLoader、模型保存与批量推理","2026-09-09",[2508,2292,2672,2644,2673],"DataLoader","模型保存","用 DataLoader 分批训练平方函数，保存并加载模型参数，再比较 1000 个样本与 1 个样本的推理耗时，最后把批量计算对应到矩阵乘法。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching\u002Fcmu-pytorch-batching-cover-20261015.webp","样本卡片按批次进入计算模块，旁边配有参数存储盒和输出矩阵面板的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的平方函数实验，这一节继续练习分批训练、保存参数和独立预测。最后做了一次计时：同一个小网络，逐个处理 ","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-batching-20261015.zip",{"slug":2681,"path":2682,"title":2683,"date":2684,"tags":2685,"description":2686,"cover":2687,"coverAlt":2688,"excerpt":2689,"draft":2280,"hidden":2280,"published":2284,"readingTime":2471,"series":2508,"seriesOrder":2261,"codeUrl":2678,"attachmentUrl":2690},"cmu-15-779-pytorch-linear-module-relu","\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu","CMU 15-779 学习记录 02：nn.Linear、张量形状与 ReLU","2026-09-06",[2508,2292,2659,2658],"从带偏置的线性回归开始，弄清 nn.Linear 和 nn.Module 的分工，再用平方函数观察线性模型的限制、ReLU 的作用和测试点的选择。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu\u002Fcmu-pytorch-relu-cover-20261015.webp","输入经过两层神经网络和 ReLU 折角模块，输出曲线拟合结果的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 上一篇里，我用一个参数拟合了 $y=2x$。这一篇继续顺着往下写，先加上偏置，再把手写参数换成 nn.Linea","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-models-20261015.zip",{"slug":2692,"path":2693,"title":2694,"date":2695,"tags":2696,"description":2699,"cover":2700,"coverAlt":2701,"excerpt":2702,"draft":2280,"hidden":2280,"published":2284,"readingTime":2501,"series":2508,"seriesOrder":1174,"codeUrl":2678,"attachmentUrl":2703},"cmu-15-779-pytorch-autograd-linear-regression","\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression","CMU 15-779 学习记录 01：PyTorch 自动求导与线性回归","2026-09-03",[2508,2292,2697,2698],"自动求导","线性回归","从自动求导和手动梯度下降开始，比较学习率，再用 SGD 训练线性回归。记录 w * 4 写错后停在 1.25 的原因和修正结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression\u002Fcmu-pytorch-autograd-cover-20261015.webp","梯度曲面上的小球沿路径下降，前侧面板展示散点与拟合直线的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 开始学习 CMU 15779 时，我会 Python，但还没用过 PyTorch。这次先在 Mac 上用 CPU","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-basics-20261013.zip",{"slug":2705,"path":2706,"title":2707,"date":2708,"tags":2709,"description":2715,"cover":2716,"coverAlt":2717,"excerpt":2718,"draft":2280,"hidden":2280,"published":2284,"readingTime":2531,"codeUrl":2719,"attachmentUrl":2720},"embodied-pickplace-lab-robustness","\u002Fposts\u002Fembodied-pickplace-lab-robustness","Pickplace Lab 项目复盘：抓取成功率之外的时延与失败分析","2026-08-29",[2710,2711,2712,2292,2713,2714],"具身智能","模仿学习","MuJoCo","机器人","鲁棒性","从 MuJoCo 抓取搬运闭环出发，记录 22,100 个评测回合。动作平滑在自建任务的 50 ms 延迟下有效，换到 Panda Lift 却没有支持同样的收益。","\u002Fimages\u002Fposts\u002Fembodied-pickplace-lab-robustness\u002Fpickplace-grasp-cover-20261016.webp","MuJoCo 实际渲染的笛卡尔夹爪抓取方块场景，桌面上标出放置目标","Pickplace Lab 从一个简单任务开始：让夹爪抓起方块，搬到指定位置，放下，再退开。先把仿真、示范采集、训练和闭环评测接起来，随后再问：干净条件下成功的策略，换一种执行条件，还能不能完成任务？ 最值得讨论的结果出现在指令延迟","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Ftree\u002Fv0.1.0","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Freleases\u002Fdownload\u002Fv0.1.0\u002Fpickplace-lab-research-v1.zip",{"slug":2722,"path":2723,"title":2724,"date":2725,"tags":2726,"description":2731,"cover":2732,"coverAlt":2733,"excerpt":2734,"draft":2280,"hidden":2280,"published":2284,"readingTime":2735},"qwen25-vl-sglang-verl-single-gpu-lab","\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab","从 Mac 到单卡 GPU：Qwen2.5-VL 部署、GRPO 与计数 LoRA 实践","2026-08-19",[2410,2351,2727,2728,2729,2730,2564],"SGLang","verl","GRPO","推理优化","在 M2 Pro 和云端单卡上部署 Qwen2.5-VL-3B，实测连续批处理、前缀缓存与 SGLang 参数调优，再用 verl 完成多模态 GRPO 更新和同条件前后评测，记录有效优化、兼容性问题与未观察到提升的结果。","\u002Fimages\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab\u002Fqvl-lab-cover-20261010.webp","笔记本与单卡计算模块通过图像和 token 光轨连接的实验台插画","这次实验从一个很具体的问题开始：M2 Pro、16 GB 统一内存，能不能在本机跑 Qwen2.5VL？跑起来之后，我又想把图文问答、推理性能测量和强化学习训练接成一条完整流程。 本机的 MLX 4bit 服务和云端的 SGLang",29,{"slug":2737,"path":2738,"title":2739,"date":2740,"tags":2741,"description":2745,"cover":2746,"coverAlt":2747,"excerpt":2748,"draft":2280,"hidden":2280,"published":2284,"readingTime":2531},"cycleresearcher-post-training","\u002Fposts\u002Fcycleresearcher-post-training","从 DeepReviewer 2.0 到 CycleResearcher：科研 Agent 如何真正做 Post-training","2026-08-07 17:33:05",[2394,2411,2352,2742,2743,2744],"SimPO","Reward Model","ICLR","区分推理时 Agent 编排与模型后训练，并拆解 CycleResearcher 的领域 SFT、生成式 Reward Model、Iterative SimPO 和评测风险。","\u002Fimages\u002Fposts\u002Fcycleresearcher-post-training\u002Fcycleresearcher-cover-20261018.webp","策略生成模块产出多份候选文稿经多维评审棱镜排序后通过偏好梯度反馈弧闭环优化的等距技术插画","看到一个系统会检索、规划、反思和调用工具，不能据此判断它做过 Posttraining。Agent 描述的是模型在运行时如何完成任务；Posttraining 描述的是预训练之后，是否继续用监督数据、偏好数据或奖励信号更新模型参数。",{"slug":2750,"path":2751,"title":2752,"date":2753,"tags":2754,"description":2756,"cover":2757,"coverAlt":2758,"excerpt":2759,"draft":2280,"hidden":2280,"published":2284,"readingTime":2387,"series":2760,"seriesOrder":1174},"cuda-fused-rmsnorm-kernel","\u002Fposts\u002Fcuda-fused-rmsnorm-kernel","从 PyTorch 到 CUDA Kernel：融合 RMSNorm 在 RTX 4090D 上加速 4.27 倍","2026-07-22 18:16:50",[2617,2322,2324,2292,2436,2755],"Kernel 优化","从零实现 PyTorch CUDA 扩展，拆解 RMSNorm 的访存瓶颈、warp 归约、向量化加载、算子融合与冷热缓存 benchmark。","\u002Fimages\u002Fposts\u002Fcuda-fused-rmsnorm-kernel\u002Fcuda-rmsnorm-cover-20261018.webp","融合残差加法与向量化 float4 访存经 Warp Shuffle 蝴蝶树归约输出归一化张量的等距硬件架构插画","为了准备 AI Infra 岗位，我做了一个尽量小而完整的 CUDA 项目：不追求堆很多算子，而是选一个真实的 LLM 推理路径，把实现、正确性、benchmark 和性能解释全部做扎实。 项目实现了两个 PyTorch CUDA ","AI Infra 实战",{"slug":2762,"path":2763,"title":2764,"date":2765,"tags":2766,"description":2768,"cover":2769,"coverAlt":2770,"excerpt":2771,"draft":2280,"hidden":2280,"published":2284,"readingTime":2311,"series":2772,"seriesOrder":1174},"agent-memory","\u002Fposts\u002Fagent-memory","Agent 上下文工程与分层记忆架构：从 Observation 剪枝、滚动摘要到向量召回闭环","2026-04-24",[2394,2352,2767,2322],"RAG","结合生产级多约束 Agent 与超 100K Token 科研智能体实践，系统拆解长程会话中的上下文膨胀与注意力稀释问题：涵盖 Observation 结构化剪枝（降低约 40% Token 开销）、滚动工作摘要、向量检索长期记忆及 Prefix Cache 友好的上下文组装。","\u002Fimages\u002Fposts\u002Fagent-memory\u002Fagent-memory-cover-20261018.webp","底层高噪数据经剪枝滤网进入中期滑动窗口与金色状态台账并由顶层向量星座按需召回的三层记忆架构插画","在构建多步工具调用（ToolUse）与长周期自主规划（Autonomous Planning）的 Agent 系统时，上下文管理（Context Engineering）往往比单纯改写 System Prompt 更决定线上稳定性。","Agent 上下文工程",{"slug":2774,"path":2775,"title":2776,"date":2777,"tags":2778,"description":2781,"cover":2782,"coverAlt":2783,"excerpt":2784,"draft":2280,"hidden":2280,"published":2284,"readingTime":2428,"series":2785,"seriesOrder":2261},"backend-stack-deep-dive","\u002Fposts\u002Fbackend-stack-deep-dive","高并发 AI 平台后端基础设施实践：FastAPI 异步网关、SQLAlchemy 2.0 Async、PostgreSQL 与 Docker 沙箱编排","2026-04-07",[2440,2437,2439,2779,2780,2322],"PostgreSQL","Docker","结合 DeepScientist 生产级 AI 科研平台实践，拆解 FastAPI ASGI 异步事件循环、SQLAlchemy 2.0 + AsyncPG 连接池调优、PostgreSQL + MinIO 存储分层与 Docker SDK 动态沙箱容器编排的职责边界与核心设计。","\u002Fimages\u002Fposts\u002Fbackend-stack-deep-dive\u002Fbackend-stack-cover-20261018.webp","ASGI 异步事件循环涡轮将非阻塞协程流分发至 PostgreSQL 连接池、MinIO 存储矩阵与动态 Docker 沙箱的等距架构插画","结合 DeepScientist 生产级 AI 科研平台（服务 7,000+ 用户）的实际工程落地，本文系统拆解支撑高并发流式推理与容器沙箱调度的后端基础设施栈——FastAPI（ASGI 异步网关 + SSE 流式推送）、SQLA","DeepScientist 工程复盘",{"slug":2787,"path":2788,"title":2789,"date":2790,"tags":2791,"description":2794,"cover":2795,"coverAlt":2796,"excerpt":2797,"draft":2280,"hidden":2280,"published":2284,"readingTime":2482,"series":2798,"seriesOrder":1174},"minicode-source-analysis","\u002Fposts\u002Fminicode-source-analysis","MiniCode 架构与工具评测深度拆解：开源终端 Agentic Coding 引擎（1k+ Stars）的 ReAct、MCP 与 30 项基准验证","2026-04-06",[2352,2792,2793,2382,2322],"TypeScript","MCP","深度剖析开源终端 AI 编程智能体 MiniCode（1k+ GitHub Stars）的核心架构：从零重型依赖的 ReAct 规划循环、声明式工具注册表与权限拦截层、MCP 协议帧自动协商，到 30 项可执行 Tool-Use 评测基准（86.7% 完成率、100% 错误恢复率）的失败分布归因与闭环优化。","\u002Fimages\u002Fposts\u002Fminicode-source-analysis\u002Fminicode-cover-20261018.webp","终端编程智能体控制台被 ReAct 状态环、Unified Diff 补丁预览门与 MCP 协议插槽环绕的等距技术插画","MiniCode 是我作为核心开发者参与构建的开源终端 Agentic Coding 引擎（目前已在 GitHub 获得 1,000+ Stars）。项目的核心设计目标是：在不引入庞大运行时框架的前提下，用约 5,000 行高内聚的","AI 项目源码导读",{"slug":2800,"path":2801,"title":2802,"date":2790,"tags":2803,"description":2804,"cover":2805,"coverAlt":2806,"excerpt":2807,"draft":2280,"hidden":2280,"published":2284,"readingTime":2501,"series":2785,"seriesOrder":1174},"deepscientist-tech-stack","\u002Fposts\u002Fdeepscientist-tech-stack","DeepScientist 架构与运行时全解析：超 100K 上下文科研智能体、MCP 沙箱与高并发流式系统",[2352,2322,2793,2780,2439,2437],"深度拆解服务 7,000+ 用户的科研自主智能体平台 DeepScientist：从一题一仓库（Quest）持久化状态机、MCP 工具栈（memory \u002F artifact \u002F bash_exec）、Docker SDK 动态容器编排，到 SSE + WebSocket 双流通信与 PostgreSQL \u002F Redis \u002F Prometheus 生产级基础设施。","\u002Fimages\u002Fposts\u002Fdeepscientist-tech-stack\u002Fdeepscientist-cover-20261018.webp","科研智能体编排核心通过 memory、artifact 与隔离容器沙箱三座协议桥连接双流工作台的等距架构插画","DeepScientist（deepscientist.cc）是我们在西湖大学自然语言处理实验室研发的自主科学发现与科研管理平台，目前累计服务 7,000+ 科研用户。与单轮对话或纯文本文献问答工具不同，科研智能体需要在数小时乃至数",1795088946009]