[{"data":1,"prerenderedAt":2965},["ShallowReactive",2],{"post-\u002Fposts\u002Fcs336-multihead-attention-shapes":3,"archive-public-posts":2454},{"id":4,"title":5,"body":6,"categories":2431,"cover":2433,"coverAlt":2434,"date":2435,"description":2436,"draft":2437,"extension":2438,"haloPublished":2437,"haloSync":2437,"hidden":2437,"legacy":2437,"meta":2439,"navigation":2441,"path":2442,"published":2437,"readingTime":2443,"seo":2444,"series":2445,"seriesOrder":543,"stem":2446,"tags":2447,"__hash__":2453},"posts\u002Fposts\u002Fcs336-multihead-attention-shapes.md","Stanford CS336 学习记录 04：多头注意力怎样拆头、计算和合头",{"type":7,"value":8,"toc":2415},"minimark",[9,18,30,37,42,52,430,437,458,461,464,468,471,799,843,846,913,916,920,926,977,988,991,999,1008,1011,1015,1018,1057,1060,1063,1069,1072,1078,1093,1099,1102,1105,1112,1399,1459,1462,1625,1628,1632,1635,1638,1690,1700,1703,1716,1720,1730,1733,1779,1782,1788,1795,1801,1805,1810,1819,1833,1844,1853,1857,1863,1873,2158,2161,2164,2168,2183,2186,2189,2251,2261,2265,2272,2278,2281,2301,2304,2324,2336,2339,2411],[10,11,12,17],"p",{},[13,14,16],"a",{"href":15},"\u002Fposts\u002Fcs336-softmax-attention-mask\u002F","上一篇","完成了缩放点积注意力。那个函数接收已经准备好的 Q、K、V，计算匹配分数，应用 mask，再用权重汇总 V。这一篇继续把它放进一个带参数的多头自注意力模块。",[10,19,20,21,25,26,29],{},"这次最需要看清的是“拆头”。代码里只是几行 ",[22,23,24],"code",{},"reshape"," 和 ",[22,27,28],{},"transpose","，但如果不知道每根轴代表什么，很容易做到形状能运行，却把不同 token 的数据拼到一起。",[10,31,32,33,36],{},"我沿用学习过程中反复使用的例子：输入形状为 ",[22,34,35],{},"(2, 5, 64)","，一批有两条序列，每条序列五个 token，每个 token 有 64 个特征，设置四个头。最后，这个不含 RoPE 的多头自注意力模块通过了对应官方测试。",[38,39,41],"h2",{"id":40},"先生成-qkv再拆头","先生成 Q、K、V，再拆头",[10,43,44,45,48,49,51],{},"“自注意力”中的“自”，在这里指 Q、K、V 都来自同一份输入 ",[22,46,47],{},"x","。但它们并不是把 ",[22,50,47],{}," 换三个名字，而是分别经过三个有独立权重的 Linear：",[53,54,57],"span",{"className":55},[56],"katex-display",[53,58,61,149],{"className":59},[60],"katex",[53,62,65],{"className":63},[64],"katex-mathml",[66,67,70],"math",{"xmlns":68,"display":69},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML","block",[71,72,73,144],"semantics",{},[74,75,76,80,84,87,99,103,107,110,112,114,122,124,126,129,131,133,141],"mrow",{},[77,78,79],"mi",{},"Q",[81,82,83],"mo",{},"=",[77,85,86],{},"X",[88,89,90,93,95],"msubsup",{},[77,91,92],{},"W",[77,94,79],{},[77,96,98],{"mathvariant":97},"normal","⊤",[81,100,102],{"separator":101},"true",",",[104,105],"mspace",{"width":106},"2em",[77,108,109],{},"K",[81,111,83],{},[77,113,86],{},[88,115,116,118,120],{},[77,117,92],{},[77,119,109],{},[77,121,98],{"mathvariant":97},[81,123,102],{"separator":101},[104,125],{"width":106},[77,127,128],{},"V",[81,130,83],{},[77,132,86],{},[88,134,135,137,139],{},[77,136,92],{},[77,138,128],{},[77,140,98],{"mathvariant":97},[77,142,143],{"mathvariant":97},".",[145,146,148],"annotation",{"encoding":147},"application\u002Fx-tex","Q=XW_Q^\\top,\\qquad\nK=XW_K^\\top,\\qquad\nV=XW_V^\\top.",[53,150,153,178,283,367],{"className":151,"ariaHidden":101},[152],"katex-html",[53,154,157,162,167,171,175],{"className":155},[156],"base",[53,158],{"className":159,"style":161},[160],"strut","height:0.8778em;vertical-align:-0.1944em;",[53,163,79],{"className":164},[165,166],"mord","mathnormal",[53,168],{"className":169,"style":170},[104],"margin-right:0.2778em;",[53,172,83],{"className":173},[174],"mrel",[53,176],{"className":177,"style":170},[104],[53,179,181,185,189,258,262,266,270,274,277,280],{"className":180},[156],[53,182],{"className":183,"style":184},[160],"height:1.2822em;vertical-align:-0.3831em;",[53,186,86],{"className":187,"style":188},[165,166],"margin-right:0.0785em;",[53,190,192,196],{"className":191},[165],[53,193,92],{"className":194,"style":195},[165,166],"margin-right:0.1389em;",[53,197,200],{"className":198},[199],"msupsub",[53,201,205,249],{"className":202},[203,204],"vlist-t","vlist-t2",[53,206,209,244],{"className":207},[208],"vlist-r",[53,210,214,232],{"className":211,"style":213},[212],"vlist","height:0.8991em;",[53,215,217,222],{"style":216},"top:-2.453em;margin-left:-0.1389em;margin-right:0.05em;",[53,218],{"className":219,"style":221},[220],"pstrut","height:2.7em;",[53,223,229],{"className":224},[225,226,227,228],"sizing","reset-size6","size3","mtight",[53,230,79],{"className":231},[165,166,228],[53,233,235,238],{"style":234},"top:-3.113em;margin-right:0.05em;",[53,236],{"className":237,"style":221},[220],[53,239,241],{"className":240},[225,226,227,228],[53,242,98],{"className":243},[165,228],[53,245,248],{"className":246},[247],"vlist-s","​",[53,250,252],{"className":251},[208],[53,253,256],{"className":254,"style":255},[212],"height:0.3831em;",[53,257],{},[53,259,102],{"className":260},[261],"mpunct",[53,263],{"className":264,"style":265},[104],"margin-right:2em;",[53,267],{"className":268,"style":269},[104],"margin-right:0.1667em;",[53,271,109],{"className":272,"style":273},[165,166],"margin-right:0.0715em;",[53,275],{"className":276,"style":170},[104],[53,278,83],{"className":279},[174],[53,281],{"className":282,"style":170},[104],[53,284,286,290,293,345,348,351,354,358,361,364],{"className":285},[156],[53,287],{"className":288,"style":289},[160],"height:1.1461em;vertical-align:-0.247em;",[53,291,86],{"className":292,"style":188},[165,166],[53,294,296,299],{"className":295},[165],[53,297,92],{"className":298,"style":195},[165,166],[53,300,302],{"className":301},[199],[53,303,305,336],{"className":304},[203,204],[53,306,308,333],{"className":307},[208],[53,309,311,322],{"className":310,"style":213},[212],[53,312,313,316],{"style":216},[53,314],{"className":315,"style":221},[220],[53,317,319],{"className":318},[225,226,227,228],[53,320,109],{"className":321,"style":273},[165,166,228],[53,323,324,327],{"style":234},[53,325],{"className":326,"style":221},[220],[53,328,330],{"className":329},[225,226,227,228],[53,331,98],{"className":332},[165,228],[53,334,248],{"className":335},[247],[53,337,339],{"className":338},[208],[53,340,343],{"className":341,"style":342},[212],"height:0.247em;",[53,344],{},[53,346,102],{"className":347},[261],[53,349],{"className":350,"style":265},[104],[53,352],{"className":353,"style":269},[104],[53,355,128],{"className":356,"style":357},[165,166],"margin-right:0.2222em;",[53,359],{"className":360,"style":170},[104],[53,362,83],{"className":363},[174],[53,365],{"className":366,"style":170},[104],[53,368,370,373,376,427],{"className":369},[156],[53,371],{"className":372,"style":289},[160],[53,374,86],{"className":375,"style":188},[165,166],[53,377,379,382],{"className":378},[165],[53,380,92],{"className":381,"style":195},[165,166],[53,383,385],{"className":384},[199],[53,386,388,419],{"className":387},[203,204],[53,389,391,416],{"className":390},[208],[53,392,394,405],{"className":393,"style":213},[212],[53,395,396,399],{"style":216},[53,397],{"className":398,"style":221},[220],[53,400,402],{"className":401},[225,226,227,228],[53,403,128],{"className":404,"style":357},[165,166,228],[53,406,407,410],{"style":234},[53,408],{"className":409,"style":221},[220],[53,411,413],{"className":412},[225,226,227,228],[53,414,98],{"className":415},[165,228],[53,417,248],{"className":418},[247],[53,420,422],{"className":421},[208],[53,423,425],{"className":424,"style":342},[212],[53,426],{},[53,428,143],{"className":429},[165],[10,431,432,433,436],{},"这里继续使用前几篇的约定：输入向量按行组织，Linear 权重按 ",[22,434,435],{},"(输出维度, 输入维度)"," 保存。",[10,438,439,440,443,444,443,447,450,451,454,455,457],{},"当前 ",[22,441,442],{},"q_proj","、",[22,445,446],{},"k_proj",[22,448,449],{},"v_proj"," 都是 ",[22,452,453],{},"64 → 64","，因此三个输出的形状仍然是 ",[22,456,35],{},"。形状相同，并不意味着它们的数值相同：三组参数可以学出不同变换。",[10,459,460],{},"为什么先生成完整的 64 维，再分头？这样每种投影都可以用一次矩阵乘法同时生成所有头需要的特征。Q、K、V 各做一次投影，一共三次。每个输出特征都能依赖输入的全部 64 个特征，随后才按输出特征分组。",[10,462,463],{},"因此，不能把这一过程理解成“把原始输入切成四块，每个头永远只能看到自己那一块原始特征”。拆分发生在可学习投影之后。",[38,465,467],{"id":466},"四个头分的是特征不是五个-token","四个头分的是特征，不是五个 token",[10,469,470],{},"当前实现采用等宽的头，满足：",[53,472,474],{"className":473},[56],[53,475,477,553],{"className":476},[60],[53,478,480],{"className":479},[64],[66,481,482],{"xmlns":68,"display":69},[71,483,484,550],{},[74,485,486,504,506,531,533,535,539,541,544,547],{},[487,488,489,492],"msub",{},[77,490,491],{},"d",[74,493,494,497,500,502],{},[77,495,496],{"mathvariant":97},"h",[77,498,499],{"mathvariant":97},"e",[77,501,13],{"mathvariant":97},[77,503,491],{"mathvariant":97},[81,505,83],{},[507,508,509,528],"mfrac",{},[487,510,511,513],{},[77,512,491],{},[74,514,515,518,521,523,525],{},[77,516,517],{"mathvariant":97},"m",[77,519,520],{"mathvariant":97},"o",[77,522,491],{"mathvariant":97},[77,524,499],{"mathvariant":97},[77,526,527],{"mathvariant":97},"l",[77,529,530],{},"H",[81,532,102],{"separator":101},[104,534],{"width":106},[536,537,538],"mn",{},"64",[81,540,83],{},[536,542,543],{},"4",[81,545,546],{},"×",[536,548,549],{},"16.",[145,551,552],{"encoding":147},"d_{\\mathrm{head}}=\\frac{d_{\\mathrm{model}}}{H},\n\\qquad\n64=4\\times16.",[53,554,556,623,769,789],{"className":555,"ariaHidden":101},[152],[53,557,559,563,614,617,620],{"className":558},[156],[53,560],{"className":561,"style":562},[160],"height:0.8444em;vertical-align:-0.15em;",[53,564,566,569],{"className":565},[165],[53,567,491],{"className":568},[165,166],[53,570,572],{"className":571},[199],[53,573,575,605],{"className":574},[203,204],[53,576,578,602],{"className":577},[208],[53,579,582],{"className":580,"style":581},[212],"height:0.3361em;",[53,583,585,588],{"style":584},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[53,586],{"className":587,"style":221},[220],[53,589,591],{"className":590},[225,226,227,228],[53,592,594],{"className":593},[165,228],[53,595,597],{"className":596},[165,228],[53,598,601],{"className":599},[165,600,228],"mathrm","head",[53,603,248],{"className":604},[247],[53,606,608],{"className":607},[208],[53,609,612],{"className":610,"style":611},[212],"height:0.15em;",[53,613],{},[53,615],{"className":616,"style":170},[104],[53,618,83],{"className":619},[174],[53,621],{"className":622,"style":170},[104],[53,624,626,630,748,751,754,757,760,763,766],{"className":625},[156],[53,627],{"className":628,"style":629},[160],"height:2.0574em;vertical-align:-0.686em;",[53,631,633,638,744],{"className":632},[165],[53,634],{"className":635},[636,637],"mopen","nulldelimiter",[53,639,641],{"className":640},[507],[53,642,644,735],{"className":643},[203,204],[53,645,647,732],{"className":646},[208],[53,648,651,665,676],{"className":649,"style":650},[212],"height:1.3714em;",[53,652,654,658],{"style":653},"top:-2.314em;",[53,655],{"className":656,"style":657},[220],"height:3em;",[53,659,661],{"className":660},[165],[53,662,530],{"className":663,"style":664},[165,166],"margin-right:0.0813em;",[53,666,668,671],{"style":667},"top:-3.23em;",[53,669],{"className":670,"style":657},[220],[53,672],{"className":673,"style":675},[674],"frac-line","border-bottom-width:0.04em;",[53,677,679,682],{"style":678},"top:-3.677em;",[53,680],{"className":681,"style":657},[220],[53,683,685],{"className":684},[165],[53,686,688,691],{"className":687},[165],[53,689,491],{"className":690},[165,166],[53,692,694],{"className":693},[199],[53,695,697,724],{"className":696},[203,204],[53,698,700,721],{"className":699},[208],[53,701,703],{"className":702,"style":581},[212],[53,704,705,708],{"style":584},[53,706],{"className":707,"style":221},[220],[53,709,711],{"className":710},[225,226,227,228],[53,712,714],{"className":713},[165,228],[53,715,717],{"className":716},[165,228],[53,718,720],{"className":719},[165,600,228],"model",[53,722,248],{"className":723},[247],[53,725,727],{"className":726},[208],[53,728,730],{"className":729,"style":611},[212],[53,731],{},[53,733,248],{"className":734},[247],[53,736,738],{"className":737},[208],[53,739,742],{"className":740,"style":741},[212],"height:0.686em;",[53,743],{},[53,745],{"className":746},[747,637],"mclose",[53,749,102],{"className":750},[261],[53,752],{"className":753,"style":265},[104],[53,755],{"className":756,"style":269},[104],[53,758,538],{"className":759},[165],[53,761],{"className":762,"style":170},[104],[53,764,83],{"className":765},[174],[53,767],{"className":768,"style":170},[104],[53,770,772,776,779,782,786],{"className":771},[156],[53,773],{"className":774,"style":775},[160],"height:0.7278em;vertical-align:-0.0833em;",[53,777,543],{"className":778},[165],[53,780],{"className":781,"style":357},[104],[53,783,546],{"className":784},[785],"mbin",[53,787],{"className":788,"style":357},[104],[53,790,792,796],{"className":791},[156],[53,793],{"className":794,"style":795},[160],"height:0.6444em;",[53,797,549],{"className":798},[165],[10,800,801,830,831,834,835,838,839,842],{},[53,802,804,817],{"className":803},[60],[53,805,807],{"className":806},[64],[66,808,809],{"xmlns":68},[71,810,811,815],{},[74,812,813],{},[77,814,530],{},[145,816,530],{"encoding":147},[53,818,820],{"className":819,"ariaHidden":101},[152],[53,821,823,827],{"className":822},[156],[53,824],{"className":825,"style":826},[160],"height:0.6833em;",[53,828,530],{"className":829,"style":664},[165,166]," 是头数。对于这个约定，头数必须是正整数，且能整除 ",[22,832,833],{},"d_model","。代码用 ",[22,836,837],{},"%"," 检查余数，再用 ",[22,840,841],{},"\u002F\u002F"," 得到每头的特征维度。这一检查对应的是能否拆成等宽的头，不能只用整除运算默默丢掉余数。",[10,844,845],{},"拆头后，每个 token 的 64 个投影特征被组织为四组，每组 16 维。五个 token 仍然都在，每个头也仍然处理完整的五个位置。",[847,848,849,866],"table",{},[850,851,852],"thead",{},[853,854,855,859,863],"tr",{},[856,857,858],"th",{},"维度",[856,860,862],{"align":861},"right","数值",[856,864,865],{},"含义",[867,868,869,881,892,902],"tbody",{},[853,870,871,875,878],{},[872,873,874],"td",{},"batch",[872,876,877],{"align":861},"2",[872,879,880],{},"两条序列",[853,882,883,886,889],{},[872,884,885],{},"sequence length",[872,887,888],{"align":861},"5",[872,890,891],{},"每条序列五个位置",[853,893,894,897,899],{},[872,895,896],{},"heads",[872,898,543],{"align":861},[872,900,901],{},"四组注意力计算",[853,903,904,907,910],{},[872,905,906],{},"head dimension",[872,908,909],{"align":861},"16",[872,911,912],{},"每组中一个 token 的特征数",[10,914,915],{},"如果把五个 token 分给不同头，就会改变每个头能够参与匹配和汇总的位置集合，不是这里要实现的多头注意力。",[38,917,919],{"id":918},"head_shape-里的加号在拼形状","head_shape 里的加号在拼形状",[10,921,439,922,925],{},[22,923,924],{},"forward"," 中，目标形状由这一行得到：",[927,928,933],"pre",{"className":929,"code":930,"language":931,"meta":932,"style":932},"language-python shiki shiki-themes github-dark-dimmed github-light houston","head_shape=x.shape[:-1]+(self.num_heads,self.d_head)\n","python","",[22,934,935],{"__ignoreMap":932},[53,936,939,943,946,949,952,956,959,962,965,969,972,974],{"class":937,"line":938},"line",1,[53,940,942],{"class":941},"sWURj","head_shape",[53,944,83],{"class":945},"sf2cM",[53,947,948],{"class":941},"x.shape[:",[53,950,951],{"class":945},"-",[53,953,955],{"class":954},"sgENW","1",[53,957,958],{"class":941},"]",[53,960,961],{"class":945},"+",[53,963,964],{"class":941},"(",[53,966,968],{"class":967},"sRGqv","self",[53,970,971],{"class":941},".num_heads,",[53,973,968],{"class":967},[53,975,976],{"class":941},".d_head)\n",[10,978,979,980,983,984,987],{},"第一次看时，要把张量的数值运算与形状对象的操作分开。",[22,981,982],{},"x.shape"," 是 ",[22,985,986],{},"torch.Size","，支持像元组一样切片和拼接。",[10,989,990],{},"对于这个例子：",[927,992,997],{"className":993,"code":995,"language":996,"meta":932},[994],"language-text","x.shape                    → (2, 5, 64)\nx.shape[:-1]               → (2, 5)\n(num_heads, d_head)        → (4, 16)\n拼接后的 head_shape        → (2, 5, 4, 16)\n","text",[22,998,995],{"__ignoreMap":932},[10,1000,1001,1004,1005,1007],{},[22,1002,1003],{},"[:-1]"," 取除了最后一维以外的形状。这里的 ",[22,1006,961],{}," 把两段形状连接起来，不是在把 2 与 4、5 与 16 分别相加，也没有修改输入张量里的数字。",[10,1009,1010],{},"这样写还保留了原输入的前导维度。虽然解释时用了一个 batch 维，表达式本身没有把这个维度的位置或长度写死。",[38,1012,1014],{"id":1013},"reshape-之后为什么还要-transpose","reshape 之后为什么还要 transpose",[10,1016,1017],{},"拆 Q 的已有代码是：",[927,1019,1021],{"className":929,"code":1020,"language":931,"meta":932,"style":932},"q=q.reshape(head_shape).transpose(-3,-2)\n",[22,1022,1023],{"__ignoreMap":932},[53,1024,1025,1028,1030,1033,1036,1039,1041,1043,1045,1048,1050,1052,1054],{"class":937,"line":938},[53,1026,1027],{"class":941},"q",[53,1029,83],{"class":945},[53,1031,1032],{"class":941},"q.",[53,1034,24],{"class":1035},"sq3c-",[53,1037,1038],{"class":941},"(head_shape).",[53,1040,28],{"class":1035},[53,1042,964],{"class":941},[53,1044,951],{"class":945},[53,1046,1047],{"class":954},"3",[53,1049,102],{"class":941},[53,1051,951],{"class":945},[53,1053,877],{"class":954},[53,1055,1056],{"class":941},")\n",[10,1058,1059],{},"这行有两个动作，不能只记最后的形状。",[10,1061,1062],{},"第一步将每个 token 的 64 个特征组织成四组 16 维：",[927,1064,1067],{"className":1065,"code":1066,"language":996,"meta":932},[994],"(2, 5, 64) → (2, 5, 4, 16)\n             B  L  H  d_head\n",[22,1068,1066],{"__ignoreMap":932},[10,1070,1071],{},"第二步交换倒数第三维和倒数第二维，即交换 token 轴与 head 轴：",[927,1073,1076],{"className":1074,"code":1075,"language":996,"meta":932},[994],"(2, 5, 4, 16) → (2, 4, 5, 16)\n B  L  H  d     B  H  L  d\n",[22,1077,1075],{"__ignoreMap":932},[10,1079,1080,1081,1084,1085,1088,1089,1092],{},"为什么不保留 ",[22,1082,1083],{},"(B, L, H, d_head)","？因为前面写好的注意力函数，把最后两维解释为“位置数、特征数”。整理为 ",[22,1086,1087],{},"(B, H, L, d_head)"," 后，batch 和 head 都成为前导维度，每个头可以独立处理自己的 ",[22,1090,1091],{},"(L, d_head)"," 矩阵。",[10,1094,1095,1096,1098],{},"这里的 ",[22,1097,28],{}," 在交换轴的含义，不是把二维矩阵转置的口诀照搬到四维张量。使用负数轴编号，是为了相对最后几维定位，在有额外前导维度时也能保留同样的关系。",[10,1100,1101],{},"K、V 做相同的整理。每个头独立计算注意力，并不意味着把它们写成四次 Python 循环；批量矩阵乘法可以一起完成这些计算。",[38,1103,1104],{"id":1104},"每个头都会产生一张五乘五的匹配表",[10,1106,1107,1108,1111],{},"拆头后，Q、K、V 都是 ",[22,1109,1110],{},"(2, 4, 5, 16)","。把最后两维代入前面的缩放点积注意力：",[53,1113,1115],{"className":1114},[56],[53,1116,1118,1167],{"className":1117},[60],[53,1119,1121],{"className":1120},[64],[66,1122,1123],{"xmlns":68,"display":69},[71,1124,1125,1164],{},[74,1126,1127,1130,1132,1162],{},[77,1128,1129],{},"S",[81,1131,83],{},[507,1133,1134,1145],{},[74,1135,1136,1138],{},[77,1137,79],{},[1139,1140,1141,1143],"msup",{},[77,1142,109],{},[77,1144,98],{"mathvariant":97},[1146,1147,1148],"msqrt",{},[487,1149,1150,1152],{},[77,1151,491],{},[74,1153,1154,1156,1158,1160],{},[77,1155,496],{"mathvariant":97},[77,1157,499],{"mathvariant":97},[77,1159,13],{"mathvariant":97},[77,1161,491],{"mathvariant":97},[77,1163,143],{"mathvariant":97},[145,1165,1166],{"encoding":147},"S=\\frac{QK^\\top}{\\sqrt{d_{\\mathrm{head}}}}.",[53,1168,1170,1189],{"className":1169,"ariaHidden":101},[152],[53,1171,1173,1176,1180,1183,1186],{"className":1172},[156],[53,1174],{"className":1175,"style":826},[160],[53,1177,1129],{"className":1178,"style":1179},[165,166],"margin-right:0.0576em;",[53,1181],{"className":1182,"style":170},[104],[53,1184,83],{"className":1185},[174],[53,1187],{"className":1188,"style":170},[104],[53,1190,1192,1196,1396],{"className":1191},[156],[53,1193],{"className":1194,"style":1195},[160],"height:2.4561em;vertical-align:-0.93em;",[53,1197,1199,1202,1393],{"className":1198},[165],[53,1200],{"className":1201},[636,637],[53,1203,1205],{"className":1204},[507],[53,1206,1208,1384],{"className":1207},[203,204],[53,1209,1211,1381],{"className":1210},[208],[53,1212,1215,1331,1339],{"className":1213,"style":1214},[212],"height:1.5261em;",[53,1216,1218,1221],{"style":1217},"top:-2.2528em;",[53,1219],{"className":1220,"style":657},[220],[53,1222,1224],{"className":1223},[165],[53,1225,1228],{"className":1226},[165,1227],"sqrt",[53,1229,1231,1322],{"className":1230},[203,204],[53,1232,1234,1319],{"className":1233},[208],[53,1235,1238,1296],{"className":1236,"style":1237},[212],"height:0.8572em;",[53,1239,1243,1246],{"className":1240,"style":1242},[1241],"svg-align","top:-3em;",[53,1244],{"className":1245,"style":657},[220],[53,1247,1250],{"className":1248,"style":1249},[165],"padding-left:0.833em;",[53,1251,1253,1256],{"className":1252},[165],[53,1254,491],{"className":1255},[165,166],[53,1257,1259],{"className":1258},[199],[53,1260,1262,1288],{"className":1261},[203,204],[53,1263,1265,1285],{"className":1264},[208],[53,1266,1268],{"className":1267,"style":581},[212],[53,1269,1270,1273],{"style":584},[53,1271],{"className":1272,"style":221},[220],[53,1274,1276],{"className":1275},[225,226,227,228],[53,1277,1279],{"className":1278},[165,228],[53,1280,1282],{"className":1281},[165,228],[53,1283,601],{"className":1284},[165,600,228],[53,1286,248],{"className":1287},[247],[53,1289,1291],{"className":1290},[208],[53,1292,1294],{"className":1293,"style":611},[212],[53,1295],{},[53,1297,1299,1302],{"style":1298},"top:-2.8172em;",[53,1300],{"className":1301,"style":657},[220],[53,1303,1307],{"className":1304,"style":1306},[1305],"hide-tail","min-width:0.853em;height:1.08em;",[1308,1309,1315],"svg",{"xmlns":1310,"width":1311,"height":1312,"viewBox":1313,"preserveAspectRatio":1314},"http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","400em","1.08em","0 0 400000 1080","xMinYMin slice",[1316,1317],"path",{"d":1318},"M95,702\nc-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14\nc0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54\nc44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10\ns173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429\nc69,-144,104.5,-217.7,106.5,-221\nl0 -0\nc5.3,-9.3,12,-14,20,-14\nH400000v40H845.2724\ns-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7\nc-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z\nM834 80h400000v40h-400000z",[53,1320,248],{"className":1321},[247],[53,1323,1325],{"className":1324},[208],[53,1326,1329],{"className":1327,"style":1328},[212],"height:0.1828em;",[53,1330],{},[53,1332,1333,1336],{"style":667},[53,1334],{"className":1335,"style":657},[220],[53,1337],{"className":1338,"style":675},[674],[53,1340,1341,1344],{"style":678},[53,1342],{"className":1343,"style":657},[220],[53,1345,1347,1350],{"className":1346},[165],[53,1348,79],{"className":1349},[165,166],[53,1351,1353,1356],{"className":1352},[165],[53,1354,109],{"className":1355,"style":273},[165,166],[53,1357,1359],{"className":1358},[199],[53,1360,1362],{"className":1361},[203],[53,1363,1365],{"className":1364},[208],[53,1366,1369],{"className":1367,"style":1368},[212],"height:0.8491em;",[53,1370,1372,1375],{"style":1371},"top:-3.063em;margin-right:0.05em;",[53,1373],{"className":1374,"style":221},[220],[53,1376,1378],{"className":1377},[225,226,227,228],[53,1379,98],{"className":1380},[165,228],[53,1382,248],{"className":1383},[247],[53,1385,1387],{"className":1386},[208],[53,1388,1391],{"className":1389,"style":1390},[212],"height:0.93em;",[53,1392],{},[53,1394],{"className":1395},[747,637],[53,1397,143],{"className":1398},[165],[847,1400,1401,1411],{},[850,1402,1403],{},[853,1404,1405,1408],{},[856,1406,1407],{},"步骤",[856,1409,1410],{},"形状",[867,1412,1413,1421,1431,1441,1450],{},[853,1414,1415,1417],{},[872,1416,79],{},[872,1418,1419],{},[22,1420,1110],{},[853,1422,1423,1426],{},[872,1424,1425],{},"K 交换最后两维",[872,1427,1428],{},[22,1429,1430],{},"(2, 4, 16, 5)",[853,1432,1433,1436],{},[872,1434,1435],{},"匹配分数",[872,1437,1438],{},[22,1439,1440],{},"(2, 4, 5, 5)",[853,1442,1443,1446],{},[872,1444,1445],{},"Softmax 权重",[872,1447,1448],{},[22,1449,1440],{},[853,1451,1452,1455],{},[872,1453,1454],{},"权重乘 V",[872,1456,1457],{},[22,1458,1110],{},[10,1460,1461],{},"每个 batch、每个头都有自己的五乘五分数矩阵。一行对应一个 query 位置，一列对应一个 key 位置。",[10,1463,1464,1465,1544,1545,1620,1621,1624],{},"缩放因子也跟着拆头后的维度变化。在这个例子中，它是 ",[53,1466,1468,1484],{"className":1467},[60],[53,1469,1471],{"className":1470},[64],[66,1472,1473],{"xmlns":68},[71,1474,1475,1481],{},[74,1476,1477],{},[1146,1478,1479],{},[536,1480,909],{},[145,1482,1483],{"encoding":147},"\\sqrt{16}",[53,1485,1487],{"className":1486,"ariaHidden":101},[152],[53,1488,1490,1494],{"className":1489},[156],[53,1491],{"className":1492,"style":1493},[160],"height:1.04em;vertical-align:-0.1328em;",[53,1495,1497],{"className":1496},[165,1227],[53,1498,1500,1535],{"className":1499},[203,204],[53,1501,1503,1532],{"className":1502},[208],[53,1504,1507,1519],{"className":1505,"style":1506},[212],"height:0.9072em;",[53,1508,1510,1513],{"className":1509,"style":1242},[1241],[53,1511],{"className":1512,"style":657},[220],[53,1514,1516],{"className":1515,"style":1249},[165],[53,1517,909],{"className":1518},[165],[53,1520,1522,1525],{"style":1521},"top:-2.8672em;",[53,1523],{"className":1524,"style":657},[220],[53,1526,1528],{"className":1527,"style":1306},[1305],[1308,1529,1530],{"xmlns":1310,"width":1311,"height":1312,"viewBox":1313,"preserveAspectRatio":1314},[1316,1531],{"d":1318},[53,1533,248],{"className":1534},[247],[53,1536,1538],{"className":1537},[208],[53,1539,1542],{"className":1540,"style":1541},[212],"height:0.1328em;",[53,1543],{},"，不是 ",[53,1546,1548,1564],{"className":1547},[60],[53,1549,1551],{"className":1550},[64],[66,1552,1553],{"xmlns":68},[71,1554,1555,1561],{},[74,1556,1557],{},[1146,1558,1559],{},[536,1560,538],{},[145,1562,1563],{"encoding":147},"\\sqrt{64}",[53,1565,1567],{"className":1566,"ariaHidden":101},[152],[53,1568,1570,1573],{"className":1569},[156],[53,1571],{"className":1572,"style":1493},[160],[53,1574,1576],{"className":1575},[165,1227],[53,1577,1579,1612],{"className":1578},[203,204],[53,1580,1582,1609],{"className":1581},[208],[53,1583,1585,1597],{"className":1584,"style":1506},[212],[53,1586,1588,1591],{"className":1587,"style":1242},[1241],[53,1589],{"className":1590,"style":657},[220],[53,1592,1594],{"className":1593,"style":1249},[165],[53,1595,538],{"className":1596},[165],[53,1598,1599,1602],{"style":1521},[53,1600],{"className":1601,"style":657},[220],[53,1603,1605],{"className":1604,"style":1306},[1305],[1308,1606,1607],{"xmlns":1310,"width":1311,"height":1312,"viewBox":1313,"preserveAspectRatio":1314},[1316,1608],{"d":1318},[53,1610,248],{"className":1611},[247],[53,1613,1615],{"className":1614},[208],[53,1616,1618],{"className":1617,"style":1541},[212],[53,1619],{},"。已有注意力函数读取 ",[22,1622,1623],{},"Q.shape[-1]","，所以整理好形状后，取到的自然就是每头维度。",[10,1626,1627],{},"这也解释了“多头”具体在哪里不同：各头使用不同的投影特征，因此可以产生不同的匹配分数和加权结果。仅仅把一个相同结果复制四份，并不会获得这些差异。",[38,1629,1631],{"id":1630},"同一张因果-mask-可以用于所有头","同一张因果 mask 可以用于所有头",[10,1633,1634],{},"自回归语言模型不能让当前位置读取未来 token。这一版模块内部根据序列长度构造布尔下三角 mask：包含对角线的下三角为 True，上三角为 False。",[10,1636,1637],{},"以三个位置说明：",[847,1639,1640,1656],{},[850,1641,1642],{},[853,1643,1644,1647,1650,1653],{},[856,1645,1646],{},"query 位置",[856,1648,1649],{},"key 1",[856,1651,1652],{},"key 2",[856,1654,1655],{},"key 3",[867,1657,1658,1670,1680],{},[853,1659,1660,1662,1665,1668],{},[872,1661,955],{},[872,1663,1664],{},"True",[872,1666,1667],{},"False",[872,1669,1667],{},[853,1671,1672,1674,1676,1678],{},[872,1673,877],{},[872,1675,1664],{},[872,1677,1664],{},[872,1679,1667],{},[853,1681,1682,1684,1686,1688],{},[872,1683,1047],{},[872,1685,1664],{},[872,1687,1664],{},[872,1689,1664],{},[10,1691,1692,1693,1696,1697,1699],{},"在五个位置的例子中，mask 的形状为 ",[22,1694,1695],{},"(5, 5)","，分数的形状为 ",[22,1698,1440],{},"。广播从尾部对齐，相当于 mask 前面有两个长度为 1 的维度，可以应用到所有 batch 和所有头。",[10,1701,1702],{},"这里共享的是“哪些位置允许关注”的规则。每个头的分数仍然不同，通过 Softmax 得到的注意力权重也可以不同。共享 mask 不等于共享注意力结果。",[10,1704,1705,1706,1709,1710,1712,1713,1715],{},"mask 使用输入的 ",[22,1707,1708],{},"device","，避免计算设备不一致；使用布尔类型，则能沿用上一节函数中 ",[22,1711,1664],{}," 允许、",[22,1714,1667],{}," 屏蔽的约定。每行至少保留对角线上的自己，因此这张正常的非空因果 mask 不会构造出整行全被屏蔽的情况。",[38,1717,1719],{"id":1718},"合头前先把同一个-token-的结果放回一起","合头前，先把同一个 token 的结果放回一起",[10,1721,1722,1723,1725,1726,1729],{},"注意力输出仍然是 ",[22,1724,1087],{},"。要回到输入形状，不能只看元素总数相同就直接压成 ",[22,1727,1728],{},"(B, L, d_model)","。",[10,1731,1732],{},"当前代码先把轴交换回来，再合并最后两维：",[927,1734,1736],{"className":929,"code":1735,"language":931,"meta":932,"style":932},"head_outputs=head_outputs.transpose(-3,-2)\nmerged=head_outputs.reshape(x.shape)\n",[22,1737,1738,1764],{"__ignoreMap":932},[53,1739,1740,1743,1745,1748,1750,1752,1754,1756,1758,1760,1762],{"class":937,"line":938},[53,1741,1742],{"class":941},"head_outputs",[53,1744,83],{"class":945},[53,1746,1747],{"class":941},"head_outputs.",[53,1749,28],{"class":1035},[53,1751,964],{"class":941},[53,1753,951],{"class":945},[53,1755,1047],{"class":954},[53,1757,102],{"class":941},[53,1759,951],{"class":945},[53,1761,877],{"class":954},[53,1763,1056],{"class":941},[53,1765,1767,1770,1772,1774,1776],{"class":937,"line":1766},2,[53,1768,1769],{"class":941},"merged",[53,1771,83],{"class":945},[53,1773,1747],{"class":941},[53,1775,24],{"class":1035},[53,1777,1778],{"class":941},"(x.shape)\n",[10,1780,1781],{},"对应的形状变化是：",[927,1783,1786],{"className":1784,"code":1785,"language":996,"meta":932},[994],"(2, 4, 5, 16) → (2, 5, 4, 16) → (2, 5, 64)\n B  H  L  d     B  L  H  d      B  L  D\n",[22,1787,1785],{"__ignoreMap":932},[10,1789,1790,1791,1794],{},"先交换回来，才能让同一个 token 的四个头在最后两维相邻。然后把 ",[22,1792,1793],{},"4 × 16"," 个特征连接为 64 维向量。",[10,1796,1797,1798,1800],{},"如果省略这次交换，直接 ",[22,1799,24],{}," 仍可能得到想要的外观尺寸，但会按照错误的轴顺序重新组织元素，把 head 与 token 的关系混淆。维度乘积一致只是必要条件；每一根轴的语义也必须对。",[38,1802,1804],{"id":1803},"reshapeview-和非连续张量","reshape、view 和非连续张量",[10,1806,1807,1809],{},[22,1808,28],{}," 通常通过交换尺寸与步长来创建视图，不必实际移动底层存储中的全部元素。逻辑上的轴顺序改变后，张量可能不再满足通常的连续布局。",[10,1811,1812,1813,25,1816,1818],{},"这时要谨慎区分 ",[22,1814,1815],{},"view",[22,1817,24],{},"：",[1820,1821,1822,1828],"ul",{},[1823,1824,1825,1827],"li",{},[22,1826,1815],{}," 要求新的形状与已有尺寸、步长兼容，不能在任意转置结果上都成功。",[1823,1829,1830,1832],{},[22,1831,24],{}," 在布局允许时返回视图，否则可以复制数据，得到符合新形状的结果。",[10,1834,1835,1836,1838,1839,1841,1842,1729],{},"所以不能说 ",[22,1837,24],{}," 永远不复制，也不能把 ",[22,1840,1815],{}," 的要求简化成“只能处理连续张量”——某些非连续张量也能在满足步长条件时使用特定的 ",[22,1843,1815],{},[10,1845,1846,1847,1849,1850,1852],{},"当前合头使用 ",[22,1848,24],{},"，可以处理转置后需要复制的情况。不过它只解决存储布局与目标形状的兼容问题，不会替我们判断轴的语义。前面该做的 ",[22,1851,28],{},"，仍然不能省略。这里没有做两种写法的性能比较，也不据此推断复制开销可以忽略。",[38,1854,1856],{"id":1855},"已经回到-64-维为什么还要输出-linear","已经回到 64 维，为什么还要输出 Linear",[10,1858,1859,1860,1862],{},"合头只是把各头结果连接起来，恢复了 ",[22,1861,35],{},"。它没有自动学会怎样组合不同头的特征。",[10,1864,1865,1866,1869,1870,1872],{},"最后的 ",[22,1867,1868],{},"output_proj"," 仍是 ",[22,1871,453],{},"，按每个 token 的完整合并向量做一次可学习线性变换：",[53,1874,1876],{"className":1875},[56],[53,1877,1879,1954],{"className":1878},[60],[53,1880,1882],{"className":1881},[64],[66,1883,1884],{"xmlns":68,"display":69},[71,1885,1886,1951],{},[74,1887,1888,1891,1893,1896,1899,1902,1916,1918,1921,1923,1937,1940,1949],{},[77,1889,1890],{},"Y",[81,1892,83],{},[77,1894,1895],{"mathvariant":97},"Concat",[81,1897,1898],{},"⁡",[81,1900,964],{"stretchy":1901},"false",[487,1903,1904,1914],{},[74,1905,1906,1908,1910,1912],{},[77,1907,496],{"mathvariant":97},[77,1909,499],{"mathvariant":97},[77,1911,13],{"mathvariant":97},[77,1913,491],{"mathvariant":97},[536,1915,955],{},[81,1917,102],{"separator":101},[81,1919,1920],{},"…",[81,1922,102],{"separator":101},[487,1924,1925,1935],{},[74,1926,1927,1929,1931,1933],{},[77,1928,496],{"mathvariant":97},[77,1930,499],{"mathvariant":97},[77,1932,13],{"mathvariant":97},[77,1934,491],{"mathvariant":97},[77,1936,530],{},[81,1938,1939],{"stretchy":1901},")",[88,1941,1942,1944,1947],{},[77,1943,92],{},[77,1945,1946],{},"O",[77,1948,98],{"mathvariant":97},[77,1950,143],{"mathvariant":97},[145,1952,1953],{"encoding":147},"Y=\\operatorname{Concat}(\\mathrm{head}_1,\\ldots,\\mathrm{head}_H)\nW_O^\\top.",[53,1955,1957,1975],{"className":1956,"ariaHidden":101},[152],[53,1958,1960,1963,1966,1969,1972],{"className":1959},[156],[53,1961],{"className":1962,"style":826},[160],[53,1964,1890],{"className":1965,"style":357},[165,166],[53,1967],{"className":1968,"style":170},[104],[53,1970,83],{"className":1971},[174],[53,1973],{"className":1974,"style":170},[104],[53,1976,1978,1982,1989,1992,2037,2040,2043,2047,2050,2053,2056,2100,2103,2155],{"className":1977},[156],[53,1979],{"className":1980,"style":1981},[160],"height:1.1491em;vertical-align:-0.25em;",[53,1983,1986],{"className":1984},[1985],"mop",[53,1987,1895],{"className":1988},[165,600],[53,1990,964],{"className":1991},[636],[53,1993,1995,2001],{"className":1994},[165],[53,1996,1998],{"className":1997},[165],[53,1999,601],{"className":2000},[165,600],[53,2002,2004],{"className":2003},[199],[53,2005,2007,2029],{"className":2006},[203,204],[53,2008,2010,2026],{"className":2009},[208],[53,2011,2014],{"className":2012,"style":2013},[212],"height:0.3011em;",[53,2015,2017,2020],{"style":2016},"top:-2.55em;margin-right:0.05em;",[53,2018],{"className":2019,"style":221},[220],[53,2021,2023],{"className":2022},[225,226,227,228],[53,2024,955],{"className":2025},[165,228],[53,2027,248],{"className":2028},[247],[53,2030,2032],{"className":2031},[208],[53,2033,2035],{"className":2034,"style":611},[212],[53,2036],{},[53,2038,102],{"className":2039},[261],[53,2041],{"className":2042,"style":269},[104],[53,2044,1920],{"className":2045},[2046],"minner",[53,2048],{"className":2049,"style":269},[104],[53,2051,102],{"className":2052},[261],[53,2054],{"className":2055,"style":269},[104],[53,2057,2059,2065],{"className":2058},[165],[53,2060,2062],{"className":2061},[165],[53,2063,601],{"className":2064},[165,600],[53,2066,2068],{"className":2067},[199],[53,2069,2071,2092],{"className":2070},[203,204],[53,2072,2074,2089],{"className":2073},[208],[53,2075,2078],{"className":2076,"style":2077},[212],"height:0.3283em;",[53,2079,2080,2083],{"style":2016},[53,2081],{"className":2082,"style":221},[220],[53,2084,2086],{"className":2085},[225,226,227,228],[53,2087,530],{"className":2088,"style":664},[165,166,228],[53,2090,248],{"className":2091},[247],[53,2093,2095],{"className":2094},[208],[53,2096,2098],{"className":2097,"style":611},[212],[53,2099],{},[53,2101,1939],{"className":2102},[747],[53,2104,2106,2109],{"className":2105},[165],[53,2107,92],{"className":2108,"style":195},[165,166],[53,2110,2112],{"className":2111},[199],[53,2113,2115,2147],{"className":2114},[203,204],[53,2116,2118,2144],{"className":2117},[208],[53,2119,2121,2133],{"className":2120,"style":213},[212],[53,2122,2123,2126],{"style":216},[53,2124],{"className":2125,"style":221},[220],[53,2127,2129],{"className":2128},[225,226,227,228],[53,2130,1946],{"className":2131,"style":2132},[165,166,228],"margin-right:0.0278em;",[53,2134,2135,2138],{"style":234},[53,2136],{"className":2137,"style":221},[220],[53,2139,2141],{"className":2140},[225,226,227,228],[53,2142,98],{"className":2143},[165,228],[53,2145,248],{"className":2146},[247],[53,2148,2150],{"className":2149},[208],[53,2151,2153],{"className":2152,"style":342},[212],[53,2154],{},[53,2156,143],{"className":2157},[165],[10,2159,2160],{},"输入输出宽度相同，不代表这是恒等变换。一个输出特征可以组合多个头提供的分量。与前面的投影一样，它在每个 token 内处理特征；这一步本身没有再次跨 token 汇总信息。",[10,2162,2163],{},"因此，完整路径可以读成：先用三组权重生成 Q、K、V，拆成头，在各头里做跨位置注意力，再合并各头，由第四组权重学习特征组合。",[38,2165,2167],{"id":2166},"这次的真实错误先发生在-class-上","这次的真实错误，先发生在 class 上",[10,2169,2170,2171,2174,2175,2178,2179,2182],{},"继续写模块时，我把定义开头写成了 ",[22,2172,2173],{},"def MultiHeadSelfAttention(nn.Module):","。这里需要定义类，应使用 ",[22,2176,2177],{},"class","；",[22,2180,2181],{},"def"," 用于定义方法或函数。",[10,2184,2185],{},"这个错误还没有进入任何注意力运算，先把“类”和“函数”的语法混在一起了。到这一篇，仍然不能因为已经理解了注意力公式，就忽略最基础的 Python 结构。",[10,2187,2188],{},"另一处需要认真对照的是测试适配器。模块里保存了四个 Linear，因此要加载四组测试给定的权重：",[847,2190,2191,2201],{},[850,2192,2193],{},[853,2194,2195,2198],{},[856,2196,2197],{},"模块内部参数名",[856,2199,2200],{},"测试传入的权重",[867,2202,2203,2215,2227,2239],{},[853,2204,2205,2210],{},[872,2206,2207],{},[22,2208,2209],{},"q_proj.weight",[872,2211,2212],{},[22,2213,2214],{},"q_proj_weight",[853,2216,2217,2222],{},[872,2218,2219],{},[22,2220,2221],{},"k_proj.weight",[872,2223,2224],{},[22,2225,2226],{},"k_proj_weight",[853,2228,2229,2234],{},[872,2230,2231],{},[22,2232,2233],{},"v_proj.weight",[872,2235,2236],{},[22,2237,2238],{},"v_proj_weight",[853,2240,2241,2246],{},[872,2242,2243],{},[22,2244,2245],{},"output_proj.weight",[872,2247,2248],{},[22,2249,2250],{},"o_proj_weight",[10,2252,2253,2254,2257,2258,2260],{},"最后一项的名字并不完全相同。",[22,2255,2256],{},"load_state_dict"," 的键要对应模块内部的成员名，所以这里是 ",[22,2259,2245],{},"。固定测试权重加载进去之后，才能用同样的输入与参考输出比较，避免随机初始化干扰结果。",[38,2262,2264],{"id":2263},"测试通过的范围以及接下来要做什么","测试通过的范围，以及接下来要做什么",[10,2266,2267,2268,2271],{},"原学习过程运行了官方 ",[22,2269,2270],{},"tests\u002Ftest_model.py::test_multihead_self_attention","，结果为：",[927,2273,2276],{"className":2274,"code":2275,"language":996,"meta":932},[994],"test_multihead_self_attention PASSED\n1 passed in 0.09s\n",[22,2277,2275],{"__ignoreMap":932},[10,2279,2280],{},"这是在上一篇 SDPA 测试之后，新增的一项模块测试。它把投影、拆头、因果 mask、注意力计算、合头与输出投影接在一起，确认当前不含 RoPE 的多头自注意力在对应官方用例中与参考结果一致。",[10,2282,2283,2284,443,2288,443,2292,2296,2297,1729],{},"下一步才是 RoPE。到这次记录结束时，只介绍了根据位置旋转 Q、K，以及把每个头的特征两两配对的概念，尚未完成实现，也没有 RoPE 测试通过的记录。后续完整的 RoPE、Pre-Norm Transformer LM、Byte-Level BPE 与 AdamW 训练闭环已分别记入 ",[13,2285,2287],{"href":2286},"\u002Fposts\u002Fcs336-rope-rotary-position-embedding\u002F","学习记录 05：RoPE 旋转位置编码",[13,2289,2291],{"href":2290},"\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm\u002F","学习记录 06：Pre-Norm Transformer LM",[13,2293,2295],{"href":2294},"\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding\u002F","学习记录 07：Byte-Level BPE 分词器"," 与 ",[13,2298,2300],{"href":2299},"\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop\u002F","学习记录 08：Cross-Entropy、AdamW 与训练闭环",[38,2302,2303],{"id":2303},"发布时的复核",[10,2305,2306,2307,2313,2314,2317,2318,2323],{},"发布前对",[13,2308,2312],{"href":2309,"rel":2310},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002F31363223498fafd1528c550a51efbe0aed231736",[2311],"nofollow","独立代码快照","重新运行了已完成组件对应的 9 项官方测试，结果为 ",[22,2315,2316],{},"9 passed","。这次新增无 RoPE 多头自注意力，累计覆盖 8 类组件。完整清单与运行环境见",[13,2319,2322],{"href":2320,"rel":2321},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Fblob\u002F31363223498fafd1528c550a51efbe0aed231736\u002Fdocs\u002Fprogress.md",[2311],"进度记录","。前三篇仍保留最初 8 项测试对应的历史代码版本。",[10,2325,2326],{},[2327,2328,2329,2330,2335],"em",{},"封面使用 ",[13,2331,2334],{"href":2332,"rel":2333},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Faigc-image-generator",[2311],"aigc-image-generator"," 辅助生成，属于概念插画；轨道数量、块数与连线不代表精确计算，张量形状和运算关系以正文为准。",[38,2337,2338],{"id":2338},"参考资料",[1820,2340,2341,2348,2355,2362,2369,2376,2383,2390,2397,2404],{},[1823,2342,2343],{},[13,2344,2347],{"href":2345,"rel":2346},"https:\u002F\u002Fcs336.stanford.edu\u002F",[2311],"Stanford CS336 官方课程页面",[1823,2349,2350],{},[13,2351,2354],{"href":2352,"rel":2353},"https:\u002F\u002Fgithub.com\u002Fstanford-cs336\u002Fassignment1-basics",[2311],"Assignment 1 官方仓库与讲义",[1823,2356,2357],{},[13,2358,2361],{"href":2359,"rel":2360},"https:\u002F\u002Fgithub.com\u002Fstanford-cs336\u002Fassignment1-basics\u002Fblob\u002Fmain\u002Ftests\u002Ftest_model.py",[2311],"Assignment 1 官方模型测试",[1823,2363,2364],{},[13,2365,2368],{"href":2366,"rel":2367},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fsize.html",[2311],"PyTorch：torch.Size",[1823,2370,2371],{},[13,2372,2375],{"href":2373,"rel":2374},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fgenerated\u002Ftorch.reshape.html",[2311],"PyTorch：torch.reshape",[1823,2377,2378],{},[13,2379,2382],{"href":2380,"rel":2381},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fgenerated\u002Ftorch.transpose.html",[2311],"PyTorch：torch.transpose",[1823,2384,2385],{},[13,2386,2389],{"href":2387,"rel":2388},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fgenerated\u002Ftorch.Tensor.view.html",[2311],"PyTorch：Tensor.view",[1823,2391,2392],{},[13,2393,2396],{"href":2394,"rel":2395},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Ftensor_view.html",[2311],"PyTorch：Tensor Views",[1823,2398,2399],{},[13,2400,2403],{"href":2401,"rel":2402},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fnotes\u002Fbroadcasting.html",[2311],"PyTorch：广播语义",[1823,2405,2406],{},[13,2407,2410],{"href":2408,"rel":2409},"https:\u002F\u002Fdocs.pytorch.org\u002Fdocs\u002Fstable\u002Fgenerated\u002Ftorch.tril.html",[2311],"PyTorch：torch.tril",[2412,2413,2414],"style",{},"html pre.shiki code .sWURj, html code.shiki .sWURj{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#EEF0F9}html pre.shiki code .sf2cM, html code.shiki .sf2cM{--shiki-default:#F47067;--shiki-light:#D73A49;--shiki-cyber:#EEF0F9}html pre.shiki code .sgENW, html code.shiki .sgENW{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#FFD493}html pre.shiki code .sRGqv, html code.shiki .sRGqv{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#ACAFFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html.cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html pre.shiki code .sq3c-, html code.shiki .sq3c-{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#00DAEF}",{"title":932,"searchDepth":1766,"depth":2416,"links":2417},3,[2418,2419,2420,2421,2422,2423,2424,2425,2426,2427,2428,2429,2430],{"id":40,"depth":1766,"text":41},{"id":466,"depth":1766,"text":467},{"id":918,"depth":1766,"text":919},{"id":1013,"depth":1766,"text":1014},{"id":1104,"depth":1766,"text":1104},{"id":1630,"depth":1766,"text":1631},{"id":1718,"depth":1766,"text":1719},{"id":1803,"depth":1766,"text":1804},{"id":1855,"depth":1766,"text":1856},{"id":2166,"depth":1766,"text":2167},{"id":2263,"depth":1766,"text":2264},{"id":2303,"depth":1766,"text":2303},{"id":2338,"depth":1766,"text":2338},[2432],"学习笔记","\u002Fimages\u002Fposts\u002Fcs336-multihead-attention-shapes\u002Fcs336-multihead-attention-shapes-cover.webp","宽向量束分成多条并行特征轨道后重新合并的 AI 辅助概念插画","2026-10-19","沿着 (2, 5, 64) 的输入追踪 QKV 投影、reshape、transpose、共享因果 mask 和合头，理解每个头仍处理完整序列，以及最后为什么还需要输出 Linear。",false,"md",{"codeUrl":2309,"attachmentUrl":2440},"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-04-multihead-attention-20261019.zip",true,"\u002Fposts\u002Fcs336-multihead-attention-shapes",null,{"title":5,"description":2436},"Stanford CS336","posts\u002Fcs336-multihead-attention-shapes",[2448,2449,2450,2451,2452],"CS336","PyTorch","Transformer","Attention","张量形状","3Ysd1AbrK1xcRmeRUYfozeO3Hx2_3fKjSsPBNKrKHVw",[2455,2471,2487,2502,2516,2530,2545,2560,2574,2586,2600,2605,2619,2632,2645,2659,2676,2691,2702,2714,2729,2742,2756,2768,2782,2794,2809,2823,2837,2848,2861,2878,2893,2906,2918,2930,2943,2956],{"slug":2456,"path":2457,"title":2458,"date":2459,"tags":2460,"description":2464,"cover":2465,"coverAlt":2466,"excerpt":2467,"draft":2437,"hidden":2437,"published":2441,"readingTime":2468,"series":2445,"seriesOrder":2468,"codeUrl":2469,"attachmentUrl":2470},"cs336-cross-entropy-adamw-training-loop","\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop","Stanford CS336 学习记录 08：数值稳定交叉熵、AdamW、余弦调度与完整训练闭环","2026-10-27",[2448,2449,2461,2462,2463],"AdamW","CrossEntropy","训练闭环","用 LogSumExp 消去交叉熵中的指数溢出，手写解耦权重衰减的 AdamW 优化器、全局 L2 梯度裁剪、Warmup 余弦退火调度、memmap 数据采样、Checkpoint 序列化与 Top-p 解码，完成 CS336 Assignment 1 全部 48 项测试。","\u002Fimages\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop\u002Fcs336-cross-entropy-adamw-training-loop-cover.webp","滑动窗口采样、余弦退火曲线、双动量优化轨迹与检查点归档构成的训练闭环 AI 辅助概念插画","作为 Stanford CS336 Assignment 1 系列的收官篇，我们已经拥有了分词器（bpe.py、tokenizer.py）和完整的语言模型架构（transformer.py）。要让模型在语料上真正训练并生成文本，还需",8,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fb5c93ad0cc73178b2b2a62e518a0900da47a0fb7","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-08-adamw-training-loop-20261027.zip",{"slug":2472,"path":2473,"title":2474,"date":2475,"tags":2476,"description":2483,"cover":2484,"coverAlt":2485,"excerpt":2486,"draft":2437,"hidden":2437,"published":2441,"readingTime":2468},"editcache-video-dit-inference-acceleration","\u002Fposts\u002Feditcache-video-dit-inference-acceleration","EditCache：视频 Diffusion Transformer (DiT) 局部编辑的跨步特征复用与区域重算加速","2026-10-25",[2477,2478,2479,2480,2481,2482],"扩散模型","视频生成","AI Infra","推理加速","GPU","论文解读","拆解视频扩散模型（Wan2.1-VACE）局部编辑中的三重算力冗余，通过可重入 Latent、跨步 Block 隐状态缓存复用、带 Halo 边界的局部区域重算与自适应推理起点，在单卡 H100 上实现 2.46× 模型计算加速与 1.49× 严格同步端到端加速。","\u002Fimages\u002Fposts\u002Feditcache-video-dit-inference-acceleration\u002Feditcache-dit-cover-20261025.webp","视频扩散 Transformer 时空三维体素网格中冻结背景缓存与暖橙色局部编辑重算通道的等距技术插画","在基于 Diffusion Transformer（DiT）的视频编辑与局部重绘（Video Inpainting \u002F Local Editing）任务中，用户的典型工作流并不是一次性生成最终成片，而是针对同一段源视频反复调整掩码区",{"slug":2488,"path":2489,"title":2490,"date":2475,"tags":2491,"description":2496,"cover":2497,"coverAlt":2498,"excerpt":2499,"draft":2437,"hidden":2437,"published":2441,"readingTime":2468,"series":2445,"seriesOrder":2500,"codeUrl":2469,"attachmentUrl":2501},"cs336-bpe-tokenizer-training-encoding","\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding","Stanford CS336 学习记录 07：Byte-Level BPE 分词器训练、倒排索引加速与流式编码",[2448,2492,2493,2494,2495],"Tokenizer","BPE","Tiktoken","算法优化","实现 GPT-2 正则预分词、特殊 token 硬边界切分、基于 pair-to-word 倒排索引的增量 BPE 合并算法（\u003C 0.25s 通过速度基准），以及与 tiktoken 完全对齐的 Tokenizer。","\u002Fimages\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding\u002Fcs336-bpe-tokenizer-training-encoding-cover.webp","UTF-8 字节块经过正则切分闸口后沿合并树聚合成子词单元的 AI 辅助概念插画","前面六篇完成了 Transformer 语言模型本身的全部神经网络组件。要让模型真正处理自然语言文本，还需要打通入口处的 ByteLevel BPE（BytePair Encoding）分词器。这一篇记录 cs336basics\u002Fb",7,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-07-bpe-tokenizer-20261025.zip",{"slug":2503,"path":2504,"title":2505,"date":2506,"tags":2507,"description":2512,"cover":2513,"coverAlt":2514,"excerpt":2515,"draft":2437,"hidden":2437,"published":2441,"readingTime":2468},"in-situ-autoguidance-editable-scientific-illustration","\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration","从原位自校正引导（In-situ Autoguidance）到可编辑科研插图生成（AutoFigure-Edit）","2026-10-24",[2477,2508,2509,2510,2511,2482],"多模态","Agent","ICML","ACL","串联两项生成式模型与多模态智能体研究：如何通过推理期局部随机微扰在单模型内构造免训练扩散自引导向量（ICML 2025 Workshop 第一作者，arXiv:2510.17136），以及如何通过五阶段视觉-符号混合流水线生成原生可编辑 SVG 科研插图（ACL 2026 Demo，FigureBench 83.0% 胜率）。","\u002Fimages\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration\u002Fautoguidance-autofigure-cover-20261024.webp","左侧扩散流形自校正轨迹矢量场与右侧五阶段可编辑 SVG 科研插图分层网格相衔接的等距技术插画","生成式视觉模型正在经历从“生成高保真像素（Pixel Synthesis）”向“可控结构化生成（Controllable & Editable Generation）”的演进。在这一进程中，我作为第一作者或核心贡献者参与了两项分别处",{"slug":2517,"path":2518,"title":2519,"date":2520,"tags":2521,"description":2524,"cover":2525,"coverAlt":2526,"excerpt":2527,"draft":2437,"hidden":2437,"published":2441,"readingTime":2468,"series":2445,"seriesOrder":2528,"codeUrl":2469,"attachmentUrl":2529},"cs336-pre-norm-transformer-block-lm","\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm","Stanford CS336 学习记录 06：Pre-Norm Transformer Block 与完整语言模型装配","2026-10-23",[2448,2449,2450,2522,2523],"Pre-Norm","FLOPs","将 RMSNorm、带 RoPE 的因果多头自注意力和 SwiGLU 组装成 Pre-Norm Transformer Block，搭建完整 TransformerLM，并核算 GPT-2 XL 的参数量与前向 FLOPs。","\u002Fimages\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm\u002Fcs336-pre-norm-transformer-block-lm-cover.webp","干净残差流主干穿过多个 Pre-Norm 子层并汇聚到词表投影层的 AI 辅助概念插画","前五篇把 Linear、Embedding、RMSNorm、SwiGLU、RotaryPositionalEmbedding 和因果 MultiHeadSelfAttention 全部单独打通了。这一篇进入 Assignment 1",6,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-06-transformer-block-lm-20261023.zip",{"slug":2531,"path":2532,"title":2533,"date":2520,"tags":2534,"description":2540,"cover":2541,"coverAlt":2542,"excerpt":2543,"draft":2437,"hidden":2437,"published":2441,"readingTime":2544},"xgrammar-concurrent-cache-generation-fix","\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix","XGrammar 结构化解码缓存剖析：Unlimited 模式并发 Clear 字节记账竞态与代际修复（PR #1011）",[2479,2535,2536,2537,2538,2539],"C++","XGrammar","结构化解码","并发编程","源码分析","拆解 XGrammar GrammarCompiler 的双层缓存架构与 ThreadSafeLRUCache 锁分离实现，定位 Unlimited 模式下 Clear() 字节残留与在途编译（In-flight Computation）跨代污染根因，给出代际号 + 读写锁原子修复与零 sleep 确定性并发 GoogleTest 验证。","\u002Fimages\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix\u002Fxgrammar-cache-fix-cover-20261022.webp","双层磨砂玻璃流道经暖橙色代际闸门过滤后汇入晶格缓存矩阵的等距技术插画","在大语言模型服务（如 SGLang、vLLM）的结构化输出场景中，JSON Schema 或 EBNF 语法需要先编译为词表级掩码缓存（CompiledGrammar），才能在每个 Decode 步以微秒级开销生成合法 token ",14,{"slug":2546,"path":2547,"title":2548,"date":2549,"tags":2550,"description":2556,"cover":2557,"coverAlt":2558,"excerpt":2559,"draft":2437,"hidden":2437,"published":2441,"readingTime":2544},"multi-constraint-agent-routing-dag-eval","\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval","多约束生产级 Agent 系统实践：两阶段工具路由、DAG 局部重规划与轨迹级评测闭环","2026-10-22",[2509,2551,2552,2553,2554,2555],"LLM","工具调用","上下文工程","DPO","强化学习","围绕跨境酒旅等强耦合多约束业务场景，拆解向量 Top-K 召回 + LLM 精排 + 跨语言实体链接与 Schema 门控（96%+ 调用合法率）、基于状态机的 DAG 局部子图重规划、Observation 冗余裁剪（节省 40% Token），以及从线上失败恢复轨迹构造 SFT\u002FDPO 偏好对的评测与训练闭环。","\u002Fimages\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval\u002Fagent-dag-routing-cover-20261020.webp","候选模块经漏斗滤网进入双分支有向无环图轨道并在受阻节点触发暖橙色局部旁路重连的等距技术插画","在玩具级 Demo 里，几个工具加上一段 ReAct 循环就能跑通大多数单步问答。但当 Agent 进入跨境酒旅、复杂交易或研发沙箱等真实业务生产环境时，单靠线性 ReAct 循环很快会撞上四堵墙： 1. 工具空间与参数耦合爆炸：候",{"slug":2561,"path":2562,"title":2563,"date":2564,"tags":2565,"description":2569,"cover":2570,"coverAlt":2571,"excerpt":2572,"draft":2437,"hidden":2437,"published":2441,"readingTime":2573},"llm-mrd-distillation-auditable-review-agents","\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents","多视角推理蒸馏（LLM-MRD）与可审计生成式评审系统（DeepReviewer 2.0）深度拆解","2026-10-21",[2566,2508,2567,2509,2568,2482],"知识蒸馏","Qwen2.5-VL","Post-training","对比大模型落地的两条互补路径：如何用 Qwen2.5-VL 离线多视角分析结合 MLP 残差校准与温度缩放 KL 蒸馏，将推理能力压入 358M 学生模型实现 17.2 ms 独立推理（LLM-MRD，DASFAA 2026 Oral）；以及如何通过主张-证据-风险台账、PASA 可比性门控与导出预算校验构建可审计的同行评审 Agent（DeepReviewer 2.0）。","\u002Fimages\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents\u002Fmrd-review-distill-cover-20261016.webp","三块半透明几何棱镜发出平行光束穿过中央校准环汇聚至右侧高密度晶体模块的等距技术插画","把大语言模型或视觉语言模型（VLM）的深度推理能力用到实际系统中，通常会面临一个两难选择： 如果场景受限于毫秒级在线延迟（如高吞吐多模态内容审核与虚假信息检测）：直接在线调用 7B～72B 的多模态大模型做长链条思维链（CoT）生成",18,{"slug":2575,"path":2576,"title":2577,"date":2564,"tags":2578,"description":2580,"cover":2581,"coverAlt":2582,"excerpt":2583,"draft":2437,"hidden":2437,"published":2441,"readingTime":2500,"series":2445,"seriesOrder":2584,"codeUrl":2469,"attachmentUrl":2585},"cs336-rope-rotary-position-embedding","\u002Fposts\u002Fcs336-rope-rotary-position-embedding","Stanford CS336 学习记录 05：RoPE 旋转位置编码与带位置的多头注意力",[2448,2449,2450,2579,2451],"RoPE","从二维平面正交旋转推导 RoPE 的奇偶特征配对公式，理解为什么内积只依赖相对位置差 (i - j)，并用 register_buffer 预计算 cos\u002Fsin 缓存接入因果多头自注意力。","\u002Fimages\u002Fposts\u002Fcs336-rope-rotary-position-embedding\u002Fcs336-rope-rotary-position-embedding-cover.webp","二维特征平面中的旋转向量对与预计算余弦正弦网格的 AI 辅助概念插画","上一篇完成了不含位置编码的因果多头自注意力。如果只看那一套矩阵乘法，把输入序列前两个 token 交换顺序（并在因果 mask 允许的范围内看同一个后缀 query 对它们的点积），点积本身并不知道哪个 key 离当前 query ",5,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-05-rope-mha-20261021.zip",{"slug":2587,"path":2588,"title":2589,"date":2435,"tags":2590,"description":2596,"cover":2597,"coverAlt":2598,"excerpt":2599,"draft":2437,"hidden":2437,"published":2441,"readingTime":2468},"llm-gateway-semantic-cache-dynamic-batching","\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching","高并发 LLM 推理网关架构实践：两级精确\u002F语义缓存、50ms 有界动态批处理与尾部延迟治理",[2479,2591,2592,2593,2594,2595],"LLM 推理","分布式系统","Redis","FastAPI","后端","深度拆解生产级大模型推理网关 LLM-Gateway 的核心系统设计：基于 SHA-256 与 Redis 向量余弦相似度的两级精确\u002F语义缓存、50ms 有界等待动态批处理调度器（Dynamic Batching）、令牌桶多租户限流与 Prometheus 全链路可观测性，将命中请求 P99 延迟从 5,307 ms 压降至 17 ms。","\u002Fimages\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching\u002Fllm-gateway-cover-20261019.webp","高并发请求流依次穿过精确哈希滤网、向量球面语义缓存层与 50ms 动态批处理时间窗拼装器的等距系统架构插画","在将大语言模型接入高并发业务系统时，如果让上游微服务直接裸连底层推理引擎（vLLM \u002F SGLang 或外部模型 API），系统很快会暴露出三个典型的生产痛点： 1. 高频重复与近义查询消耗巨量算力：大量用户请求在语义上高度等价（例",{"slug":2601,"path":2442,"title":5,"date":2435,"tags":2602,"description":2436,"cover":2433,"coverAlt":2434,"excerpt":2603,"draft":2437,"hidden":2437,"published":2441,"readingTime":2544,"series":2445,"seriesOrder":2604,"codeUrl":2309,"attachmentUrl":2440},"cs336-multihead-attention-shapes",[2448,2449,2450,2451,2452],"上一篇完成了缩放点积注意力。那个函数接收已经准备好的 Q、K、V，计算匹配分数，应用 mask，再用权重汇总 V。这一篇继续把它放进一个带参数的多头自注意力模块。 这次最需要看清的是“拆头”。代码里只是几行 reshape 和 tr",4,{"slug":2606,"path":2607,"title":2608,"date":2609,"tags":2610,"description":2614,"cover":2615,"coverAlt":2616,"excerpt":2617,"draft":2437,"hidden":2437,"published":2441,"readingTime":2618},"qwen25-sft-dpo-alignment-tax-ablation","\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation","大模型后训练偏好对齐实录：Qwen2.5 SFT 多维消融、动态参考 DPO 与对齐税（Alignment Tax）量化分析","2026-10-18",[2568,2554,2611,2612,2555,2613],"SFT","LoRA","Qwen2.5","基于 Qwen2.5 完整复现 Instruct SFT → Bradley-Terry DPO 两阶段后训练管线：通过 3 随机种子消融 LoRA Rank、学习率与数据规模（GSM8K +12.73pp），实现免双倍权重的动态 Adapter 切换 DPO 训练，并定量揭示偏好对齐对确定性数学推理的对齐税效应。","\u002Fimages\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation\u002Fqwen25-sft-dpo-cover-20261018.webp","左侧监督微调轨迹流经中央 Bradley-Terry 偏好分流天平分离出正负样本通道并连接右侧对齐税监控仪表盘的等距技术插画","在大语言模型后训练（PostTraining）管线中，监督微调（SFT）负责教会模型任务格式与领域推理模式，而直接偏好优化（DPO, Direct Preference Optimization）则利用成对偏好数据 $(x, yw,",9,{"slug":2620,"path":2621,"title":2622,"date":2609,"tags":2623,"description":2625,"cover":2626,"coverAlt":2627,"excerpt":2628,"draft":2437,"hidden":2437,"published":2441,"readingTime":2629,"series":2445,"seriesOrder":2416,"codeUrl":2630,"attachmentUrl":2631},"cs336-softmax-attention-mask","\u002Fposts\u002Fcs336-softmax-attention-mask","Stanford CS336 学习记录 03：Softmax 的数值稳定与 Attention Mask",[2448,2449,2450,2624,2451],"Softmax","从减去最大值为何不改变 Softmax 开始，理清函数与有参数层的区别，再沿着 Q、K、V 的形状理解缩放点积注意力，以及 mask 为什么必须填负无穷。","\u002Fimages\u002Fposts\u002Fcs336-softmax-attention-mask\u002Fcs336-softmax-attention-mask-cover.webp","注意力分数、权重与因果遮罩的 AI 辅助概念插画","上一篇写到 SwiGLU，接下来进入 Attention。真正开始相乘之前，先做了 Softmax：它把一组分数变成总和为 1 的权重，后面用这些权重汇总不同 token 提供的信息。 这段学习里，有两个很具体的问题值得记下来。一个",11,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fa39eb684710817f7f389626423fcb26ca0f6d629","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-03-softmax-sdpa-20261018.zip",{"slug":2633,"path":2634,"title":2635,"date":2636,"tags":2637,"description":2640,"cover":2641,"coverAlt":2642,"excerpt":2643,"draft":2437,"hidden":2437,"published":2441,"readingTime":2618,"series":2445,"seriesOrder":1766,"codeUrl":2630,"attachmentUrl":2644},"cs336-rmsnorm-swiglu","\u002Fposts\u002Fcs336-rmsnorm-swiglu","Stanford CS336 学习记录 02：RMSNorm 的广播与 SwiGLU 的门控","2026-10-16",[2448,2449,2450,2638,2639],"RMSNorm","SwiGLU","用一个 token 向量理解均方根归一化，拆开 dim=-1 与 keepdim=True 的作用，再沿着两条分支读懂 SiLU、逐元素乘法和 SwiGLU。","\u002Fimages\u002Fposts\u002Fcs336-rmsnorm-swiglu\u002Fcs336-rmsnorm-swiglu-cover.webp","特征尺度整理和双分支门控的 AI 辅助概念插画","上一篇把 Linear 和 Embedding 的参数形状理顺了。继续往下，最先遇到的是 RMSNorm：公式不算长，但把它落到一批 token 上时，dim=1、keepdim=True 和广播必须一起理解。 然后是 SiLU 和","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-02-rmsnorm-swiglu-20261016.zip",{"slug":2646,"path":2647,"title":2648,"date":2649,"tags":2650,"description":2653,"cover":2654,"coverAlt":2655,"excerpt":2656,"draft":2437,"hidden":2437,"published":2441,"readingTime":2657,"series":2445,"seriesOrder":938,"codeUrl":2630,"attachmentUrl":2658},"cs336-linear-embedding-shapes","\u002Fposts\u002Fcs336-linear-embedding-shapes","Stanford CS336 学习记录 01：从 Linear、Embedding 搞清张量形状","2026-10-14",[2448,2449,2450,2651,2652],"Linear","Embedding","从 PyTorch 类、参数与 forward 的关系开始，理解 Linear 为什么保存两行三列的权重、计算时为什么要转置，再区分 Embedding 查表与线性变换。","\u002Fimages\u002Fposts\u002Fcs336-linear-embedding-shapes\u002Fcs336-linear-embedding-shapes-cover.webp","token 方块、向量表与矩阵变换的 AI 辅助概念插画","开始沿着 Stanford CS336 的 Assignment 1 一步步实现。BPE 之前已经接触过，这轮先跳过分词器，从 Transformer 的基础层开始。这里记录的是当时写代码时真正卡住的问题，以及对应测试能确认到的范围",10,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-01-linear-embedding-20261014.zip",{"slug":2660,"path":2661,"title":2662,"date":2663,"tags":2664,"description":2669,"cover":2670,"coverAlt":2671,"excerpt":2672,"draft":2437,"hidden":2437,"published":2441,"readingTime":2673,"series":2665,"seriesOrder":2544,"codeUrl":2674,"attachmentUrl":2675},"cmu-15-779-kv-cache-granularity-study","\u002Fposts\u002Fcmu-15-779-kv-cache-granularity-study","CMU 15-779 学习记录 14：KV 页大小、共享边界与 CPU 实现成本","2026-10-12",[2665,2666,2667,2668],"CMU 15-779","KV cache","PagedAttention","系统实验","用 210 组 CPU 配置比较连续与分页 KV cache，观察 31\u002F32-token 共享前缀的页边界，分别记录活跃槽位、池容量、复制计数和完整调用耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling\u002Fcmu-serving-cover-20261019.webp","浅色手绘分页存储抽屉与共享前缀卡片，旁边连接不同长度的请求队列","个人课程项目中的 CPU 实验记录，延续记录 11 的分页 KV 实现。已完成 210 组配置与 1470 次计时，输出均通过连续存储参考对照；本篇不代表正式作业提交、整门课程或项目全部交付完成。 同样是共享 31 个 token，",12,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F200faf569eccf94fb125e65110bb5d14094d454a","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fkv-cache-granularity-20261019.zip",{"slug":2677,"path":2678,"title":2679,"date":2680,"tags":2681,"description":2684,"cover":2685,"coverAlt":2686,"excerpt":2687,"draft":2437,"hidden":2437,"published":2441,"readingTime":2584,"series":2665,"seriesOrder":2688,"codeUrl":2689,"attachmentUrl":2690},"cmu-15-779-moe-routing-capacity","\u002Fposts\u002Fcmu-15-779-moe-routing-capacity","CMU 15-779 学习记录 13：MoE 路由、稀疏分发与容量溢出","2026-10-09",[2665,2682,2683],"MoE","Routing","用 CPU top-2 MoE 检查稀疏分发的输出与梯度，统计 expert-token 行数，并保留容量丢弃会改变输出的反例。","\u002Fimages\u002Fposts\u002Fcmu-15-779-moe-routing-capacity\u002Fcmu-moe-cover-20261019.webp","浅色手绘分拣器将小珠分配到四个专家工作屋，队列呈现负载不均","第 18 讲的个人配套实验。36 号脚本串行运行 CPU 专家，检查 topk 路由后的分发与合并；没有训练 router，也没有执行 GPU grouped GEMM 或跨设备 alltoall。 上一节记录低秩适配，这次转向每个",13,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fc4851e0cedadf13af7c421867e83c8ff57244fa0","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-parallel-serving-lora-moe-20261019.zip",{"slug":2692,"path":2693,"title":2694,"date":2695,"tags":2696,"description":2698,"cover":2699,"coverAlt":2700,"excerpt":2701,"draft":2437,"hidden":2437,"published":2441,"readingTime":2584,"series":2665,"seriesOrder":2673,"codeUrl":2689,"attachmentUrl":2690},"cmu-15-779-lora-uniform-quantization","\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization","CMU 15-779 学习记录 12：LoRA 首步梯度、权重合并与均匀 4-bit 打包","2026-10-06",[2665,2612,2697],"Quantization","用一次 CPU SGD 更新检查 LoRA 初始化、冻结基座与合并等价，再分别记录均匀 4-bit 存储字节数和量化误差。","\u002Fimages\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization\u002Fcmu-lora-cover-20261019.webp","浅色手绘的大计算板、两块细长适配板和成对打包的小方块","第 17 讲的个人配套实验，35 号脚本实际执行一次 CPU LoRA 更新，并实现均匀 4bit 打包。实验检查机制与数值误差，不评估微调任务效果，也没有实现完整 QLoRA。 这次把两个问题分开记录：低秩分支是否能合并进原权重，",{"slug":2703,"path":2704,"title":2705,"date":2706,"tags":2707,"description":2711,"cover":2670,"coverAlt":2712,"excerpt":2713,"draft":2437,"hidden":2437,"published":2441,"readingTime":2500,"series":2665,"seriesOrder":2629,"codeUrl":2689,"attachmentUrl":2690},"cmu-15-779-kv-serving-speculative-sampling","\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling","CMU 15-779 学习记录 11：分页 KV、连续批处理与推测采样","2026-10-03",[2665,2708,2709,2710],"LLM Serving","KV Cache","Speculative Decoding","在 CPU 上验证 KV 页共享和写时复制，用单位成本模型检查连续批处理，再用条件概率小模型核对推测采样的接受与补采样。","浅色手绘的共享卡片抽屉与不断补入小珠子的请求托盘","第 15、16 讲的个人配套实验。33 号实现 CPU 分页 KV 存储，37 号模拟请求准入，34 号运行小概率模型的推测采样。它们分别验证存储、调度和分布问题，没有测量真实 LLM 服务吞吐或 GPU 延迟。 记录 08已经对照",{"slug":2715,"path":2716,"title":2717,"date":2718,"tags":2719,"description":2724,"cover":2725,"coverAlt":2726,"excerpt":2727,"draft":2437,"hidden":2437,"published":2441,"readingTime":2728},"qwen3-mid-post-train-mlx","\u002Fposts\u002Fqwen3-mid-post-train-mlx","在 M2 Pro 上完成 Qwen3 两阶段训练：Mid Train、SFT 与可复核评估","2026-09-30 11:19:10",[2551,2720,2721,2612,2722,2611,2723],"Qwen3","MLX","持续预训练","Apple Silicon","用 Qwen3 0.6B Base 和 MLX 完成领域持续训练与监督微调，记录数据划分、聊天模板、损失掩码、数值稳定性修复和独立测试结果。","\u002Fimages\u002Fposts\u002Fqwen3-mid-post-train-mlx\u002Fqwen3-mlx-cover-20261018.webp","Apple Silicon 统一内存芯片底座支撑两阶段持续预训练与带精确 Loss 掩码滤网的监督微调桥接插画","在一台 M2 Pro、16 GB 内存的 Mac 上，能否把一个 Base 语言模型接续训练成输出 JSON 的运维事件分级助手？这次项目完成了领域持续训练、监督微调、检查点选择、重新加载和独立测试，并保留了权重、日志和数据哈希。 ",16,{"slug":2730,"path":2731,"title":2732,"date":2733,"tags":2734,"description":2738,"cover":2739,"coverAlt":2740,"excerpt":2741,"draft":2437,"hidden":2437,"published":2441,"readingTime":2528,"series":2665,"seriesOrder":2657,"codeUrl":2689,"attachmentUrl":2690},"cmu-15-779-cpu-parallelism-pipeline","\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline","CMU 15-779 学习记录 10：加权梯度、状态分片与同步流水线","2026-09-30",[2665,2735,2736,2737],"Parallel Training","Gloo","Pipeline","用双 CPU 进程 Gloo 检查不等数据分片的梯度，推导持久状态字节数，再用依赖模型比较 GPipe 与同步 1F1B 的激活存活。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline\u002Fcmu-parallel-cover-20261019.webp","浅色手绘的四阶段桌面流水线与按数量汇总的托盘","第 12 至 14 讲的个人配套实验，脚本编号 31、32 不对应官方作业。Gloo 实际运行两个 CPU 进程；张量切分、状态字节数和流水线分别按下文的数学或成本模型解释。真实 GPU 实测按学习安排跳过。 上一节检查单个计算图的",{"slug":2743,"path":2744,"title":2745,"date":2746,"tags":2747,"description":2750,"cover":2751,"coverAlt":2752,"excerpt":2753,"draft":2437,"hidden":2437,"published":2441,"readingTime":2500,"series":2665,"seriesOrder":2618,"codeUrl":2754,"attachmentUrl":2755},"cmu-15-779-cpu-compiler-optimizations","\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations","CMU 15-779 学习记录 09：CPU 编译优化、合法重写与候选搜索","2026-09-27",[2665,2748,2749,2449],"ML Compiler","TorchInductor","用 CPU 实验验证 BatchNorm 折叠、RMSNorm 重排与非线性反例，测量分块候选，并运行 TorchInductor，分别记录首调用与预热后的时间。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations\u002Fcmu-cpu-compiler-cover-20261019.webp","淡紫计算积木经过薄荷绿折叠模具变成浅蓝积木，前方摆放四种候选托盘的浅色手绘插画","编译章节的个人配套实验，编号 30 不对应官方作业。图重写、分块候选和 TorchInductor 均在 CPU 上运行；真实 GPU 实验按学习安排跳过。 上一节记录了分块 Attention 和 KV cache。这次转到编译优","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F9261bea6aedf31cdccbbab2541ca693fc97b3743","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-compiler-optimizations-20261019.zip",{"slug":2757,"path":2758,"title":2759,"date":2760,"tags":2761,"description":2763,"cover":2764,"coverAlt":2765,"excerpt":2766,"draft":2437,"hidden":2437,"published":2441,"readingTime":2673,"series":2665,"seriesOrder":2468,"codeUrl":2674,"attachmentUrl":2767},"cmu-15-779-attention-online-softmax","\u002Fposts\u002Fcmu-15-779-attention-online-softmax","CMU 15-779 学习记录 08：分块 Attention、在线 Softmax 与 KV cache","2026-09-24",[2665,2451,2624,2762,2449],"FlashAttention","在 CPU 上对照完整 Attention、在线 Softmax 分块实现与 PyTorch SDPA，检查因果掩码、浮点误差和分数块大小，理解 FlashAttention 的一部分算法基础。","\u002Fimages\u002Fposts\u002Fcmu-15-779-attention-online-softmax\u002Fcmu-online-attention-cover-20261016.webp","浅紫色卡片分批进入绿色计算托盘，旁边三个状态容器保存累积量的浅色手绘插画","第 5 讲 Transformers & Attention 的个人配套实验。28\u002F29 号是 CPU 单头前向与 KV cache；后续补入 38 号 CUDA 多头合成张量的显式 SDPA MATH\u002FFLASH 对照。没有手写 ","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcuda-triton-sdpa-20261019.zip",{"slug":2769,"path":2770,"title":2771,"date":2772,"tags":2773,"description":2778,"cover":2779,"coverAlt":2780,"excerpt":2781,"draft":2437,"hidden":2437,"published":2441,"readingTime":2657,"series":2665,"seriesOrder":2500,"codeUrl":2674,"attachmentUrl":2767},"cmu-15-779-matmul-tiling-tree-reduction","\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction","CMU 15-779 学习记录 07：矩阵分块、数据复用与树形归约","2026-09-21",[2665,2774,2775,2776,2777],"CUDA","矩阵乘法","数据复用","归约","用 CPU 模型统计矩阵分块的输入读取次数，再比较串行求和和树形归约的依赖深度，记录补零、同步与浮点加法顺序的边界。","\u002Fimages\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction\u002Fcmu-matmul-reduction-cover-20261016.webp","矩阵积木在共享托盘中复用，旁边的小珠沿树形管道逐层合并的浅色手绘插画","GPU Architecture & CUDA Programming 的个人配套实验。首次记录用 CPU 模型检查数据复用与依赖深度；后续补入单卡 GPU 原生 CUDA 矩阵验证，以及 Triton 两阶段归约和矩阵乘法实测。 ",{"slug":2783,"path":2784,"title":2785,"date":2786,"tags":2787,"description":2790,"cover":2791,"coverAlt":2792,"excerpt":2793,"draft":2437,"hidden":2437,"published":2441,"readingTime":2657,"series":2665,"seriesOrder":2528,"codeUrl":2674,"attachmentUrl":2767},"cmu-15-779-cuda-thread-indexing","\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing","CMU 15-779 学习记录 06：CUDA 线程索引与边界检查","2026-09-18",[2665,2774,2481,2788,2789],"线程索引","边界检查","用 CPU 演示 CUDA 的 block、thread 与全局下标：10 个元素如何分给 3 个线程块，为什么要跳过越界线程，以及这些验证能说明什么。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing\u002Fcmu-cuda-indexing-cover-20261015.webp","分组的浅紫色计算方块与末端边界挡板组成线程分配教具的浅色插画","GPU Architecture & CUDA Programming 的个人配套练习。首次记录为 CPU 索引演示；后续在单卡 GPU 环境补入原生 CUDA 编译、运行与 memcheck，以及 Triton 向量加法实测。 上",{"slug":2795,"path":2796,"title":2797,"date":2798,"tags":2799,"description":2803,"cover":2804,"coverAlt":2805,"excerpt":2806,"draft":2437,"hidden":2437,"published":2441,"readingTime":2657,"series":2665,"seriesOrder":2584,"codeUrl":2807,"attachmentUrl":2808},"cmu-15-779-profiler-operator-batching","\u002Fposts\u002Fcmu-15-779-profiler-operator-batching","CMU 15-779 学习记录 05：Profiler、算子调用与批量推理","2026-09-15",[2665,2449,2800,2801,2802],"Profiler","批量推理","性能分析","对照 ML Systems 基础，用 Profiler 拆开两层网络的执行过程，再固定 1000 个样本扫描 batch size，观察算子调用次数、尾批和 CPU 耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-profiler-operator-batching\u002Fcmu-pytorch-profiling-cover-20261015.webp","放大镜观察线性层与 ReLU 模块，旁边面板呈现执行片段的浅色技术教具插画","本篇对照课程官网的 Introduction to ML Systems 整理，是个人配套 CPU 实验，不是课程官方作业。脚本编号是练习顺序，不对应课程讲次。 前几篇一直在补 PyTorch 基础。到了这一步，我在学习聊天里问：现","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fcecf5110d8a95161bb5763eadfca94cd7a0cc50f","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-profiler-batching-20261015.zip",{"slug":2810,"path":2811,"title":2812,"date":2813,"tags":2814,"description":2817,"cover":2818,"coverAlt":2819,"excerpt":2820,"draft":2437,"hidden":2437,"published":2441,"readingTime":2618,"series":2665,"seriesOrder":2604,"codeUrl":2821,"attachmentUrl":2822},"cmu-15-779-pytorch-linear-relu-absolute-value","\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value","CMU 15-779 学习记录 04：多输出线性层、ReLU 与绝对值网络","2026-09-12",[2665,2449,2775,2815,2816],"ReLU","神经网络","把 nn.Linear 对应到矩阵乘法，弄清多个输出怎样计算，再逐层拆开一个绝对值网络，比较手动设置参数与训练得到的结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value\u002Fcmu-pytorch-absolute-cover-20261015.webp","双分支经过折角模块后汇入紫色 V 形曲线面板的浅色技术教具插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的批量推理与矩阵乘法，我继续把 model(x) 拆开看：一层里有多个输出时，每个数是怎么算出来的？两","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F4cc63904a3a583da4a2122c246a14b43aac38528","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-linear-relu-abs-20261015.zip",{"slug":2824,"path":2825,"title":2826,"date":2827,"tags":2828,"description":2831,"cover":2832,"coverAlt":2833,"excerpt":2834,"draft":2437,"hidden":2437,"published":2441,"readingTime":2618,"series":2665,"seriesOrder":2416,"codeUrl":2835,"attachmentUrl":2836},"cmu-15-779-pytorch-dataloader-checkpoints-batching","\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching","CMU 15-779 学习记录 03：DataLoader、模型保存与批量推理","2026-09-09",[2665,2449,2829,2801,2830],"DataLoader","模型保存","用 DataLoader 分批训练平方函数，保存并加载模型参数，再比较 1000 个样本与 1 个样本的推理耗时，最后把批量计算对应到矩阵乘法。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching\u002Fcmu-pytorch-batching-cover-20261015.webp","样本卡片按批次进入计算模块，旁边配有参数存储盒和输出矩阵面板的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的平方函数实验，这一节继续练习分批训练、保存参数和独立预测。最后做了一次计时：同一个小网络，逐个处理 ","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-batching-20261015.zip",{"slug":2838,"path":2839,"title":2840,"date":2841,"tags":2842,"description":2843,"cover":2844,"coverAlt":2845,"excerpt":2846,"draft":2437,"hidden":2437,"published":2441,"readingTime":2618,"series":2665,"seriesOrder":1766,"codeUrl":2835,"attachmentUrl":2847},"cmu-15-779-pytorch-linear-module-relu","\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu","CMU 15-779 学习记录 02：nn.Linear、张量形状与 ReLU","2026-09-06",[2665,2449,2816,2815],"从带偏置的线性回归开始，弄清 nn.Linear 和 nn.Module 的分工，再用平方函数观察线性模型的限制、ReLU 的作用和测试点的选择。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu\u002Fcmu-pytorch-relu-cover-20261015.webp","输入经过两层神经网络和 ReLU 折角模块，输出曲线拟合结果的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 上一篇里，我用一个参数拟合了 $y=2x$。这一篇继续顺着往下写，先加上偏置，再把手写参数换成 nn.Linea","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-models-20261015.zip",{"slug":2849,"path":2850,"title":2851,"date":2852,"tags":2853,"description":2856,"cover":2857,"coverAlt":2858,"excerpt":2859,"draft":2437,"hidden":2437,"published":2441,"readingTime":2657,"series":2665,"seriesOrder":938,"codeUrl":2835,"attachmentUrl":2860},"cmu-15-779-pytorch-autograd-linear-regression","\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression","CMU 15-779 学习记录 01：PyTorch 自动求导与线性回归","2026-09-03",[2665,2449,2854,2855],"自动求导","线性回归","从自动求导和手动梯度下降开始，比较学习率，再用 SGD 训练线性回归。记录 w * 4 写错后停在 1.25 的原因和修正结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression\u002Fcmu-pytorch-autograd-cover-20261015.webp","梯度曲面上的小球沿路径下降，前侧面板展示散点与拟合直线的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 开始学习 CMU 15779 时，我会 Python，但还没用过 PyTorch。这次先在 Mac 上用 CPU","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-basics-20261013.zip",{"slug":2862,"path":2863,"title":2864,"date":2865,"tags":2866,"description":2872,"cover":2873,"coverAlt":2874,"excerpt":2875,"draft":2437,"hidden":2437,"published":2441,"readingTime":2688,"codeUrl":2876,"attachmentUrl":2877},"embodied-pickplace-lab-robustness","\u002Fposts\u002Fembodied-pickplace-lab-robustness","Pickplace Lab 项目复盘：抓取成功率之外的时延与失败分析","2026-08-29",[2867,2868,2869,2449,2870,2871],"具身智能","模仿学习","MuJoCo","机器人","鲁棒性","从 MuJoCo 抓取搬运闭环出发，记录 22,100 个评测回合。动作平滑在自建任务的 50 ms 延迟下有效，换到 Panda Lift 却没有支持同样的收益。","\u002Fimages\u002Fposts\u002Fembodied-pickplace-lab-robustness\u002Fpickplace-grasp-cover-20261016.webp","MuJoCo 实际渲染的笛卡尔夹爪抓取方块场景，桌面上标出放置目标","Pickplace Lab 从一个简单任务开始：让夹爪抓起方块，搬到指定位置，放下，再退开。先把仿真、示范采集、训练和闭环评测接起来，随后再问：干净条件下成功的策略，换一种执行条件，还能不能完成任务？ 最值得讨论的结果出现在指令延迟","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Ftree\u002Fv0.1.0","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Freleases\u002Fdownload\u002Fv0.1.0\u002Fpickplace-lab-research-v1.zip",{"slug":2879,"path":2880,"title":2881,"date":2882,"tags":2883,"description":2888,"cover":2889,"coverAlt":2890,"excerpt":2891,"draft":2437,"hidden":2437,"published":2441,"readingTime":2892},"qwen25-vl-sglang-verl-single-gpu-lab","\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab","从 Mac 到单卡 GPU：Qwen2.5-VL 部署、GRPO 与计数 LoRA 实践","2026-08-19",[2567,2508,2884,2885,2886,2887,2721],"SGLang","verl","GRPO","推理优化","在 M2 Pro 和云端单卡上部署 Qwen2.5-VL-3B，实测连续批处理、前缀缓存与 SGLang 参数调优，再用 verl 完成多模态 GRPO 更新和同条件前后评测，记录有效优化、兼容性问题与未观察到提升的结果。","\u002Fimages\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab\u002Fqvl-lab-cover-20261010.webp","笔记本与单卡计算模块通过图像和 token 光轨连接的实验台插画","这次实验从一个很具体的问题开始：M2 Pro、16 GB 统一内存，能不能在本机跑 Qwen2.5VL？跑起来之后，我又想把图文问答、推理性能测量和强化学习训练接成一条完整流程。 本机的 MLX 4bit 服务和云端的 SGLang",29,{"slug":2894,"path":2895,"title":2896,"date":2897,"tags":2898,"description":2902,"cover":2903,"coverAlt":2904,"excerpt":2905,"draft":2437,"hidden":2437,"published":2441,"readingTime":2688},"cycleresearcher-post-training","\u002Fposts\u002Fcycleresearcher-post-training","从 DeepReviewer 2.0 到 CycleResearcher：科研 Agent 如何真正做 Post-training","2026-08-07 17:33:05",[2551,2568,2509,2899,2900,2901],"SimPO","Reward Model","ICLR","区分推理时 Agent 编排与模型后训练，并拆解 CycleResearcher 的领域 SFT、生成式 Reward Model、Iterative SimPO 和评测风险。","\u002Fimages\u002Fposts\u002Fcycleresearcher-post-training\u002Fcycleresearcher-cover-20261018.webp","策略生成模块产出多份候选文稿经多维评审棱镜排序后通过偏好梯度反馈弧闭环优化的等距技术插画","看到一个系统会检索、规划、反思和调用工具，不能据此判断它做过 Posttraining。Agent 描述的是模型在运行时如何完成任务；Posttraining 描述的是预训练之后，是否继续用监督数据、偏好数据或奖励信号更新模型参数。",{"slug":2907,"path":2908,"title":2909,"date":2910,"tags":2911,"description":2913,"cover":2914,"coverAlt":2915,"excerpt":2916,"draft":2437,"hidden":2437,"published":2441,"readingTime":2544,"series":2917,"seriesOrder":938},"cuda-fused-rmsnorm-kernel","\u002Fposts\u002Fcuda-fused-rmsnorm-kernel","从 PyTorch 到 CUDA Kernel：融合 RMSNorm 在 RTX 4090D 上加速 4.27 倍","2026-07-22 18:16:50",[2774,2479,2481,2449,2591,2912],"Kernel 优化","从零实现 PyTorch CUDA 扩展，拆解 RMSNorm 的访存瓶颈、warp 归约、向量化加载、算子融合与冷热缓存 benchmark。","\u002Fimages\u002Fposts\u002Fcuda-fused-rmsnorm-kernel\u002Fcuda-rmsnorm-cover-20261018.webp","融合残差加法与向量化 float4 访存经 Warp Shuffle 蝴蝶树归约输出归一化张量的等距硬件架构插画","为了准备 AI Infra 岗位，我做了一个尽量小而完整的 CUDA 项目：不追求堆很多算子，而是选一个真实的 LLM 推理路径，把实现、正确性、benchmark 和性能解释全部做扎实。 项目实现了两个 PyTorch CUDA ","AI Infra 实战",{"slug":2919,"path":2920,"title":2921,"date":2922,"tags":2923,"description":2925,"cover":2926,"coverAlt":2927,"excerpt":2928,"draft":2437,"hidden":2437,"published":2441,"readingTime":2468,"series":2929,"seriesOrder":938},"agent-memory","\u002Fposts\u002Fagent-memory","Agent 上下文工程与分层记忆架构：从 Observation 剪枝、滚动摘要到向量召回闭环","2026-04-24",[2551,2509,2924,2479],"RAG","结合生产级多约束 Agent 与超 100K Token 科研智能体实践，系统拆解长程会话中的上下文膨胀与注意力稀释问题：涵盖 Observation 结构化剪枝（降低约 40% Token 开销）、滚动工作摘要、向量检索长期记忆及 Prefix Cache 友好的上下文组装。","\u002Fimages\u002Fposts\u002Fagent-memory\u002Fagent-memory-cover-20261018.webp","底层高噪数据经剪枝滤网进入中期滑动窗口与金色状态台账并由顶层向量星座按需召回的三层记忆架构插画","在构建多步工具调用（ToolUse）与长周期自主规划（Autonomous Planning）的 Agent 系统时，上下文管理（Context Engineering）往往比单纯改写 System Prompt 更决定线上稳定性。","Agent 上下文工程",{"slug":2931,"path":2932,"title":2933,"date":2934,"tags":2935,"description":2938,"cover":2939,"coverAlt":2940,"excerpt":2941,"draft":2437,"hidden":2437,"published":2441,"readingTime":2584,"series":2942,"seriesOrder":1766},"backend-stack-deep-dive","\u002Fposts\u002Fbackend-stack-deep-dive","高并发 AI 平台后端基础设施实践：FastAPI 异步网关、SQLAlchemy 2.0 Async、PostgreSQL 与 Docker 沙箱编排","2026-04-07",[2595,2592,2594,2936,2937,2479],"PostgreSQL","Docker","结合 DeepScientist 生产级 AI 科研平台实践，拆解 FastAPI ASGI 异步事件循环、SQLAlchemy 2.0 + AsyncPG 连接池调优、PostgreSQL + MinIO 存储分层与 Docker SDK 动态沙箱容器编排的职责边界与核心设计。","\u002Fimages\u002Fposts\u002Fbackend-stack-deep-dive\u002Fbackend-stack-cover-20261018.webp","ASGI 异步事件循环涡轮将非阻塞协程流分发至 PostgreSQL 连接池、MinIO 存储矩阵与动态 Docker 沙箱的等距架构插画","结合 DeepScientist 生产级 AI 科研平台（服务 7,000+ 用户）的实际工程落地，本文系统拆解支撑高并发流式推理与容器沙箱调度的后端基础设施栈——FastAPI（ASGI 异步网关 + SSE 流式推送）、SQLA","DeepScientist 工程复盘",{"slug":2944,"path":2945,"title":2946,"date":2947,"tags":2948,"description":2951,"cover":2952,"coverAlt":2953,"excerpt":2954,"draft":2437,"hidden":2437,"published":2441,"readingTime":2629,"series":2955,"seriesOrder":938},"minicode-source-analysis","\u002Fposts\u002Fminicode-source-analysis","MiniCode 架构与工具评测深度拆解：开源终端 Agentic Coding 引擎（1k+ Stars）的 ReAct、MCP 与 30 项基准验证","2026-04-06",[2509,2949,2950,2539,2479],"TypeScript","MCP","深度剖析开源终端 AI 编程智能体 MiniCode（1k+ GitHub Stars）的核心架构：从零重型依赖的 ReAct 规划循环、声明式工具注册表与权限拦截层、MCP 协议帧自动协商，到 30 项可执行 Tool-Use 评测基准（86.7% 完成率、100% 错误恢复率）的失败分布归因与闭环优化。","\u002Fimages\u002Fposts\u002Fminicode-source-analysis\u002Fminicode-cover-20261018.webp","终端编程智能体控制台被 ReAct 状态环、Unified Diff 补丁预览门与 MCP 协议插槽环绕的等距技术插画","MiniCode 是我作为核心开发者参与构建的开源终端 Agentic Coding 引擎（目前已在 GitHub 获得 1,000+ Stars）。项目的核心设计目标是：在不引入庞大运行时框架的前提下，用约 5,000 行高内聚的","AI 项目源码导读",{"slug":2957,"path":2958,"title":2959,"date":2947,"tags":2960,"description":2961,"cover":2962,"coverAlt":2963,"excerpt":2964,"draft":2437,"hidden":2437,"published":2441,"readingTime":2657,"series":2942,"seriesOrder":938},"deepscientist-tech-stack","\u002Fposts\u002Fdeepscientist-tech-stack","DeepScientist 架构与运行时全解析：超 100K 上下文科研智能体、MCP 沙箱与高并发流式系统",[2509,2479,2950,2937,2594,2592],"深度拆解服务 7,000+ 用户的科研自主智能体平台 DeepScientist：从一题一仓库（Quest）持久化状态机、MCP 工具栈（memory \u002F artifact \u002F bash_exec）、Docker SDK 动态容器编排，到 SSE + WebSocket 双流通信与 PostgreSQL \u002F Redis \u002F Prometheus 生产级基础设施。","\u002Fimages\u002Fposts\u002Fdeepscientist-tech-stack\u002Fdeepscientist-cover-20261018.webp","科研智能体编排核心通过 memory、artifact 与隔离容器沙箱三座协议桥连接双流工作台的等距架构插画","DeepScientist（deepscientist.cc）是我们在西湖大学自然语言处理实验室研发的自主科学发现与科研管理平台，目前累计服务 7,000+ 科研用户。与单轮对话或纯文本文献问答工具不同，科研智能体需要在数小时乃至数",1795088946074]