[{"data":1,"prerenderedAt":3300},["ShallowReactive",2],{"post-\u002Fposts\u002Fcmu-15-779-attention-online-softmax":3,"archive-public-posts":2796},{"id":4,"title":5,"body":6,"categories":2775,"cover":2777,"coverAlt":2778,"date":2779,"description":2780,"draft":2781,"extension":2782,"haloPublished":2781,"haloSync":2781,"hidden":2781,"legacy":2781,"meta":2783,"navigation":2784,"path":2785,"published":2781,"readingTime":2786,"seo":2787,"series":2788,"seriesOrder":2247,"stem":2789,"tags":2790,"__hash__":2795},"posts\u002Fposts\u002Fcmu-15-779-attention-online-softmax.md","CMU 15-779 学习记录 08：分块 Attention、在线 Softmax 与 KV cache",{"type":7,"value":8,"toc":2763},"minimark",[9,16,24,31,44,49,63,471,478,493,497,500,503,1647,1654,1661,1846,1855,1858,1861,1906,1917,1920,1923,1927,1934,1990,1993,2000,2003,2007,2014,2017,2067,2074,2077,2082,2085,2205,2216,2219,2222,2380,2383,2390,2393,2400,2555,2558,2577,2585,2589,2596,2599,2677,2688,2691,2727,2730,2733,2748,2759],[10,11,12],"blockquote",{},[13,14,15],"p",{},"第 5 讲 Transformers & Attention 的个人配套实验。28\u002F29 号是 CPU 单头前向与 KV cache；后续补入 38 号 CUDA 多头合成张量的显式 SDPA MATH\u002FFLASH 对照。没有手写 FlashAttention kernel，也没有总内存峰值测量。",[13,17,18,23],{},[19,20,22],"a",{"href":21},"\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction\u002F","上一节","通过分块共用矩阵输入。这次接到 Attention：能不能一块一块读取分数，同时得到与完整 Softmax 一致的输出？",[13,25,26,30],{},[27,28,29],"code",{},"28_attention_online_softmax.py"," 已保存完整计算、分块计算和 PyTorch SDPA MATH 的 CPU 对照结果。整理时在冻结副本中复跑了默认配置和因果配置，并补入 29 号 KV cache 实验在独立冻结副本中复验。",[13,32,33,39,40],{},[19,34,38],{"href":35,"rel":36},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F9261bea6aedf31cdccbbab2541ca693fc97b3743",[37],"nofollow","历史 CPU 代码版本"," · ",[19,41,43],{"href":42},"\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fattention-online-softmax-kv-cache-20261019.zip","下载代码与结果 ZIP",[45,46,48],"h2",{"id":47},"完整-attention-会生成什么","完整 Attention 会生成什么",[13,50,51,52,55,56,55,59,62],{},"本次只看一个 attention head，输入形状为 ",[27,53,54],{},"Q[N,d]","、",[27,57,58],{},"K[N,d]",[27,60,61],{},"V[N,dv]","：",[64,65,68],"span",{"className":66},[67],"katex-display",[64,69,72,170],{"className":70},[71],"katex",[64,73,76],{"className":74},[75],"katex-mathml",[77,78,81],"math",{"xmlns":79,"display":80},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML","block",[82,83,84,165],"semantics",{},[85,86,87,91,95,119,123,127,130,132,136,139,143,145,148,150,152,155,157,159,162],"mrow",{},[88,89,90],"mi",{},"S",[92,93,94],"mo",{},"=",[96,97,98,113],"mfrac",{},[85,99,100,103],{},[88,101,102],{},"Q",[104,105,106,109],"msup",{},[88,107,108],{},"K",[88,110,112],{"mathvariant":111},"sans-serif","T",[114,115,116],"msqrt",{},[88,117,118],{},"d",[92,120,122],{"separator":121},"true",",",[124,125],"mspace",{"width":126},"2em",[88,128,129],{},"P",[92,131,94],{},[88,133,135],{"mathvariant":134},"normal","softmax",[92,137,138],{},"⁡",[92,140,142],{"stretchy":141},"false","(",[88,144,90],{},[92,146,147],{"stretchy":141},")",[92,149,122],{"separator":121},[124,151],{"width":126},[88,153,154],{},"O",[92,156,94],{},[88,158,129],{},[88,160,161],{},"V",[88,163,164],{"mathvariant":134},".",[166,167,169],"annotation",{"encoding":168},"application\u002Fx-tex","S=\\frac{QK^\\mathsf{T}}{\\sqrt d},\\qquad P=\\operatorname{softmax}(S),\\qquad O=PV.",[64,171,174,200,409,455],{"className":172,"ariaHidden":121},[173],"katex-html",[64,175,178,183,189,193,197],{"className":176},[177],"base",[64,179],{"className":180,"style":182},[181],"strut","height:0.6833em;",[64,184,90],{"className":185,"style":188},[186,187],"mord","mathnormal","margin-right:0.0576em;",[64,190],{"className":191,"style":192},[124],"margin-right:0.2778em;",[64,194,94],{"className":195},[196],"mrel",[64,198],{"className":199,"style":192},[124],[64,201,203,207,384,388,392,396,400,403,406],{"className":202},[177],[64,204],{"className":205,"style":206},[181],"height:2.4561em;vertical-align:-0.93em;",[64,208,210,215,380],{"className":209},[186],[64,211],{"className":212},[213,214],"mopen","nulldelimiter",[64,216,218],{"className":217},[96],[64,219,223,371],{"className":220},[221,222],"vlist-t","vlist-t2",[64,224,227,368],{"className":225},[226],"vlist-r",[64,228,232,306,317],{"className":229,"style":231},[230],"vlist","height:1.5261em;",[64,233,235,240],{"style":234},"top:-2.1778em;",[64,236],{"className":237,"style":239},[238],"pstrut","height:3em;",[64,241,243],{"className":242},[186],[64,244,247],{"className":245},[186,246],"sqrt",[64,248,250,297],{"className":249},[221,222],[64,251,253,292],{"className":252},[226],[64,254,257,269],{"className":255,"style":256},[230],"height:0.9322em;",[64,258,262,265],{"className":259,"style":261},[260],"svg-align","top:-3em;",[64,263],{"className":264,"style":239},[238],[64,266,118],{"className":267,"style":268},[186,187],"padding-left:0.833em;",[64,270,272,275],{"style":271},"top:-2.8922em;",[64,273],{"className":274,"style":239},[238],[64,276,280],{"className":277,"style":279},[278],"hide-tail","min-width:0.853em;height:1.08em;",[281,282,288],"svg",{"xmlns":283,"width":284,"height":285,"viewBox":286,"preserveAspectRatio":287},"http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","400em","1.08em","0 0 400000 1080","xMinYMin slice",[289,290],"path",{"d":291},"M95,702\nc-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14\nc0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54\nc44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10\ns173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429\nc69,-144,104.5,-217.7,106.5,-221\nl0 -0\nc5.3,-9.3,12,-14,20,-14\nH400000v40H845.2724\ns-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7\nc-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z\nM834 80h400000v40h-400000z",[64,293,296],{"className":294},[295],"vlist-s","​",[64,298,300],{"className":299},[226],[64,301,304],{"className":302,"style":303},[230],"height:0.1078em;",[64,305],{},[64,307,309,312],{"style":308},"top:-3.23em;",[64,310],{"className":311,"style":239},[238],[64,313],{"className":314,"style":316},[315],"frac-line","border-bottom-width:0.04em;",[64,318,320,323],{"style":319},"top:-3.677em;",[64,321],{"className":322,"style":239},[238],[64,324,326,329],{"className":325},[186],[64,327,102],{"className":328},[186,187],[64,330,332,336],{"className":331},[186],[64,333,108],{"className":334,"style":335},[186,187],"margin-right:0.0715em;",[64,337,340],{"className":338},[339],"msupsub",[64,341,343],{"className":342},[221],[64,344,346],{"className":345},[226],[64,347,350],{"className":348,"style":349},[230],"height:0.8491em;",[64,351,353,357],{"style":352},"top:-3.063em;margin-right:0.05em;",[64,354],{"className":355,"style":356},[238],"height:2.7em;",[64,358,364],{"className":359},[360,361,362,363],"sizing","reset-size6","size3","mtight",[64,365,112],{"className":366},[186,367,363],"mathsf",[64,369,296],{"className":370},[295],[64,372,374],{"className":373},[226],[64,375,378],{"className":376,"style":377},[230],"height:0.93em;",[64,379],{},[64,381],{"className":382},[383,214],"mclose",[64,385,122],{"className":386},[387],"mpunct",[64,389],{"className":390,"style":391},[124],"margin-right:2em;",[64,393],{"className":394,"style":395},[124],"margin-right:0.1667em;",[64,397,129],{"className":398,"style":399},[186,187],"margin-right:0.1389em;",[64,401],{"className":402,"style":192},[124],[64,404,94],{"className":405},[196],[64,407],{"className":408,"style":192},[124],[64,410,412,416,424,427,430,433,436,439,442,446,449,452],{"className":411},[177],[64,413],{"className":414,"style":415},[181],"height:1em;vertical-align:-0.25em;",[64,417,420],{"className":418},[419],"mop",[64,421,135],{"className":422},[186,423],"mathrm",[64,425,142],{"className":426},[213],[64,428,90],{"className":429,"style":188},[186,187],[64,431,147],{"className":432},[383],[64,434,122],{"className":435},[387],[64,437],{"className":438,"style":391},[124],[64,440],{"className":441,"style":395},[124],[64,443,154],{"className":444,"style":445},[186,187],"margin-right:0.0278em;",[64,447],{"className":448,"style":192},[124],[64,450,94],{"className":451},[196],[64,453],{"className":454,"style":192},[124],[64,456,458,461,464,468],{"className":457},[177],[64,459],{"className":460,"style":182},[181],[64,462,129],{"className":463,"style":399},[186,187],[64,465,161],{"className":466,"style":467},[186,187],"margin-right:0.2222em;",[64,469,164],{"className":470},[186],[13,472,473,474,477],{},"Softmax 沿每一行的 key 方向归一化。完整实现先得到 ",[27,475,476],{},"N×N"," 分数矩阵，再计算权重和输出。因果模式先把未来位置的分数设为负无穷，确保当前位置看不到后面的 K、V。",[13,479,480,481,484,485,488,489,492],{},"默认配置为 ",[27,482,483],{},"N=64, d=16, dv=8","，使用 float64，输出形状为 ",[27,486,487],{},"[64,8]","。SDPA 参考明确指定 ",[27,490,491],{},"SDPBackend.MATH","，输入位于 CPU，不能把这个对照称为调用了 GPU FlashAttention。",[45,494,496],{"id":495},"为什么不能分别做-softmax-再拼起来","为什么不能分别做 Softmax 再拼起来",[13,498,499],{},"一行中的所有分数共享同一个归一化分母。若每个小块各自做 Softmax，它们各自的权重和都等于 1；把这些权重拼接，无法得到整行的权重。",[13,501,502],{},"分块实现需要保存每一行已经看过的最大分数 m、指数和 l，以及未归一化的输出向量 u。遇到新分数块 s 和对应值向量块 V 后，更新为：",[64,504,506],{"className":505},[67],[64,507,509,769],{"className":508},[71],[64,510,512],{"className":511},[75],[77,513,514],{"xmlns":79,"display":80},[82,515,516,766],{},[517,518,522,585,626,672,717],"mtable",{"rowspacing":519,"columnalign":520,"columnspacing":521},"0.25em","right left","0em",[523,524,525,540],"mtr",{},[526,527,528],"mtd",{},[529,530,532],"mstyle",{"scriptlevel":531,"displaystyle":121},"0",[104,533,534,537],{},[88,535,536],{},"m",[92,538,539],{"mathvariant":134,"lspace":521,"rspace":521},"′",[526,541,542],{},[529,543,544],{"scriptlevel":531,"displaystyle":121},[85,545,546,548,550,553,555,557,559,561,573,581,583],{},[85,547],{},[92,549,94],{},[88,551,552],{},"max",[92,554,138],{},[92,556,142],{"stretchy":141},[88,558,536],{},[92,560,122],{"separator":121},[562,563,564,570],"munder",{},[85,565,566,568],{},[88,567,552],{},[92,569,138],{},[88,571,572],{},"j",[574,575,576,579],"msub",{},[88,577,578],{},"s",[88,580,572],{},[92,582,147],{"stretchy":141},[92,584,122],{"separator":121},[523,586,587,594],{},[526,588,589],{},[529,590,591],{"scriptlevel":531,"displaystyle":121},[88,592,593],{},"α",[526,595,596],{},[529,597,598],{"scriptlevel":531,"displaystyle":121},[85,599,600,602,604,607,609,611,613,616,622,624],{},[85,601],{},[92,603,94],{},[88,605,606],{},"exp",[92,608,138],{},[92,610,142],{"stretchy":141},[88,612,536],{},[92,614,615],{},"−",[104,617,618,620],{},[88,619,536],{},[92,621,539],{"mathvariant":134,"lspace":521,"rspace":521},[92,623,147],{"stretchy":141},[92,625,122],{"separator":121},[523,627,628,638],{},[526,629,630],{},[529,631,632],{"scriptlevel":531,"displaystyle":121},[574,633,634,636],{},[88,635,13],{},[88,637,572],{},[526,639,640],{},[529,641,642],{"scriptlevel":531,"displaystyle":121},[85,643,644,646,648,650,652,654,660,662,668,670],{},[85,645],{},[92,647,94],{},[88,649,606],{},[92,651,138],{},[92,653,142],{"stretchy":141},[574,655,656,658],{},[88,657,578],{},[88,659,572],{},[92,661,615],{},[104,663,664,666],{},[88,665,536],{},[92,667,539],{"mathvariant":134,"lspace":521,"rspace":521},[92,669,147],{"stretchy":141},[92,671,122],{"separator":121},[523,673,674,685],{},[526,675,676],{},[529,677,678],{"scriptlevel":531,"displaystyle":121},[104,679,680,683],{},[88,681,682],{},"l",[92,684,539],{"mathvariant":134,"lspace":521,"rspace":521},[526,686,687],{},[529,688,689],{"scriptlevel":531,"displaystyle":121},[85,690,691,693,695,697,699,702,709,715],{},[85,692],{},[92,694,94],{},[88,696,593],{},[88,698,682],{},[92,700,701],{},"+",[562,703,704,707],{},[92,705,706],{},"∑",[88,708,572],{},[574,710,711,713],{},[88,712,13],{},[88,714,572],{},[92,716,122],{"separator":121},[523,718,719,730],{},[526,720,721],{},[529,722,723],{"scriptlevel":531,"displaystyle":121},[104,724,725,728],{},[88,726,727],{},"u",[92,729,539],{"mathvariant":134,"lspace":521,"rspace":521},[526,731,732],{},[529,733,734],{"scriptlevel":531,"displaystyle":121},[85,735,736,738,740,742,744,746,752,758,764],{},[85,737],{},[92,739,94],{},[88,741,593],{},[88,743,727],{},[92,745,701],{},[562,747,748,750],{},[92,749,706],{},[88,751,572],{},[574,753,754,756],{},[88,755,13],{},[88,757,572],{},[574,759,760,762],{},[88,761,161],{},[88,763,572],{},[88,765,164],{"mathvariant":134},[166,767,768],{"encoding":168},"\\begin{aligned}\nm'&=\\max(m,\\max_j s_j),\\\\\n\\alpha&=\\exp(m-m'),\\\\\np_j&=\\exp(s_j-m'),\\\\\nl'&=\\alpha l+\\sum_j p_j,\\\\\nu'&=\\alpha u+\\sum_j p_jV_j.\n\\end{aligned}",[64,770,772],{"className":771,"ariaHidden":121},[173],[64,773,775,779],{"className":774},[177],[64,776],{"className":777,"style":778},[181],"height:10.2313em;vertical-align:-4.8657em;",[64,780,782],{"className":781},[186],[64,783,785,1004],{"className":784},[517],[64,786,789],{"className":787},[788],"col-align-r",[64,790,792,995],{"className":791},[221,222],[64,793,795,992],{"className":794},[226],[64,796,799,843,856,909,951],{"className":797,"style":798},[230],"height:5.3657em;",[64,800,802,806],{"style":801},"top:-7.5757em;",[64,803],{"className":804,"style":805},[238],"height:3.05em;",[64,807,809],{"className":808},[186],[64,810,812,815],{"className":811},[186],[64,813,536],{"className":814},[186,187],[64,816,818],{"className":817},[339],[64,819,821],{"className":820},[221],[64,822,824],{"className":823},[226],[64,825,828],{"className":826,"style":827},[230],"height:0.8019em;",[64,829,831,834],{"style":830},"top:-3.113em;margin-right:0.05em;",[64,832],{"className":833,"style":356},[238],[64,835,837],{"className":836},[360,361,362,363],[64,838,840],{"className":839},[186,363],[64,841,539],{"className":842},[186,363],[64,844,846,849],{"style":845},"top:-5.5719em;",[64,847],{"className":848,"style":805},[238],[64,850,852],{"className":851},[186],[64,853,593],{"className":854,"style":855},[186,187],"margin-right:0.0037em;",[64,857,859,862],{"style":858},"top:-4.0719em;",[64,860],{"className":861,"style":805},[238],[64,863,865],{"className":864},[186],[64,866,868,871],{"className":867},[186],[64,869,13],{"className":870},[186,187],[64,872,874],{"className":873},[339],[64,875,877,900],{"className":876},[221,222],[64,878,880,897],{"className":879},[226],[64,881,884],{"className":882,"style":883},[230],"height:0.3117em;",[64,885,887,890],{"style":886},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[64,888],{"className":889,"style":356},[238],[64,891,893],{"className":892},[360,361,362,363],[64,894,572],{"className":895,"style":896},[186,187,363],"margin-right:0.0572em;",[64,898,296],{"className":899},[295],[64,901,903],{"className":902},[226],[64,904,907],{"className":905,"style":906},[230],"height:0.2861em;",[64,908],{},[64,910,912,915],{"style":911},"top:-2.3619em;",[64,913],{"className":914,"style":805},[238],[64,916,918],{"className":917},[186],[64,919,921,925],{"className":920},[186],[64,922,682],{"className":923,"style":924},[186,187],"margin-right:0.0197em;",[64,926,928],{"className":927},[339],[64,929,931],{"className":930},[221],[64,932,934],{"className":933},[226],[64,935,937],{"className":936,"style":827},[230],[64,938,939,942],{"style":830},[64,940],{"className":941,"style":356},[238],[64,943,945],{"className":944},[360,361,362,363],[64,946,948],{"className":947},[186,363],[64,949,539],{"className":950},[186,363],[64,952,954,957],{"style":953},"top:0.4019em;",[64,955],{"className":956,"style":805},[238],[64,958,960],{"className":959},[186],[64,961,963,966],{"className":962},[186],[64,964,727],{"className":965},[186,187],[64,967,969],{"className":968},[339],[64,970,972],{"className":971},[221],[64,973,975],{"className":974},[226],[64,976,978],{"className":977,"style":827},[230],[64,979,980,983],{"style":830},[64,981],{"className":982,"style":356},[238],[64,984,986],{"className":985},[360,361,362,363],[64,987,989],{"className":988},[186,363],[64,990,539],{"className":991},[186,363],[64,993,296],{"className":994},[295],[64,996,998],{"className":997},[226],[64,999,1002],{"className":1000,"style":1001},[230],"height:4.8657em;",[64,1003],{},[64,1005,1008],{"className":1006},[1007],"col-align-l",[64,1009,1011,1639],{"className":1010},[221,222],[64,1012,1014,1636],{"className":1013},[226],[64,1015,1017,1149,1226,1339,1470],{"className":1016,"style":798},[230],[64,1018,1019,1022],{"style":801},[64,1020],{"className":1021,"style":805},[238],[64,1023,1025,1028,1031,1034,1037,1040,1043,1046,1049,1052,1100,1103,1143,1146],{"className":1024},[186],[64,1026],{"className":1027},[186],[64,1029],{"className":1030,"style":192},[124],[64,1032,94],{"className":1033},[196],[64,1035],{"className":1036,"style":192},[124],[64,1038,552],{"className":1039},[419],[64,1041,142],{"className":1042},[213],[64,1044,536],{"className":1045},[186,187],[64,1047,122],{"className":1048},[387],[64,1050],{"className":1051,"style":395},[124],[64,1053,1056],{"className":1054},[419,1055],"op-limits",[64,1057,1059,1091],{"className":1058},[221,222],[64,1060,1062,1088],{"className":1061},[226],[64,1063,1066,1078],{"className":1064,"style":1065},[230],"height:0.4306em;",[64,1067,1069,1072],{"style":1068},"top:-2.3723em;margin-left:0em;",[64,1070],{"className":1071,"style":239},[238],[64,1073,1075],{"className":1074},[360,361,362,363],[64,1076,572],{"className":1077,"style":896},[186,187,363],[64,1079,1080,1083],{"style":261},[64,1081],{"className":1082,"style":239},[238],[64,1084,1085],{},[64,1086,552],{"className":1087},[419],[64,1089,296],{"className":1090},[295],[64,1092,1094],{"className":1093},[226],[64,1095,1098],{"className":1096,"style":1097},[230],"height:0.8638em;",[64,1099],{},[64,1101],{"className":1102,"style":395},[124],[64,1104,1106,1109],{"className":1105},[186],[64,1107,578],{"className":1108},[186,187],[64,1110,1112],{"className":1111},[339],[64,1113,1115,1135],{"className":1114},[221,222],[64,1116,1118,1132],{"className":1117},[226],[64,1119,1121],{"className":1120,"style":883},[230],[64,1122,1123,1126],{"style":886},[64,1124],{"className":1125,"style":356},[238],[64,1127,1129],{"className":1128},[360,361,362,363],[64,1130,572],{"className":1131,"style":896},[186,187,363],[64,1133,296],{"className":1134},[295],[64,1136,1138],{"className":1137},[226],[64,1139,1141],{"className":1140,"style":906},[230],[64,1142],{},[64,1144,147],{"className":1145},[383],[64,1147,122],{"className":1148},[387],[64,1150,1151,1154],{"style":845},[64,1152],{"className":1153,"style":805},[238],[64,1155,1157,1160,1163,1166,1169,1172,1175,1178,1181,1185,1188,1220,1223],{"className":1156},[186],[64,1158],{"className":1159},[186],[64,1161],{"className":1162,"style":192},[124],[64,1164,94],{"className":1165},[196],[64,1167],{"className":1168,"style":192},[124],[64,1170,606],{"className":1171},[419],[64,1173,142],{"className":1174},[213],[64,1176,536],{"className":1177},[186,187],[64,1179],{"className":1180,"style":467},[124],[64,1182,615],{"className":1183},[1184],"mbin",[64,1186],{"className":1187,"style":467},[124],[64,1189,1191,1194],{"className":1190},[186],[64,1192,536],{"className":1193},[186,187],[64,1195,1197],{"className":1196},[339],[64,1198,1200],{"className":1199},[221],[64,1201,1203],{"className":1202},[226],[64,1204,1206],{"className":1205,"style":827},[230],[64,1207,1208,1211],{"style":830},[64,1209],{"className":1210,"style":356},[238],[64,1212,1214],{"className":1213},[360,361,362,363],[64,1215,1217],{"className":1216},[186,363],[64,1218,539],{"className":1219},[186,363],[64,1221,147],{"className":1222},[383],[64,1224,122],{"className":1225},[387],[64,1227,1228,1231],{"style":858},[64,1229],{"className":1230,"style":805},[238],[64,1232,1234,1237,1240,1243,1246,1249,1252,1292,1295,1298,1301,1333,1336],{"className":1233},[186],[64,1235],{"className":1236},[186],[64,1238],{"className":1239,"style":192},[124],[64,1241,94],{"className":1242},[196],[64,1244],{"className":1245,"style":192},[124],[64,1247,606],{"className":1248},[419],[64,1250,142],{"className":1251},[213],[64,1253,1255,1258],{"className":1254},[186],[64,1256,578],{"className":1257},[186,187],[64,1259,1261],{"className":1260},[339],[64,1262,1264,1284],{"className":1263},[221,222],[64,1265,1267,1281],{"className":1266},[226],[64,1268,1270],{"className":1269,"style":883},[230],[64,1271,1272,1275],{"style":886},[64,1273],{"className":1274,"style":356},[238],[64,1276,1278],{"className":1277},[360,361,362,363],[64,1279,572],{"className":1280,"style":896},[186,187,363],[64,1282,296],{"className":1283},[295],[64,1285,1287],{"className":1286},[226],[64,1288,1290],{"className":1289,"style":906},[230],[64,1291],{},[64,1293],{"className":1294,"style":467},[124],[64,1296,615],{"className":1297},[1184],[64,1299],{"className":1300,"style":467},[124],[64,1302,1304,1307],{"className":1303},[186],[64,1305,536],{"className":1306},[186,187],[64,1308,1310],{"className":1309},[339],[64,1311,1313],{"className":1312},[221],[64,1314,1316],{"className":1315},[226],[64,1317,1319],{"className":1318,"style":827},[230],[64,1320,1321,1324],{"style":830},[64,1322],{"className":1323,"style":356},[238],[64,1325,1327],{"className":1326},[360,361,362,363],[64,1328,1330],{"className":1329},[186,363],[64,1331,539],{"className":1332},[186,363],[64,1334,147],{"className":1335},[383],[64,1337,122],{"className":1338},[387],[64,1340,1341,1344],{"style":911},[64,1342],{"className":1343,"style":805},[238],[64,1345,1347,1350,1353,1356,1359,1362,1365,1368,1371,1374,1424,1427,1467],{"className":1346},[186],[64,1348],{"className":1349},[186],[64,1351],{"className":1352,"style":192},[124],[64,1354,94],{"className":1355},[196],[64,1357],{"className":1358,"style":192},[124],[64,1360,593],{"className":1361,"style":855},[186,187],[64,1363,682],{"className":1364,"style":924},[186,187],[64,1366],{"className":1367,"style":467},[124],[64,1369,701],{"className":1370},[1184],[64,1372],{"className":1373,"style":467},[124],[64,1375,1377],{"className":1376},[419,1055],[64,1378,1380,1415],{"className":1379},[221,222],[64,1381,1383,1412],{"className":1382},[226],[64,1384,1387,1399],{"className":1385,"style":1386},[230],"height:1.05em;",[64,1388,1390,1393],{"style":1389},"top:-1.8723em;margin-left:0em;",[64,1391],{"className":1392,"style":805},[238],[64,1394,1396],{"className":1395},[360,361,362,363],[64,1397,572],{"className":1398,"style":896},[186,187,363],[64,1400,1402,1405],{"style":1401},"top:-3.05em;",[64,1403],{"className":1404,"style":805},[238],[64,1406,1407],{},[64,1408,706],{"className":1409},[419,1410,1411],"op-symbol","large-op",[64,1413,296],{"className":1414},[295],[64,1416,1418],{"className":1417},[226],[64,1419,1422],{"className":1420,"style":1421},[230],"height:1.4138em;",[64,1423],{},[64,1425],{"className":1426,"style":395},[124],[64,1428,1430,1433],{"className":1429},[186],[64,1431,13],{"className":1432},[186,187],[64,1434,1436],{"className":1435},[339],[64,1437,1439,1459],{"className":1438},[221,222],[64,1440,1442,1456],{"className":1441},[226],[64,1443,1445],{"className":1444,"style":883},[230],[64,1446,1447,1450],{"style":886},[64,1448],{"className":1449,"style":356},[238],[64,1451,1453],{"className":1452},[360,361,362,363],[64,1454,572],{"className":1455,"style":896},[186,187,363],[64,1457,296],{"className":1458},[295],[64,1460,1462],{"className":1461},[226],[64,1463,1465],{"className":1464,"style":906},[230],[64,1466],{},[64,1468,122],{"className":1469},[387],[64,1471,1472,1475],{"style":953},[64,1473],{"className":1474,"style":805},[238],[64,1476,1478,1481,1484,1487,1490,1493,1496,1499,1502,1505,1549,1552,1592,1633],{"className":1477},[186],[64,1479],{"className":1480},[186],[64,1482],{"className":1483,"style":192},[124],[64,1485,94],{"className":1486},[196],[64,1488],{"className":1489,"style":192},[124],[64,1491,593],{"className":1492,"style":855},[186,187],[64,1494,727],{"className":1495},[186,187],[64,1497],{"className":1498,"style":467},[124],[64,1500,701],{"className":1501},[1184],[64,1503],{"className":1504,"style":467},[124],[64,1506,1508],{"className":1507},[419,1055],[64,1509,1511,1541],{"className":1510},[221,222],[64,1512,1514,1538],{"className":1513},[226],[64,1515,1517,1528],{"className":1516,"style":1386},[230],[64,1518,1519,1522],{"style":1389},[64,1520],{"className":1521,"style":805},[238],[64,1523,1525],{"className":1524},[360,361,362,363],[64,1526,572],{"className":1527,"style":896},[186,187,363],[64,1529,1530,1533],{"style":1401},[64,1531],{"className":1532,"style":805},[238],[64,1534,1535],{},[64,1536,706],{"className":1537},[419,1410,1411],[64,1539,296],{"className":1540},[295],[64,1542,1544],{"className":1543},[226],[64,1545,1547],{"className":1546,"style":1421},[230],[64,1548],{},[64,1550],{"className":1551,"style":395},[124],[64,1553,1555,1558],{"className":1554},[186],[64,1556,13],{"className":1557},[186,187],[64,1559,1561],{"className":1560},[339],[64,1562,1564,1584],{"className":1563},[221,222],[64,1565,1567,1581],{"className":1566},[226],[64,1568,1570],{"className":1569,"style":883},[230],[64,1571,1572,1575],{"style":886},[64,1573],{"className":1574,"style":356},[238],[64,1576,1578],{"className":1577},[360,361,362,363],[64,1579,572],{"className":1580,"style":896},[186,187,363],[64,1582,296],{"className":1583},[295],[64,1585,1587],{"className":1586},[226],[64,1588,1590],{"className":1589,"style":906},[230],[64,1591],{},[64,1593,1595,1598],{"className":1594},[186],[64,1596,161],{"className":1597,"style":467},[186,187],[64,1599,1601],{"className":1600},[339],[64,1602,1604,1625],{"className":1603},[221,222],[64,1605,1607,1622],{"className":1606},[226],[64,1608,1610],{"className":1609,"style":883},[230],[64,1611,1613,1616],{"style":1612},"top:-2.55em;margin-left:-0.2222em;margin-right:0.05em;",[64,1614],{"className":1615,"style":356},[238],[64,1617,1619],{"className":1618},[360,361,362,363],[64,1620,572],{"className":1621,"style":896},[186,187,363],[64,1623,296],{"className":1624},[295],[64,1626,1628],{"className":1627},[226],[64,1629,1631],{"className":1630,"style":906},[230],[64,1632],{},[64,1634,164],{"className":1635},[186],[64,1637,296],{"className":1638},[295],[64,1640,1642],{"className":1641},[226],[64,1643,1645],{"className":1644,"style":1001},[230],[64,1646],{},[13,1648,1649,1650,1653],{},"所有块处理完，再计算 ",[27,1651,1652],{},"O = u \u002F l","。初始最大值 m 取负无穷，l 和 u 均置零。",[13,1655,1656,1657,1660],{},"这里容易漏掉的是旧状态修正：新块如果出现更大的分数，旧的 l 和 u 必须一起乘上 ",[27,1658,1659],{},"exp(m-m')","，才能在同一个最大值基准下相加。实际代码对应：",[1662,1663,1668],"pre",{"className":1664,"code":1665,"language":1666,"meta":1667,"style":1667},"language-python shiki shiki-themes github-dark-dimmed github-light houston","next_max = torch.maximum(row_max, scores.max(dim=-1, keepdim=True).values)\ncorrection = torch.exp(row_max - next_max)\nblock_weights = torch.exp(scores - next_max)\ndenominator = correction * denominator + block_weights.sum(dim=-1, keepdim=True)\nv_tile = v[k_start:k_start + key_block]\nnumerator = correction * numerator + block_weights @ v_tile\n","python","",[27,1669,1670,1720,1741,1760,1804,1820],{"__ignoreMap":1667},[64,1671,1674,1678,1681,1684,1688,1691,1693,1695,1699,1702,1706,1709,1712,1714,1717],{"class":1672,"line":1673},"line",1,[64,1675,1677],{"class":1676},"sWURj","next_max ",[64,1679,94],{"class":1680},"sf2cM",[64,1682,1683],{"class":1676}," torch.",[64,1685,1687],{"class":1686},"sq3c-","maximum",[64,1689,1690],{"class":1676},"(row_max, scores.",[64,1692,552],{"class":1686},[64,1694,142],{"class":1676},[64,1696,1698],{"class":1697},"sA9aF","dim",[64,1700,1701],{"class":1680},"=-",[64,1703,1705],{"class":1704},"sgENW","1",[64,1707,1708],{"class":1676},", ",[64,1710,1711],{"class":1697},"keepdim",[64,1713,94],{"class":1680},[64,1715,1716],{"class":1704},"True",[64,1718,1719],{"class":1676},").values)\n",[64,1721,1723,1726,1728,1730,1732,1735,1738],{"class":1672,"line":1722},2,[64,1724,1725],{"class":1676},"correction ",[64,1727,94],{"class":1680},[64,1729,1683],{"class":1676},[64,1731,606],{"class":1686},[64,1733,1734],{"class":1676},"(row_max ",[64,1736,1737],{"class":1680},"-",[64,1739,1740],{"class":1676}," next_max)\n",[64,1742,1744,1747,1749,1751,1753,1756,1758],{"class":1672,"line":1743},3,[64,1745,1746],{"class":1676},"block_weights ",[64,1748,94],{"class":1680},[64,1750,1683],{"class":1676},[64,1752,606],{"class":1686},[64,1754,1755],{"class":1676},"(scores ",[64,1757,1737],{"class":1680},[64,1759,1740],{"class":1676},[64,1761,1763,1766,1768,1771,1774,1777,1779,1782,1785,1787,1789,1791,1793,1795,1797,1799,1801],{"class":1672,"line":1762},4,[64,1764,1765],{"class":1676},"denominator ",[64,1767,94],{"class":1680},[64,1769,1770],{"class":1676}," correction ",[64,1772,1773],{"class":1680},"*",[64,1775,1776],{"class":1676}," denominator ",[64,1778,701],{"class":1680},[64,1780,1781],{"class":1676}," block_weights.",[64,1783,1784],{"class":1686},"sum",[64,1786,142],{"class":1676},[64,1788,1698],{"class":1697},[64,1790,1701],{"class":1680},[64,1792,1705],{"class":1704},[64,1794,1708],{"class":1676},[64,1796,1711],{"class":1697},[64,1798,94],{"class":1680},[64,1800,1716],{"class":1704},[64,1802,1803],{"class":1676},")\n",[64,1805,1807,1810,1812,1815,1817],{"class":1672,"line":1806},5,[64,1808,1809],{"class":1676},"v_tile ",[64,1811,94],{"class":1680},[64,1813,1814],{"class":1676}," v[k_start:k_start ",[64,1816,701],{"class":1680},[64,1818,1819],{"class":1676}," key_block]\n",[64,1821,1823,1826,1828,1830,1832,1835,1837,1840,1843],{"class":1672,"line":1822},6,[64,1824,1825],{"class":1676},"numerator ",[64,1827,94],{"class":1680},[64,1829,1770],{"class":1676},[64,1831,1773],{"class":1680},[64,1833,1834],{"class":1676}," numerator ",[64,1836,701],{"class":1680},[64,1838,1839],{"class":1676}," block_weights ",[64,1841,1842],{"class":1680},"@",[64,1844,1845],{"class":1676}," v_tile\n",[13,1847,1848,1849,1854],{},"这个分块递推对应 ",[19,1850,1853],{"href":1851,"rel":1852},"https:\u002F\u002Fpapers.nips.cc\u002Fpaper_files\u002Fpaper\u002F2022\u002Ffile\u002F67d57c32e20fd0a7a302cb81d36e40d5-Paper-Conference.pdf",[37],"FlashAttention 论文","讨论的在线归一化思想。本次实现用普通 PyTorch CPU 操作验证公式，不包含论文中的 GPU 内存调度或反向重计算。",[45,1856,1857],{"id":1857},"对照结果与因果检查",[13,1859,1860],{},"两份原始报告得到以下最大绝对误差，参考均为 CPU SDPA MATH：",[1862,1863,1864,1881],"table",{},[1865,1866,1867],"thead",{},[1868,1869,1870,1874,1878],"tr",{},[1871,1872,1873],"th",{},"配置",[1871,1875,1877],{"align":1876},"right","完整实现误差",[1871,1879,1880],{"align":1876},"分块实现误差",[1882,1883,1884,1895],"tbody",{},[1868,1885,1886,1890,1892],{},[1887,1888,1889],"td",{},"N=64，float64，非因果，块 16×16",[1887,1891,531],{"align":1876},[1887,1893,1894],{"align":1876},"4.44×10⁻¹⁶",[1868,1896,1897,1900,1903],{},[1887,1898,1899],{},"N=17，float32，因果，块 8×5",[1887,1901,1902],{"align":1876},"3.58×10⁻⁷",[1887,1904,1905],{"align":1876},"2.98×10⁻⁷",[13,1907,1908,1909,1912,1913,1916],{},"两份报告中的默认例子、六组边界配置均通过数值容差检查，完整权重的每行和为 1。float64 使用 ",[27,1910,1911],{},"rtol=atol=1e-12","；float32 使用 ",[27,1914,1915],{},"2e-5","。计算顺序不同会带来浮点舍入差异，因此这里比较容差内一致。",[13,1918,1919],{},"测试还覆盖了这些容易出错的位置：新块改变运行最大值后的旧状态修正；非整除分块；float32\u002Ffloat64；非连续输入；零分数对应均值；较大有限分数的稳定性。因果测试修改后半段的 K、V，前半段输出保持不变，验证未来输入不会影响前缀。",[13,1921,1922],{},"当前实现按 key 从 0 开始遍历，所以每一行在第一个块中至少能看见 key 0；后续完全被因果掩码遮住的块贡献为零。这不等于已经支持任意 padding mask 或整行全被遮蔽的情况。",[45,1924,1926],{"id":1925},"分数块变小不等于测出了总内存下降","分数块变小，不等于测出了总内存下降",[13,1928,1929,1930,1933],{},"固定 query\u002Fkey 块为 ",[27,1931,1932],{},"16×16","，默认 float64 扫描结果为：",[1862,1935,1936,1952],{},[1865,1937,1938],{},[1868,1939,1940,1943,1946,1949],{},[1871,1941,1942],{"align":1876},"N",[1871,1944,1945],{"align":1876},"完整分数矩阵元素数",[1871,1947,1948],{"align":1876},"最大单块分数元素数",[1871,1950,1951],{"align":1876},"实际计算的分数位置数",[1882,1953,1954,1966,1978],{},[1868,1955,1956,1959,1962,1964],{},[1887,1957,1958],{"align":1876},"16",[1887,1960,1961],{"align":1876},"256",[1887,1963,1961],{"align":1876},[1887,1965,1961],{"align":1876},[1868,1967,1968,1971,1974,1976],{},[1887,1969,1970],{"align":1876},"64",[1887,1972,1973],{"align":1876},"4096",[1887,1975,1961],{"align":1876},[1887,1977,1973],{"align":1876},[1868,1979,1980,1983,1986,1988],{},[1887,1981,1982],{"align":1876},"128",[1887,1984,1985],{"align":1876},"16384",[1887,1987,1961],{"align":1876},[1887,1989,1985],{"align":1876},[13,1991,1992],{},"N=64 时，单张完整分数矩阵是 32768 字节，最大单块分数是 2048 字节。这里记录的是单个分数张量的尺寸；Q、K、V、输出、权重块和运行状态仍然占内存，不能将这些数字当作程序的峰值内存。",[13,1994,1995,1996,1999],{},"这个模型仍计算所有 ",[27,1997,1998],{},"N²"," 个 query\u002Fkey 位置。因果模式也先计算块内分数，再应用掩码，没有跳过未来块来减少计算量。N 不超过块尺寸时，最大单块也可能就是整张分数矩阵。",[13,2001,2002],{},"本次结果说明分块递推可以避免在较长序列配置中保存完整分数矩阵；它没有测量 GPU IO、运行速度或总内存峰值，也没有把 Attention 的算术复杂度降成线性。",[45,2004,2006],{"id":2005},"kv-cache-省下的是哪一部分重复工作","KV cache 省下的是哪一部分重复工作",[13,2008,2009,2010,2013],{},"Prefill 一次处理整段 prompt；decode 每步只有一个新 query，却仍要读取合法历史 K\u002FV。",[27,2011,2012],{},"29_kv_cache.py"," 对固定输入表示、固定投影权重的一层 Attention 做了两条路径：每个前缀重算全部 Q\u002FK\u002FV 和完整因果 Attention，再取最后一行；或者 prompt 做一次 prefill，之后只投影新增位置，把 K\u002FV 追加到缓存。",[13,2015,2016],{},"默认 prompt 为 8 个位置，再加入 4 个位置。长度 8、9、10、11、12 的最后一行输出均通过 float64 容差检查：",[1862,2018,2019,2032],{},[1865,2020,2021],{},[1868,2022,2023,2026,2029],{},[1871,2024,2025],{},"观察项",[1871,2027,2028],{"align":1876},"前缀重算",[1871,2030,2031],{"align":1876},"KV cache",[1882,2033,2034,2045,2056],{},[1868,2035,2036,2039,2042],{},[1887,2037,2038],{},"每项投影行数",[1887,2040,2041],{"align":1876},"50",[1887,2043,2044],{"align":1876},"12",[1868,2046,2047,2050,2053],{},[1887,2048,2049],{},"分数对数",[1887,2051,2052],{"align":1876},"510",[1887,2054,2055],{"align":1876},"106",[1868,2057,2058,2061,2064],{},[1887,2059,2060],{},"最大绝对误差",[1887,2062,2063],{"align":1876},"基准",[1887,2065,2066],{"align":1876},"2.22×10⁻¹⁶",[13,2068,2069,2070,2073],{},"缓存路径的分数对为 ",[27,2071,2072],{},"8² + 9 + 10 + 11 + 12 = 106","，其中 prefill 计算完整分数矩阵，decode 每次只计算新 query 的一行。原始报告和本次复跑的数值、计数一致；另有无 decode、单位置和较长前缀等边界配置。",[13,2075,2076],{},"这些计数依赖本实验选定的完整前缀基线。若基线保存了其他中间状态或只计算最后一行 Attention，工作量会不同。表中的比值不代表通用加速倍数，也没有测量 FLOPs、耗时或峰值内存。",[2078,2079,2081],"h3",{"id":2080},"单-query-不能照搬左上三角掩码","单 query 不能照搬左上三角掩码",[13,2083,2084],{},"当前 query 位于最后一个位置，而缓存只含过去和当前位置的 key。因此这一行中的每个 key 都合法：",[1662,2086,2088],{"className":1664,"code":2087,"language":1666,"meta":1667,"style":1667},"query, key, value = project(x[position:position + 1], weights)\ncached_k = torch.cat((cached_k, key), dim=0)\ncached_v = torch.cat((cached_v, value), dim=0)\noutput = torch.softmax(query @ cached_k.T \u002F math.sqrt(query.shape[1]), dim=-1) @ cached_v\n",[27,2089,2090,2111,2134,2156],{"__ignoreMap":1667},[64,2091,2092,2095,2097,2100,2103,2105,2108],{"class":1672,"line":1673},[64,2093,2094],{"class":1676},"query, key, value ",[64,2096,94],{"class":1680},[64,2098,2099],{"class":1686}," project",[64,2101,2102],{"class":1676},"(x[position:position ",[64,2104,701],{"class":1680},[64,2106,2107],{"class":1704}," 1",[64,2109,2110],{"class":1676},"], weights)\n",[64,2112,2113,2116,2118,2120,2123,2126,2128,2130,2132],{"class":1672,"line":1722},[64,2114,2115],{"class":1676},"cached_k ",[64,2117,94],{"class":1680},[64,2119,1683],{"class":1676},[64,2121,2122],{"class":1686},"cat",[64,2124,2125],{"class":1676},"((cached_k, key), ",[64,2127,1698],{"class":1697},[64,2129,94],{"class":1680},[64,2131,531],{"class":1704},[64,2133,1803],{"class":1676},[64,2135,2136,2139,2141,2143,2145,2148,2150,2152,2154],{"class":1672,"line":1743},[64,2137,2138],{"class":1676},"cached_v ",[64,2140,94],{"class":1680},[64,2142,1683],{"class":1676},[64,2144,2122],{"class":1686},[64,2146,2147],{"class":1676},"((cached_v, value), ",[64,2149,1698],{"class":1697},[64,2151,94],{"class":1680},[64,2153,531],{"class":1704},[64,2155,1803],{"class":1676},[64,2157,2158,2161,2163,2165,2167,2170,2172,2175,2178,2181,2183,2186,2188,2191,2193,2195,2197,2200,2202],{"class":1672,"line":1762},[64,2159,2160],{"class":1676},"output ",[64,2162,94],{"class":1680},[64,2164,1683],{"class":1676},[64,2166,135],{"class":1686},[64,2168,2169],{"class":1676},"(query ",[64,2171,1842],{"class":1680},[64,2173,2174],{"class":1676}," cached_k.T ",[64,2176,2177],{"class":1680},"\u002F",[64,2179,2180],{"class":1676}," math.",[64,2182,246],{"class":1686},[64,2184,2185],{"class":1676},"(query.shape[",[64,2187,1705],{"class":1704},[64,2189,2190],{"class":1676},"]), ",[64,2192,1698],{"class":1697},[64,2194,1701],{"class":1680},[64,2196,1705],{"class":1704},[64,2198,2199],{"class":1676},") ",[64,2201,1842],{"class":1680},[64,2203,2204],{"class":1676}," cached_v\n",[13,2206,2207,2208,2211,2212,2215],{},"如果对 ",[27,2209,2210],{},"1×prefix_length"," 分数套用左上对齐的三角掩码，会只留下第 0 个 key。测试用零分数和 V 值 ",[27,2213,2214],{},"[1,2,6]"," 检查这一点：正确输出应为 3。另一项测试修改未来输入，较早位置的因果输出保持不变。",[2078,2217,2218],{"id":2218},"缓存会随上下文增长",[13,2220,2221],{},"本例 K、V 宽度均为 8，使用 float64，到第 12 个位置时保存的 K\u002FV 数据为：",[64,2223,2225],{"className":2224},[67],[64,2226,2228,2272],{"className":2227},[71],[64,2229,2231],{"className":2230},[75],[77,2232,2233],{"xmlns":79,"display":80},[82,2234,2235,2269],{},[85,2236,2237,2240,2243,2245,2248,2250,2252,2254,2256,2258,2260,2263,2267],{},[2238,2239,2044],"mn",{},[92,2241,2242],{},"×",[92,2244,142],{"stretchy":141},[2238,2246,2247],{},"8",[92,2249,701],{},[2238,2251,2247],{},[92,2253,147],{"stretchy":141},[92,2255,2242],{},[2238,2257,2247],{},[92,2259,94],{},[2238,2261,2262],{},"1536",[2264,2265,2266],"mtext",{}," bytes",[88,2268,164],{"mathvariant":134},[166,2270,2271],{"encoding":168},"12\\times(8+8)\\times8=1536\\ \\text{bytes}.",[64,2273,2275,2294,2315,2336,2355],{"className":2274,"ariaHidden":121},[173],[64,2276,2278,2282,2285,2288,2291],{"className":2277},[177],[64,2279],{"className":2280,"style":2281},[181],"height:0.7278em;vertical-align:-0.0833em;",[64,2283,2044],{"className":2284},[186],[64,2286],{"className":2287,"style":467},[124],[64,2289,2242],{"className":2290},[1184],[64,2292],{"className":2293,"style":467},[124],[64,2295,2297,2300,2303,2306,2309,2312],{"className":2296},[177],[64,2298],{"className":2299,"style":415},[181],[64,2301,142],{"className":2302},[213],[64,2304,2247],{"className":2305},[186],[64,2307],{"className":2308,"style":467},[124],[64,2310,701],{"className":2311},[1184],[64,2313],{"className":2314,"style":467},[124],[64,2316,2318,2321,2324,2327,2330,2333],{"className":2317},[177],[64,2319],{"className":2320,"style":415},[181],[64,2322,2247],{"className":2323},[186],[64,2325,147],{"className":2326},[383],[64,2328],{"className":2329,"style":467},[124],[64,2331,2242],{"className":2332},[1184],[64,2334],{"className":2335,"style":467},[124],[64,2337,2339,2343,2346,2349,2352],{"className":2338},[177],[64,2340],{"className":2341,"style":2342},[181],"height:0.6444em;",[64,2344,2247],{"className":2345},[186],[64,2347],{"className":2348,"style":192},[124],[64,2350,94],{"className":2351},[196],[64,2353],{"className":2354,"style":192},[124],[64,2356,2358,2362,2365,2369,2377],{"className":2357},[177],[64,2359],{"className":2360,"style":2361},[181],"height:0.8889em;vertical-align:-0.1944em;",[64,2363,2262],{"className":2364},[186],[64,2366,2368],{"className":2367},[124]," ",[64,2370,2373],{"className":2371},[186,2372],"text",[64,2374,2376],{"className":2375},[186],"bytes",[64,2378,164],{"className":2379},[186],[13,2381,2382],{},"这是最终缓存数据量，没有包含追加时的临时副本或整个程序内存。多层、多请求和多个 KV 头还需分别计入相应维度；GQA\u002FMQA 中的 KV 头数也可能少于 query 头数。",[13,2384,2385,2386,2389],{},"脚本用 ",[27,2387,2388],{},"torch.cat"," 便于逐步核对，增长时会复制旧内容。它不是优化后的推理服务，也不生成文字。KV cache 避免重投影历史位置，当前 query 的注意力仍要读取历史 K\u002FV，开销不会变成与上下文长度无关。",[45,2391,2392],{"id":2392},"复现与范围",[13,2394,2395,2396,2399],{},"更新后的 ZIP 包含 28\u002F29 号脚本、三份原始 JSON、依赖和 18 项归约\u002FAttention\u002FKV cache 测试。共享测试还加载归约模块，因此附上 ",[27,2397,2398],{},"27_tree_reduction.py","。完整发布仓库连同矩阵和编译实验共 30 项测试，本次冻结副本中全部通过；ZIP 自带的测试为其中 18 项。",[1662,2401,2405],{"className":2402,"code":2403,"language":2404,"meta":1667,"style":1667},"language-bash shiki shiki-themes github-dark-dimmed github-light houston","python3.11 -m venv .venv\n.venv\u002Fbin\u002Fpython -m pip install -r requirements.txt\n.venv\u002Fbin\u002Fpython 28_attention_online_softmax.py \\\n  --output results\u002Fattention-rerun.json\n.venv\u002Fbin\u002Fpython 28_attention_online_softmax.py \\\n  --n 17 --d 8 --value-dim 4 --query-block 8 --key-block 5 \\\n  --causal --dtype float32 --output results\u002Fcausal-rerun.json\n.venv\u002Fbin\u002Fpython 29_kv_cache.py --output results\u002Fkv-cache-rerun.json\n.venv\u002Fbin\u002Fpython -m unittest discover -s tests -v\n","bash",[27,2406,2407,2423,2442,2453,2461,2469,2502,2520,2533],{"__ignoreMap":1667},[64,2408,2409,2413,2416,2420],{"class":1672,"line":1673},[64,2410,2412],{"class":2411},"sLsJe","python3.11",[64,2414,2415],{"class":1704}," -m",[64,2417,2419],{"class":2418},"sWqY2"," venv",[64,2421,2422],{"class":2418}," .venv\n",[64,2424,2425,2428,2430,2433,2436,2439],{"class":1672,"line":1722},[64,2426,2427],{"class":2411},".venv\u002Fbin\u002Fpython",[64,2429,2415],{"class":1704},[64,2431,2432],{"class":2418}," pip",[64,2434,2435],{"class":2418}," install",[64,2437,2438],{"class":1704}," -r",[64,2440,2441],{"class":2418}," requirements.txt\n",[64,2443,2444,2446,2449],{"class":1672,"line":1743},[64,2445,2427],{"class":2411},[64,2447,2448],{"class":2418}," 28_attention_online_softmax.py",[64,2450,2452],{"class":2451},"sxpgZ"," \\\n",[64,2454,2455,2458],{"class":1672,"line":1762},[64,2456,2457],{"class":1704},"  --output",[64,2459,2460],{"class":2418}," results\u002Fattention-rerun.json\n",[64,2462,2463,2465,2467],{"class":1672,"line":1806},[64,2464,2427],{"class":2411},[64,2466,2448],{"class":2418},[64,2468,2452],{"class":2451},[64,2470,2471,2474,2477,2480,2483,2486,2489,2492,2494,2497,2500],{"class":1672,"line":1822},[64,2472,2473],{"class":1704},"  --n",[64,2475,2476],{"class":1704}," 17",[64,2478,2479],{"class":1704}," --d",[64,2481,2482],{"class":1704}," 8",[64,2484,2485],{"class":1704}," --value-dim",[64,2487,2488],{"class":1704}," 4",[64,2490,2491],{"class":1704}," --query-block",[64,2493,2482],{"class":1704},[64,2495,2496],{"class":1704}," --key-block",[64,2498,2499],{"class":1704}," 5",[64,2501,2452],{"class":2451},[64,2503,2505,2508,2511,2514,2517],{"class":1672,"line":2504},7,[64,2506,2507],{"class":1704},"  --causal",[64,2509,2510],{"class":1704}," --dtype",[64,2512,2513],{"class":2418}," float32",[64,2515,2516],{"class":1704}," --output",[64,2518,2519],{"class":2418}," results\u002Fcausal-rerun.json\n",[64,2521,2523,2525,2528,2530],{"class":1672,"line":2522},8,[64,2524,2427],{"class":2411},[64,2526,2527],{"class":2418}," 29_kv_cache.py",[64,2529,2516],{"class":1704},[64,2531,2532],{"class":2418}," results\u002Fkv-cache-rerun.json\n",[64,2534,2536,2538,2540,2543,2546,2549,2552],{"class":1672,"line":2535},9,[64,2537,2427],{"class":2411},[64,2539,2415],{"class":1704},[64,2541,2542],{"class":2418}," unittest",[64,2544,2545],{"class":2418}," discover",[64,2547,2548],{"class":1704}," -s",[64,2550,2551],{"class":2418}," tests",[64,2553,2554],{"class":1704}," -v\n",[13,2556,2557],{},"本次验证只包含固定输入的一层单头前向，不包含训练、反向传播、dropout、padding mask、多头模型或完整 Transformer。这份 CPU 快照当时按学习安排跳过真实 GPU；后续 SDPA 实测另列如下，没有测量 GPU prefill\u002Fdecode 服务性能。",[13,2559,2560,2561,55,2566,2571,2572,2576],{},"材料可对照",[19,2562,2565],{"href":2563,"rel":2564},"https:\u002F\u002Fwww.cs.cmu.edu\u002F~zhihaoj2\u002F15-779\u002Fslides\u002F05-transformers-attention.pdf",[37],"第 5 讲原课幻灯片",[19,2567,2570],{"href":2568,"rel":2569},"https:\u002F\u002Fpapers.nips.cc\u002Fpaper_files\u002Fpaper\u002F2017\u002Ffile\u002F3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf",[37],"Attention Is All You Need","和站内",[19,2573,2575],{"href":2574},"\u002Flearning\u002Fcmu-15-779\u002Fmaterials\u002Ftransformer-paper-zh\u002F","Transformer 中文学习资料","。这些是实验的阅读入口，不表示整篇论文评述或整门课程已经完成。",[13,2578,2579,39,2583],{},[19,2580,2582],{"href":35,"rel":2581},[37],"历史 CPU GitHub 代码",[19,2584,43],{"href":42},[45,2586,2588],{"id":2587},"后续补充显式选择-cuda-math-与-flash-后端","后续补充：显式选择 CUDA MATH 与 FLASH 后端",[13,2590,2591,2592,2595],{},"38 号脚本在学习聊天获授权的单卡环境运行，设备报告 RTX 4080 SUPER，PyTorch 2.7.0+cu128、CUDA Toolkit 12.8。输入改为 float16，形状为 ",[27,2593,2594],{},"[B=2,H=4,N,D=64]","，N 取 64、129、512，分别验证因果与非因果前向。",[13,2597,2598],{},"为了确定实际调用哪条路径，代码分别只启用一个后端：",[1662,2600,2602],{"className":1664,"code":2601,"language":1666,"meta":1667,"style":1667},"with sdpa_kernel(SDPBackend.MATH):\n    output = F.scaled_dot_product_attention(q, k, v, is_causal=causal)\nwith sdpa_kernel(SDPBackend.FLASH_ATTENTION):\n    output = F.scaled_dot_product_attention(q, k, v, is_causal=causal)\n",[27,2603,2604,2622,2646,2659],{"__ignoreMap":1667},[64,2605,2606,2610,2613,2616,2619],{"class":1672,"line":1673},[64,2607,2609],{"class":2608},"slEfO","with",[64,2611,2612],{"class":1686}," sdpa_kernel",[64,2614,2615],{"class":1676},"(SDPBackend.",[64,2617,2618],{"class":1704},"MATH",[64,2620,2621],{"class":1676},"):\n",[64,2623,2624,2627,2629,2632,2635,2638,2641,2643],{"class":1672,"line":1722},[64,2625,2626],{"class":1676},"    output ",[64,2628,94],{"class":1680},[64,2630,2631],{"class":1676}," F.",[64,2633,2634],{"class":1686},"scaled_dot_product_attention",[64,2636,2637],{"class":1676},"(q, k, v, ",[64,2639,2640],{"class":1697},"is_causal",[64,2642,94],{"class":1680},[64,2644,2645],{"class":1676},"causal)\n",[64,2647,2648,2650,2652,2654,2657],{"class":1672,"line":1743},[64,2649,2609],{"class":2608},[64,2651,2612],{"class":1686},[64,2653,2615],{"class":1676},[64,2655,2656],{"class":1704},"FLASH_ATTENTION",[64,2658,2621],{"class":1676},[64,2660,2661,2663,2665,2667,2669,2671,2673,2675],{"class":1672,"line":1762},[64,2662,2626],{"class":1676},[64,2664,94],{"class":1680},[64,2666,2631],{"class":1676},[64,2668,2634],{"class":1686},[64,2670,2637],{"class":1676},[64,2672,2640],{"class":1697},[64,2674,94],{"class":1680},[64,2676,2645],{"class":1676},[13,2678,2679,2680,2683,2684,2687],{},"没有让自动选择器在不支持 FLASH 时静默换成其他后端。两条路径分别与手工 float32 dense Attention 对照，全部通过 ",[27,2681,2682],{},"atol=rtol=3e-3","；所有记录中的最大绝对误差约为 ",[27,2685,2686],{},"8.24e-4","。",[13,2689,2690],{},"N=512、因果模式的原始结果为：",[1862,2692,2693,2706],{},[1865,2694,2695],{},[1868,2696,2697,2700,2703],{},[1871,2698,2699],{},"显式后端",[1871,2701,2702],{"align":1876},"相对 float32 dense 的最大绝对误差",[1871,2704,2705],{"align":1876},"event 中位数（ms）",[1882,2707,2708,2718],{},[1868,2709,2710,2712,2715],{},[1887,2711,2618],{},[1887,2713,2714],{"align":1876},"0.0008056164",[1887,2716,2717],{"align":1876},"0.272794",[1868,2719,2720,2722,2724],{},[1887,2721,2656],{},[1887,2723,2714],{"align":1876},[1887,2725,2726],{"align":1876},"0.038816",[13,2728,2729],{},"本组 FLASH 调用区间更短。这里调用的是 PyTorch 提供的 FLASH 后端，没有将前文 CPU 在线 Softmax 移植成手写 GPU kernel；也没有测量反向、dropout、模型训练或完整 LLM。",[13,2731,2732],{},"计时先 warmup 5 次，再取 15 个样本，每个样本平均 10 次调用。CUDA event 区间排除初次 JIT 和主机\u002F设备传输，但可能含 Python 提交和分配造成的 GPU 空闲，不能把表中数字直接视为隔离 kernel 延迟或服务加速比。原始报告保存所有样本及环境，未测量 Attention 的峰值内存或真实 HBM 流量。",[13,2734,2735,2736,2739,2740,2744,2745,2687],{},"整理时核对了原始 GPU 报告、脚本 SHA-256 和逐次样本中位数，没有在 Mac 上复跑 GPU。最新共享 ZIP 包含 38 号脚本、原生 CUDA runner 与报告，GPU 依赖见 ",[27,2737,2738],{},"requirements-gpu.txt","；28\u002F29 号 CPU 代码及前文 ZIP 继续保留。具体命令见",[19,2741,2743],{"href":2742},"\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing\u002F","记录 06 的续验","及包内 ",[27,2746,2747],{},"GPU_REPRODUCE.md",[13,2749,2750,39,2755],{},[19,2751,2754],{"href":2752,"rel":2753},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F200faf569eccf94fb125e65110bb5d14094d454a",[37],"最新 GPU 代码版本",[19,2756,2758],{"href":2757},"\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcuda-triton-sdpa-20261019.zip","下载 GPU 代码与原始结果 ZIP",[2760,2761,2762],"style",{},"html pre.shiki code .sWURj, html code.shiki .sWURj{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#EEF0F9}html pre.shiki code .sf2cM, html code.shiki .sf2cM{--shiki-default:#F47067;--shiki-light:#D73A49;--shiki-cyber:#EEF0F9}html pre.shiki code .sq3c-, html code.shiki .sq3c-{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#00DAEF}html pre.shiki code .sA9aF, html code.shiki .sA9aF{--shiki-default:#F69D50;--shiki-default-font-style:inherit;--shiki-light:#E36209;--shiki-light-font-style:inherit;--shiki-cyber:#4BF3C8;--shiki-cyber-font-style:italic}html pre.shiki code .sgENW, html code.shiki .sgENW{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#FFD493}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html.cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html pre.shiki code .sLsJe, html code.shiki .sLsJe{--shiki-default:#F69D50;--shiki-light:#6F42C1;--shiki-cyber:#00DAEF}html pre.shiki code .sWqY2, html code.shiki .sWqY2{--shiki-default:#96D0FF;--shiki-light:#032F62;--shiki-cyber:#FFD493}html pre.shiki code .sxpgZ, html code.shiki .sxpgZ{--shiki-default:#F47067;--shiki-light:#005CC5;--shiki-cyber:#EEF0F9}html pre.shiki code .slEfO, html code.shiki .slEfO{--shiki-default:#F47067;--shiki-default-font-style:inherit;--shiki-light:#D73A49;--shiki-light-font-style:inherit;--shiki-cyber:#54B9FF;--shiki-cyber-font-style:italic}",{"title":1667,"searchDepth":1722,"depth":1743,"links":2764},[2765,2766,2767,2768,2769,2773,2774],{"id":47,"depth":1722,"text":48},{"id":495,"depth":1722,"text":496},{"id":1857,"depth":1722,"text":1857},{"id":1925,"depth":1722,"text":1926},{"id":2005,"depth":1722,"text":2006,"children":2770},[2771,2772],{"id":2080,"depth":1743,"text":2081},{"id":2218,"depth":1743,"text":2218},{"id":2392,"depth":1722,"text":2392},{"id":2587,"depth":1722,"text":2588},[2776],"技术","\u002Fimages\u002Fposts\u002Fcmu-15-779-attention-online-softmax\u002Fcmu-online-attention-cover-20261016.webp","浅紫色卡片分批进入绿色计算托盘，旁边三个状态容器保存累积量的浅色手绘插画","2026-09-24","在 CPU 上对照完整 Attention、在线 Softmax 分块实现与 PyTorch SDPA，检查因果掩码、浮点误差和分数块大小，理解 FlashAttention 的一部分算法基础。",false,"md",{"codeUrl":2752,"attachmentUrl":2757},true,"\u002Fposts\u002Fcmu-15-779-attention-online-softmax",null,{"title":5,"description":2780},"CMU 15-779","posts\u002Fcmu-15-779-attention-online-softmax",[2788,2791,2792,2793,2794],"Attention","Softmax","FlashAttention","PyTorch","88AyfSFNqPsMEpmVoPsos72JtDPKHSdq3IxQ_Uz9o-Y",[2797,2814,2830,2844,2858,2872,2887,2902,2916,2927,2942,2954,2967,2979,2992,3006,3020,3035,3046,3058,3073,3086,3099,3103,3117,3129,3144,3158,3172,3183,3196,3213,3228,3241,3253,3265,3278,3291],{"slug":2798,"path":2799,"title":2800,"date":2801,"tags":2802,"description":2807,"cover":2808,"coverAlt":2809,"excerpt":2810,"draft":2781,"hidden":2781,"published":2784,"readingTime":2522,"series":2811,"seriesOrder":2522,"codeUrl":2812,"attachmentUrl":2813},"cs336-cross-entropy-adamw-training-loop","\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop","Stanford CS336 学习记录 08：数值稳定交叉熵、AdamW、余弦调度与完整训练闭环","2026-10-27",[2803,2794,2804,2805,2806],"CS336","AdamW","CrossEntropy","训练闭环","用 LogSumExp 消去交叉熵中的指数溢出，手写解耦权重衰减的 AdamW 优化器、全局 L2 梯度裁剪、Warmup 余弦退火调度、memmap 数据采样、Checkpoint 序列化与 Top-p 解码，完成 CS336 Assignment 1 全部 48 项测试。","\u002Fimages\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop\u002Fcs336-cross-entropy-adamw-training-loop-cover.webp","滑动窗口采样、余弦退火曲线、双动量优化轨迹与检查点归档构成的训练闭环 AI 辅助概念插画","作为 Stanford CS336 Assignment 1 系列的收官篇，我们已经拥有了分词器（bpe.py、tokenizer.py）和完整的语言模型架构（transformer.py）。要让模型在语料上真正训练并生成文本，还需","Stanford CS336","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fb5c93ad0cc73178b2b2a62e518a0900da47a0fb7","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-08-adamw-training-loop-20261027.zip",{"slug":2815,"path":2816,"title":2817,"date":2818,"tags":2819,"description":2826,"cover":2827,"coverAlt":2828,"excerpt":2829,"draft":2781,"hidden":2781,"published":2784,"readingTime":2522},"editcache-video-dit-inference-acceleration","\u002Fposts\u002Feditcache-video-dit-inference-acceleration","EditCache：视频 Diffusion Transformer (DiT) 局部编辑的跨步特征复用与区域重算加速","2026-10-25",[2820,2821,2822,2823,2824,2825],"扩散模型","视频生成","AI Infra","推理加速","GPU","论文解读","拆解视频扩散模型（Wan2.1-VACE）局部编辑中的三重算力冗余，通过可重入 Latent、跨步 Block 隐状态缓存复用、带 Halo 边界的局部区域重算与自适应推理起点，在单卡 H100 上实现 2.46× 模型计算加速与 1.49× 严格同步端到端加速。","\u002Fimages\u002Fposts\u002Feditcache-video-dit-inference-acceleration\u002Feditcache-dit-cover-20261025.webp","视频扩散 Transformer 时空三维体素网格中冻结背景缓存与暖橙色局部编辑重算通道的等距技术插画","在基于 Diffusion Transformer（DiT）的视频编辑与局部重绘（Video Inpainting \u002F Local Editing）任务中，用户的典型工作流并不是一次性生成最终成片，而是针对同一段源视频反复调整掩码区",{"slug":2831,"path":2832,"title":2833,"date":2818,"tags":2834,"description":2839,"cover":2840,"coverAlt":2841,"excerpt":2842,"draft":2781,"hidden":2781,"published":2784,"readingTime":2522,"series":2811,"seriesOrder":2504,"codeUrl":2812,"attachmentUrl":2843},"cs336-bpe-tokenizer-training-encoding","\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding","Stanford CS336 学习记录 07：Byte-Level BPE 分词器训练、倒排索引加速与流式编码",[2803,2835,2836,2837,2838],"Tokenizer","BPE","Tiktoken","算法优化","实现 GPT-2 正则预分词、特殊 token 硬边界切分、基于 pair-to-word 倒排索引的增量 BPE 合并算法（\u003C 0.25s 通过速度基准），以及与 tiktoken 完全对齐的 Tokenizer。","\u002Fimages\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding\u002Fcs336-bpe-tokenizer-training-encoding-cover.webp","UTF-8 字节块经过正则切分闸口后沿合并树聚合成子词单元的 AI 辅助概念插画","前面六篇完成了 Transformer 语言模型本身的全部神经网络组件。要让模型真正处理自然语言文本，还需要打通入口处的 ByteLevel BPE（BytePair Encoding）分词器。这一篇记录 cs336basics\u002Fb","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-07-bpe-tokenizer-20261025.zip",{"slug":2845,"path":2846,"title":2847,"date":2848,"tags":2849,"description":2854,"cover":2855,"coverAlt":2856,"excerpt":2857,"draft":2781,"hidden":2781,"published":2784,"readingTime":2522},"in-situ-autoguidance-editable-scientific-illustration","\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration","从原位自校正引导（In-situ Autoguidance）到可编辑科研插图生成（AutoFigure-Edit）","2026-10-24",[2820,2850,2851,2852,2853,2825],"多模态","Agent","ICML","ACL","串联两项生成式模型与多模态智能体研究：如何通过推理期局部随机微扰在单模型内构造免训练扩散自引导向量（ICML 2025 Workshop 第一作者，arXiv:2510.17136），以及如何通过五阶段视觉-符号混合流水线生成原生可编辑 SVG 科研插图（ACL 2026 Demo，FigureBench 83.0% 胜率）。","\u002Fimages\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration\u002Fautoguidance-autofigure-cover-20261024.webp","左侧扩散流形自校正轨迹矢量场与右侧五阶段可编辑 SVG 科研插图分层网格相衔接的等距技术插画","生成式视觉模型正在经历从“生成高保真像素（Pixel Synthesis）”向“可控结构化生成（Controllable & Editable Generation）”的演进。在这一进程中，我作为第一作者或核心贡献者参与了两项分别处",{"slug":2859,"path":2860,"title":2861,"date":2862,"tags":2863,"description":2867,"cover":2868,"coverAlt":2869,"excerpt":2870,"draft":2781,"hidden":2781,"published":2784,"readingTime":2522,"series":2811,"seriesOrder":1822,"codeUrl":2812,"attachmentUrl":2871},"cs336-pre-norm-transformer-block-lm","\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm","Stanford CS336 学习记录 06：Pre-Norm Transformer Block 与完整语言模型装配","2026-10-23",[2803,2794,2864,2865,2866],"Transformer","Pre-Norm","FLOPs","将 RMSNorm、带 RoPE 的因果多头自注意力和 SwiGLU 组装成 Pre-Norm Transformer Block，搭建完整 TransformerLM，并核算 GPT-2 XL 的参数量与前向 FLOPs。","\u002Fimages\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm\u002Fcs336-pre-norm-transformer-block-lm-cover.webp","干净残差流主干穿过多个 Pre-Norm 子层并汇聚到词表投影层的 AI 辅助概念插画","前五篇把 Linear、Embedding、RMSNorm、SwiGLU、RotaryPositionalEmbedding 和因果 MultiHeadSelfAttention 全部单独打通了。这一篇进入 Assignment 1","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-06-transformer-block-lm-20261023.zip",{"slug":2873,"path":2874,"title":2875,"date":2862,"tags":2876,"description":2882,"cover":2883,"coverAlt":2884,"excerpt":2885,"draft":2781,"hidden":2781,"published":2784,"readingTime":2886},"xgrammar-concurrent-cache-generation-fix","\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix","XGrammar 结构化解码缓存剖析：Unlimited 模式并发 Clear 字节记账竞态与代际修复（PR #1011）",[2822,2877,2878,2879,2880,2881],"C++","XGrammar","结构化解码","并发编程","源码分析","拆解 XGrammar GrammarCompiler 的双层缓存架构与 ThreadSafeLRUCache 锁分离实现，定位 Unlimited 模式下 Clear() 字节残留与在途编译（In-flight Computation）跨代污染根因，给出代际号 + 读写锁原子修复与零 sleep 确定性并发 GoogleTest 验证。","\u002Fimages\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix\u002Fxgrammar-cache-fix-cover-20261022.webp","双层磨砂玻璃流道经暖橙色代际闸门过滤后汇入晶格缓存矩阵的等距技术插画","在大语言模型服务（如 SGLang、vLLM）的结构化输出场景中，JSON Schema 或 EBNF 语法需要先编译为词表级掩码缓存（CompiledGrammar），才能在每个 Decode 步以微秒级开销生成合法 token ",14,{"slug":2888,"path":2889,"title":2890,"date":2891,"tags":2892,"description":2898,"cover":2899,"coverAlt":2900,"excerpt":2901,"draft":2781,"hidden":2781,"published":2784,"readingTime":2886},"multi-constraint-agent-routing-dag-eval","\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval","多约束生产级 Agent 系统实践：两阶段工具路由、DAG 局部重规划与轨迹级评测闭环","2026-10-22",[2851,2893,2894,2895,2896,2897],"LLM","工具调用","上下文工程","DPO","强化学习","围绕跨境酒旅等强耦合多约束业务场景，拆解向量 Top-K 召回 + LLM 精排 + 跨语言实体链接与 Schema 门控（96%+ 调用合法率）、基于状态机的 DAG 局部子图重规划、Observation 冗余裁剪（节省 40% Token），以及从线上失败恢复轨迹构造 SFT\u002FDPO 偏好对的评测与训练闭环。","\u002Fimages\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval\u002Fagent-dag-routing-cover-20261020.webp","候选模块经漏斗滤网进入双分支有向无环图轨道并在受阻节点触发暖橙色局部旁路重连的等距技术插画","在玩具级 Demo 里，几个工具加上一段 ReAct 循环就能跑通大多数单步问答。但当 Agent 进入跨境酒旅、复杂交易或研发沙箱等真实业务生产环境时，单靠线性 ReAct 循环很快会撞上四堵墙： 1. 工具空间与参数耦合爆炸：候",{"slug":2903,"path":2904,"title":2905,"date":2906,"tags":2907,"description":2911,"cover":2912,"coverAlt":2913,"excerpt":2914,"draft":2781,"hidden":2781,"published":2784,"readingTime":2915},"llm-mrd-distillation-auditable-review-agents","\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents","多视角推理蒸馏（LLM-MRD）与可审计生成式评审系统（DeepReviewer 2.0）深度拆解","2026-10-21",[2908,2850,2909,2851,2910,2825],"知识蒸馏","Qwen2.5-VL","Post-training","对比大模型落地的两条互补路径：如何用 Qwen2.5-VL 离线多视角分析结合 MLP 残差校准与温度缩放 KL 蒸馏，将推理能力压入 358M 学生模型实现 17.2 ms 独立推理（LLM-MRD，DASFAA 2026 Oral）；以及如何通过主张-证据-风险台账、PASA 可比性门控与导出预算校验构建可审计的同行评审 Agent（DeepReviewer 2.0）。","\u002Fimages\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents\u002Fmrd-review-distill-cover-20261016.webp","三块半透明几何棱镜发出平行光束穿过中央校准环汇聚至右侧高密度晶体模块的等距技术插画","把大语言模型或视觉语言模型（VLM）的深度推理能力用到实际系统中，通常会面临一个两难选择： 如果场景受限于毫秒级在线延迟（如高吞吐多模态内容审核与虚假信息检测）：直接在线调用 7B～72B 的多模态大模型做长链条思维链（CoT）生成",18,{"slug":2917,"path":2918,"title":2919,"date":2906,"tags":2920,"description":2922,"cover":2923,"coverAlt":2924,"excerpt":2925,"draft":2781,"hidden":2781,"published":2784,"readingTime":2504,"series":2811,"seriesOrder":1806,"codeUrl":2812,"attachmentUrl":2926},"cs336-rope-rotary-position-embedding","\u002Fposts\u002Fcs336-rope-rotary-position-embedding","Stanford CS336 学习记录 05：RoPE 旋转位置编码与带位置的多头注意力",[2803,2794,2864,2921,2791],"RoPE","从二维平面正交旋转推导 RoPE 的奇偶特征配对公式，理解为什么内积只依赖相对位置差 (i - j)，并用 register_buffer 预计算 cos\u002Fsin 缓存接入因果多头自注意力。","\u002Fimages\u002Fposts\u002Fcs336-rope-rotary-position-embedding\u002Fcs336-rope-rotary-position-embedding-cover.webp","二维特征平面中的旋转向量对与预计算余弦正弦网格的 AI 辅助概念插画","上一篇完成了不含位置编码的因果多头自注意力。如果只看那一套矩阵乘法，把输入序列前两个 token 交换顺序（并在因果 mask 允许的范围内看同一个后缀 query 对它们的点积），点积本身并不知道哪个 key 离当前 query ","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-05-rope-mha-20261021.zip",{"slug":2928,"path":2929,"title":2930,"date":2931,"tags":2932,"description":2938,"cover":2939,"coverAlt":2940,"excerpt":2941,"draft":2781,"hidden":2781,"published":2784,"readingTime":2522},"llm-gateway-semantic-cache-dynamic-batching","\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching","高并发 LLM 推理网关架构实践：两级精确\u002F语义缓存、50ms 有界动态批处理与尾部延迟治理","2026-10-19",[2822,2933,2934,2935,2936,2937],"LLM 推理","分布式系统","Redis","FastAPI","后端","深度拆解生产级大模型推理网关 LLM-Gateway 的核心系统设计：基于 SHA-256 与 Redis 向量余弦相似度的两级精确\u002F语义缓存、50ms 有界等待动态批处理调度器（Dynamic Batching）、令牌桶多租户限流与 Prometheus 全链路可观测性，将命中请求 P99 延迟从 5,307 ms 压降至 17 ms。","\u002Fimages\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching\u002Fllm-gateway-cover-20261019.webp","高并发请求流依次穿过精确哈希滤网、向量球面语义缓存层与 50ms 动态批处理时间窗拼装器的等距系统架构插画","在将大语言模型接入高并发业务系统时，如果让上游微服务直接裸连底层推理引擎（vLLM \u002F SGLang 或外部模型 API），系统很快会暴露出三个典型的生产痛点： 1. 高频重复与近义查询消耗巨量算力：大量用户请求在语义上高度等价（例",{"slug":2943,"path":2944,"title":2945,"date":2931,"tags":2946,"description":2948,"cover":2949,"coverAlt":2950,"excerpt":2951,"draft":2781,"hidden":2781,"published":2784,"readingTime":2886,"series":2811,"seriesOrder":1762,"codeUrl":2952,"attachmentUrl":2953},"cs336-multihead-attention-shapes","\u002Fposts\u002Fcs336-multihead-attention-shapes","Stanford CS336 学习记录 04：多头注意力怎样拆头、计算和合头",[2803,2794,2864,2791,2947],"张量形状","沿着 (2, 5, 64) 的输入追踪 QKV 投影、reshape、transpose、共享因果 mask 和合头，理解每个头仍处理完整序列，以及最后为什么还需要输出 Linear。","\u002Fimages\u002Fposts\u002Fcs336-multihead-attention-shapes\u002Fcs336-multihead-attention-shapes-cover.webp","宽向量束分成多条并行特征轨道后重新合并的 AI 辅助概念插画","上一篇完成了缩放点积注意力。那个函数接收已经准备好的 Q、K、V，计算匹配分数，应用 mask，再用权重汇总 V。这一篇继续把它放进一个带参数的多头自注意力模块。 这次最需要看清的是“拆头”。代码里只是几行 reshape 和 tr","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002F31363223498fafd1528c550a51efbe0aed231736","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-04-multihead-attention-20261019.zip",{"slug":2955,"path":2956,"title":2957,"date":2958,"tags":2959,"description":2963,"cover":2964,"coverAlt":2965,"excerpt":2966,"draft":2781,"hidden":2781,"published":2784,"readingTime":2535},"qwen25-sft-dpo-alignment-tax-ablation","\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation","大模型后训练偏好对齐实录：Qwen2.5 SFT 多维消融、动态参考 DPO 与对齐税（Alignment Tax）量化分析","2026-10-18",[2910,2896,2960,2961,2897,2962],"SFT","LoRA","Qwen2.5","基于 Qwen2.5 完整复现 Instruct SFT → Bradley-Terry DPO 两阶段后训练管线：通过 3 随机种子消融 LoRA Rank、学习率与数据规模（GSM8K +12.73pp），实现免双倍权重的动态 Adapter 切换 DPO 训练，并定量揭示偏好对齐对确定性数学推理的对齐税效应。","\u002Fimages\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation\u002Fqwen25-sft-dpo-cover-20261018.webp","左侧监督微调轨迹流经中央 Bradley-Terry 偏好分流天平分离出正负样本通道并连接右侧对齐税监控仪表盘的等距技术插画","在大语言模型后训练（PostTraining）管线中，监督微调（SFT）负责教会模型任务格式与领域推理模式，而直接偏好优化（DPO, Direct Preference Optimization）则利用成对偏好数据 $(x, yw,",{"slug":2968,"path":2969,"title":2970,"date":2958,"tags":2971,"description":2972,"cover":2973,"coverAlt":2974,"excerpt":2975,"draft":2781,"hidden":2781,"published":2784,"readingTime":2976,"series":2811,"seriesOrder":1743,"codeUrl":2977,"attachmentUrl":2978},"cs336-softmax-attention-mask","\u002Fposts\u002Fcs336-softmax-attention-mask","Stanford CS336 学习记录 03：Softmax 的数值稳定与 Attention Mask",[2803,2794,2864,2792,2791],"从减去最大值为何不改变 Softmax 开始，理清函数与有参数层的区别，再沿着 Q、K、V 的形状理解缩放点积注意力，以及 mask 为什么必须填负无穷。","\u002Fimages\u002Fposts\u002Fcs336-softmax-attention-mask\u002Fcs336-softmax-attention-mask-cover.webp","注意力分数、权重与因果遮罩的 AI 辅助概念插画","上一篇写到 SwiGLU，接下来进入 Attention。真正开始相乘之前，先做了 Softmax：它把一组分数变成总和为 1 的权重，后面用这些权重汇总不同 token 提供的信息。 这段学习里，有两个很具体的问题值得记下来。一个",11,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fa39eb684710817f7f389626423fcb26ca0f6d629","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-03-softmax-sdpa-20261018.zip",{"slug":2980,"path":2981,"title":2982,"date":2983,"tags":2984,"description":2987,"cover":2988,"coverAlt":2989,"excerpt":2990,"draft":2781,"hidden":2781,"published":2784,"readingTime":2535,"series":2811,"seriesOrder":1722,"codeUrl":2977,"attachmentUrl":2991},"cs336-rmsnorm-swiglu","\u002Fposts\u002Fcs336-rmsnorm-swiglu","Stanford CS336 学习记录 02：RMSNorm 的广播与 SwiGLU 的门控","2026-10-16",[2803,2794,2864,2985,2986],"RMSNorm","SwiGLU","用一个 token 向量理解均方根归一化，拆开 dim=-1 与 keepdim=True 的作用，再沿着两条分支读懂 SiLU、逐元素乘法和 SwiGLU。","\u002Fimages\u002Fposts\u002Fcs336-rmsnorm-swiglu\u002Fcs336-rmsnorm-swiglu-cover.webp","特征尺度整理和双分支门控的 AI 辅助概念插画","上一篇把 Linear 和 Embedding 的参数形状理顺了。继续往下，最先遇到的是 RMSNorm：公式不算长，但把它落到一批 token 上时，dim=1、keepdim=True 和广播必须一起理解。 然后是 SiLU 和","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-02-rmsnorm-swiglu-20261016.zip",{"slug":2993,"path":2994,"title":2995,"date":2996,"tags":2997,"description":3000,"cover":3001,"coverAlt":3002,"excerpt":3003,"draft":2781,"hidden":2781,"published":2784,"readingTime":3004,"series":2811,"seriesOrder":1673,"codeUrl":2977,"attachmentUrl":3005},"cs336-linear-embedding-shapes","\u002Fposts\u002Fcs336-linear-embedding-shapes","Stanford CS336 学习记录 01：从 Linear、Embedding 搞清张量形状","2026-10-14",[2803,2794,2864,2998,2999],"Linear","Embedding","从 PyTorch 类、参数与 forward 的关系开始，理解 Linear 为什么保存两行三列的权重、计算时为什么要转置，再区分 Embedding 查表与线性变换。","\u002Fimages\u002Fposts\u002Fcs336-linear-embedding-shapes\u002Fcs336-linear-embedding-shapes-cover.webp","token 方块、向量表与矩阵变换的 AI 辅助概念插画","开始沿着 Stanford CS336 的 Assignment 1 一步步实现。BPE 之前已经接触过，这轮先跳过分词器，从 Transformer 的基础层开始。这里记录的是当时写代码时真正卡住的问题，以及对应测试能确认到的范围",10,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-01-linear-embedding-20261014.zip",{"slug":3007,"path":3008,"title":3009,"date":3010,"tags":3011,"description":3014,"cover":3015,"coverAlt":3016,"excerpt":3017,"draft":2781,"hidden":2781,"published":2784,"readingTime":3018,"series":2788,"seriesOrder":2886,"codeUrl":2752,"attachmentUrl":3019},"cmu-15-779-kv-cache-granularity-study","\u002Fposts\u002Fcmu-15-779-kv-cache-granularity-study","CMU 15-779 学习记录 14：KV 页大小、共享边界与 CPU 实现成本","2026-10-12",[2788,2031,3012,3013],"PagedAttention","系统实验","用 210 组 CPU 配置比较连续与分页 KV cache，观察 31\u002F32-token 共享前缀的页边界，分别记录活跃槽位、池容量、复制计数和完整调用耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling\u002Fcmu-serving-cover-20261019.webp","浅色手绘分页存储抽屉与共享前缀卡片，旁边连接不同长度的请求队列","个人课程项目中的 CPU 实验记录，延续记录 11 的分页 KV 实现。已完成 210 组配置与 1470 次计时，输出均通过连续存储参考对照；本篇不代表正式作业提交、整门课程或项目全部交付完成。 同样是共享 31 个 token，",12,"\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fkv-cache-granularity-20261019.zip",{"slug":3021,"path":3022,"title":3023,"date":3024,"tags":3025,"description":3028,"cover":3029,"coverAlt":3030,"excerpt":3031,"draft":2781,"hidden":2781,"published":2784,"readingTime":1806,"series":2788,"seriesOrder":3032,"codeUrl":3033,"attachmentUrl":3034},"cmu-15-779-moe-routing-capacity","\u002Fposts\u002Fcmu-15-779-moe-routing-capacity","CMU 15-779 学习记录 13：MoE 路由、稀疏分发与容量溢出","2026-10-09",[2788,3026,3027],"MoE","Routing","用 CPU top-2 MoE 检查稀疏分发的输出与梯度，统计 expert-token 行数，并保留容量丢弃会改变输出的反例。","\u002Fimages\u002Fposts\u002Fcmu-15-779-moe-routing-capacity\u002Fcmu-moe-cover-20261019.webp","浅色手绘分拣器将小珠分配到四个专家工作屋，队列呈现负载不均","第 18 讲的个人配套实验。36 号脚本串行运行 CPU 专家，检查 topk 路由后的分发与合并；没有训练 router，也没有执行 GPU grouped GEMM 或跨设备 alltoall。 上一节记录低秩适配，这次转向每个",13,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fc4851e0cedadf13af7c421867e83c8ff57244fa0","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-parallel-serving-lora-moe-20261019.zip",{"slug":3036,"path":3037,"title":3038,"date":3039,"tags":3040,"description":3042,"cover":3043,"coverAlt":3044,"excerpt":3045,"draft":2781,"hidden":2781,"published":2784,"readingTime":1806,"series":2788,"seriesOrder":3018,"codeUrl":3033,"attachmentUrl":3034},"cmu-15-779-lora-uniform-quantization","\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization","CMU 15-779 学习记录 12：LoRA 首步梯度、权重合并与均匀 4-bit 打包","2026-10-06",[2788,2961,3041],"Quantization","用一次 CPU SGD 更新检查 LoRA 初始化、冻结基座与合并等价，再分别记录均匀 4-bit 存储字节数和量化误差。","\u002Fimages\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization\u002Fcmu-lora-cover-20261019.webp","浅色手绘的大计算板、两块细长适配板和成对打包的小方块","第 17 讲的个人配套实验，35 号脚本实际执行一次 CPU LoRA 更新，并实现均匀 4bit 打包。实验检查机制与数值误差，不评估微调任务效果，也没有实现完整 QLoRA。 这次把两个问题分开记录：低秩分支是否能合并进原权重，",{"slug":3047,"path":3048,"title":3049,"date":3050,"tags":3051,"description":3055,"cover":3015,"coverAlt":3056,"excerpt":3057,"draft":2781,"hidden":2781,"published":2784,"readingTime":2504,"series":2788,"seriesOrder":2976,"codeUrl":3033,"attachmentUrl":3034},"cmu-15-779-kv-serving-speculative-sampling","\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling","CMU 15-779 学习记录 11：分页 KV、连续批处理与推测采样","2026-10-03",[2788,3052,3053,3054],"LLM Serving","KV Cache","Speculative Decoding","在 CPU 上验证 KV 页共享和写时复制，用单位成本模型检查连续批处理，再用条件概率小模型核对推测采样的接受与补采样。","浅色手绘的共享卡片抽屉与不断补入小珠子的请求托盘","第 15、16 讲的个人配套实验。33 号实现 CPU 分页 KV 存储，37 号模拟请求准入，34 号运行小概率模型的推测采样。它们分别验证存储、调度和分布问题，没有测量真实 LLM 服务吞吐或 GPU 延迟。 记录 08已经对照",{"slug":3059,"path":3060,"title":3061,"date":3062,"tags":3063,"description":3068,"cover":3069,"coverAlt":3070,"excerpt":3071,"draft":2781,"hidden":2781,"published":2784,"readingTime":3072},"qwen3-mid-post-train-mlx","\u002Fposts\u002Fqwen3-mid-post-train-mlx","在 M2 Pro 上完成 Qwen3 两阶段训练：Mid Train、SFT 与可复核评估","2026-09-30 11:19:10",[2893,3064,3065,2961,3066,2960,3067],"Qwen3","MLX","持续预训练","Apple Silicon","用 Qwen3 0.6B Base 和 MLX 完成领域持续训练与监督微调，记录数据划分、聊天模板、损失掩码、数值稳定性修复和独立测试结果。","\u002Fimages\u002Fposts\u002Fqwen3-mid-post-train-mlx\u002Fqwen3-mlx-cover-20261018.webp","Apple Silicon 统一内存芯片底座支撑两阶段持续预训练与带精确 Loss 掩码滤网的监督微调桥接插画","在一台 M2 Pro、16 GB 内存的 Mac 上，能否把一个 Base 语言模型接续训练成输出 JSON 的运维事件分级助手？这次项目完成了领域持续训练、监督微调、检查点选择、重新加载和独立测试，并保留了权重、日志和数据哈希。 ",16,{"slug":3074,"path":3075,"title":3076,"date":3077,"tags":3078,"description":3082,"cover":3083,"coverAlt":3084,"excerpt":3085,"draft":2781,"hidden":2781,"published":2784,"readingTime":1822,"series":2788,"seriesOrder":3004,"codeUrl":3033,"attachmentUrl":3034},"cmu-15-779-cpu-parallelism-pipeline","\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline","CMU 15-779 学习记录 10：加权梯度、状态分片与同步流水线","2026-09-30",[2788,3079,3080,3081],"Parallel Training","Gloo","Pipeline","用双 CPU 进程 Gloo 检查不等数据分片的梯度，推导持久状态字节数，再用依赖模型比较 GPipe 与同步 1F1B 的激活存活。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline\u002Fcmu-parallel-cover-20261019.webp","浅色手绘的四阶段桌面流水线与按数量汇总的托盘","第 12 至 14 讲的个人配套实验，脚本编号 31、32 不对应官方作业。Gloo 实际运行两个 CPU 进程；张量切分、状态字节数和流水线分别按下文的数学或成本模型解释。真实 GPU 实测按学习安排跳过。 上一节检查单个计算图的",{"slug":3087,"path":3088,"title":3089,"date":3090,"tags":3091,"description":3094,"cover":3095,"coverAlt":3096,"excerpt":3097,"draft":2781,"hidden":2781,"published":2784,"readingTime":2504,"series":2788,"seriesOrder":2535,"codeUrl":35,"attachmentUrl":3098},"cmu-15-779-cpu-compiler-optimizations","\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations","CMU 15-779 学习记录 09：CPU 编译优化、合法重写与候选搜索","2026-09-27",[2788,3092,3093,2794],"ML Compiler","TorchInductor","用 CPU 实验验证 BatchNorm 折叠、RMSNorm 重排与非线性反例，测量分块候选，并运行 TorchInductor，分别记录首调用与预热后的时间。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations\u002Fcmu-cpu-compiler-cover-20261019.webp","淡紫计算积木经过薄荷绿折叠模具变成浅蓝积木，前方摆放四种候选托盘的浅色手绘插画","编译章节的个人配套实验，编号 30 不对应官方作业。图重写、分块候选和 TorchInductor 均在 CPU 上运行；真实 GPU 实验按学习安排跳过。 上一节记录了分块 Attention 和 KV cache。这次转到编译优","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-compiler-optimizations-20261019.zip",{"slug":3100,"path":2785,"title":5,"date":2779,"tags":3101,"description":2780,"cover":2777,"coverAlt":2778,"excerpt":3102,"draft":2781,"hidden":2781,"published":2784,"readingTime":3018,"series":2788,"seriesOrder":2522,"codeUrl":2752,"attachmentUrl":2757},"cmu-15-779-attention-online-softmax",[2788,2791,2792,2793,2794],"第 5 讲 Transformers & Attention 的个人配套实验。28\u002F29 号是 CPU 单头前向与 KV cache；后续补入 38 号 CUDA 多头合成张量的显式 SDPA MATH\u002FFLASH 对照。没有手写 ",{"slug":3104,"path":3105,"title":3106,"date":3107,"tags":3108,"description":3113,"cover":3114,"coverAlt":3115,"excerpt":3116,"draft":2781,"hidden":2781,"published":2784,"readingTime":3004,"series":2788,"seriesOrder":2504,"codeUrl":2752,"attachmentUrl":2757},"cmu-15-779-matmul-tiling-tree-reduction","\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction","CMU 15-779 学习记录 07：矩阵分块、数据复用与树形归约","2026-09-21",[2788,3109,3110,3111,3112],"CUDA","矩阵乘法","数据复用","归约","用 CPU 模型统计矩阵分块的输入读取次数，再比较串行求和和树形归约的依赖深度，记录补零、同步与浮点加法顺序的边界。","\u002Fimages\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction\u002Fcmu-matmul-reduction-cover-20261016.webp","矩阵积木在共享托盘中复用，旁边的小珠沿树形管道逐层合并的浅色手绘插画","GPU Architecture & CUDA Programming 的个人配套实验。首次记录用 CPU 模型检查数据复用与依赖深度；后续补入单卡 GPU 原生 CUDA 矩阵验证，以及 Triton 两阶段归约和矩阵乘法实测。 ",{"slug":3118,"path":3119,"title":3120,"date":3121,"tags":3122,"description":3125,"cover":3126,"coverAlt":3127,"excerpt":3128,"draft":2781,"hidden":2781,"published":2784,"readingTime":3004,"series":2788,"seriesOrder":1822,"codeUrl":2752,"attachmentUrl":2757},"cmu-15-779-cuda-thread-indexing","\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing","CMU 15-779 学习记录 06：CUDA 线程索引与边界检查","2026-09-18",[2788,3109,2824,3123,3124],"线程索引","边界检查","用 CPU 演示 CUDA 的 block、thread 与全局下标：10 个元素如何分给 3 个线程块，为什么要跳过越界线程，以及这些验证能说明什么。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing\u002Fcmu-cuda-indexing-cover-20261015.webp","分组的浅紫色计算方块与末端边界挡板组成线程分配教具的浅色插画","GPU Architecture & CUDA Programming 的个人配套练习。首次记录为 CPU 索引演示；后续在单卡 GPU 环境补入原生 CUDA 编译、运行与 memcheck，以及 Triton 向量加法实测。 上",{"slug":3130,"path":3131,"title":3132,"date":3133,"tags":3134,"description":3138,"cover":3139,"coverAlt":3140,"excerpt":3141,"draft":2781,"hidden":2781,"published":2784,"readingTime":3004,"series":2788,"seriesOrder":1806,"codeUrl":3142,"attachmentUrl":3143},"cmu-15-779-profiler-operator-batching","\u002Fposts\u002Fcmu-15-779-profiler-operator-batching","CMU 15-779 学习记录 05：Profiler、算子调用与批量推理","2026-09-15",[2788,2794,3135,3136,3137],"Profiler","批量推理","性能分析","对照 ML Systems 基础，用 Profiler 拆开两层网络的执行过程，再固定 1000 个样本扫描 batch size，观察算子调用次数、尾批和 CPU 耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-profiler-operator-batching\u002Fcmu-pytorch-profiling-cover-20261015.webp","放大镜观察线性层与 ReLU 模块，旁边面板呈现执行片段的浅色技术教具插画","本篇对照课程官网的 Introduction to ML Systems 整理，是个人配套 CPU 实验，不是课程官方作业。脚本编号是练习顺序，不对应课程讲次。 前几篇一直在补 PyTorch 基础。到了这一步，我在学习聊天里问：现","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fcecf5110d8a95161bb5763eadfca94cd7a0cc50f","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-profiler-batching-20261015.zip",{"slug":3145,"path":3146,"title":3147,"date":3148,"tags":3149,"description":3152,"cover":3153,"coverAlt":3154,"excerpt":3155,"draft":2781,"hidden":2781,"published":2784,"readingTime":2535,"series":2788,"seriesOrder":1762,"codeUrl":3156,"attachmentUrl":3157},"cmu-15-779-pytorch-linear-relu-absolute-value","\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value","CMU 15-779 学习记录 04：多输出线性层、ReLU 与绝对值网络","2026-09-12",[2788,2794,3110,3150,3151],"ReLU","神经网络","把 nn.Linear 对应到矩阵乘法，弄清多个输出怎样计算，再逐层拆开一个绝对值网络，比较手动设置参数与训练得到的结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value\u002Fcmu-pytorch-absolute-cover-20261015.webp","双分支经过折角模块后汇入紫色 V 形曲线面板的浅色技术教具插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的批量推理与矩阵乘法，我继续把 model(x) 拆开看：一层里有多个输出时，每个数是怎么算出来的？两","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F4cc63904a3a583da4a2122c246a14b43aac38528","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-linear-relu-abs-20261015.zip",{"slug":3159,"path":3160,"title":3161,"date":3162,"tags":3163,"description":3166,"cover":3167,"coverAlt":3168,"excerpt":3169,"draft":2781,"hidden":2781,"published":2784,"readingTime":2535,"series":2788,"seriesOrder":1743,"codeUrl":3170,"attachmentUrl":3171},"cmu-15-779-pytorch-dataloader-checkpoints-batching","\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching","CMU 15-779 学习记录 03：DataLoader、模型保存与批量推理","2026-09-09",[2788,2794,3164,3136,3165],"DataLoader","模型保存","用 DataLoader 分批训练平方函数，保存并加载模型参数，再比较 1000 个样本与 1 个样本的推理耗时，最后把批量计算对应到矩阵乘法。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching\u002Fcmu-pytorch-batching-cover-20261015.webp","样本卡片按批次进入计算模块，旁边配有参数存储盒和输出矩阵面板的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的平方函数实验，这一节继续练习分批训练、保存参数和独立预测。最后做了一次计时：同一个小网络，逐个处理 ","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-batching-20261015.zip",{"slug":3173,"path":3174,"title":3175,"date":3176,"tags":3177,"description":3178,"cover":3179,"coverAlt":3180,"excerpt":3181,"draft":2781,"hidden":2781,"published":2784,"readingTime":2535,"series":2788,"seriesOrder":1722,"codeUrl":3170,"attachmentUrl":3182},"cmu-15-779-pytorch-linear-module-relu","\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu","CMU 15-779 学习记录 02：nn.Linear、张量形状与 ReLU","2026-09-06",[2788,2794,3151,3150],"从带偏置的线性回归开始，弄清 nn.Linear 和 nn.Module 的分工，再用平方函数观察线性模型的限制、ReLU 的作用和测试点的选择。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu\u002Fcmu-pytorch-relu-cover-20261015.webp","输入经过两层神经网络和 ReLU 折角模块，输出曲线拟合结果的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 上一篇里，我用一个参数拟合了 $y=2x$。这一篇继续顺着往下写，先加上偏置，再把手写参数换成 nn.Linea","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-models-20261015.zip",{"slug":3184,"path":3185,"title":3186,"date":3187,"tags":3188,"description":3191,"cover":3192,"coverAlt":3193,"excerpt":3194,"draft":2781,"hidden":2781,"published":2784,"readingTime":3004,"series":2788,"seriesOrder":1673,"codeUrl":3170,"attachmentUrl":3195},"cmu-15-779-pytorch-autograd-linear-regression","\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression","CMU 15-779 学习记录 01：PyTorch 自动求导与线性回归","2026-09-03",[2788,2794,3189,3190],"自动求导","线性回归","从自动求导和手动梯度下降开始，比较学习率，再用 SGD 训练线性回归。记录 w * 4 写错后停在 1.25 的原因和修正结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression\u002Fcmu-pytorch-autograd-cover-20261015.webp","梯度曲面上的小球沿路径下降，前侧面板展示散点与拟合直线的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 开始学习 CMU 15779 时，我会 Python，但还没用过 PyTorch。这次先在 Mac 上用 CPU","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-basics-20261013.zip",{"slug":3197,"path":3198,"title":3199,"date":3200,"tags":3201,"description":3207,"cover":3208,"coverAlt":3209,"excerpt":3210,"draft":2781,"hidden":2781,"published":2784,"readingTime":3032,"codeUrl":3211,"attachmentUrl":3212},"embodied-pickplace-lab-robustness","\u002Fposts\u002Fembodied-pickplace-lab-robustness","Pickplace Lab 项目复盘：抓取成功率之外的时延与失败分析","2026-08-29",[3202,3203,3204,2794,3205,3206],"具身智能","模仿学习","MuJoCo","机器人","鲁棒性","从 MuJoCo 抓取搬运闭环出发，记录 22,100 个评测回合。动作平滑在自建任务的 50 ms 延迟下有效，换到 Panda Lift 却没有支持同样的收益。","\u002Fimages\u002Fposts\u002Fembodied-pickplace-lab-robustness\u002Fpickplace-grasp-cover-20261016.webp","MuJoCo 实际渲染的笛卡尔夹爪抓取方块场景，桌面上标出放置目标","Pickplace Lab 从一个简单任务开始：让夹爪抓起方块，搬到指定位置，放下，再退开。先把仿真、示范采集、训练和闭环评测接起来，随后再问：干净条件下成功的策略，换一种执行条件，还能不能完成任务？ 最值得讨论的结果出现在指令延迟","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Ftree\u002Fv0.1.0","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Freleases\u002Fdownload\u002Fv0.1.0\u002Fpickplace-lab-research-v1.zip",{"slug":3214,"path":3215,"title":3216,"date":3217,"tags":3218,"description":3223,"cover":3224,"coverAlt":3225,"excerpt":3226,"draft":2781,"hidden":2781,"published":2784,"readingTime":3227},"qwen25-vl-sglang-verl-single-gpu-lab","\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab","从 Mac 到单卡 GPU：Qwen2.5-VL 部署、GRPO 与计数 LoRA 实践","2026-08-19",[2909,2850,3219,3220,3221,3222,3065],"SGLang","verl","GRPO","推理优化","在 M2 Pro 和云端单卡上部署 Qwen2.5-VL-3B，实测连续批处理、前缀缓存与 SGLang 参数调优，再用 verl 完成多模态 GRPO 更新和同条件前后评测，记录有效优化、兼容性问题与未观察到提升的结果。","\u002Fimages\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab\u002Fqvl-lab-cover-20261010.webp","笔记本与单卡计算模块通过图像和 token 光轨连接的实验台插画","这次实验从一个很具体的问题开始：M2 Pro、16 GB 统一内存，能不能在本机跑 Qwen2.5VL？跑起来之后，我又想把图文问答、推理性能测量和强化学习训练接成一条完整流程。 本机的 MLX 4bit 服务和云端的 SGLang",29,{"slug":3229,"path":3230,"title":3231,"date":3232,"tags":3233,"description":3237,"cover":3238,"coverAlt":3239,"excerpt":3240,"draft":2781,"hidden":2781,"published":2784,"readingTime":3032},"cycleresearcher-post-training","\u002Fposts\u002Fcycleresearcher-post-training","从 DeepReviewer 2.0 到 CycleResearcher：科研 Agent 如何真正做 Post-training","2026-08-07 17:33:05",[2893,2910,2851,3234,3235,3236],"SimPO","Reward Model","ICLR","区分推理时 Agent 编排与模型后训练，并拆解 CycleResearcher 的领域 SFT、生成式 Reward Model、Iterative SimPO 和评测风险。","\u002Fimages\u002Fposts\u002Fcycleresearcher-post-training\u002Fcycleresearcher-cover-20261018.webp","策略生成模块产出多份候选文稿经多维评审棱镜排序后通过偏好梯度反馈弧闭环优化的等距技术插画","看到一个系统会检索、规划、反思和调用工具，不能据此判断它做过 Posttraining。Agent 描述的是模型在运行时如何完成任务；Posttraining 描述的是预训练之后，是否继续用监督数据、偏好数据或奖励信号更新模型参数。",{"slug":3242,"path":3243,"title":3244,"date":3245,"tags":3246,"description":3248,"cover":3249,"coverAlt":3250,"excerpt":3251,"draft":2781,"hidden":2781,"published":2784,"readingTime":2886,"series":3252,"seriesOrder":1673},"cuda-fused-rmsnorm-kernel","\u002Fposts\u002Fcuda-fused-rmsnorm-kernel","从 PyTorch 到 CUDA Kernel：融合 RMSNorm 在 RTX 4090D 上加速 4.27 倍","2026-07-22 18:16:50",[3109,2822,2824,2794,2933,3247],"Kernel 优化","从零实现 PyTorch CUDA 扩展，拆解 RMSNorm 的访存瓶颈、warp 归约、向量化加载、算子融合与冷热缓存 benchmark。","\u002Fimages\u002Fposts\u002Fcuda-fused-rmsnorm-kernel\u002Fcuda-rmsnorm-cover-20261018.webp","融合残差加法与向量化 float4 访存经 Warp Shuffle 蝴蝶树归约输出归一化张量的等距硬件架构插画","为了准备 AI Infra 岗位，我做了一个尽量小而完整的 CUDA 项目：不追求堆很多算子，而是选一个真实的 LLM 推理路径，把实现、正确性、benchmark 和性能解释全部做扎实。 项目实现了两个 PyTorch CUDA ","AI Infra 实战",{"slug":3254,"path":3255,"title":3256,"date":3257,"tags":3258,"description":3260,"cover":3261,"coverAlt":3262,"excerpt":3263,"draft":2781,"hidden":2781,"published":2784,"readingTime":2522,"series":3264,"seriesOrder":1673},"agent-memory","\u002Fposts\u002Fagent-memory","Agent 上下文工程与分层记忆架构：从 Observation 剪枝、滚动摘要到向量召回闭环","2026-04-24",[2893,2851,3259,2822],"RAG","结合生产级多约束 Agent 与超 100K Token 科研智能体实践，系统拆解长程会话中的上下文膨胀与注意力稀释问题：涵盖 Observation 结构化剪枝（降低约 40% Token 开销）、滚动工作摘要、向量检索长期记忆及 Prefix Cache 友好的上下文组装。","\u002Fimages\u002Fposts\u002Fagent-memory\u002Fagent-memory-cover-20261018.webp","底层高噪数据经剪枝滤网进入中期滑动窗口与金色状态台账并由顶层向量星座按需召回的三层记忆架构插画","在构建多步工具调用（ToolUse）与长周期自主规划（Autonomous Planning）的 Agent 系统时，上下文管理（Context Engineering）往往比单纯改写 System Prompt 更决定线上稳定性。","Agent 上下文工程",{"slug":3266,"path":3267,"title":3268,"date":3269,"tags":3270,"description":3273,"cover":3274,"coverAlt":3275,"excerpt":3276,"draft":2781,"hidden":2781,"published":2784,"readingTime":1806,"series":3277,"seriesOrder":1722},"backend-stack-deep-dive","\u002Fposts\u002Fbackend-stack-deep-dive","高并发 AI 平台后端基础设施实践：FastAPI 异步网关、SQLAlchemy 2.0 Async、PostgreSQL 与 Docker 沙箱编排","2026-04-07",[2937,2934,2936,3271,3272,2822],"PostgreSQL","Docker","结合 DeepScientist 生产级 AI 科研平台实践，拆解 FastAPI ASGI 异步事件循环、SQLAlchemy 2.0 + AsyncPG 连接池调优、PostgreSQL + MinIO 存储分层与 Docker SDK 动态沙箱容器编排的职责边界与核心设计。","\u002Fimages\u002Fposts\u002Fbackend-stack-deep-dive\u002Fbackend-stack-cover-20261018.webp","ASGI 异步事件循环涡轮将非阻塞协程流分发至 PostgreSQL 连接池、MinIO 存储矩阵与动态 Docker 沙箱的等距架构插画","结合 DeepScientist 生产级 AI 科研平台（服务 7,000+ 用户）的实际工程落地，本文系统拆解支撑高并发流式推理与容器沙箱调度的后端基础设施栈——FastAPI（ASGI 异步网关 + SSE 流式推送）、SQLA","DeepScientist 工程复盘",{"slug":3279,"path":3280,"title":3281,"date":3282,"tags":3283,"description":3286,"cover":3287,"coverAlt":3288,"excerpt":3289,"draft":2781,"hidden":2781,"published":2784,"readingTime":2976,"series":3290,"seriesOrder":1673},"minicode-source-analysis","\u002Fposts\u002Fminicode-source-analysis","MiniCode 架构与工具评测深度拆解：开源终端 Agentic Coding 引擎（1k+ Stars）的 ReAct、MCP 与 30 项基准验证","2026-04-06",[2851,3284,3285,2881,2822],"TypeScript","MCP","深度剖析开源终端 AI 编程智能体 MiniCode（1k+ GitHub Stars）的核心架构：从零重型依赖的 ReAct 规划循环、声明式工具注册表与权限拦截层、MCP 协议帧自动协商，到 30 项可执行 Tool-Use 评测基准（86.7% 完成率、100% 错误恢复率）的失败分布归因与闭环优化。","\u002Fimages\u002Fposts\u002Fminicode-source-analysis\u002Fminicode-cover-20261018.webp","终端编程智能体控制台被 ReAct 状态环、Unified Diff 补丁预览门与 MCP 协议插槽环绕的等距技术插画","MiniCode 是我作为核心开发者参与构建的开源终端 Agentic Coding 引擎（目前已在 GitHub 获得 1,000+ Stars）。项目的核心设计目标是：在不引入庞大运行时框架的前提下，用约 5,000 行高内聚的","AI 项目源码导读",{"slug":3292,"path":3293,"title":3294,"date":3282,"tags":3295,"description":3296,"cover":3297,"coverAlt":3298,"excerpt":3299,"draft":2781,"hidden":2781,"published":2784,"readingTime":3004,"series":3277,"seriesOrder":1673},"deepscientist-tech-stack","\u002Fposts\u002Fdeepscientist-tech-stack","DeepScientist 架构与运行时全解析：超 100K 上下文科研智能体、MCP 沙箱与高并发流式系统",[2851,2822,3285,3272,2936,2934],"深度拆解服务 7,000+ 用户的科研自主智能体平台 DeepScientist：从一题一仓库（Quest）持久化状态机、MCP 工具栈（memory \u002F artifact \u002F bash_exec）、Docker SDK 动态容器编排，到 SSE + WebSocket 双流通信与 PostgreSQL \u002F Redis \u002F Prometheus 生产级基础设施。","\u002Fimages\u002Fposts\u002Fdeepscientist-tech-stack\u002Fdeepscientist-cover-20261018.webp","科研智能体编排核心通过 memory、artifact 与隔离容器沙箱三座协议桥连接双流工作台的等距架构插画","DeepScientist（deepscientist.cc）是我们在西湖大学自然语言处理实验室研发的自主科学发现与科研管理平台，目前累计服务 7,000+ 科研用户。与单轮对话或纯文本文献问答工具不同，科研智能体需要在数小时乃至数",1795088942939]