[{"data":1,"prerenderedAt":2751},["ShallowReactive",2],{"post-\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations":3,"archive-public-posts":2240},{"id":4,"title":5,"body":6,"categories":2220,"cover":2222,"coverAlt":2223,"date":2224,"description":2225,"draft":2226,"extension":2227,"haloPublished":2226,"haloSync":2226,"hidden":2226,"legacy":2226,"meta":2228,"navigation":1979,"path":2229,"published":2226,"readingTime":2230,"seo":2231,"series":2232,"seriesOrder":2233,"stem":2234,"tags":2235,"__hash__":2239},"posts\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations.md","CMU 15-779 学习记录 09：CPU 编译优化、合法重写与候选搜索",{"type":7,"value":8,"toc":2213},"minimark",[9,16,24,37,42,50,698,705,712,716,719,1675,1682,1689,1692,1802,1805,1809,1820,1827,1903,1906,1909,1913,1924,2019,2034,2072,2075,2078,2081,2084,2169,2180,2201,2209],[10,11,12],"blockquote",{},[13,14,15],"p",{},"编译章节的个人配套实验，编号 30 不对应官方作业。图重写、分块候选和 TorchInductor 均在 CPU 上运行；真实 GPU 实验按学习安排跳过。",[13,17,18,23],{},[19,20,22],"a",{"href":21},"\u002Fposts\u002Fcmu-15-779-attention-online-softmax\u002F","上一节","记录了分块 Attention 和 KV cache。这次转到编译优化：先检查计算改写是否成立，再测量不同执行方式。源码和原始报告保存在实验快照中，整理时从独立副本复跑。",[13,25,26,32,33],{},[19,27,31],{"href":28,"rel":29},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F9261bea6aedf31cdccbbab2541ca693fc97b3743",[30],"nofollow","查看代码版本"," · ",[19,34,36],{"href":35},"\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-compiler-optimizations-20261019.zip","下载代码与结果 ZIP",[38,39,41],"h2",{"id":40},"linear-后的-batchnorm-什么时候能折叠","Linear 后的 BatchNorm 什么时候能折叠",[13,43,44,45,49],{},"PyTorch 的 Linear 使用 ",[46,47,48],"code",{},"Y = X @ W.T + b","。如果 BatchNorm 处于 eval 模式，使用固定的运行均值 μ、运行方差 v 和参数 γ、β，可以先计算：",[51,52,55],"span",{"className":53},[54],"katex-display",[51,56,59,226],{"className":57},[58],"katex",[51,60,63],{"className":61},[62],"katex-mathml",[64,65,68],"math",{"xmlns":66,"display":67},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML","block",[69,70,71,221],"semantics",{},[72,73,77,128,173],"mtable",{"rowspacing":74,"columnalign":75,"columnspacing":76},"0.25em","right left","0em",[78,79,80,92],"mtr",{},[81,82,83],"mtd",{},[84,85,88],"mstyle",{"scriptlevel":86,"displaystyle":87},"0","true",[89,90,91],"mi",{},"s",[81,93,94],{},[84,95,96],{"scriptlevel":86,"displaystyle":87},[97,98,99,101,105,125],"mrow",{},[97,100],{},[102,103,104],"mo",{},"=",[106,107,108,111],"mfrac",{},[89,109,110],{},"γ",[112,113,114],"msqrt",{},[97,115,116,119,122],{},[89,117,118],{},"v",[102,120,121],{},"+",[89,123,124],{},"ϵ",[102,126,127],{"separator":87},",",[78,129,130,144],{},[81,131,132],{},[84,133,134],{"scriptlevel":86,"displaystyle":87},[135,136,137,140],"msup",{},[89,138,139],{},"W",[102,141,143],{"mathvariant":142,"lspace":76,"rspace":76},"normal","′",[81,145,146],{},[84,147,148],{"scriptlevel":86,"displaystyle":87},[97,149,150,152,154,157,160,164,166,169,171],{},[97,151],{},[102,153,104],{},[89,155,156],{"mathvariant":142},"diag",[102,158,159],{},"⁡",[102,161,163],{"stretchy":162},"false","(",[89,165,91],{},[102,167,168],{"stretchy":162},")",[89,170,139],{},[102,172,127],{"separator":87},[78,174,175,186],{},[81,176,177],{},[84,178,179],{"scriptlevel":86,"displaystyle":87},[135,180,181,184],{},[89,182,183],{},"b",[102,185,143],{"mathvariant":142,"lspace":76,"rspace":76},[81,187,188],{},[84,189,190],{"scriptlevel":86,"displaystyle":87},[97,191,192,194,196,198,201,203,205,208,211,213,215,218],{},[97,193],{},[102,195,104],{},[89,197,91],{},[102,199,200],{},"⊙",[102,202,163],{"stretchy":162},[89,204,183],{},[102,206,207],{},"−",[89,209,210],{},"μ",[102,212,168],{"stretchy":162},[102,214,121],{},[89,216,217],{},"β",[89,219,220],{"mathvariant":142},".",[222,223,225],"annotation",{"encoding":224},"application\u002Fx-tex","\\begin{aligned}\ns&=\\frac{\\gamma}{\\sqrt{v+\\epsilon}},\\\\\nW'&=\\operatorname{diag}(s)W,\\\\\nb'&=s\\odot(b-\\mu)+\\beta.\n\\end{aligned}",[51,227,230],{"className":228,"ariaHidden":87},[229],"katex-html",[51,231,234,239],{"className":232},[233],"base",[51,235],{"className":236,"style":238},[237],"strut","height:5.0376em;vertical-align:-2.2688em;",[51,240,243],{"className":241},[242],"mord",[51,244,246,384],{"className":245},[72],[51,247,250],{"className":248},[249],"col-align-r",[51,251,255,375],{"className":252},[253,254],"vlist-t","vlist-t2",[51,256,259,370],{"className":257},[258],"vlist-r",[51,260,264,279,329],{"className":261,"style":263},[262],"vlist","height:2.7688em;",[51,265,267,272],{"style":266},"top:-4.7688em;",[51,268],{"className":269,"style":271},[270],"pstrut","height:3.1076em;",[51,273,275],{"className":274},[242],[51,276,91],{"className":277},[242,278],"mathnormal",[51,280,282,285],{"style":281},"top:-2.6988em;",[51,283],{"className":284,"style":271},[270],[51,286,288],{"className":287},[242],[51,289,291,295],{"className":290},[242],[51,292,139],{"className":293,"style":294},[242,278],"margin-right:0.1389em;",[51,296,299],{"className":297},[298],"msupsub",[51,300,302],{"className":301},[253],[51,303,305],{"className":304},[258],[51,306,309],{"className":307,"style":308},[262],"height:0.8019em;",[51,310,312,316],{"style":311},"top:-3.113em;margin-right:0.05em;",[51,313],{"className":314,"style":315},[270],"height:2.7em;",[51,317,323],{"className":318},[319,320,321,322],"sizing","reset-size6","size3","mtight",[51,324,326],{"className":325},[242,322],[51,327,143],{"className":328},[242,322],[51,330,332,335],{"style":331},"top:-1.1988em;",[51,333],{"className":334,"style":271},[270],[51,336,338],{"className":337},[242],[51,339,341,344],{"className":340},[242],[51,342,183],{"className":343},[242,278],[51,345,347],{"className":346},[298],[51,348,350],{"className":349},[253],[51,351,353],{"className":352},[258],[51,354,356],{"className":355,"style":308},[262],[51,357,358,361],{"style":311},[51,359],{"className":360,"style":315},[270],[51,362,364],{"className":363},[319,320,321,322],[51,365,367],{"className":366},[242,322],[51,368,143],{"className":369},[242,322],[51,371,374],{"className":372},[373],"vlist-s","​",[51,376,378],{"className":377},[258],[51,379,382],{"className":380,"style":381},[262],"height:2.2688em;",[51,383],{},[51,385,388],{"className":386},[387],"col-align-l",[51,389,391,690],{"className":390},[253,254],[51,392,394,687],{"className":393},[258],[51,395,397,574,618],{"className":396,"style":263},[262],[51,398,399,402],{"style":266},[51,400],{"className":401,"style":271},[270],[51,403,405,408,413,417,420,570],{"className":404},[242],[51,406],{"className":407},[242],[51,409],{"className":410,"style":412},[411],"mspace","margin-right:0.2778em;",[51,414,104],{"className":415},[416],"mrel",[51,418],{"className":419,"style":412},[411],[51,421,423,428,566],{"className":422},[242],[51,424],{"className":425},[426,427],"mopen","nulldelimiter",[51,429,431],{"className":430},[106],[51,432,434,557],{"className":433},[253,254],[51,435,437,554],{"className":436},[258],[51,438,441,530,541],{"className":439,"style":440},[262],"height:1.1076em;",[51,442,444,448],{"style":443},"top:-2.275em;",[51,445],{"className":446,"style":447},[270],"height:3em;",[51,449,451],{"className":450},[242],[51,452,455],{"className":453},[242,454],"sqrt",[51,456,458,521],{"className":457},[253,254],[51,459,461,518],{"className":460},[258],[51,462,465,495],{"className":463,"style":464},[262],"height:0.835em;",[51,466,470,473],{"className":467,"style":469},[468],"svg-align","top:-3em;",[51,471],{"className":472,"style":447},[270],[51,474,477,481,485,489,492],{"className":475,"style":476},[242],"padding-left:0.833em;",[51,478,118],{"className":479,"style":480},[242,278],"margin-right:0.0359em;",[51,482],{"className":483,"style":484},[411],"margin-right:0.2222em;",[51,486,121],{"className":487},[488],"mbin",[51,490],{"className":491,"style":484},[411],[51,493,124],{"className":494},[242,278],[51,496,498,501],{"style":497},"top:-2.795em;",[51,499],{"className":500,"style":447},[270],[51,502,506],{"className":503,"style":505},[504],"hide-tail","min-width:0.853em;height:1.08em;",[507,508,514],"svg",{"xmlns":509,"width":510,"height":511,"viewBox":512,"preserveAspectRatio":513},"http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","400em","1.08em","0 0 400000 1080","xMinYMin slice",[515,516],"path",{"d":517},"M95,702\nc-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14\nc0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54\nc44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10\ns173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429\nc69,-144,104.5,-217.7,106.5,-221\nl0 -0\nc5.3,-9.3,12,-14,20,-14\nH400000v40H845.2724\ns-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7\nc-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z\nM834 80h400000v40h-400000z",[51,519,374],{"className":520},[373],[51,522,524],{"className":523},[258],[51,525,528],{"className":526,"style":527},[262],"height:0.205em;",[51,529],{},[51,531,533,536],{"style":532},"top:-3.23em;",[51,534],{"className":535,"style":447},[270],[51,537],{"className":538,"style":540},[539],"frac-line","border-bottom-width:0.04em;",[51,542,544,547],{"style":543},"top:-3.677em;",[51,545],{"className":546,"style":447},[270],[51,548,550],{"className":549},[242],[51,551,110],{"className":552,"style":553},[242,278],"margin-right:0.0556em;",[51,555,374],{"className":556},[373],[51,558,560],{"className":559},[258],[51,561,564],{"className":562,"style":563},[262],"height:0.93em;",[51,565],{},[51,567],{"className":568},[569,427],"mclose",[51,571,127],{"className":572},[573],"mpunct",[51,575,576,579],{"style":281},[51,577],{"className":578,"style":271},[270],[51,580,582,585,588,591,594,603,606,609,612,615],{"className":581},[242],[51,583],{"className":584},[242],[51,586],{"className":587,"style":412},[411],[51,589,104],{"className":590},[416],[51,592],{"className":593,"style":412},[411],[51,595,598],{"className":596},[597],"mop",[51,599,156],{"className":600,"style":602},[242,601],"mathrm","margin-right:0.0139em;",[51,604,163],{"className":605},[426],[51,607,91],{"className":608},[242,278],[51,610,168],{"className":611},[569],[51,613,139],{"className":614,"style":294},[242,278],[51,616,127],{"className":617},[573],[51,619,620,623],{"style":331},[51,621],{"className":622,"style":271},[270],[51,624,626,629,632,635,638,641,644,647,650,653,656,659,662,665,668,671,674,677,680,684],{"className":625},[242],[51,627],{"className":628},[242],[51,630],{"className":631,"style":412},[411],[51,633,104],{"className":634},[416],[51,636],{"className":637,"style":412},[411],[51,639,91],{"className":640},[242,278],[51,642],{"className":643,"style":484},[411],[51,645,200],{"className":646},[488],[51,648],{"className":649,"style":484},[411],[51,651,163],{"className":652},[426],[51,654,183],{"className":655},[242,278],[51,657],{"className":658,"style":484},[411],[51,660,207],{"className":661},[488],[51,663],{"className":664,"style":484},[411],[51,666,210],{"className":667},[242,278],[51,669,168],{"className":670},[569],[51,672],{"className":673,"style":484},[411],[51,675,121],{"className":676},[488],[51,678],{"className":679,"style":484},[411],[51,681,217],{"className":682,"style":683},[242,278],"margin-right:0.0528em;",[51,685,220],{"className":686},[242],[51,688,374],{"className":689},[373],[51,691,693],{"className":692},[258],[51,694,696],{"className":695,"style":381},[262],[51,697],{},[13,699,700,701,704],{},"随后只运行 ",[46,702,703],{},"X @ W'.T + b'","。训练模式中的统计量随输入变化，不能直接套用这个固定参数折叠。脚本拒绝训练模式、未跟踪运行统计量和特征维度不匹配的输入。",[13,706,707,708,711],{},"原始 float64 例子的最大绝对误差为 ",[46,709,710],{},"8.88e-16","。测试也覆盖了 Linear 无偏置、BatchNorm 无 affine 参数以及不同 batch size。报告里的模块操作数由 2 变为 1，只表示显式模块结构变化，没有统计底层 kernel 数量或证明运行时间下降。",[38,713,715],{"id":714},"rmsnorm-的行标量可以跨过无偏置线性映射","RMSNorm 的行标量可以跨过无偏置线性映射",[13,717,718],{},"脚本中的 RMSNorm 为每行计算一个逆均方根 r，再乘固定的 γ。对无偏置矩阵映射 W：",[51,720,722],{"className":721},[54],[51,723,725,924],{"className":724},[58],[51,726,728],{"className":727},[62],[64,729,730],{"xmlns":66,"display":67},[69,731,732,921],{},[72,733,734,817,853],{"rowspacing":74,"columnalign":75,"columnspacing":76},[78,735,736,749],{},[81,737,738],{},[84,739,740],{"scriptlevel":86,"displaystyle":87},[741,742,743,746],"msub",{},[89,744,745],{},"r",[89,747,748],{},"i",[81,750,751],{},[84,752,753],{"scriptlevel":86,"displaystyle":87},[97,754,755,757,759,815],{},[97,756],{},[102,758,104],{},[135,760,761,804],{},[97,762,763,765,774,783,798,800,802],{},[102,764,163],{"fence":87},[106,766,767,771],{},[768,769,770],"mn",{},"1",[89,772,773],{},"d",[775,776,777,780],"munder",{},[102,778,779],{},"∑",[89,781,782],{},"j",[784,785,786,789,795],"msubsup",{},[89,787,788],{},"x",[97,790,791,793],{},[89,792,748],{},[89,794,782],{},[768,796,797],{},"2",[102,799,121],{},[89,801,124],{},[102,803,168],{"fence":87},[97,805,806,808,810,813],{},[102,807,207],{},[768,809,770],{},[89,811,812],{"mathvariant":142},"\u002F",[768,814,797],{},[102,816,127],{"separator":87},[78,818,819,829],{},[81,820,821],{},[84,822,823],{"scriptlevel":86,"displaystyle":87},[741,824,825,827],{},[89,826,139],{},[89,828,110],{},[81,830,831],{},[84,832,833],{"scriptlevel":86,"displaystyle":87},[97,834,835,837,839,841,843,845,847,849,851],{},[97,836],{},[102,838,104],{},[89,840,156],{"mathvariant":142},[102,842,159],{},[102,844,163],{"stretchy":162},[89,846,110],{},[102,848,168],{"stretchy":162},[89,850,139],{},[102,852,127],{"separator":87},[78,854,855,887],{},[81,856,857],{},[84,858,859],{"scriptlevel":86,"displaystyle":87},[97,860,861,863,865,871,877,879,881,883,885],{},[102,862,163],{"stretchy":162},[102,864,163],{"stretchy":162},[741,866,867,869],{},[89,868,788],{},[89,870,748],{},[741,872,873,875],{},[89,874,745],{},[89,876,748],{},[102,878,168],{"stretchy":162},[102,880,200],{},[89,882,110],{},[102,884,168],{"stretchy":162},[89,886,139],{},[81,888,889],{},[84,890,891],{"scriptlevel":86,"displaystyle":87},[97,892,893,895,897,899,905,911,913,919],{},[97,894],{},[102,896,104],{},[102,898,163],{"stretchy":162},[741,900,901,903],{},[89,902,788],{},[89,904,748],{},[741,906,907,909],{},[89,908,139],{},[89,910,110],{},[102,912,168],{"stretchy":162},[741,914,915,917],{},[89,916,745],{},[89,918,748],{},[89,920,220],{"mathvariant":142},[222,922,923],{"encoding":224},"\\begin{aligned}\nr_i&=\\left(\\frac{1}{d}\\sum_j x_{ij}^2+\\epsilon\\right)^{-1\u002F2},\\\\\nW_\\gamma&=\\operatorname{diag}(\\gamma)W,\\\\\n((x_i r_i)\\odot\\gamma)W&=(x_iW_\\gamma)r_i.\n\\end{aligned}",[51,925,927],{"className":926,"ariaHidden":87},[229],[51,928,930,934],{"className":929},[233],[51,931],{"className":932,"style":933},[237],"height:6.4417em;vertical-align:-2.9708em;",[51,935,937],{"className":936},[242],[51,938,940,1186],{"className":939},[72],[51,941,943],{"className":942},[249],[51,944,946,1177],{"className":945},[253,254],[51,947,949,1174],{"className":948},[258],[51,950,953,1007,1059],{"className":951,"style":952},[262],"height:3.4708em;",[51,954,956,960],{"style":955},"top:-5.4708em;",[51,957],{"className":958,"style":959},[270],"height:4.0279em;",[51,961,963],{"className":962},[242],[51,964,966,970],{"className":965},[242],[51,967,745],{"className":968,"style":969},[242,278],"margin-right:0.0278em;",[51,971,973],{"className":972},[298],[51,974,976,998],{"className":975},[253,254],[51,977,979,995],{"className":978},[258],[51,980,983],{"className":981,"style":982},[262],"height:0.3117em;",[51,984,986,989],{"style":985},"top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;",[51,987],{"className":988,"style":315},[270],[51,990,992],{"className":991},[319,320,321,322],[51,993,748],{"className":994},[242,278,322],[51,996,374],{"className":997},[373],[51,999,1001],{"className":1000},[258],[51,1002,1005],{"className":1003,"style":1004},[262],"height:0.15em;",[51,1006],{},[51,1008,1010,1013],{"style":1009},"top:-2.9171em;",[51,1011],{"className":1012,"style":959},[270],[51,1014,1016],{"className":1015},[242],[51,1017,1019,1022],{"className":1018},[242],[51,1020,139],{"className":1021,"style":294},[242,278],[51,1023,1025],{"className":1024},[298],[51,1026,1028,1050],{"className":1027},[253,254],[51,1029,1031,1047],{"className":1030},[258],[51,1032,1035],{"className":1033,"style":1034},[262],"height:0.1514em;",[51,1036,1038,1041],{"style":1037},"top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;",[51,1039],{"className":1040,"style":315},[270],[51,1042,1044],{"className":1043},[319,320,321,322],[51,1045,110],{"className":1046,"style":553},[242,278,322],[51,1048,374],{"className":1049},[373],[51,1051,1053],{"className":1052},[258],[51,1054,1057],{"className":1055,"style":1056},[262],"height:0.2861em;",[51,1058],{},[51,1060,1062,1065],{"style":1061},"top:-1.4171em;",[51,1063],{"className":1064,"style":959},[270],[51,1066,1068,1072,1113,1153,1156,1159,1162,1165,1168,1171],{"className":1067},[242],[51,1069,1071],{"className":1070},[426],"((",[51,1073,1075,1078],{"className":1074},[242],[51,1076,788],{"className":1077},[242,278],[51,1079,1081],{"className":1080},[298],[51,1082,1084,1105],{"className":1083},[253,254],[51,1085,1087,1102],{"className":1086},[258],[51,1088,1090],{"className":1089,"style":982},[262],[51,1091,1093,1096],{"style":1092},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[51,1094],{"className":1095,"style":315},[270],[51,1097,1099],{"className":1098},[319,320,321,322],[51,1100,748],{"className":1101},[242,278,322],[51,1103,374],{"className":1104},[373],[51,1106,1108],{"className":1107},[258],[51,1109,1111],{"className":1110,"style":1004},[262],[51,1112],{},[51,1114,1116,1119],{"className":1115},[242],[51,1117,745],{"className":1118,"style":969},[242,278],[51,1120,1122],{"className":1121},[298],[51,1123,1125,1145],{"className":1124},[253,254],[51,1126,1128,1142],{"className":1127},[258],[51,1129,1131],{"className":1130,"style":982},[262],[51,1132,1133,1136],{"style":985},[51,1134],{"className":1135,"style":315},[270],[51,1137,1139],{"className":1138},[319,320,321,322],[51,1140,748],{"className":1141},[242,278,322],[51,1143,374],{"className":1144},[373],[51,1146,1148],{"className":1147},[258],[51,1149,1151],{"className":1150,"style":1004},[262],[51,1152],{},[51,1154,168],{"className":1155},[569],[51,1157],{"className":1158,"style":484},[411],[51,1160,200],{"className":1161},[488],[51,1163],{"className":1164,"style":484},[411],[51,1166,110],{"className":1167,"style":553},[242,278],[51,1169,168],{"className":1170},[569],[51,1172,139],{"className":1173,"style":294},[242,278],[51,1175,374],{"className":1176},[373],[51,1178,1180],{"className":1179},[258],[51,1181,1184],{"className":1182,"style":1183},[262],"height:2.9708em;",[51,1185],{},[51,1187,1189],{"className":1188},[387],[51,1190,1192,1667],{"className":1191},[253,254],[51,1193,1195,1664],{"className":1194},[258],[51,1196,1198,1474,1515],{"className":1197,"style":952},[262],[51,1199,1200,1203],{"style":955},[51,1201],{"className":1202,"style":959},[270],[51,1204,1206,1209,1212,1215,1218,1468,1471],{"className":1205},[242],[51,1207],{"className":1208},[242],[51,1210],{"className":1211,"style":412},[411],[51,1213,104],{"className":1214},[416],[51,1216],{"className":1217,"style":412},[411],[51,1219,1222,1436],{"className":1220},[1221],"minner",[51,1223,1225,1235,1300,1304,1357,1360,1418,1421,1424,1427,1430],{"className":1224},[1221],[51,1226,1230],{"className":1227,"style":1229},[426,1228],"delimcenter","top:0em;",[51,1231,163],{"className":1232},[1233,1234],"delimsizing","size4",[51,1236,1238,1241,1297],{"className":1237},[242],[51,1239],{"className":1240},[426,427],[51,1242,1244],{"className":1243},[106],[51,1245,1247,1288],{"className":1246},[253,254],[51,1248,1250,1285],{"className":1249},[258],[51,1251,1254,1266,1274],{"className":1252,"style":1253},[262],"height:1.3214em;",[51,1255,1257,1260],{"style":1256},"top:-2.314em;",[51,1258],{"className":1259,"style":447},[270],[51,1261,1263],{"className":1262},[242],[51,1264,773],{"className":1265},[242,278],[51,1267,1268,1271],{"style":532},[51,1269],{"className":1270,"style":447},[270],[51,1272],{"className":1273,"style":540},[539],[51,1275,1276,1279],{"style":543},[51,1277],{"className":1278,"style":447},[270],[51,1280,1282],{"className":1281},[242],[51,1283,770],{"className":1284},[242],[51,1286,374],{"className":1287},[373],[51,1289,1291],{"className":1290},[258],[51,1292,1295],{"className":1293,"style":1294},[262],"height:0.686em;",[51,1296],{},[51,1298],{"className":1299},[569,427],[51,1301],{"className":1302,"style":1303},[411],"margin-right:0.1667em;",[51,1305,1308],{"className":1306},[597,1307],"op-limits",[51,1309,1311,1348],{"className":1310},[253,254],[51,1312,1314,1345],{"className":1313},[258],[51,1315,1318,1332],{"className":1316,"style":1317},[262],"height:1.05em;",[51,1319,1321,1325],{"style":1320},"top:-1.8723em;margin-left:0em;",[51,1322],{"className":1323,"style":1324},[270],"height:3.05em;",[51,1326,1328],{"className":1327},[319,320,321,322],[51,1329,782],{"className":1330,"style":1331},[242,278,322],"margin-right:0.0572em;",[51,1333,1335,1338],{"style":1334},"top:-3.05em;",[51,1336],{"className":1337,"style":1324},[270],[51,1339,1340],{},[51,1341,779],{"className":1342},[597,1343,1344],"op-symbol","large-op",[51,1346,374],{"className":1347},[373],[51,1349,1351],{"className":1350},[258],[51,1352,1355],{"className":1353,"style":1354},[262],"height:1.4138em;",[51,1356],{},[51,1358],{"className":1359,"style":1303},[411],[51,1361,1363,1366],{"className":1362},[242],[51,1364,788],{"className":1365},[242,278],[51,1367,1369],{"className":1368},[298],[51,1370,1372,1409],{"className":1371},[253,254],[51,1373,1375,1406],{"className":1374},[258],[51,1376,1379,1395],{"className":1377,"style":1378},[262],"height:0.8641em;",[51,1380,1382,1385],{"style":1381},"top:-2.453em;margin-left:0em;margin-right:0.05em;",[51,1383],{"className":1384,"style":315},[270],[51,1386,1388],{"className":1387},[319,320,321,322],[51,1389,1391],{"className":1390},[242,322],[51,1392,1394],{"className":1393,"style":1331},[242,278,322],"ij",[51,1396,1397,1400],{"style":311},[51,1398],{"className":1399,"style":315},[270],[51,1401,1403],{"className":1402},[319,320,321,322],[51,1404,797],{"className":1405},[242,322],[51,1407,374],{"className":1408},[373],[51,1410,1412],{"className":1411},[258],[51,1413,1416],{"className":1414,"style":1415},[262],"height:0.3831em;",[51,1417],{},[51,1419],{"className":1420,"style":484},[411],[51,1422,121],{"className":1423},[488],[51,1425],{"className":1426,"style":484},[411],[51,1428,124],{"className":1429},[242,278],[51,1431,1433],{"className":1432,"style":1229},[569,1228],[51,1434,168],{"className":1435},[1233,1234],[51,1437,1439],{"className":1438},[298],[51,1440,1442],{"className":1441},[253],[51,1443,1445],{"className":1444},[258],[51,1446,1449],{"className":1447,"style":1448},[262],"height:2.0279em;",[51,1450,1452,1455],{"style":1451},"top:-4.2029em;margin-right:0.05em;",[51,1453],{"className":1454,"style":315},[270],[51,1456,1458],{"className":1457},[319,320,321,322],[51,1459,1461,1464],{"className":1460},[242,322],[51,1462,207],{"className":1463},[242,322],[51,1465,1467],{"className":1466},[242,322],"1\u002F2",[51,1469],{"className":1470,"style":1303},[411],[51,1472,127],{"className":1473},[573],[51,1475,1476,1479],{"style":1009},[51,1477],{"className":1478,"style":959},[270],[51,1480,1482,1485,1488,1491,1494,1500,1503,1506,1509,1512],{"className":1481},[242],[51,1483],{"className":1484},[242],[51,1486],{"className":1487,"style":412},[411],[51,1489,104],{"className":1490},[416],[51,1492],{"className":1493,"style":412},[411],[51,1495,1497],{"className":1496},[597],[51,1498,156],{"className":1499,"style":602},[242,601],[51,1501,163],{"className":1502},[426],[51,1504,110],{"className":1505,"style":553},[242,278],[51,1507,168],{"className":1508},[569],[51,1510,139],{"className":1511,"style":294},[242,278],[51,1513,127],{"className":1514},[573],[51,1516,1517,1520],{"style":1061},[51,1518],{"className":1519,"style":959},[270],[51,1521,1523,1526,1529,1532,1535,1538,1578,1618,1621,1661],{"className":1522},[242],[51,1524],{"className":1525},[242],[51,1527],{"className":1528,"style":412},[411],[51,1530,104],{"className":1531},[416],[51,1533],{"className":1534,"style":412},[411],[51,1536,163],{"className":1537},[426],[51,1539,1541,1544],{"className":1540},[242],[51,1542,788],{"className":1543},[242,278],[51,1545,1547],{"className":1546},[298],[51,1548,1550,1570],{"className":1549},[253,254],[51,1551,1553,1567],{"className":1552},[258],[51,1554,1556],{"className":1555,"style":982},[262],[51,1557,1558,1561],{"style":1092},[51,1559],{"className":1560,"style":315},[270],[51,1562,1564],{"className":1563},[319,320,321,322],[51,1565,748],{"className":1566},[242,278,322],[51,1568,374],{"className":1569},[373],[51,1571,1573],{"className":1572},[258],[51,1574,1576],{"className":1575,"style":1004},[262],[51,1577],{},[51,1579,1581,1584],{"className":1580},[242],[51,1582,139],{"className":1583,"style":294},[242,278],[51,1585,1587],{"className":1586},[298],[51,1588,1590,1610],{"className":1589},[253,254],[51,1591,1593,1607],{"className":1592},[258],[51,1594,1596],{"className":1595,"style":1034},[262],[51,1597,1598,1601],{"style":1037},[51,1599],{"className":1600,"style":315},[270],[51,1602,1604],{"className":1603},[319,320,321,322],[51,1605,110],{"className":1606,"style":553},[242,278,322],[51,1608,374],{"className":1609},[373],[51,1611,1613],{"className":1612},[258],[51,1614,1616],{"className":1615,"style":1056},[262],[51,1617],{},[51,1619,168],{"className":1620},[569],[51,1622,1624,1627],{"className":1623},[242],[51,1625,745],{"className":1626,"style":969},[242,278],[51,1628,1630],{"className":1629},[298],[51,1631,1633,1653],{"className":1632},[253,254],[51,1634,1636,1650],{"className":1635},[258],[51,1637,1639],{"className":1638,"style":982},[262],[51,1640,1641,1644],{"style":985},[51,1642],{"className":1643,"style":315},[270],[51,1645,1647],{"className":1646},[319,320,321,322],[51,1648,748],{"className":1649},[242,278,322],[51,1651,374],{"className":1652},[373],[51,1654,1656],{"className":1655},[258],[51,1657,1659],{"className":1658,"style":1004},[262],[51,1660],{},[51,1662,220],{"className":1663},[242],[51,1665,374],{"className":1666},[373],[51,1668,1670],{"className":1669},[258],[51,1671,1673],{"className":1672,"style":1183},[262],[51,1674],{},[13,1676,1677,1678,1681],{},"因此可预先得到 ",[46,1679,1680],{},"preweighted = gamma[:, None] * weight","，先做矩阵乘法，最后乘每行的 r。r 仍依赖当前输入，不能预先变成一个固定常量。这里也没有中间非线性或待处理的线性层偏置。",[13,1683,1684,1685,1688],{},"原始例子的最大绝对误差为 ",[46,1686,1687],{},"1.33e-15","，零输入行、带负号或为零的 γ 也通过了测试。浮点计算顺序变化后，结果按容差比较。",[13,1690,1691],{},"并非相邻操作都能移动。脚本保留了一个具体反例：",[1693,1694,1699],"pre",{"className":1695,"code":1696,"language":1697,"meta":1698,"style":1698},"language-python shiki shiki-themes github-dark-dimmed github-light houston","x = torch.tensor([[-1.0, 1.0]])\nw = torch.tensor([[-1.0], [1.0]])\ntorch.relu(x @ w)       # 2.0\ntorch.relu(x) @ w       # 1.0\n","python","",[46,1700,1701,1737,1762,1784],{"__ignoreMap":1698},[51,1702,1705,1709,1712,1715,1719,1722,1725,1729,1732,1734],{"class":1703,"line":1704},"line",1,[51,1706,1708],{"class":1707},"sWURj","x ",[51,1710,104],{"class":1711},"sf2cM",[51,1713,1714],{"class":1707}," torch.",[51,1716,1718],{"class":1717},"sq3c-","tensor",[51,1720,1721],{"class":1707},"([[",[51,1723,1724],{"class":1711},"-",[51,1726,1728],{"class":1727},"sgENW","1.0",[51,1730,1731],{"class":1707},", ",[51,1733,1728],{"class":1727},[51,1735,1736],{"class":1707},"]])\n",[51,1738,1740,1743,1745,1747,1749,1751,1753,1755,1758,1760],{"class":1703,"line":1739},2,[51,1741,1742],{"class":1707},"w ",[51,1744,104],{"class":1711},[51,1746,1714],{"class":1707},[51,1748,1718],{"class":1717},[51,1750,1721],{"class":1707},[51,1752,1724],{"class":1711},[51,1754,1728],{"class":1727},[51,1756,1757],{"class":1707},"], [",[51,1759,1728],{"class":1727},[51,1761,1736],{"class":1707},[51,1763,1765,1768,1771,1774,1777,1780],{"class":1703,"line":1764},3,[51,1766,1767],{"class":1707},"torch.",[51,1769,1770],{"class":1717},"relu",[51,1772,1773],{"class":1707},"(x ",[51,1775,1776],{"class":1711},"@",[51,1778,1779],{"class":1707}," w)       ",[51,1781,1783],{"class":1782},"sCxhR","# 2.0\n",[51,1785,1787,1789,1791,1794,1796,1799],{"class":1703,"line":1786},4,[51,1788,1767],{"class":1707},[51,1790,1770],{"class":1717},[51,1792,1793],{"class":1707},"(x) ",[51,1795,1776],{"class":1711},[51,1797,1798],{"class":1707}," w       ",[51,1800,1801],{"class":1782},"# 1.0\n",[13,1803,1804],{},"测试给出数值对照和规则成立的前提；这些有限输入的检查本身不是形式化等价证明。",[38,1806,1808],{"id":1807},"搜索候选前每个候选都要通过正确性检查","搜索候选前，每个候选都要通过正确性检查",[13,1810,1811,1812,1815,1816,1819],{},"矩阵尺寸固定为 ",[46,1813,1814],{},"A[97,65] @ B[65,81]","，float64、单 CPU 线程。候选分块尺寸为 16、32、64、128，按固定种子打乱为 32、16、128、64，再逐一与 ",[46,1817,1818],{},"a @ b"," 对照并计时。",[13,1821,1822,1823,1826],{},"原始 ",[46,1824,1825],{},"30_cpu_compiler.json"," 中的中位时间如下，单位为毫秒：",[1828,1829,1830,1847],"table",{},[1831,1832,1833],"thead",{},[1834,1835,1836,1840,1844],"tr",{},[1837,1838,1839],"th",{},"执行方式",[1837,1841,1843],{"align":1842},"right","中位时间",[1837,1845,1846],{},"输出容差检查",[1848,1849,1850,1862,1872,1882,1892],"tbody",{},[1834,1851,1852,1856,1859],{},[1853,1854,1855],"td",{},"分块 T=16",[1853,1857,1858],{"align":1842},"1.445975",[1853,1860,1861],{},"通过",[1834,1863,1864,1867,1870],{},[1853,1865,1866],{},"分块 T=32",[1853,1868,1869],{"align":1842},"0.288637",[1853,1871,1861],{},[1834,1873,1874,1877,1880],{},[1853,1875,1876],{},"分块 T=64",[1853,1878,1879],{"align":1842},"0.076659",[1853,1881,1861],{},[1834,1883,1884,1887,1890],{},[1853,1885,1886],{},"分块 T=128",[1853,1888,1889],{"align":1842},"0.017947",[1853,1891,1861],{},[1834,1893,1894,1897,1900],{},[1853,1895,1896],{},"直接 a @ b",[1853,1898,1899],{"align":1842},"0.007439",[1853,1901,1902],{},"基准",[13,1904,1905],{},"这份测量选择 T=128；直接调用 matmul 仍更快。T=128 已覆盖三个矩阵维度，Python 循环只需一次局部矩阵乘法，但仍包含输出分配、切片和累加。小块会增加这些开销，不能据此推断 GPU 共享内存分块应选最大的块。",[13,1907,1908],{},"候选使用普通 Python 循环调用 PyTorch CPU matmul，没有生成 Triton 或 TVM kernel。每个 Timer 的最短测量窗口为 0.04 秒，独立批次只有 1 至 3 个，JSON 同时保存 IQR 和每批调用次数。这个结果只适用于当时的形状、机器、线程与候选集合，不能作为稳定排名或全局最优的证据。",[38,1910,1912],{"id":1911},"torchinductor-的首调用需要单独记录","TorchInductor 的首调用需要单独记录",[13,1914,1915,1916,1919,1920,1923],{},"实际编译的是这个 float32 函数，输入 ",[46,1917,1918],{},"x[1024,64]"," 和 ",[46,1921,1922],{},"bias[64]","：",[1693,1925,1927],{"className":1695,"code":1926,"language":1697,"meta":1698,"style":1698},"def elementwise(x, bias):\n    return torch.relu(x + bias) * 0.5\n\ncompiled = torch.compile(elementwise, backend=\"inductor\", fullgraph=True)\n",[46,1928,1929,1952,1975,1981],{"__ignoreMap":1698},[51,1930,1931,1935,1939,1941,1944,1946,1949],{"class":1703,"line":1704},[51,1932,1934],{"class":1933},"sST-V","def",[51,1936,1938],{"class":1937},"sS_0i"," elementwise",[51,1940,163],{"class":1707},[51,1942,788],{"class":1943},"s9NhK",[51,1945,1731],{"class":1707},[51,1947,1948],{"class":1943},"bias",[51,1950,1951],{"class":1707},"):\n",[51,1953,1954,1958,1960,1962,1964,1966,1969,1972],{"class":1703,"line":1739},[51,1955,1957],{"class":1956},"slEfO","    return",[51,1959,1714],{"class":1707},[51,1961,1770],{"class":1717},[51,1963,1773],{"class":1707},[51,1965,121],{"class":1711},[51,1967,1968],{"class":1707}," bias) ",[51,1970,1971],{"class":1711},"*",[51,1973,1974],{"class":1727}," 0.5\n",[51,1976,1977],{"class":1703,"line":1764},[51,1978,1980],{"emptyLinePlaceholder":1979},true,"\n",[51,1982,1983,1986,1988,1990,1993,1996,2000,2002,2006,2008,2011,2013,2016],{"class":1703,"line":1786},[51,1984,1985],{"class":1707},"compiled ",[51,1987,104],{"class":1711},[51,1989,1714],{"class":1707},[51,1991,1992],{"class":1717},"compile",[51,1994,1995],{"class":1707},"(elementwise, ",[51,1997,1999],{"class":1998},"sA9aF","backend",[51,2001,104],{"class":1711},[51,2003,2005],{"class":2004},"sWqY2","\"inductor\"",[51,2007,1731],{"class":1707},[51,2009,2010],{"class":1998},"fullgraph",[51,2012,104],{"class":1711},[51,2014,2015],{"class":1727},"True",[51,2017,2018],{"class":1707},")\n",[13,2020,2021,2022,2025,2026,2029,2030,2033],{},"原始运行使用 Python 3.11.15、PyTorch 2.14.1、arm64、单 CPU 线程，",[46,2023,2024],{},"inductor.status"," 为 ",[46,2027,2028],{},"executed","。第一次调用返回后，脚本检查结果；另换一组输入再次比较，两组均通过 ",[46,2031,2032],{},"rtol=atol=1e-6"," 的检查。",[1828,2035,2036,2046],{},[1831,2037,2038],{},[1834,2039,2040,2043],{},[1837,2041,2042],{},"原始记录的阶段",[1837,2044,2045],{"align":1842},"时间",[1848,2047,2048,2056,2064],{},[1834,2049,2050,2053],{},[1853,2051,2052],{},"编译路径首调用",[1853,2054,2055],{"align":1842},"8628.540 ms",[1834,2057,2058,2061],{},[1853,2059,2060],{},"eager 预热后中位数",[1853,2062,2063],{"align":1842},"0.021665 ms",[1834,2065,2066,2069],{},[1853,2067,2068],{},"compiled 预热后中位数",[1853,2070,2071],{"align":1842},"0.018640 ms",[13,2073,2074],{},"首调用包含编译或编译缓存查询及执行；后两项只测预热后的调用。整理时使用单独的编译缓存目录复跑，首调用为 5471.626 ms，eager 与 compiled 的预热后中位数分别为 0.022108 和 0.017297 ms，正确性再次通过。编译成本和短窗口计时都会波动，这两次结果不足以证明普遍的加速比例。",[13,2076,2077],{},"本实验确认了 CPU TorchInductor 编译路径执行成功。脚本没有检查生成代码或统计 kernel，因此不额外声称某几项操作必然融合成了一个 kernel。",[38,2079,2080],{"id":2080},"复现与课程位置",[13,2082,2083],{},"ZIP 包含冻结脚本、原始 JSON、5 项编译测试、30 项测试通过的日志和依赖说明。完整发布仓库中的 30 项测试也在本次冻结副本中全部通过，涵盖此前的矩阵、归约、Attention 和 KV cache。",[1693,2085,2089],{"className":2086,"code":2087,"language":2088,"meta":1698,"style":1698},"language-bash shiki shiki-themes github-dark-dimmed github-light houston","python3.11 -m venv .venv\n.venv\u002Fbin\u002Fpython -m pip install -r requirements.txt\n.venv\u002Fbin\u002Fpython 30_cpu_compiler.py --compile \\\n  --output results\u002Fcompiler-rerun.json\n.venv\u002Fbin\u002Fpython -m unittest discover -s tests -v\n","bash",[46,2090,2091,2106,2125,2139,2147],{"__ignoreMap":1698},[51,2092,2093,2097,2100,2103],{"class":1703,"line":1704},[51,2094,2096],{"class":2095},"sLsJe","python3.11",[51,2098,2099],{"class":1727}," -m",[51,2101,2102],{"class":2004}," venv",[51,2104,2105],{"class":2004}," .venv\n",[51,2107,2108,2111,2113,2116,2119,2122],{"class":1703,"line":1739},[51,2109,2110],{"class":2095},".venv\u002Fbin\u002Fpython",[51,2112,2099],{"class":1727},[51,2114,2115],{"class":2004}," pip",[51,2117,2118],{"class":2004}," install",[51,2120,2121],{"class":1727}," -r",[51,2123,2124],{"class":2004}," requirements.txt\n",[51,2126,2127,2129,2132,2135],{"class":1703,"line":1764},[51,2128,2110],{"class":2095},[51,2130,2131],{"class":2004}," 30_cpu_compiler.py",[51,2133,2134],{"class":1727}," --compile",[51,2136,2138],{"class":2137},"sxpgZ"," \\\n",[51,2140,2141,2144],{"class":1703,"line":1786},[51,2142,2143],{"class":1727},"  --output",[51,2145,2146],{"class":2004}," results\u002Fcompiler-rerun.json\n",[51,2148,2150,2152,2154,2157,2160,2163,2166],{"class":1703,"line":2149},5,[51,2151,2110],{"class":2095},[51,2153,2099],{"class":1727},[51,2155,2156],{"class":2004}," unittest",[51,2158,2159],{"class":2004}," discover",[51,2161,2162],{"class":1727}," -s",[51,2164,2165],{"class":2004}," tests",[51,2167,2168],{"class":1727}," -v\n",[13,2170,2171,2172,2175,2176,2179],{},"运行 ",[46,2173,2174],{},"--compile"," 还需要 TorchInductor 可用的本机编译工具链。不加该选项时，脚本只运行重写与候选搜索，并把编译状态记为 ",[46,2177,2178],{},"not_requested","，不能把这种输出当作已执行编译。",[13,2181,2182,2183,2188,2189,2194,2195,2200],{},"可对照原课的 ",[19,2184,2187],{"href":2185,"rel":2186},"https:\u002F\u002Fwww.cs.cmu.edu\u002F~zhihaoj2\u002F15-779\u002Fslides\u002F08-ML-compilers-part-1.pdf",[30],"Tile-based DSL","、",[19,2190,2193],{"href":2191,"rel":2192},"https:\u002F\u002Fwww.cs.cmu.edu\u002F~zhihaoj2\u002F15-779\u002Fslides\u002F10-graph-level-optimizations.pdf",[30],"图级优化","和 ",[19,2196,2199],{"href":2197,"rel":2198},"https:\u002F\u002Fwww.cs.cmu.edu\u002F~zhihaoj2\u002F15-779\u002Fslides\u002F11-kernel-superoptimization.pdf",[30],"ML Superoptimization","材料。本篇提供几个可运行的配套例子，没有实现这些课程系统的完整优化器，也不表示论文阅读、GPU 调优或整门课程已经完成。",[13,2202,2203,32,2207],{},[19,2204,2206],{"href":28,"rel":2205},[30],"本篇 GitHub 代码",[19,2208,36],{"href":35},[2210,2211,2212],"style",{},"html pre.shiki code .sWURj, html code.shiki .sWURj{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#EEF0F9}html pre.shiki code .sf2cM, html code.shiki .sf2cM{--shiki-default:#F47067;--shiki-light:#D73A49;--shiki-cyber:#EEF0F9}html pre.shiki code .sq3c-, html code.shiki .sq3c-{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#00DAEF}html pre.shiki code .sgENW, html code.shiki .sgENW{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#FFD493}html pre.shiki code .sCxhR, html code.shiki .sCxhR{--shiki-default:#768390;--shiki-default-font-style:inherit;--shiki-light:#6A737D;--shiki-light-font-style:inherit;--shiki-cyber:#EEF0F98F;--shiki-cyber-font-style:italic}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html.cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html pre.shiki code .sST-V, html code.shiki .sST-V{--shiki-default:#F47067;--shiki-light:#D73A49;--shiki-cyber:#54B9FF}html pre.shiki code .sS_0i, html code.shiki .sS_0i{--shiki-default:#DCBDFB;--shiki-light:#6F42C1;--shiki-cyber:#00DAEF}html pre.shiki code .s9NhK, html code.shiki .s9NhK{--shiki-default:#ADBAC7;--shiki-default-font-style:inherit;--shiki-light:#24292E;--shiki-light-font-style:inherit;--shiki-cyber:#FFD493;--shiki-cyber-font-style:italic}html pre.shiki code .slEfO, html code.shiki .slEfO{--shiki-default:#F47067;--shiki-default-font-style:inherit;--shiki-light:#D73A49;--shiki-light-font-style:inherit;--shiki-cyber:#54B9FF;--shiki-cyber-font-style:italic}html pre.shiki code .sA9aF, html code.shiki .sA9aF{--shiki-default:#F69D50;--shiki-default-font-style:inherit;--shiki-light:#E36209;--shiki-light-font-style:inherit;--shiki-cyber:#4BF3C8;--shiki-cyber-font-style:italic}html pre.shiki code .sWqY2, html code.shiki .sWqY2{--shiki-default:#96D0FF;--shiki-light:#032F62;--shiki-cyber:#FFD493}html pre.shiki code .sLsJe, html code.shiki .sLsJe{--shiki-default:#F69D50;--shiki-light:#6F42C1;--shiki-cyber:#00DAEF}html pre.shiki code .sxpgZ, html code.shiki .sxpgZ{--shiki-default:#F47067;--shiki-light:#005CC5;--shiki-cyber:#EEF0F9}",{"title":1698,"searchDepth":1739,"depth":1764,"links":2214},[2215,2216,2217,2218,2219],{"id":40,"depth":1739,"text":41},{"id":714,"depth":1739,"text":715},{"id":1807,"depth":1739,"text":1808},{"id":1911,"depth":1739,"text":1912},{"id":2080,"depth":1739,"text":2080},[2221],"技术","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations\u002Fcmu-cpu-compiler-cover-20261019.webp","淡紫计算积木经过薄荷绿折叠模具变成浅蓝积木，前方摆放四种候选托盘的浅色手绘插画","2026-09-27","用 CPU 实验验证 BatchNorm 折叠、RMSNorm 重排与非线性反例，测量分块候选，并运行 TorchInductor，分别记录首调用与预热后的时间。",false,"md",{"codeUrl":28,"attachmentUrl":35},"\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations",null,{"title":5,"description":2225},"CMU 15-779","9","posts\u002Fcmu-15-779-cpu-compiler-optimizations",[2232,2236,2237,2238],"ML Compiler","TorchInductor","PyTorch","RKndeQcpF2_8gVd0pZ57DcZk89pWeH4WtYcTlVRYfqM",[2241,2259,2275,2290,2304,2319,2334,2349,2363,2375,2390,2402,2416,2429,2442,2456,2472,2487,2498,2510,2525,2538,2542,2554,2568,2580,2595,2609,2623,2634,2647,2664,2679,2692,2704,2716,2729,2742],{"slug":2242,"path":2243,"title":2244,"date":2245,"tags":2246,"description":2251,"cover":2252,"coverAlt":2253,"excerpt":2254,"draft":2226,"hidden":2226,"published":1979,"readingTime":2255,"series":2256,"seriesOrder":2255,"codeUrl":2257,"attachmentUrl":2258},"cs336-cross-entropy-adamw-training-loop","\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop","Stanford CS336 学习记录 08：数值稳定交叉熵、AdamW、余弦调度与完整训练闭环","2026-10-27",[2247,2238,2248,2249,2250],"CS336","AdamW","CrossEntropy","训练闭环","用 LogSumExp 消去交叉熵中的指数溢出，手写解耦权重衰减的 AdamW 优化器、全局 L2 梯度裁剪、Warmup 余弦退火调度、memmap 数据采样、Checkpoint 序列化与 Top-p 解码，完成 CS336 Assignment 1 全部 48 项测试。","\u002Fimages\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop\u002Fcs336-cross-entropy-adamw-training-loop-cover.webp","滑动窗口采样、余弦退火曲线、双动量优化轨迹与检查点归档构成的训练闭环 AI 辅助概念插画","作为 Stanford CS336 Assignment 1 系列的收官篇，我们已经拥有了分词器（bpe.py、tokenizer.py）和完整的语言模型架构（transformer.py）。要让模型在语料上真正训练并生成文本，还需",8,"Stanford CS336","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fb5c93ad0cc73178b2b2a62e518a0900da47a0fb7","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-08-adamw-training-loop-20261027.zip",{"slug":2260,"path":2261,"title":2262,"date":2263,"tags":2264,"description":2271,"cover":2272,"coverAlt":2273,"excerpt":2274,"draft":2226,"hidden":2226,"published":1979,"readingTime":2255},"editcache-video-dit-inference-acceleration","\u002Fposts\u002Feditcache-video-dit-inference-acceleration","EditCache：视频 Diffusion Transformer (DiT) 局部编辑的跨步特征复用与区域重算加速","2026-10-25",[2265,2266,2267,2268,2269,2270],"扩散模型","视频生成","AI Infra","推理加速","GPU","论文解读","拆解视频扩散模型（Wan2.1-VACE）局部编辑中的三重算力冗余，通过可重入 Latent、跨步 Block 隐状态缓存复用、带 Halo 边界的局部区域重算与自适应推理起点，在单卡 H100 上实现 2.46× 模型计算加速与 1.49× 严格同步端到端加速。","\u002Fimages\u002Fposts\u002Feditcache-video-dit-inference-acceleration\u002Feditcache-dit-cover-20261025.webp","视频扩散 Transformer 时空三维体素网格中冻结背景缓存与暖橙色局部编辑重算通道的等距技术插画","在基于 Diffusion Transformer（DiT）的视频编辑与局部重绘（Video Inpainting \u002F Local Editing）任务中，用户的典型工作流并不是一次性生成最终成片，而是针对同一段源视频反复调整掩码区",{"slug":2276,"path":2277,"title":2278,"date":2263,"tags":2279,"description":2284,"cover":2285,"coverAlt":2286,"excerpt":2287,"draft":2226,"hidden":2226,"published":1979,"readingTime":2255,"series":2256,"seriesOrder":2288,"codeUrl":2257,"attachmentUrl":2289},"cs336-bpe-tokenizer-training-encoding","\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding","Stanford CS336 学习记录 07：Byte-Level BPE 分词器训练、倒排索引加速与流式编码",[2247,2280,2281,2282,2283],"Tokenizer","BPE","Tiktoken","算法优化","实现 GPT-2 正则预分词、特殊 token 硬边界切分、基于 pair-to-word 倒排索引的增量 BPE 合并算法（\u003C 0.25s 通过速度基准），以及与 tiktoken 完全对齐的 Tokenizer。","\u002Fimages\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding\u002Fcs336-bpe-tokenizer-training-encoding-cover.webp","UTF-8 字节块经过正则切分闸口后沿合并树聚合成子词单元的 AI 辅助概念插画","前面六篇完成了 Transformer 语言模型本身的全部神经网络组件。要让模型真正处理自然语言文本，还需要打通入口处的 ByteLevel BPE（BytePair Encoding）分词器。这一篇记录 cs336basics\u002Fb",7,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-07-bpe-tokenizer-20261025.zip",{"slug":2291,"path":2292,"title":2293,"date":2294,"tags":2295,"description":2300,"cover":2301,"coverAlt":2302,"excerpt":2303,"draft":2226,"hidden":2226,"published":1979,"readingTime":2255},"in-situ-autoguidance-editable-scientific-illustration","\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration","从原位自校正引导（In-situ Autoguidance）到可编辑科研插图生成（AutoFigure-Edit）","2026-10-24",[2265,2296,2297,2298,2299,2270],"多模态","Agent","ICML","ACL","串联两项生成式模型与多模态智能体研究：如何通过推理期局部随机微扰在单模型内构造免训练扩散自引导向量（ICML 2025 Workshop 第一作者，arXiv:2510.17136），以及如何通过五阶段视觉-符号混合流水线生成原生可编辑 SVG 科研插图（ACL 2026 Demo，FigureBench 83.0% 胜率）。","\u002Fimages\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration\u002Fautoguidance-autofigure-cover-20261024.webp","左侧扩散流形自校正轨迹矢量场与右侧五阶段可编辑 SVG 科研插图分层网格相衔接的等距技术插画","生成式视觉模型正在经历从“生成高保真像素（Pixel Synthesis）”向“可控结构化生成（Controllable & Editable Generation）”的演进。在这一进程中，我作为第一作者或核心贡献者参与了两项分别处",{"slug":2305,"path":2306,"title":2307,"date":2308,"tags":2309,"description":2313,"cover":2314,"coverAlt":2315,"excerpt":2316,"draft":2226,"hidden":2226,"published":1979,"readingTime":2255,"series":2256,"seriesOrder":2317,"codeUrl":2257,"attachmentUrl":2318},"cs336-pre-norm-transformer-block-lm","\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm","Stanford CS336 学习记录 06：Pre-Norm Transformer Block 与完整语言模型装配","2026-10-23",[2247,2238,2310,2311,2312],"Transformer","Pre-Norm","FLOPs","将 RMSNorm、带 RoPE 的因果多头自注意力和 SwiGLU 组装成 Pre-Norm Transformer Block，搭建完整 TransformerLM，并核算 GPT-2 XL 的参数量与前向 FLOPs。","\u002Fimages\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm\u002Fcs336-pre-norm-transformer-block-lm-cover.webp","干净残差流主干穿过多个 Pre-Norm 子层并汇聚到词表投影层的 AI 辅助概念插画","前五篇把 Linear、Embedding、RMSNorm、SwiGLU、RotaryPositionalEmbedding 和因果 MultiHeadSelfAttention 全部单独打通了。这一篇进入 Assignment 1",6,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-06-transformer-block-lm-20261023.zip",{"slug":2320,"path":2321,"title":2322,"date":2308,"tags":2323,"description":2329,"cover":2330,"coverAlt":2331,"excerpt":2332,"draft":2226,"hidden":2226,"published":1979,"readingTime":2333},"xgrammar-concurrent-cache-generation-fix","\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix","XGrammar 结构化解码缓存剖析：Unlimited 模式并发 Clear 字节记账竞态与代际修复（PR #1011）",[2267,2324,2325,2326,2327,2328],"C++","XGrammar","结构化解码","并发编程","源码分析","拆解 XGrammar GrammarCompiler 的双层缓存架构与 ThreadSafeLRUCache 锁分离实现，定位 Unlimited 模式下 Clear() 字节残留与在途编译（In-flight Computation）跨代污染根因，给出代际号 + 读写锁原子修复与零 sleep 确定性并发 GoogleTest 验证。","\u002Fimages\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix\u002Fxgrammar-cache-fix-cover-20261022.webp","双层磨砂玻璃流道经暖橙色代际闸门过滤后汇入晶格缓存矩阵的等距技术插画","在大语言模型服务（如 SGLang、vLLM）的结构化输出场景中，JSON Schema 或 EBNF 语法需要先编译为词表级掩码缓存（CompiledGrammar），才能在每个 Decode 步以微秒级开销生成合法 token ",14,{"slug":2335,"path":2336,"title":2337,"date":2338,"tags":2339,"description":2345,"cover":2346,"coverAlt":2347,"excerpt":2348,"draft":2226,"hidden":2226,"published":1979,"readingTime":2333},"multi-constraint-agent-routing-dag-eval","\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval","多约束生产级 Agent 系统实践：两阶段工具路由、DAG 局部重规划与轨迹级评测闭环","2026-10-22",[2297,2340,2341,2342,2343,2344],"LLM","工具调用","上下文工程","DPO","强化学习","围绕跨境酒旅等强耦合多约束业务场景，拆解向量 Top-K 召回 + LLM 精排 + 跨语言实体链接与 Schema 门控（96%+ 调用合法率）、基于状态机的 DAG 局部子图重规划、Observation 冗余裁剪（节省 40% Token），以及从线上失败恢复轨迹构造 SFT\u002FDPO 偏好对的评测与训练闭环。","\u002Fimages\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval\u002Fagent-dag-routing-cover-20261020.webp","候选模块经漏斗滤网进入双分支有向无环图轨道并在受阻节点触发暖橙色局部旁路重连的等距技术插画","在玩具级 Demo 里，几个工具加上一段 ReAct 循环就能跑通大多数单步问答。但当 Agent 进入跨境酒旅、复杂交易或研发沙箱等真实业务生产环境时，单靠线性 ReAct 循环很快会撞上四堵墙： 1. 工具空间与参数耦合爆炸：候",{"slug":2350,"path":2351,"title":2352,"date":2353,"tags":2354,"description":2358,"cover":2359,"coverAlt":2360,"excerpt":2361,"draft":2226,"hidden":2226,"published":1979,"readingTime":2362},"llm-mrd-distillation-auditable-review-agents","\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents","多视角推理蒸馏（LLM-MRD）与可审计生成式评审系统（DeepReviewer 2.0）深度拆解","2026-10-21",[2355,2296,2356,2297,2357,2270],"知识蒸馏","Qwen2.5-VL","Post-training","对比大模型落地的两条互补路径：如何用 Qwen2.5-VL 离线多视角分析结合 MLP 残差校准与温度缩放 KL 蒸馏，将推理能力压入 358M 学生模型实现 17.2 ms 独立推理（LLM-MRD，DASFAA 2026 Oral）；以及如何通过主张-证据-风险台账、PASA 可比性门控与导出预算校验构建可审计的同行评审 Agent（DeepReviewer 2.0）。","\u002Fimages\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents\u002Fmrd-review-distill-cover-20261016.webp","三块半透明几何棱镜发出平行光束穿过中央校准环汇聚至右侧高密度晶体模块的等距技术插画","把大语言模型或视觉语言模型（VLM）的深度推理能力用到实际系统中，通常会面临一个两难选择： 如果场景受限于毫秒级在线延迟（如高吞吐多模态内容审核与虚假信息检测）：直接在线调用 7B～72B 的多模态大模型做长链条思维链（CoT）生成",18,{"slug":2364,"path":2365,"title":2366,"date":2353,"tags":2367,"description":2370,"cover":2371,"coverAlt":2372,"excerpt":2373,"draft":2226,"hidden":2226,"published":1979,"readingTime":2288,"series":2256,"seriesOrder":2149,"codeUrl":2257,"attachmentUrl":2374},"cs336-rope-rotary-position-embedding","\u002Fposts\u002Fcs336-rope-rotary-position-embedding","Stanford CS336 学习记录 05：RoPE 旋转位置编码与带位置的多头注意力",[2247,2238,2310,2368,2369],"RoPE","Attention","从二维平面正交旋转推导 RoPE 的奇偶特征配对公式，理解为什么内积只依赖相对位置差 (i - j)，并用 register_buffer 预计算 cos\u002Fsin 缓存接入因果多头自注意力。","\u002Fimages\u002Fposts\u002Fcs336-rope-rotary-position-embedding\u002Fcs336-rope-rotary-position-embedding-cover.webp","二维特征平面中的旋转向量对与预计算余弦正弦网格的 AI 辅助概念插画","上一篇完成了不含位置编码的因果多头自注意力。如果只看那一套矩阵乘法，把输入序列前两个 token 交换顺序（并在因果 mask 允许的范围内看同一个后缀 query 对它们的点积），点积本身并不知道哪个 key 离当前 query ","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-05-rope-mha-20261021.zip",{"slug":2376,"path":2377,"title":2378,"date":2379,"tags":2380,"description":2386,"cover":2387,"coverAlt":2388,"excerpt":2389,"draft":2226,"hidden":2226,"published":1979,"readingTime":2255},"llm-gateway-semantic-cache-dynamic-batching","\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching","高并发 LLM 推理网关架构实践：两级精确\u002F语义缓存、50ms 有界动态批处理与尾部延迟治理","2026-10-19",[2267,2381,2382,2383,2384,2385],"LLM 推理","分布式系统","Redis","FastAPI","后端","深度拆解生产级大模型推理网关 LLM-Gateway 的核心系统设计：基于 SHA-256 与 Redis 向量余弦相似度的两级精确\u002F语义缓存、50ms 有界等待动态批处理调度器（Dynamic Batching）、令牌桶多租户限流与 Prometheus 全链路可观测性，将命中请求 P99 延迟从 5,307 ms 压降至 17 ms。","\u002Fimages\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching\u002Fllm-gateway-cover-20261019.webp","高并发请求流依次穿过精确哈希滤网、向量球面语义缓存层与 50ms 动态批处理时间窗拼装器的等距系统架构插画","在将大语言模型接入高并发业务系统时，如果让上游微服务直接裸连底层推理引擎（vLLM \u002F SGLang 或外部模型 API），系统很快会暴露出三个典型的生产痛点： 1. 高频重复与近义查询消耗巨量算力：大量用户请求在语义上高度等价（例",{"slug":2391,"path":2392,"title":2393,"date":2379,"tags":2394,"description":2396,"cover":2397,"coverAlt":2398,"excerpt":2399,"draft":2226,"hidden":2226,"published":1979,"readingTime":2333,"series":2256,"seriesOrder":1786,"codeUrl":2400,"attachmentUrl":2401},"cs336-multihead-attention-shapes","\u002Fposts\u002Fcs336-multihead-attention-shapes","Stanford CS336 学习记录 04：多头注意力怎样拆头、计算和合头",[2247,2238,2310,2369,2395],"张量形状","沿着 (2, 5, 64) 的输入追踪 QKV 投影、reshape、transpose、共享因果 mask 和合头，理解每个头仍处理完整序列，以及最后为什么还需要输出 Linear。","\u002Fimages\u002Fposts\u002Fcs336-multihead-attention-shapes\u002Fcs336-multihead-attention-shapes-cover.webp","宽向量束分成多条并行特征轨道后重新合并的 AI 辅助概念插画","上一篇完成了缩放点积注意力。那个函数接收已经准备好的 Q、K、V，计算匹配分数，应用 mask，再用权重汇总 V。这一篇继续把它放进一个带参数的多头自注意力模块。 这次最需要看清的是“拆头”。代码里只是几行 reshape 和 tr","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002F31363223498fafd1528c550a51efbe0aed231736","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-04-multihead-attention-20261019.zip",{"slug":2403,"path":2404,"title":2405,"date":2406,"tags":2407,"description":2411,"cover":2412,"coverAlt":2413,"excerpt":2414,"draft":2226,"hidden":2226,"published":1979,"readingTime":2415},"qwen25-sft-dpo-alignment-tax-ablation","\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation","大模型后训练偏好对齐实录：Qwen2.5 SFT 多维消融、动态参考 DPO 与对齐税（Alignment Tax）量化分析","2026-10-18",[2357,2343,2408,2409,2344,2410],"SFT","LoRA","Qwen2.5","基于 Qwen2.5 完整复现 Instruct SFT → Bradley-Terry DPO 两阶段后训练管线：通过 3 随机种子消融 LoRA Rank、学习率与数据规模（GSM8K +12.73pp），实现免双倍权重的动态 Adapter 切换 DPO 训练，并定量揭示偏好对齐对确定性数学推理的对齐税效应。","\u002Fimages\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation\u002Fqwen25-sft-dpo-cover-20261018.webp","左侧监督微调轨迹流经中央 Bradley-Terry 偏好分流天平分离出正负样本通道并连接右侧对齐税监控仪表盘的等距技术插画","在大语言模型后训练（PostTraining）管线中，监督微调（SFT）负责教会模型任务格式与领域推理模式，而直接偏好优化（DPO, Direct Preference Optimization）则利用成对偏好数据 $(x, yw,",9,{"slug":2417,"path":2418,"title":2419,"date":2406,"tags":2420,"description":2422,"cover":2423,"coverAlt":2424,"excerpt":2425,"draft":2226,"hidden":2226,"published":1979,"readingTime":2426,"series":2256,"seriesOrder":1764,"codeUrl":2427,"attachmentUrl":2428},"cs336-softmax-attention-mask","\u002Fposts\u002Fcs336-softmax-attention-mask","Stanford CS336 学习记录 03：Softmax 的数值稳定与 Attention Mask",[2247,2238,2310,2421,2369],"Softmax","从减去最大值为何不改变 Softmax 开始，理清函数与有参数层的区别，再沿着 Q、K、V 的形状理解缩放点积注意力，以及 mask 为什么必须填负无穷。","\u002Fimages\u002Fposts\u002Fcs336-softmax-attention-mask\u002Fcs336-softmax-attention-mask-cover.webp","注意力分数、权重与因果遮罩的 AI 辅助概念插画","上一篇写到 SwiGLU，接下来进入 Attention。真正开始相乘之前，先做了 Softmax：它把一组分数变成总和为 1 的权重，后面用这些权重汇总不同 token 提供的信息。 这段学习里，有两个很具体的问题值得记下来。一个",11,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fa39eb684710817f7f389626423fcb26ca0f6d629","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-03-softmax-sdpa-20261018.zip",{"slug":2430,"path":2431,"title":2432,"date":2433,"tags":2434,"description":2437,"cover":2438,"coverAlt":2439,"excerpt":2440,"draft":2226,"hidden":2226,"published":1979,"readingTime":2415,"series":2256,"seriesOrder":1739,"codeUrl":2427,"attachmentUrl":2441},"cs336-rmsnorm-swiglu","\u002Fposts\u002Fcs336-rmsnorm-swiglu","Stanford CS336 学习记录 02：RMSNorm 的广播与 SwiGLU 的门控","2026-10-16",[2247,2238,2310,2435,2436],"RMSNorm","SwiGLU","用一个 token 向量理解均方根归一化，拆开 dim=-1 与 keepdim=True 的作用，再沿着两条分支读懂 SiLU、逐元素乘法和 SwiGLU。","\u002Fimages\u002Fposts\u002Fcs336-rmsnorm-swiglu\u002Fcs336-rmsnorm-swiglu-cover.webp","特征尺度整理和双分支门控的 AI 辅助概念插画","上一篇把 Linear 和 Embedding 的参数形状理顺了。继续往下，最先遇到的是 RMSNorm：公式不算长，但把它落到一批 token 上时，dim=1、keepdim=True 和广播必须一起理解。 然后是 SiLU 和","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-02-rmsnorm-swiglu-20261016.zip",{"slug":2443,"path":2444,"title":2445,"date":2446,"tags":2447,"description":2450,"cover":2451,"coverAlt":2452,"excerpt":2453,"draft":2226,"hidden":2226,"published":1979,"readingTime":2454,"series":2256,"seriesOrder":1704,"codeUrl":2427,"attachmentUrl":2455},"cs336-linear-embedding-shapes","\u002Fposts\u002Fcs336-linear-embedding-shapes","Stanford CS336 学习记录 01：从 Linear、Embedding 搞清张量形状","2026-10-14",[2247,2238,2310,2448,2449],"Linear","Embedding","从 PyTorch 类、参数与 forward 的关系开始，理解 Linear 为什么保存两行三列的权重、计算时为什么要转置，再区分 Embedding 查表与线性变换。","\u002Fimages\u002Fposts\u002Fcs336-linear-embedding-shapes\u002Fcs336-linear-embedding-shapes-cover.webp","token 方块、向量表与矩阵变换的 AI 辅助概念插画","开始沿着 Stanford CS336 的 Assignment 1 一步步实现。BPE 之前已经接触过，这轮先跳过分词器，从 Transformer 的基础层开始。这里记录的是当时写代码时真正卡住的问题，以及对应测试能确认到的范围",10,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-01-linear-embedding-20261014.zip",{"slug":2457,"path":2458,"title":2459,"date":2460,"tags":2461,"description":2465,"cover":2466,"coverAlt":2467,"excerpt":2468,"draft":2226,"hidden":2226,"published":1979,"readingTime":2469,"series":2232,"seriesOrder":2333,"codeUrl":2470,"attachmentUrl":2471},"cmu-15-779-kv-cache-granularity-study","\u002Fposts\u002Fcmu-15-779-kv-cache-granularity-study","CMU 15-779 学习记录 14：KV 页大小、共享边界与 CPU 实现成本","2026-10-12",[2232,2462,2463,2464],"KV cache","PagedAttention","系统实验","用 210 组 CPU 配置比较连续与分页 KV cache，观察 31\u002F32-token 共享前缀的页边界，分别记录活跃槽位、池容量、复制计数和完整调用耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling\u002Fcmu-serving-cover-20261019.webp","浅色手绘分页存储抽屉与共享前缀卡片，旁边连接不同长度的请求队列","个人课程项目中的 CPU 实验记录，延续记录 11 的分页 KV 实现。已完成 210 组配置与 1470 次计时，输出均通过连续存储参考对照；本篇不代表正式作业提交、整门课程或项目全部交付完成。 同样是共享 31 个 token，",12,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F200faf569eccf94fb125e65110bb5d14094d454a","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fkv-cache-granularity-20261019.zip",{"slug":2473,"path":2474,"title":2475,"date":2476,"tags":2477,"description":2480,"cover":2481,"coverAlt":2482,"excerpt":2483,"draft":2226,"hidden":2226,"published":1979,"readingTime":2149,"series":2232,"seriesOrder":2484,"codeUrl":2485,"attachmentUrl":2486},"cmu-15-779-moe-routing-capacity","\u002Fposts\u002Fcmu-15-779-moe-routing-capacity","CMU 15-779 学习记录 13：MoE 路由、稀疏分发与容量溢出","2026-10-09",[2232,2478,2479],"MoE","Routing","用 CPU top-2 MoE 检查稀疏分发的输出与梯度，统计 expert-token 行数，并保留容量丢弃会改变输出的反例。","\u002Fimages\u002Fposts\u002Fcmu-15-779-moe-routing-capacity\u002Fcmu-moe-cover-20261019.webp","浅色手绘分拣器将小珠分配到四个专家工作屋，队列呈现负载不均","第 18 讲的个人配套实验。36 号脚本串行运行 CPU 专家，检查 topk 路由后的分发与合并；没有训练 router，也没有执行 GPU grouped GEMM 或跨设备 alltoall。 上一节记录低秩适配，这次转向每个",13,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fc4851e0cedadf13af7c421867e83c8ff57244fa0","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-parallel-serving-lora-moe-20261019.zip",{"slug":2488,"path":2489,"title":2490,"date":2491,"tags":2492,"description":2494,"cover":2495,"coverAlt":2496,"excerpt":2497,"draft":2226,"hidden":2226,"published":1979,"readingTime":2149,"series":2232,"seriesOrder":2469,"codeUrl":2485,"attachmentUrl":2486},"cmu-15-779-lora-uniform-quantization","\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization","CMU 15-779 学习记录 12：LoRA 首步梯度、权重合并与均匀 4-bit 打包","2026-10-06",[2232,2409,2493],"Quantization","用一次 CPU SGD 更新检查 LoRA 初始化、冻结基座与合并等价，再分别记录均匀 4-bit 存储字节数和量化误差。","\u002Fimages\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization\u002Fcmu-lora-cover-20261019.webp","浅色手绘的大计算板、两块细长适配板和成对打包的小方块","第 17 讲的个人配套实验，35 号脚本实际执行一次 CPU LoRA 更新，并实现均匀 4bit 打包。实验检查机制与数值误差，不评估微调任务效果，也没有实现完整 QLoRA。 这次把两个问题分开记录：低秩分支是否能合并进原权重，",{"slug":2499,"path":2500,"title":2501,"date":2502,"tags":2503,"description":2507,"cover":2466,"coverAlt":2508,"excerpt":2509,"draft":2226,"hidden":2226,"published":1979,"readingTime":2288,"series":2232,"seriesOrder":2426,"codeUrl":2485,"attachmentUrl":2486},"cmu-15-779-kv-serving-speculative-sampling","\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling","CMU 15-779 学习记录 11：分页 KV、连续批处理与推测采样","2026-10-03",[2232,2504,2505,2506],"LLM Serving","KV Cache","Speculative Decoding","在 CPU 上验证 KV 页共享和写时复制，用单位成本模型检查连续批处理，再用条件概率小模型核对推测采样的接受与补采样。","浅色手绘的共享卡片抽屉与不断补入小珠子的请求托盘","第 15、16 讲的个人配套实验。33 号实现 CPU 分页 KV 存储，37 号模拟请求准入，34 号运行小概率模型的推测采样。它们分别验证存储、调度和分布问题，没有测量真实 LLM 服务吞吐或 GPU 延迟。 记录 08已经对照",{"slug":2511,"path":2512,"title":2513,"date":2514,"tags":2515,"description":2520,"cover":2521,"coverAlt":2522,"excerpt":2523,"draft":2226,"hidden":2226,"published":1979,"readingTime":2524},"qwen3-mid-post-train-mlx","\u002Fposts\u002Fqwen3-mid-post-train-mlx","在 M2 Pro 上完成 Qwen3 两阶段训练：Mid Train、SFT 与可复核评估","2026-09-30 11:19:10",[2340,2516,2517,2409,2518,2408,2519],"Qwen3","MLX","持续预训练","Apple Silicon","用 Qwen3 0.6B Base 和 MLX 完成领域持续训练与监督微调，记录数据划分、聊天模板、损失掩码、数值稳定性修复和独立测试结果。","\u002Fimages\u002Fposts\u002Fqwen3-mid-post-train-mlx\u002Fqwen3-mlx-cover-20261018.webp","Apple Silicon 统一内存芯片底座支撑两阶段持续预训练与带精确 Loss 掩码滤网的监督微调桥接插画","在一台 M2 Pro、16 GB 内存的 Mac 上，能否把一个 Base 语言模型接续训练成输出 JSON 的运维事件分级助手？这次项目完成了领域持续训练、监督微调、检查点选择、重新加载和独立测试，并保留了权重、日志和数据哈希。 ",16,{"slug":2526,"path":2527,"title":2528,"date":2529,"tags":2530,"description":2534,"cover":2535,"coverAlt":2536,"excerpt":2537,"draft":2226,"hidden":2226,"published":1979,"readingTime":2317,"series":2232,"seriesOrder":2454,"codeUrl":2485,"attachmentUrl":2486},"cmu-15-779-cpu-parallelism-pipeline","\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline","CMU 15-779 学习记录 10：加权梯度、状态分片与同步流水线","2026-09-30",[2232,2531,2532,2533],"Parallel Training","Gloo","Pipeline","用双 CPU 进程 Gloo 检查不等数据分片的梯度，推导持久状态字节数，再用依赖模型比较 GPipe 与同步 1F1B 的激活存活。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline\u002Fcmu-parallel-cover-20261019.webp","浅色手绘的四阶段桌面流水线与按数量汇总的托盘","第 12 至 14 讲的个人配套实验，脚本编号 31、32 不对应官方作业。Gloo 实际运行两个 CPU 进程；张量切分、状态字节数和流水线分别按下文的数学或成本模型解释。真实 GPU 实测按学习安排跳过。 上一节检查单个计算图的",{"slug":2539,"path":2229,"title":5,"date":2224,"tags":2540,"description":2225,"cover":2222,"coverAlt":2223,"excerpt":2541,"draft":2226,"hidden":2226,"published":1979,"readingTime":2288,"series":2232,"seriesOrder":2415,"codeUrl":28,"attachmentUrl":35},"cmu-15-779-cpu-compiler-optimizations",[2232,2236,2237,2238],"编译章节的个人配套实验，编号 30 不对应官方作业。图重写、分块候选和 TorchInductor 均在 CPU 上运行；真实 GPU 实验按学习安排跳过。 上一节记录了分块 Attention 和 KV cache。这次转到编译优",{"slug":2543,"path":2544,"title":2545,"date":2546,"tags":2547,"description":2549,"cover":2550,"coverAlt":2551,"excerpt":2552,"draft":2226,"hidden":2226,"published":1979,"readingTime":2469,"series":2232,"seriesOrder":2255,"codeUrl":2470,"attachmentUrl":2553},"cmu-15-779-attention-online-softmax","\u002Fposts\u002Fcmu-15-779-attention-online-softmax","CMU 15-779 学习记录 08：分块 Attention、在线 Softmax 与 KV cache","2026-09-24",[2232,2369,2421,2548,2238],"FlashAttention","在 CPU 上对照完整 Attention、在线 Softmax 分块实现与 PyTorch SDPA，检查因果掩码、浮点误差和分数块大小，理解 FlashAttention 的一部分算法基础。","\u002Fimages\u002Fposts\u002Fcmu-15-779-attention-online-softmax\u002Fcmu-online-attention-cover-20261016.webp","浅紫色卡片分批进入绿色计算托盘，旁边三个状态容器保存累积量的浅色手绘插画","第 5 讲 Transformers & Attention 的个人配套实验。28\u002F29 号是 CPU 单头前向与 KV cache；后续补入 38 号 CUDA 多头合成张量的显式 SDPA MATH\u002FFLASH 对照。没有手写 ","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcuda-triton-sdpa-20261019.zip",{"slug":2555,"path":2556,"title":2557,"date":2558,"tags":2559,"description":2564,"cover":2565,"coverAlt":2566,"excerpt":2567,"draft":2226,"hidden":2226,"published":1979,"readingTime":2454,"series":2232,"seriesOrder":2288,"codeUrl":2470,"attachmentUrl":2553},"cmu-15-779-matmul-tiling-tree-reduction","\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction","CMU 15-779 学习记录 07：矩阵分块、数据复用与树形归约","2026-09-21",[2232,2560,2561,2562,2563],"CUDA","矩阵乘法","数据复用","归约","用 CPU 模型统计矩阵分块的输入读取次数，再比较串行求和和树形归约的依赖深度，记录补零、同步与浮点加法顺序的边界。","\u002Fimages\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction\u002Fcmu-matmul-reduction-cover-20261016.webp","矩阵积木在共享托盘中复用，旁边的小珠沿树形管道逐层合并的浅色手绘插画","GPU Architecture & CUDA Programming 的个人配套实验。首次记录用 CPU 模型检查数据复用与依赖深度；后续补入单卡 GPU 原生 CUDA 矩阵验证，以及 Triton 两阶段归约和矩阵乘法实测。 ",{"slug":2569,"path":2570,"title":2571,"date":2572,"tags":2573,"description":2576,"cover":2577,"coverAlt":2578,"excerpt":2579,"draft":2226,"hidden":2226,"published":1979,"readingTime":2454,"series":2232,"seriesOrder":2317,"codeUrl":2470,"attachmentUrl":2553},"cmu-15-779-cuda-thread-indexing","\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing","CMU 15-779 学习记录 06：CUDA 线程索引与边界检查","2026-09-18",[2232,2560,2269,2574,2575],"线程索引","边界检查","用 CPU 演示 CUDA 的 block、thread 与全局下标：10 个元素如何分给 3 个线程块，为什么要跳过越界线程，以及这些验证能说明什么。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing\u002Fcmu-cuda-indexing-cover-20261015.webp","分组的浅紫色计算方块与末端边界挡板组成线程分配教具的浅色插画","GPU Architecture & CUDA Programming 的个人配套练习。首次记录为 CPU 索引演示；后续在单卡 GPU 环境补入原生 CUDA 编译、运行与 memcheck，以及 Triton 向量加法实测。 上",{"slug":2581,"path":2582,"title":2583,"date":2584,"tags":2585,"description":2589,"cover":2590,"coverAlt":2591,"excerpt":2592,"draft":2226,"hidden":2226,"published":1979,"readingTime":2454,"series":2232,"seriesOrder":2149,"codeUrl":2593,"attachmentUrl":2594},"cmu-15-779-profiler-operator-batching","\u002Fposts\u002Fcmu-15-779-profiler-operator-batching","CMU 15-779 学习记录 05：Profiler、算子调用与批量推理","2026-09-15",[2232,2238,2586,2587,2588],"Profiler","批量推理","性能分析","对照 ML Systems 基础，用 Profiler 拆开两层网络的执行过程，再固定 1000 个样本扫描 batch size，观察算子调用次数、尾批和 CPU 耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-profiler-operator-batching\u002Fcmu-pytorch-profiling-cover-20261015.webp","放大镜观察线性层与 ReLU 模块，旁边面板呈现执行片段的浅色技术教具插画","本篇对照课程官网的 Introduction to ML Systems 整理，是个人配套 CPU 实验，不是课程官方作业。脚本编号是练习顺序，不对应课程讲次。 前几篇一直在补 PyTorch 基础。到了这一步，我在学习聊天里问：现","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fcecf5110d8a95161bb5763eadfca94cd7a0cc50f","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-profiler-batching-20261015.zip",{"slug":2596,"path":2597,"title":2598,"date":2599,"tags":2600,"description":2603,"cover":2604,"coverAlt":2605,"excerpt":2606,"draft":2226,"hidden":2226,"published":1979,"readingTime":2415,"series":2232,"seriesOrder":1786,"codeUrl":2607,"attachmentUrl":2608},"cmu-15-779-pytorch-linear-relu-absolute-value","\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value","CMU 15-779 学习记录 04：多输出线性层、ReLU 与绝对值网络","2026-09-12",[2232,2238,2561,2601,2602],"ReLU","神经网络","把 nn.Linear 对应到矩阵乘法，弄清多个输出怎样计算，再逐层拆开一个绝对值网络，比较手动设置参数与训练得到的结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value\u002Fcmu-pytorch-absolute-cover-20261015.webp","双分支经过折角模块后汇入紫色 V 形曲线面板的浅色技术教具插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的批量推理与矩阵乘法，我继续把 model(x) 拆开看：一层里有多个输出时，每个数是怎么算出来的？两","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F4cc63904a3a583da4a2122c246a14b43aac38528","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-linear-relu-abs-20261015.zip",{"slug":2610,"path":2611,"title":2612,"date":2613,"tags":2614,"description":2617,"cover":2618,"coverAlt":2619,"excerpt":2620,"draft":2226,"hidden":2226,"published":1979,"readingTime":2415,"series":2232,"seriesOrder":1764,"codeUrl":2621,"attachmentUrl":2622},"cmu-15-779-pytorch-dataloader-checkpoints-batching","\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching","CMU 15-779 学习记录 03：DataLoader、模型保存与批量推理","2026-09-09",[2232,2238,2615,2587,2616],"DataLoader","模型保存","用 DataLoader 分批训练平方函数，保存并加载模型参数，再比较 1000 个样本与 1 个样本的推理耗时，最后把批量计算对应到矩阵乘法。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching\u002Fcmu-pytorch-batching-cover-20261015.webp","样本卡片按批次进入计算模块，旁边配有参数存储盒和输出矩阵面板的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的平方函数实验，这一节继续练习分批训练、保存参数和独立预测。最后做了一次计时：同一个小网络，逐个处理 ","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-batching-20261015.zip",{"slug":2624,"path":2625,"title":2626,"date":2627,"tags":2628,"description":2629,"cover":2630,"coverAlt":2631,"excerpt":2632,"draft":2226,"hidden":2226,"published":1979,"readingTime":2415,"series":2232,"seriesOrder":1739,"codeUrl":2621,"attachmentUrl":2633},"cmu-15-779-pytorch-linear-module-relu","\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu","CMU 15-779 学习记录 02：nn.Linear、张量形状与 ReLU","2026-09-06",[2232,2238,2602,2601],"从带偏置的线性回归开始，弄清 nn.Linear 和 nn.Module 的分工，再用平方函数观察线性模型的限制、ReLU 的作用和测试点的选择。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu\u002Fcmu-pytorch-relu-cover-20261015.webp","输入经过两层神经网络和 ReLU 折角模块，输出曲线拟合结果的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 上一篇里，我用一个参数拟合了 $y=2x$。这一篇继续顺着往下写，先加上偏置，再把手写参数换成 nn.Linea","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-models-20261015.zip",{"slug":2635,"path":2636,"title":2637,"date":2638,"tags":2639,"description":2642,"cover":2643,"coverAlt":2644,"excerpt":2645,"draft":2226,"hidden":2226,"published":1979,"readingTime":2454,"series":2232,"seriesOrder":1704,"codeUrl":2621,"attachmentUrl":2646},"cmu-15-779-pytorch-autograd-linear-regression","\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression","CMU 15-779 学习记录 01：PyTorch 自动求导与线性回归","2026-09-03",[2232,2238,2640,2641],"自动求导","线性回归","从自动求导和手动梯度下降开始，比较学习率，再用 SGD 训练线性回归。记录 w * 4 写错后停在 1.25 的原因和修正结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression\u002Fcmu-pytorch-autograd-cover-20261015.webp","梯度曲面上的小球沿路径下降，前侧面板展示散点与拟合直线的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 开始学习 CMU 15779 时，我会 Python，但还没用过 PyTorch。这次先在 Mac 上用 CPU","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-basics-20261013.zip",{"slug":2648,"path":2649,"title":2650,"date":2651,"tags":2652,"description":2658,"cover":2659,"coverAlt":2660,"excerpt":2661,"draft":2226,"hidden":2226,"published":1979,"readingTime":2484,"codeUrl":2662,"attachmentUrl":2663},"embodied-pickplace-lab-robustness","\u002Fposts\u002Fembodied-pickplace-lab-robustness","Pickplace Lab 项目复盘：抓取成功率之外的时延与失败分析","2026-08-29",[2653,2654,2655,2238,2656,2657],"具身智能","模仿学习","MuJoCo","机器人","鲁棒性","从 MuJoCo 抓取搬运闭环出发，记录 22,100 个评测回合。动作平滑在自建任务的 50 ms 延迟下有效，换到 Panda Lift 却没有支持同样的收益。","\u002Fimages\u002Fposts\u002Fembodied-pickplace-lab-robustness\u002Fpickplace-grasp-cover-20261016.webp","MuJoCo 实际渲染的笛卡尔夹爪抓取方块场景，桌面上标出放置目标","Pickplace Lab 从一个简单任务开始：让夹爪抓起方块，搬到指定位置，放下，再退开。先把仿真、示范采集、训练和闭环评测接起来，随后再问：干净条件下成功的策略，换一种执行条件，还能不能完成任务？ 最值得讨论的结果出现在指令延迟","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Ftree\u002Fv0.1.0","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Freleases\u002Fdownload\u002Fv0.1.0\u002Fpickplace-lab-research-v1.zip",{"slug":2665,"path":2666,"title":2667,"date":2668,"tags":2669,"description":2674,"cover":2675,"coverAlt":2676,"excerpt":2677,"draft":2226,"hidden":2226,"published":1979,"readingTime":2678},"qwen25-vl-sglang-verl-single-gpu-lab","\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab","从 Mac 到单卡 GPU：Qwen2.5-VL 部署、GRPO 与计数 LoRA 实践","2026-08-19",[2356,2296,2670,2671,2672,2673,2517],"SGLang","verl","GRPO","推理优化","在 M2 Pro 和云端单卡上部署 Qwen2.5-VL-3B，实测连续批处理、前缀缓存与 SGLang 参数调优，再用 verl 完成多模态 GRPO 更新和同条件前后评测，记录有效优化、兼容性问题与未观察到提升的结果。","\u002Fimages\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab\u002Fqvl-lab-cover-20261010.webp","笔记本与单卡计算模块通过图像和 token 光轨连接的实验台插画","这次实验从一个很具体的问题开始：M2 Pro、16 GB 统一内存，能不能在本机跑 Qwen2.5VL？跑起来之后，我又想把图文问答、推理性能测量和强化学习训练接成一条完整流程。 本机的 MLX 4bit 服务和云端的 SGLang",29,{"slug":2680,"path":2681,"title":2682,"date":2683,"tags":2684,"description":2688,"cover":2689,"coverAlt":2690,"excerpt":2691,"draft":2226,"hidden":2226,"published":1979,"readingTime":2484},"cycleresearcher-post-training","\u002Fposts\u002Fcycleresearcher-post-training","从 DeepReviewer 2.0 到 CycleResearcher：科研 Agent 如何真正做 Post-training","2026-08-07 17:33:05",[2340,2357,2297,2685,2686,2687],"SimPO","Reward Model","ICLR","区分推理时 Agent 编排与模型后训练，并拆解 CycleResearcher 的领域 SFT、生成式 Reward Model、Iterative SimPO 和评测风险。","\u002Fimages\u002Fposts\u002Fcycleresearcher-post-training\u002Fcycleresearcher-cover-20261018.webp","策略生成模块产出多份候选文稿经多维评审棱镜排序后通过偏好梯度反馈弧闭环优化的等距技术插画","看到一个系统会检索、规划、反思和调用工具，不能据此判断它做过 Posttraining。Agent 描述的是模型在运行时如何完成任务；Posttraining 描述的是预训练之后，是否继续用监督数据、偏好数据或奖励信号更新模型参数。",{"slug":2693,"path":2694,"title":2695,"date":2696,"tags":2697,"description":2699,"cover":2700,"coverAlt":2701,"excerpt":2702,"draft":2226,"hidden":2226,"published":1979,"readingTime":2333,"series":2703,"seriesOrder":1704},"cuda-fused-rmsnorm-kernel","\u002Fposts\u002Fcuda-fused-rmsnorm-kernel","从 PyTorch 到 CUDA Kernel：融合 RMSNorm 在 RTX 4090D 上加速 4.27 倍","2026-07-22 18:16:50",[2560,2267,2269,2238,2381,2698],"Kernel 优化","从零实现 PyTorch CUDA 扩展，拆解 RMSNorm 的访存瓶颈、warp 归约、向量化加载、算子融合与冷热缓存 benchmark。","\u002Fimages\u002Fposts\u002Fcuda-fused-rmsnorm-kernel\u002Fcuda-rmsnorm-cover-20261018.webp","融合残差加法与向量化 float4 访存经 Warp Shuffle 蝴蝶树归约输出归一化张量的等距硬件架构插画","为了准备 AI Infra 岗位，我做了一个尽量小而完整的 CUDA 项目：不追求堆很多算子，而是选一个真实的 LLM 推理路径，把实现、正确性、benchmark 和性能解释全部做扎实。 项目实现了两个 PyTorch CUDA ","AI Infra 实战",{"slug":2705,"path":2706,"title":2707,"date":2708,"tags":2709,"description":2711,"cover":2712,"coverAlt":2713,"excerpt":2714,"draft":2226,"hidden":2226,"published":1979,"readingTime":2255,"series":2715,"seriesOrder":1704},"agent-memory","\u002Fposts\u002Fagent-memory","Agent 上下文工程与分层记忆架构：从 Observation 剪枝、滚动摘要到向量召回闭环","2026-04-24",[2340,2297,2710,2267],"RAG","结合生产级多约束 Agent 与超 100K Token 科研智能体实践，系统拆解长程会话中的上下文膨胀与注意力稀释问题：涵盖 Observation 结构化剪枝（降低约 40% Token 开销）、滚动工作摘要、向量检索长期记忆及 Prefix Cache 友好的上下文组装。","\u002Fimages\u002Fposts\u002Fagent-memory\u002Fagent-memory-cover-20261018.webp","底层高噪数据经剪枝滤网进入中期滑动窗口与金色状态台账并由顶层向量星座按需召回的三层记忆架构插画","在构建多步工具调用（ToolUse）与长周期自主规划（Autonomous Planning）的 Agent 系统时，上下文管理（Context Engineering）往往比单纯改写 System Prompt 更决定线上稳定性。","Agent 上下文工程",{"slug":2717,"path":2718,"title":2719,"date":2720,"tags":2721,"description":2724,"cover":2725,"coverAlt":2726,"excerpt":2727,"draft":2226,"hidden":2226,"published":1979,"readingTime":2149,"series":2728,"seriesOrder":1739},"backend-stack-deep-dive","\u002Fposts\u002Fbackend-stack-deep-dive","高并发 AI 平台后端基础设施实践：FastAPI 异步网关、SQLAlchemy 2.0 Async、PostgreSQL 与 Docker 沙箱编排","2026-04-07",[2385,2382,2384,2722,2723,2267],"PostgreSQL","Docker","结合 DeepScientist 生产级 AI 科研平台实践，拆解 FastAPI ASGI 异步事件循环、SQLAlchemy 2.0 + AsyncPG 连接池调优、PostgreSQL + MinIO 存储分层与 Docker SDK 动态沙箱容器编排的职责边界与核心设计。","\u002Fimages\u002Fposts\u002Fbackend-stack-deep-dive\u002Fbackend-stack-cover-20261018.webp","ASGI 异步事件循环涡轮将非阻塞协程流分发至 PostgreSQL 连接池、MinIO 存储矩阵与动态 Docker 沙箱的等距架构插画","结合 DeepScientist 生产级 AI 科研平台（服务 7,000+ 用户）的实际工程落地，本文系统拆解支撑高并发流式推理与容器沙箱调度的后端基础设施栈——FastAPI（ASGI 异步网关 + SSE 流式推送）、SQLA","DeepScientist 工程复盘",{"slug":2730,"path":2731,"title":2732,"date":2733,"tags":2734,"description":2737,"cover":2738,"coverAlt":2739,"excerpt":2740,"draft":2226,"hidden":2226,"published":1979,"readingTime":2426,"series":2741,"seriesOrder":1704},"minicode-source-analysis","\u002Fposts\u002Fminicode-source-analysis","MiniCode 架构与工具评测深度拆解：开源终端 Agentic Coding 引擎（1k+ Stars）的 ReAct、MCP 与 30 项基准验证","2026-04-06",[2297,2735,2736,2328,2267],"TypeScript","MCP","深度剖析开源终端 AI 编程智能体 MiniCode（1k+ GitHub Stars）的核心架构：从零重型依赖的 ReAct 规划循环、声明式工具注册表与权限拦截层、MCP 协议帧自动协商，到 30 项可执行 Tool-Use 评测基准（86.7% 完成率、100% 错误恢复率）的失败分布归因与闭环优化。","\u002Fimages\u002Fposts\u002Fminicode-source-analysis\u002Fminicode-cover-20261018.webp","终端编程智能体控制台被 ReAct 状态环、Unified Diff 补丁预览门与 MCP 协议插槽环绕的等距技术插画","MiniCode 是我作为核心开发者参与构建的开源终端 Agentic Coding 引擎（目前已在 GitHub 获得 1,000+ Stars）。项目的核心设计目标是：在不引入庞大运行时框架的前提下，用约 5,000 行高内聚的","AI 项目源码导读",{"slug":2743,"path":2744,"title":2745,"date":2733,"tags":2746,"description":2747,"cover":2748,"coverAlt":2749,"excerpt":2750,"draft":2226,"hidden":2226,"published":1979,"readingTime":2454,"series":2728,"seriesOrder":1704},"deepscientist-tech-stack","\u002Fposts\u002Fdeepscientist-tech-stack","DeepScientist 架构与运行时全解析：超 100K 上下文科研智能体、MCP 沙箱与高并发流式系统",[2297,2267,2736,2723,2384,2382],"深度拆解服务 7,000+ 用户的科研自主智能体平台 DeepScientist：从一题一仓库（Quest）持久化状态机、MCP 工具栈（memory \u002F artifact \u002F bash_exec）、Docker SDK 动态容器编排，到 SSE + WebSocket 双流通信与 PostgreSQL \u002F Redis \u002F Prometheus 生产级基础设施。","\u002Fimages\u002Fposts\u002Fdeepscientist-tech-stack\u002Fdeepscientist-cover-20261018.webp","科研智能体编排核心通过 memory、artifact 与隔离容器沙箱三座协议桥连接双流工作台的等距架构插画","DeepScientist（deepscientist.cc）是我们在西湖大学自然语言处理实验室研发的自主科学发现与科研管理平台，目前累计服务 7,000+ 科研用户。与单轮对话或纯文本文献问答工具不同，科研智能体需要在数小时乃至数",1795088943051]