[{"data":1,"prerenderedAt":7343},["ShallowReactive",2],{"post-\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm":3,"archive-public-posts":6848},{"id":4,"title":5,"body":6,"categories":6827,"cover":6829,"coverAlt":6830,"date":6831,"description":6832,"draft":6833,"extension":6834,"haloPublished":6833,"haloSync":6833,"hidden":6833,"legacy":6833,"meta":6835,"navigation":6836,"path":6837,"published":6833,"readingTime":6838,"seo":6839,"series":6840,"seriesOrder":3956,"stem":6841,"tags":6842,"__hash__":6847},"posts\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm.md","Stanford CS336 学习记录 06：Pre-Norm Transformer Block 与完整语言模型装配",{"type":7,"value":8,"toc":6812},"minimark",[9,49,63,68,229,697,890,901,916,922,1045,1061,1313,1340,1347,1362,1371,1384,1580,1583,1659,1684,1692,1699,2142,2147,2153,3322,3325,3612,3617,3911,4057,4115,4267,6101,6104,6743,6746,6753,6760,6770,6782,6785,6808],[10,11,12,13,17,18,17,21,17,24,17,27,30,31,34,35,42,43,48],"p",{},"前五篇把 ",[14,15,16],"code",{},"Linear","、",[14,19,20],{},"Embedding",[14,22,23],{},"RMSNorm",[14,25,26],{},"SwiGLU",[14,28,29],{},"RotaryPositionalEmbedding"," 和因果 ",[14,32,33],{},"MultiHeadSelfAttention"," 全部单独打通了。这一篇进入 Assignment 1 第 3.5 节：把这些积木拼成 ",[36,37,38,39],"strong",{},"Pre-Norm ",[14,40,41],{},"TransformerBlock"," 和完整的 ",[36,44,45],{},[14,46,47],{},"TransformerLM","，并完成课程要求的资源核算（参数量与前向 FLOPs 拆解）。",[10,50,51,58,59],{},[52,53,57],"a",{"href":54,"rel":55},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fb5c93ad0cc73178b2b2a62e518a0900da47a0fb7",[56],"nofollow","查看代码仓库"," · ",[52,60,62],{"href":61},"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-06-transformer-block-lm-20261023.zip","下载本节代码与测试 ZIP",[64,65,67],"h2",{"id":66},"pre-norm-与-post-norm为什么残差主干上不放归一化","Pre-Norm 与 Post-Norm：为什么残差主干上不放归一化",[10,69,70,71,220,221,224,225,228],{},"原始 Vaswani et al. (2017) 采用的是 Post-Norm 结构：",[72,73,76,133],"span",{"className":74},[75],"katex",[72,77,80],{"className":78},[79],"katex-mathml",[81,82,84],"math",{"xmlns":83},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[85,86,87,128],"semantics",{},[88,89,90,94,98,102,105,109,111,114,117,119,121,123,126],"mrow",{},[91,92,93],"mi",{},"x",[95,96,97],"mo",{},"←",[91,99,101],{"mathvariant":100},"normal","Norm",[95,103,104],{},"⁡",[95,106,108],{"stretchy":107},"false","(",[91,110,93],{},[95,112,113],{},"+",[91,115,116],{"mathvariant":100},"SubLayer",[95,118,104],{},[95,120,108],{"stretchy":107},[91,122,93],{},[95,124,125],{"stretchy":107},")",[95,127,125],{"stretchy":107},[129,130,132],"annotation",{"encoding":131},"application\u002Fx-tex","x \\leftarrow \\operatorname{Norm}(x + \\operatorname{SubLayer}(x))",[72,134,138,164,197],{"className":135,"ariaHidden":137},[136],"katex-html","true",[72,139,142,147,152,157,161],{"className":140},[141],"base",[72,143],{"className":144,"style":146},[145],"strut","height:0.4306em;",[72,148,93],{"className":149},[150,151],"mord","mathnormal",[72,153],{"className":154,"style":156},[155],"mspace","margin-right:0.2778em;",[72,158,97],{"className":159},[160],"mrel",[72,162],{"className":163,"style":156},[155],[72,165,167,171,179,183,186,190,194],{"className":166},[141],[72,168],{"className":169,"style":170},[145],"height:1em;vertical-align:-0.25em;",[72,172,175],{"className":173},[174],"mop",[72,176,101],{"className":177},[150,178],"mathrm",[72,180,108],{"className":181},[182],"mopen",[72,184,93],{"className":185},[150,151],[72,187],{"className":188,"style":189},[155],"margin-right:0.2222em;",[72,191,113],{"className":192},[193],"mbin",[72,195],{"className":196,"style":189},[155],[72,198,200,203,209,212,215],{"className":199},[141],[72,201],{"className":202,"style":170},[145],[72,204,206],{"className":205},[174],[72,207,116],{"className":208},[150,178],[72,210,108],{"className":211},[182],[72,213,93],{"className":214},[150,151],[72,216,219],{"className":217},[218],"mclose","))","，每次残差相加之后都要经过一次归一化。而在深层网络中，把归一化移到每个子层",[36,222,223],{},"入口","的 ",[36,226,227],{},"Pre-Norm"," 结构能保持一条从底层 Embedding 直通顶层输出的干净“残差流（Residual Stream）”：",[72,230,233],{"className":231},[232],"katex-display",[72,234,236,363],{"className":235},[75],[72,237,239],{"className":238},[79],[81,240,242],{"xmlns":83,"display":241},"block",[85,243,244,360],{},[245,246,250,309],"mtable",{"rowspacing":247,"columnalign":248,"columnspacing":249},"0.25em","right left","0em",[251,252,253,263],"mtr",{},[254,255,256],"mtd",{},[257,258,260],"mstyle",{"scriptlevel":259,"displaystyle":137},"0",[91,261,262],{},"h",[254,264,265],{},[257,266,267],{"scriptlevel":259,"displaystyle":137},[88,268,269,271,274,276,278,280,282,285,298,300,302,304,306],{},[88,270],{},[95,272,273],{},"=",[91,275,93],{},[95,277,113],{},[91,279,33],{"mathvariant":100},[95,281,104],{},[95,283,108],{"fence":107,"stretchy":137,"minsize":284,"maxsize":284},"1.2em",[286,287,288,294],"msub",{},[88,289,290,292],{},[91,291,23],{"mathvariant":100},[95,293,104],{},[295,296,297],"mn",{},"1",[95,299,108],{"stretchy":107},[91,301,93],{},[95,303,125],{"stretchy":107},[95,305,125],{"fence":107,"stretchy":137,"minsize":284,"maxsize":284},[95,307,308],{"separator":137},",",[251,310,311,318],{},[254,312,313],{},[257,314,315],{"scriptlevel":259,"displaystyle":137},[91,316,317],{},"y",[254,319,320],{},[257,321,322],{"scriptlevel":259,"displaystyle":137},[88,323,324,326,328,330,332,334,336,338,349,351,353,355,357],{},[88,325],{},[95,327,273],{},[91,329,262],{},[95,331,113],{},[91,333,26],{"mathvariant":100},[95,335,104],{},[95,337,108],{"fence":107,"stretchy":137,"minsize":284,"maxsize":284},[286,339,340,346],{},[88,341,342,344],{},[91,343,23],{"mathvariant":100},[95,345,104],{},[295,347,348],{},"2",[95,350,108],{"stretchy":107},[91,352,262],{},[95,354,125],{"stretchy":107},[95,356,125],{"fence":107,"stretchy":137,"minsize":284,"maxsize":284},[91,358,359],{"mathvariant":100},".",[129,361,362],{"encoding":131},"\\begin{aligned}\nh &= x + \\operatorname{MultiHeadSelfAttention}\\big(\\operatorname{RMSNorm}_1(x)\\big), \\\\\ny &= h + \\operatorname{SwiGLU}\\big(\\operatorname{RMSNorm}_2(h)\\big).\n\\end{aligned}",[72,364,366],{"className":365,"ariaHidden":137},[136],[72,367,369,373],{"className":368},[141],[72,370],{"className":371,"style":372},[145],"height:2.72em;vertical-align:-1.11em;",[72,374,376],{"className":375},[150],[72,377,379,438],{"className":378},[245],[72,380,383],{"className":381},[382],"col-align-r",[72,384,388,429],{"className":385},[386,387],"vlist-t","vlist-t2",[72,389,392,424],{"className":390},[391],"vlist-r",[72,393,397,411],{"className":394,"style":396},[395],"vlist","height:1.61em;",[72,398,400,405],{"style":399},"top:-3.76em;",[72,401],{"className":402,"style":404},[403],"pstrut","height:3em;",[72,406,408],{"className":407},[150],[72,409,262],{"className":410},[150,151],[72,412,414,417],{"style":413},"top:-2.25em;",[72,415],{"className":416,"style":404},[403],[72,418,420],{"className":419},[150],[72,421,317],{"className":422,"style":423},[150,151],"margin-right:0.0359em;",[72,425,428],{"className":426},[427],"vlist-s","​",[72,430,432],{"className":431},[391],[72,433,436],{"className":434,"style":435},[395],"height:1.11em;",[72,437],{},[72,439,442],{"className":440},[441],"col-align-l",[72,443,445,689],{"className":444},[386,387],[72,446,448,686],{"className":447},[391],[72,449,451,575],{"className":450,"style":396},[395],[72,452,453,456],{"style":399},[72,454],{"className":455,"style":404},[403],[72,457,459,462,465,468,471,474,477,480,483,489,493,501,504,556,559,562,565,571],{"className":458},[150],[72,460],{"className":461},[150],[72,463],{"className":464,"style":156},[155],[72,466,273],{"className":467},[160],[72,469],{"className":470,"style":156},[155],[72,472,93],{"className":473},[150,151],[72,475],{"className":476,"style":189},[155],[72,478,113],{"className":479},[193],[72,481],{"className":482,"style":189},[155],[72,484,486],{"className":485},[174],[72,487,33],{"className":488},[150,178],[72,490],{"className":491,"style":492},[155],"margin-right:0.1667em;",[72,494,496],{"className":495},[150],[72,497,108],{"className":498},[499,500],"delimsizing","size1",[72,502],{"className":503,"style":492},[155],[72,505,507,513],{"className":506},[174],[72,508,510],{"className":509},[174],[72,511,23],{"className":512},[150,178],[72,514,517],{"className":515},[516],"msupsub",[72,518,520,547],{"className":519},[386,387],[72,521,523,544],{"className":522},[391],[72,524,527],{"className":525,"style":526},[395],"height:0.3011em;",[72,528,530,534],{"style":529},"top:-2.55em;margin-right:0.05em;",[72,531],{"className":532,"style":533},[403],"height:2.7em;",[72,535,541],{"className":536},[537,538,539,540],"sizing","reset-size6","size3","mtight",[72,542,297],{"className":543},[150,540],[72,545,428],{"className":546},[427],[72,548,550],{"className":549},[391],[72,551,554],{"className":552,"style":553},[395],"height:0.15em;",[72,555],{},[72,557,108],{"className":558},[182],[72,560,93],{"className":561},[150,151],[72,563,125],{"className":564},[218],[72,566,568],{"className":567},[150],[72,569,125],{"className":570},[499,500],[72,572,308],{"className":573},[574],"mpunct",[72,576,577,580],{"style":413},[72,578],{"className":579,"style":404},[403],[72,581,583,586,589,592,595,598,601,604,607,613,616,622,625,668,671,674,677,683],{"className":582},[150],[72,584],{"className":585},[150],[72,587],{"className":588,"style":156},[155],[72,590,273],{"className":591},[160],[72,593],{"className":594,"style":156},[155],[72,596,262],{"className":597},[150,151],[72,599],{"className":600,"style":189},[155],[72,602,113],{"className":603},[193],[72,605],{"className":606,"style":189},[155],[72,608,610],{"className":609},[174],[72,611,26],{"className":612},[150,178],[72,614],{"className":615,"style":492},[155],[72,617,619],{"className":618},[150],[72,620,108],{"className":621},[499,500],[72,623],{"className":624,"style":492},[155],[72,626,628,634],{"className":627},[174],[72,629,631],{"className":630},[174],[72,632,23],{"className":633},[150,178],[72,635,637],{"className":636},[516],[72,638,640,660],{"className":639},[386,387],[72,641,643,657],{"className":642},[391],[72,644,646],{"className":645,"style":526},[395],[72,647,648,651],{"style":529},[72,649],{"className":650,"style":533},[403],[72,652,654],{"className":653},[537,538,539,540],[72,655,348],{"className":656},[150,540],[72,658,428],{"className":659},[427],[72,661,663],{"className":662},[391],[72,664,666],{"className":665,"style":553},[395],[72,667],{},[72,669,108],{"className":670},[182],[72,672,262],{"className":673},[150,151],[72,675,125],{"className":676},[218],[72,678,680],{"className":679},[150],[72,681,125],{"className":682},[499,500],[72,684,359],{"className":685},[150],[72,687,428],{"className":688},[427],[72,690,692],{"className":691},[391],[72,693,695],{"className":694,"style":435},[395],[72,696],{},[10,698,699,700,874,875,878,879,882,883,889],{},"反向传播时，梯度可以沿着恒等映射项 ",[72,701,703,741],{"className":702},[75],[72,704,706],{"className":705},[79],[81,707,708],{"xmlns":83},[85,709,710,738],{},[88,711,712,728,730,733,735],{},[713,714,715,722],"mfrac",{},[88,716,717,720],{},[91,718,719],{"mathvariant":100},"∂",[91,721,317],{},[88,723,724,726],{},[91,725,719],{"mathvariant":100},[91,727,93],{},[95,729,273],{},[91,731,732],{},"I",[95,734,113],{},[95,736,737],{},"…",[129,739,740],{"encoding":131},"\\frac{\\partial y}{\\partial x} = I + \\dots",[72,742,744,843,863],{"className":743,"ariaHidden":137},[136],[72,745,747,751,834,837,840],{"className":746},[141],[72,748],{"className":749,"style":750},[145],"height:1.2772em;vertical-align:-0.345em;",[72,752,754,758,831],{"className":753},[150],[72,755],{"className":756},[182,757],"nulldelimiter",[72,759,761],{"className":760},[713],[72,762,764,822],{"className":763},[386,387],[72,765,767,819],{"className":766},[391],[72,768,771,790,801],{"className":769,"style":770},[395],"height:0.9322em;",[72,772,774,777],{"style":773},"top:-2.655em;",[72,775],{"className":776,"style":404},[403],[72,778,780],{"className":779},[537,538,539,540],[72,781,783,787],{"className":782},[150,540],[72,784,719],{"className":785,"style":786},[150,540],"margin-right:0.0556em;",[72,788,93],{"className":789},[150,151,540],[72,791,793,796],{"style":792},"top:-3.23em;",[72,794],{"className":795,"style":404},[403],[72,797],{"className":798,"style":800},[799],"frac-line","border-bottom-width:0.04em;",[72,802,804,807],{"style":803},"top:-3.4461em;",[72,805],{"className":806,"style":404},[403],[72,808,810],{"className":809},[537,538,539,540],[72,811,813,816],{"className":812},[150,540],[72,814,719],{"className":815,"style":786},[150,540],[72,817,317],{"className":818,"style":423},[150,151,540],[72,820,428],{"className":821},[427],[72,823,825],{"className":824},[391],[72,826,829],{"className":827,"style":828},[395],"height:0.345em;",[72,830],{},[72,832],{"className":833},[218,757],[72,835],{"className":836,"style":156},[155],[72,838,273],{"className":839},[160],[72,841],{"className":842,"style":156},[155],[72,844,846,850,854,857,860],{"className":845},[141],[72,847],{"className":848,"style":849},[145],"height:0.7667em;vertical-align:-0.0833em;",[72,851,732],{"className":852,"style":853},[150,151],"margin-right:0.0785em;",[72,855],{"className":856,"style":189},[155],[72,858,113],{"className":859},[193],[72,861],{"className":862,"style":189},[155],[72,864,866,870],{"className":865},[141],[72,867],{"className":868,"style":869},[145],"height:0.123em;",[72,871,737],{"className":872},[873],"minner"," 直接回传到浅层，显著缓解深层 Transformer 训练初期的梯度消失与不稳定问题。作为代价，经过 ",[14,876,877],{},"num_layers"," 层累加后的残差流激活幅度会随层数增长，因此在进入最后的词表线性头 ",[14,880,881],{},"lm_head"," 之前，必须额外加一层 ",[36,884,885,888],{},[14,886,887],{},"ln_final","（最终 RMSNorm）","。",[64,891,893,894,897,898],{"id":892},"对齐官方-state_dict-键名与-nnmodulelist","对齐官方 ",[14,895,896],{},"state_dict"," 键名与 ",[14,899,900],{},"nn.ModuleList",[10,902,903,904,907,908,911,912,915],{},"在编写 ",[14,905,906],{},"cs336_basics\u002Ftransformer.py"," 时，除了数学公式正确，模块属性名还必须与官方参考权重字典 ",[14,909,910],{},"ts_state_dict"," 严格一致，这样 ",[14,913,914],{},"load_state_dict"," 才能直接加载而无需手写繁琐的键名重映射。",[10,917,918,919,921],{},"单个 ",[14,920,41],{}," 的子模块命名对照如下：",[923,924,925,944],"table",{},[926,927,928],"thead",{},[929,930,931,935,938,941],"tr",{},[932,933,934],"th",{},"属性名",[932,936,937],{},"模块类型",[932,939,940],{},"内部参数键名",[932,942,943],{},"参数形状",[945,946,947,970,992,1012],"tbody",{},[929,948,949,955,960,965],{},[950,951,952],"td",{},[14,953,954],{},"self.ln1",[950,956,957],{},[14,958,959],{},"RMSNorm(d_model)",[950,961,962],{},[14,963,964],{},"ln1.weight",[950,966,967],{},[14,968,969],{},"(d_model,)",[929,971,972,977,981,986],{},[950,973,974],{},[14,975,976],{},"self.attn",[950,978,979],{},[14,980,33],{},[950,982,983],{},[14,984,985],{},"attn.{q,k,v,output}_proj.weight",[950,987,988,989],{},"各为 ",[14,990,991],{},"(d_model, d_model)",[929,993,994,999,1003,1008],{},[950,995,996],{},[14,997,998],{},"self.ln2",[950,1000,1001],{},[14,1002,959],{},[950,1004,1005],{},[14,1006,1007],{},"ln2.weight",[950,1009,1010],{},[14,1011,969],{},[929,1013,1014,1019,1024,1035],{},[950,1015,1016],{},[14,1017,1018],{},"self.ffn",[950,1020,1021],{},[14,1022,1023],{},"SwiGLU(d_model, d_ff)",[950,1025,1026,17,1029,17,1032],{},[14,1027,1028],{},"ffn.w1.weight",[14,1030,1031],{},"ffn.w2.weight",[14,1033,1034],{},"ffn.w3.weight",[950,1036,1037,17,1040,17,1043],{},[14,1038,1039],{},"(d_ff, d_model)",[14,1041,1042],{},"(d_model, d_ff)",[14,1044,1039],{},[10,1046,1047,1048,1050,1051,1053,1054,1056,1057,1060],{},"而在拼装顶层 ",[14,1049,47],{}," 时，有多层 ",[14,1052,41],{},"。这里必须使用 ",[14,1055,900],{}," 而不是普通 Python ",[14,1058,1059],{},"list","：",[1062,1063,1068],"pre",{"className":1064,"code":1065,"language":1066,"meta":1067,"style":1067},"language-python shiki shiki-themes github-dark-dimmed github-light houston","self.token_embeddings = Embedding(vocab_size, d_model, device=device, dtype=dtype)\nself.layers = nn.ModuleList(\n    [\n        TransformerBlock(\n            d_model=d_model,\n            num_heads=num_heads,\n            d_ff=d_ff,\n            max_seq_len=context_length,\n            theta=rope_theta,\n            device=device,\n            dtype=dtype,\n        )\n        for _ in range(num_layers)\n    ]\n)\nself.ln_final = RMSNorm(d_model, device=device, dtype=dtype)\nself.lm_head = Linear(d_model, vocab_size, device=device, dtype=dtype)\n","python","",[14,1069,1070,1109,1128,1134,1142,1153,1164,1175,1186,1197,1208,1219,1225,1245,1251,1257,1285],{"__ignoreMap":1067},[72,1071,1074,1078,1082,1085,1089,1092,1096,1098,1101,1104,1106],{"class":1072,"line":1073},"line",1,[72,1075,1077],{"class":1076},"sRGqv","self",[72,1079,1081],{"class":1080},"sWURj",".token_embeddings ",[72,1083,273],{"class":1084},"sf2cM",[72,1086,1088],{"class":1087},"sq3c-"," Embedding",[72,1090,1091],{"class":1080},"(vocab_size, d_model, ",[72,1093,1095],{"class":1094},"sA9aF","device",[72,1097,273],{"class":1084},[72,1099,1100],{"class":1080},"device, ",[72,1102,1103],{"class":1094},"dtype",[72,1105,273],{"class":1084},[72,1107,1108],{"class":1080},"dtype)\n",[72,1110,1112,1114,1117,1119,1122,1125],{"class":1072,"line":1111},2,[72,1113,1077],{"class":1076},[72,1115,1116],{"class":1080},".layers ",[72,1118,273],{"class":1084},[72,1120,1121],{"class":1080}," nn.",[72,1123,1124],{"class":1087},"ModuleList",[72,1126,1127],{"class":1080},"(\n",[72,1129,1131],{"class":1072,"line":1130},3,[72,1132,1133],{"class":1080},"    [\n",[72,1135,1137,1140],{"class":1072,"line":1136},4,[72,1138,1139],{"class":1087},"        TransformerBlock",[72,1141,1127],{"class":1080},[72,1143,1145,1148,1150],{"class":1072,"line":1144},5,[72,1146,1147],{"class":1094},"            d_model",[72,1149,273],{"class":1084},[72,1151,1152],{"class":1080},"d_model,\n",[72,1154,1156,1159,1161],{"class":1072,"line":1155},6,[72,1157,1158],{"class":1094},"            num_heads",[72,1160,273],{"class":1084},[72,1162,1163],{"class":1080},"num_heads,\n",[72,1165,1167,1170,1172],{"class":1072,"line":1166},7,[72,1168,1169],{"class":1094},"            d_ff",[72,1171,273],{"class":1084},[72,1173,1174],{"class":1080},"d_ff,\n",[72,1176,1178,1181,1183],{"class":1072,"line":1177},8,[72,1179,1180],{"class":1094},"            max_seq_len",[72,1182,273],{"class":1084},[72,1184,1185],{"class":1080},"context_length,\n",[72,1187,1189,1192,1194],{"class":1072,"line":1188},9,[72,1190,1191],{"class":1094},"            theta",[72,1193,273],{"class":1084},[72,1195,1196],{"class":1080},"rope_theta,\n",[72,1198,1200,1203,1205],{"class":1072,"line":1199},10,[72,1201,1202],{"class":1094},"            device",[72,1204,273],{"class":1084},[72,1206,1207],{"class":1080},"device,\n",[72,1209,1211,1214,1216],{"class":1072,"line":1210},11,[72,1212,1213],{"class":1094},"            dtype",[72,1215,273],{"class":1084},[72,1217,1218],{"class":1080},"dtype,\n",[72,1220,1222],{"class":1072,"line":1221},12,[72,1223,1224],{"class":1080},"        )\n",[72,1226,1228,1232,1235,1238,1242],{"class":1072,"line":1227},13,[72,1229,1231],{"class":1230},"slEfO","        for",[72,1233,1234],{"class":1080}," _ ",[72,1236,1237],{"class":1230},"in",[72,1239,1241],{"class":1240},"svkvy"," range",[72,1243,1244],{"class":1080},"(num_layers)\n",[72,1246,1248],{"class":1072,"line":1247},14,[72,1249,1250],{"class":1080},"    ]\n",[72,1252,1254],{"class":1072,"line":1253},15,[72,1255,1256],{"class":1080},")\n",[72,1258,1260,1262,1265,1267,1270,1273,1275,1277,1279,1281,1283],{"class":1072,"line":1259},16,[72,1261,1077],{"class":1076},[72,1263,1264],{"class":1080},".ln_final ",[72,1266,273],{"class":1084},[72,1268,1269],{"class":1087}," RMSNorm",[72,1271,1272],{"class":1080},"(d_model, ",[72,1274,1095],{"class":1094},[72,1276,273],{"class":1084},[72,1278,1100],{"class":1080},[72,1280,1103],{"class":1094},[72,1282,273],{"class":1084},[72,1284,1108],{"class":1080},[72,1286,1288,1290,1293,1295,1298,1301,1303,1305,1307,1309,1311],{"class":1072,"line":1287},17,[72,1289,1077],{"class":1076},[72,1291,1292],{"class":1080},".lm_head ",[72,1294,273],{"class":1084},[72,1296,1297],{"class":1087}," Linear",[72,1299,1300],{"class":1080},"(d_model, vocab_size, ",[72,1302,1095],{"class":1094},[72,1304,273],{"class":1084},[72,1306,1100],{"class":1080},[72,1308,1103],{"class":1094},[72,1310,273],{"class":1084},[72,1312,1108],{"class":1080},[10,1314,1315,1316,1319,1320,1323,1324,1327,1328,1331,1332,1335,1336,1339],{},"如果把 ",[14,1317,1318],{},"self.layers"," 写成普通 Python 列表 ",[14,1321,1322],{},"[...]","，PyTorch 的 ",[14,1325,1326],{},"nn.Module"," 不会把列表里的子模块注册进参数树——",[14,1329,1330],{},"model.parameters()"," 会漏掉所有 Transformer 层的权重，",[14,1333,1334],{},"state_dict()"," 也不会生成 ",[14,1337,1338],{},"layers.0.attn.q_proj.weight"," 这样的层级键名。",[64,1341,1343,1344,1346],{"id":1342},"为什么-transformerlm-返回未归一化的-logits","为什么 ",[14,1345,47],{}," 返回未归一化的 Logits",[10,1348,1349,1350,1353,1354,1357,1358,1361],{},"阅读讲义图 1 时，示意图最顶端画了一个 ",[14,1351,1352],{},"Softmax -> Output Probabilities","。但对照 ",[14,1355,1356],{},"adapters.py"," 中 ",[14,1359,1360],{},"run_transformer_lm"," 的返回值契约：",[1363,1364,1365],"blockquote",{},[10,1366,1367,1370],{},[14,1368,1369],{},"Float[Tensor, \"batch_size sequence_length vocab_size\"]",": Tensor with the predicted unnormalized next-word distribution for each token.",[10,1372,1373,1376,1377,1380,1381,1060],{},[14,1374,1375],{},"TransformerLM.forward"," 的最后一步是 ",[14,1378,1379],{},"return self.lm_head(x)","，直接返回未归一化的 logits，",[36,1382,1383],{},"不在模型内部调用 Softmax",[1062,1385,1387],{"className":1064,"code":1386,"language":1066,"meta":1067,"style":1067},"def forward(self, in_indices: torch.Tensor, token_positions: torch.Tensor | None = None) -> torch.Tensor:\n    seq_len = in_indices.shape[-1]\n    if token_positions is None:\n        token_positions = torch.arange(seq_len, device=in_indices.device, dtype=torch.long)\n    x = self.token_embeddings(in_indices)\n    for layer in self.layers:\n        x = layer(x, token_positions=token_positions)\n    x = self.ln_final(x)\n    return self.lm_head(x)\n",[14,1388,1389,1435,1453,1469,1499,1517,1532,1552,1567],{"__ignoreMap":1067},[72,1390,1391,1395,1399,1401,1404,1407,1411,1414,1417,1420,1423,1427,1430,1432],{"class":1072,"line":1073},[72,1392,1394],{"class":1393},"sST-V","def",[72,1396,1398],{"class":1397},"sS_0i"," forward",[72,1400,108],{"class":1080},[72,1402,1077],{"class":1403},"sbU-9",[72,1405,1406],{"class":1080},", ",[72,1408,1410],{"class":1409},"s9NhK","in_indices",[72,1412,1413],{"class":1080},": torch.Tensor, ",[72,1415,1416],{"class":1409},"token_positions",[72,1418,1419],{"class":1080},": torch.Tensor ",[72,1421,1422],{"class":1084},"|",[72,1424,1426],{"class":1425},"sgENW"," None",[72,1428,1429],{"class":1084}," =",[72,1431,1426],{"class":1425},[72,1433,1434],{"class":1080},") -> torch.Tensor:\n",[72,1436,1437,1440,1442,1445,1448,1450],{"class":1072,"line":1111},[72,1438,1439],{"class":1080},"    seq_len ",[72,1441,273],{"class":1084},[72,1443,1444],{"class":1080}," in_indices.shape[",[72,1446,1447],{"class":1084},"-",[72,1449,297],{"class":1425},[72,1451,1452],{"class":1080},"]\n",[72,1454,1455,1458,1461,1464,1466],{"class":1072,"line":1130},[72,1456,1457],{"class":1230},"    if",[72,1459,1460],{"class":1080}," token_positions ",[72,1462,1463],{"class":1393},"is",[72,1465,1426],{"class":1425},[72,1467,1468],{"class":1080},":\n",[72,1470,1471,1474,1476,1479,1482,1485,1487,1489,1492,1494,1496],{"class":1072,"line":1136},[72,1472,1473],{"class":1080},"        token_positions ",[72,1475,273],{"class":1084},[72,1477,1478],{"class":1080}," torch.",[72,1480,1481],{"class":1087},"arange",[72,1483,1484],{"class":1080},"(seq_len, ",[72,1486,1095],{"class":1094},[72,1488,273],{"class":1084},[72,1490,1491],{"class":1080},"in_indices.device, ",[72,1493,1103],{"class":1094},[72,1495,273],{"class":1084},[72,1497,1498],{"class":1080},"torch.long)\n",[72,1500,1501,1504,1506,1509,1511,1514],{"class":1072,"line":1144},[72,1502,1503],{"class":1080},"    x ",[72,1505,273],{"class":1084},[72,1507,1508],{"class":1076}," self",[72,1510,359],{"class":1080},[72,1512,1513],{"class":1087},"token_embeddings",[72,1515,1516],{"class":1080},"(in_indices)\n",[72,1518,1519,1522,1525,1527,1529],{"class":1072,"line":1155},[72,1520,1521],{"class":1230},"    for",[72,1523,1524],{"class":1080}," layer ",[72,1526,1237],{"class":1230},[72,1528,1508],{"class":1076},[72,1530,1531],{"class":1080},".layers:\n",[72,1533,1534,1537,1539,1542,1545,1547,1549],{"class":1072,"line":1166},[72,1535,1536],{"class":1080},"        x ",[72,1538,273],{"class":1084},[72,1540,1541],{"class":1087}," layer",[72,1543,1544],{"class":1080},"(x, ",[72,1546,1416],{"class":1094},[72,1548,273],{"class":1084},[72,1550,1551],{"class":1080},"token_positions)\n",[72,1553,1554,1556,1558,1560,1562,1564],{"class":1072,"line":1177},[72,1555,1503],{"class":1080},[72,1557,273],{"class":1084},[72,1559,1508],{"class":1076},[72,1561,359],{"class":1080},[72,1563,887],{"class":1087},[72,1565,1566],{"class":1080},"(x)\n",[72,1568,1569,1572,1574,1576,1578],{"class":1072,"line":1188},[72,1570,1571],{"class":1230},"    return",[72,1573,1508],{"class":1076},[72,1575,359],{"class":1080},[72,1577,881],{"class":1087},[72,1579,1566],{"class":1080},[10,1581,1582],{},"这样设计有两个工程原因：",[1584,1585,1586,1620],"ol",{},[1587,1588,1589,1590,1593,1594,1597,1598,1601,1602,1605,1606,1609,1610,1612,1613,1616,1617,889],"li",{},"训练阶段计算交叉熵损失时，需要将 ",[14,1591,1592],{},"log"," 与 ",[14,1595,1596],{},"softmax"," 合并为数值稳定的 ",[14,1599,1600],{},"LogSumExp"," 消去指数项；如果模型先输出了概率 ",[14,1603,1604],{},"softmax(o)","，一旦某个目标概率下溢为 ",[14,1607,1608],{},"0.0","，再取 ",[14,1611,1592],{}," 就会直接产生 ",[14,1614,1615],{},"-inf"," \u002F ",[14,1618,1619],{},"NaN",[1587,1621,1622,1623,1654,1655,1658],{},"推理采样阶段往往要先对 logits 除以温度 ",[72,1624,1626,1641],{"className":1625},[75],[72,1627,1629],{"className":1628},[79],[81,1630,1631],{"xmlns":83},[85,1632,1633,1638],{},[88,1634,1635],{},[91,1636,1637],{},"τ",[129,1639,1640],{"encoding":131},"\\tau",[72,1642,1644],{"className":1643,"ariaHidden":137},[136],[72,1645,1647,1650],{"className":1646},[141],[72,1648],{"className":1649,"style":146},[145],[72,1651,1637],{"className":1652,"style":1653},[150,151],"margin-right:0.1132em;","（",[14,1656,1657],{},"logits \u002F temperature","）再做 Softmax，直接输出 logits 最灵活。",[10,1660,1661,1662,1665,1666,1669,1670,1672,1673,1676,1677,1680,1681,889],{},"此外，当输入序列长度 ",[14,1663,1664],{},"seq_len \u003C context_length","（对应官方测试 ",[14,1667,1668],{},"test_transformer_lm_truncated_input","）时，",[14,1671,1416],{}," 按实际 ",[14,1674,1675],{},"seq_len"," 生成 ",[14,1678,1679],{},"0 .. seq_len - 1","，RoPE 缓存和因果 mask 自动按当前序列长度切片，无需把输入强行 padding 到 ",[14,1682,1683],{},"context_length",[64,1685,1687,1688,1691],{"id":1686},"资源核算problem-transformer_accounting以-gpt-2-xl-为例","资源核算（Problem ",[14,1689,1690],{},"transformer_accounting","）：以 GPT-2 XL 为例",[10,1693,1694,1695,1698],{},"课程讲义第 3.5 节要求对采用本作业架构（RMSNorm + RoPE + SwiGLU + 无偏置 Linear + 非绑定词表权重）的 ",[36,1696,1697],{},"GPT-2 XL"," 配置进行精确的参数量与前向 FLOPs 核算：",[1700,1701,1702,1841],"ul",{},[1587,1703,1704,1707,1708,1774,1775,1707,1777],{},[14,1705,1706],{},"vocab_size"," ",[72,1709,1711,1731],{"className":1710},[75],[72,1712,1714],{"className":1713},[79],[81,1715,1716],{"xmlns":83},[85,1717,1718,1728],{},[88,1719,1720,1723,1725],{},[91,1721,1722],{},"V",[95,1724,273],{},[295,1726,1727],{},"50,257",[129,1729,1730],{"encoding":131},"V = 50{,}257",[72,1732,1734,1753],{"className":1733,"ariaHidden":137},[136],[72,1735,1737,1741,1744,1747,1750],{"className":1736},[141],[72,1738],{"className":1739,"style":1740},[145],"height:0.6833em;",[72,1742,1722],{"className":1743,"style":189},[150,151],[72,1745],{"className":1746,"style":156},[155],[72,1748,273],{"className":1749},[160],[72,1751],{"className":1752,"style":156},[155],[72,1754,1756,1760,1764,1770],{"className":1755},[141],[72,1757],{"className":1758,"style":1759},[145],"height:0.8389em;vertical-align:-0.1944em;",[72,1761,1763],{"className":1762},[150],"50",[72,1765,1767],{"className":1766},[150],[72,1768,308],{"className":1769},[574],[72,1771,1773],{"className":1772},[150],"257","，",[14,1776,1683],{},[72,1778,1780,1800],{"className":1779},[75],[72,1781,1783],{"className":1782},[79],[81,1784,1785],{"xmlns":83},[85,1786,1787,1797],{},[88,1788,1789,1792,1794],{},[91,1790,1791],{},"S",[95,1793,273],{},[295,1795,1796],{},"1,024",[129,1798,1799],{"encoding":131},"S = 1{,}024",[72,1801,1803,1822],{"className":1802,"ariaHidden":137},[136],[72,1804,1806,1809,1813,1816,1819],{"className":1805},[141],[72,1807],{"className":1808,"style":1740},[145],[72,1810,1791],{"className":1811,"style":1812},[150,151],"margin-right:0.0576em;",[72,1814],{"className":1815,"style":156},[155],[72,1817,273],{"className":1818},[160],[72,1820],{"className":1821,"style":156},[155],[72,1823,1825,1828,1831,1837],{"className":1824},[141],[72,1826],{"className":1827,"style":1759},[145],[72,1829,297],{"className":1830},[150],[72,1832,1834],{"className":1833},[150],[72,1835,308],{"className":1836},[574],[72,1838,1840],{"className":1839},[150],"024",[1587,1842,1843,1707,1845,1774,1899,1707,1902,1774,1966,1707,1969,1774,2023,1707,2026],{},[14,1844,877],{},[72,1846,1848,1868],{"className":1847},[75],[72,1849,1851],{"className":1850},[79],[81,1852,1853],{"xmlns":83},[85,1854,1855,1865],{},[88,1856,1857,1860,1862],{},[91,1858,1859],{},"L",[95,1861,273],{},[295,1863,1864],{},"48",[129,1866,1867],{"encoding":131},"L = 48",[72,1869,1871,1889],{"className":1870,"ariaHidden":137},[136],[72,1872,1874,1877,1880,1883,1886],{"className":1873},[141],[72,1875],{"className":1876,"style":1740},[145],[72,1878,1859],{"className":1879},[150,151],[72,1881],{"className":1882,"style":156},[155],[72,1884,273],{"className":1885},[160],[72,1887],{"className":1888,"style":156},[155],[72,1890,1892,1896],{"className":1891},[141],[72,1893],{"className":1894,"style":1895},[145],"height:0.6444em;",[72,1897,1864],{"className":1898},[150],[14,1900,1901],{},"d_model",[72,1903,1905,1925],{"className":1904},[75],[72,1906,1908],{"className":1907},[79],[81,1909,1910],{"xmlns":83},[85,1911,1912,1922],{},[88,1913,1914,1917,1919],{},[91,1915,1916],{},"d",[95,1918,273],{},[295,1920,1921],{},"1,600",[129,1923,1924],{"encoding":131},"d = 1{,}600",[72,1926,1928,1947],{"className":1927,"ariaHidden":137},[136],[72,1929,1931,1935,1938,1941,1944],{"className":1930},[141],[72,1932],{"className":1933,"style":1934},[145],"height:0.6944em;",[72,1936,1916],{"className":1937},[150,151],[72,1939],{"className":1940,"style":156},[155],[72,1942,273],{"className":1943},[160],[72,1945],{"className":1946,"style":156},[155],[72,1948,1950,1953,1956,1962],{"className":1949},[141],[72,1951],{"className":1952,"style":1759},[145],[72,1954,297],{"className":1955},[150],[72,1957,1959],{"className":1958},[150],[72,1960,308],{"className":1961},[574],[72,1963,1965],{"className":1964},[150],"600",[14,1967,1968],{},"num_heads",[72,1970,1972,1992],{"className":1971},[75],[72,1973,1975],{"className":1974},[79],[81,1976,1977],{"xmlns":83},[85,1978,1979,1989],{},[88,1980,1981,1984,1986],{},[91,1982,1983],{},"H",[95,1985,273],{},[295,1987,1988],{},"25",[129,1990,1991],{"encoding":131},"H = 25",[72,1993,1995,2014],{"className":1994,"ariaHidden":137},[136],[72,1996,1998,2001,2005,2008,2011],{"className":1997},[141],[72,1999],{"className":2000,"style":1740},[145],[72,2002,1983],{"className":2003,"style":2004},[150,151],"margin-right:0.0813em;",[72,2006],{"className":2007,"style":156},[155],[72,2009,273],{"className":2010},[160],[72,2012],{"className":2013,"style":156},[155],[72,2015,2017,2020],{"className":2016},[141],[72,2018],{"className":2019,"style":1895},[145],[72,2021,1988],{"className":2022},[150],[14,2024,2025],{},"d_ff",[72,2027,2029,2054],{"className":2028},[75],[72,2030,2032],{"className":2031},[79],[81,2033,2034],{"xmlns":83},[85,2035,2036,2051],{},[88,2037,2038,2046,2048],{},[286,2039,2040,2042],{},[91,2041,1916],{},[2043,2044,2045],"mtext",{},"ff",[95,2047,273],{},[295,2049,2050],{},"4,288",[129,2052,2053],{"encoding":131},"d_{\\text{ff}} = 4{,}288",[72,2055,2057,2122],{"className":2056,"ariaHidden":137},[136],[72,2058,2060,2064,2113,2116,2119],{"className":2059},[141],[72,2061],{"className":2062,"style":2063},[145],"height:0.8444em;vertical-align:-0.15em;",[72,2065,2067,2070],{"className":2066},[150],[72,2068,1916],{"className":2069},[150,151],[72,2071,2073],{"className":2072},[516],[72,2074,2076,2105],{"className":2075},[386,387],[72,2077,2079,2102],{"className":2078},[391],[72,2080,2083],{"className":2081,"style":2082},[395],"height:0.3361em;",[72,2084,2086,2089],{"style":2085},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[72,2087],{"className":2088,"style":533},[403],[72,2090,2092],{"className":2091},[537,538,539,540],[72,2093,2095],{"className":2094},[150,540],[72,2096,2099],{"className":2097},[150,2098,540],"text",[72,2100,2045],{"className":2101},[150,540],[72,2103,428],{"className":2104},[427],[72,2106,2108],{"className":2107},[391],[72,2109,2111],{"className":2110,"style":553},[395],[72,2112],{},[72,2114],{"className":2115,"style":156},[155],[72,2117,273],{"className":2118},[160],[72,2120],{"className":2121,"style":156},[155],[72,2123,2125,2128,2132,2138],{"className":2124},[141],[72,2126],{"className":2127,"style":1759},[145],[72,2129,2131],{"className":2130},[150],"4",[72,2133,2135],{"className":2134},[150],[72,2136,308],{"className":2137},[574],[72,2139,2141],{"className":2140},[150],"288",[2143,2144,2146],"h3",{"id":2145},"_1-可训练参数量与-fp32-权重显存","1. 可训练参数量与 FP32 权重显存",[10,2148,2149,2150,2152],{},"每层 ",[14,2151,41],{}," 的参数包含：",[1700,2154,2155,2619,3051,3145],{},[1587,2156,2157,2158,1060,2401],{},"Attention 四个投影矩阵 ",[72,2159,2161,2207],{"className":2160},[75],[72,2162,2164],{"className":2163},[79],[81,2165,2166],{"xmlns":83},[85,2167,2168,2204],{},[88,2169,2170,2178,2180,2187,2189,2195,2197],{},[286,2171,2172,2175],{},[91,2173,2174],{},"W",[91,2176,2177],{},"Q",[95,2179,308],{"separator":137},[286,2181,2182,2184],{},[91,2183,2174],{},[91,2185,2186],{},"K",[95,2188,308],{"separator":137},[286,2190,2191,2193],{},[91,2192,2174],{},[91,2194,1722],{},[95,2196,308],{"separator":137},[286,2198,2199,2201],{},[91,2200,2174],{},[91,2202,2203],{},"O",[129,2205,2206],{"encoding":131},"W_Q, W_K, W_V, W_O",[72,2208,2210],{"className":2209,"ariaHidden":137},[136],[72,2211,2213,2217,2261,2264,2267,2308,2311,2314,2354,2357,2360],{"className":2212},[141],[72,2214],{"className":2215,"style":2216},[145],"height:0.9694em;vertical-align:-0.2861em;",[72,2218,2220,2224],{"className":2219},[150],[72,2221,2174],{"className":2222,"style":2223},[150,151],"margin-right:0.1389em;",[72,2225,2227],{"className":2226},[516],[72,2228,2230,2252],{"className":2229},[386,387],[72,2231,2233,2249],{"className":2232},[391],[72,2234,2237],{"className":2235,"style":2236},[395],"height:0.3283em;",[72,2238,2240,2243],{"style":2239},"top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;",[72,2241],{"className":2242,"style":533},[403],[72,2244,2246],{"className":2245},[537,538,539,540],[72,2247,2177],{"className":2248},[150,151,540],[72,2250,428],{"className":2251},[427],[72,2253,2255],{"className":2254},[391],[72,2256,2259],{"className":2257,"style":2258},[395],"height:0.2861em;",[72,2260],{},[72,2262,308],{"className":2263},[574],[72,2265],{"className":2266,"style":492},[155],[72,2268,2270,2273],{"className":2269},[150],[72,2271,2174],{"className":2272,"style":2223},[150,151],[72,2274,2276],{"className":2275},[516],[72,2277,2279,2300],{"className":2278},[386,387],[72,2280,2282,2297],{"className":2281},[391],[72,2283,2285],{"className":2284,"style":2236},[395],[72,2286,2287,2290],{"style":2239},[72,2288],{"className":2289,"style":533},[403],[72,2291,2293],{"className":2292},[537,538,539,540],[72,2294,2186],{"className":2295,"style":2296},[150,151,540],"margin-right:0.0715em;",[72,2298,428],{"className":2299},[427],[72,2301,2303],{"className":2302},[391],[72,2304,2306],{"className":2305,"style":553},[395],[72,2307],{},[72,2309,308],{"className":2310},[574],[72,2312],{"className":2313,"style":492},[155],[72,2315,2317,2320],{"className":2316},[150],[72,2318,2174],{"className":2319,"style":2223},[150,151],[72,2321,2323],{"className":2322},[516],[72,2324,2326,2346],{"className":2325},[386,387],[72,2327,2329,2343],{"className":2328},[391],[72,2330,2332],{"className":2331,"style":2236},[395],[72,2333,2334,2337],{"style":2239},[72,2335],{"className":2336,"style":533},[403],[72,2338,2340],{"className":2339},[537,538,539,540],[72,2341,1722],{"className":2342,"style":189},[150,151,540],[72,2344,428],{"className":2345},[427],[72,2347,2349],{"className":2348},[391],[72,2350,2352],{"className":2351,"style":553},[395],[72,2353],{},[72,2355,308],{"className":2356},[574],[72,2358],{"className":2359,"style":492},[155],[72,2361,2363,2366],{"className":2362},[150],[72,2364,2174],{"className":2365,"style":2223},[150,151],[72,2367,2369],{"className":2368},[516],[72,2370,2372,2393],{"className":2371},[386,387],[72,2373,2375,2390],{"className":2374},[391],[72,2376,2378],{"className":2377,"style":2236},[395],[72,2379,2380,2383],{"style":2239},[72,2381],{"className":2382,"style":533},[403],[72,2384,2386],{"className":2385},[537,538,539,540],[72,2387,2203],{"className":2388,"style":2389},[150,151,540],"margin-right:0.0278em;",[72,2391,428],{"className":2392},[427],[72,2394,2396],{"className":2395},[391],[72,2397,2399],{"className":2398,"style":553},[395],[72,2400],{},[72,2402,2404,2445],{"className":2403},[75],[72,2405,2407],{"className":2406},[79],[81,2408,2409],{"xmlns":83},[85,2410,2411,2442],{},[88,2412,2413,2415,2418,2425,2427,2429,2431,2437,2439],{},[295,2414,2131],{},[95,2416,2417],{},"×",[2419,2420,2421,2423],"msup",{},[91,2422,1916],{},[295,2424,348],{},[95,2426,273],{},[295,2428,2131],{},[95,2430,2417],{},[2419,2432,2433,2435],{},[295,2434,1921],{},[295,2436,348],{},[95,2438,273],{},[295,2440,2441],{},"10,240,000",[129,2443,2444],{"encoding":131},"4 \\times d^2 = 4 \\times 1{,}600^2 = 10{,}240{,}000",[72,2446,2448,2467,2513,2531,2589],{"className":2447,"ariaHidden":137},[136],[72,2449,2451,2455,2458,2461,2464],{"className":2450},[141],[72,2452],{"className":2453,"style":2454},[145],"height:0.7278em;vertical-align:-0.0833em;",[72,2456,2131],{"className":2457},[150],[72,2459],{"className":2460,"style":189},[155],[72,2462,2417],{"className":2463},[193],[72,2465],{"className":2466,"style":189},[155],[72,2468,2470,2474,2504,2507,2510],{"className":2469},[141],[72,2471],{"className":2472,"style":2473},[145],"height:0.8141em;",[72,2475,2477,2480],{"className":2476},[150],[72,2478,1916],{"className":2479},[150,151],[72,2481,2483],{"className":2482},[516],[72,2484,2486],{"className":2485},[386],[72,2487,2489],{"className":2488},[391],[72,2490,2492],{"className":2491,"style":2473},[395],[72,2493,2495,2498],{"style":2494},"top:-3.063em;margin-right:0.05em;",[72,2496],{"className":2497,"style":533},[403],[72,2499,2501],{"className":2500},[537,538,539,540],[72,2502,348],{"className":2503},[150,540],[72,2505],{"className":2506,"style":156},[155],[72,2508,273],{"className":2509},[160],[72,2511],{"className":2512,"style":156},[155],[72,2514,2516,2519,2522,2525,2528],{"className":2515},[141],[72,2517],{"className":2518,"style":2454},[145],[72,2520,2131],{"className":2521},[150],[72,2523],{"className":2524,"style":189},[155],[72,2526,2417],{"className":2527},[193],[72,2529],{"className":2530,"style":189},[155],[72,2532,2534,2538,2541,2547,2551,2580,2583,2586],{"className":2533},[141],[72,2535],{"className":2536,"style":2537},[145],"height:1.0085em;vertical-align:-0.1944em;",[72,2539,297],{"className":2540},[150],[72,2542,2544],{"className":2543},[150],[72,2545,308],{"className":2546},[574],[72,2548,2550],{"className":2549},[150],"60",[72,2552,2554,2557],{"className":2553},[150],[72,2555,259],{"className":2556},[150],[72,2558,2560],{"className":2559},[516],[72,2561,2563],{"className":2562},[386],[72,2564,2566],{"className":2565},[391],[72,2567,2569],{"className":2568,"style":2473},[395],[72,2570,2571,2574],{"style":2494},[72,2572],{"className":2573,"style":533},[403],[72,2575,2577],{"className":2576},[537,538,539,540],[72,2578,348],{"className":2579},[150,540],[72,2581],{"className":2582,"style":156},[155],[72,2584,273],{"className":2585},[160],[72,2587],{"className":2588,"style":156},[155],[72,2590,2592,2595,2599,2605,2609,2615],{"className":2591},[141],[72,2593],{"className":2594,"style":1759},[145],[72,2596,2598],{"className":2597},[150],"10",[72,2600,2602],{"className":2601},[150],[72,2603,308],{"className":2604},[574],[72,2606,2608],{"className":2607},[150],"240",[72,2610,2612],{"className":2611},[150],[72,2613,308],{"className":2614},[574],[72,2616,2618],{"className":2617},[150],"000",[1587,2620,2621,2622,1060,2802],{},"SwiGLU 三个矩阵 ",[72,2623,2625,2660],{"className":2624},[75],[72,2626,2628],{"className":2627},[79],[81,2629,2630],{"xmlns":83},[85,2631,2632,2657],{},[88,2633,2634,2640,2642,2648,2650],{},[286,2635,2636,2638],{},[91,2637,2174],{},[295,2639,297],{},[95,2641,308],{"separator":137},[286,2643,2644,2646],{},[91,2645,2174],{},[295,2647,348],{},[95,2649,308],{"separator":137},[286,2651,2652,2654],{},[91,2653,2174],{},[295,2655,2656],{},"3",[129,2658,2659],{"encoding":131},"W_1, W_2, W_3",[72,2661,2663],{"className":2662,"ariaHidden":137},[136],[72,2664,2666,2670,2710,2713,2716,2756,2759,2762],{"className":2665},[141],[72,2667],{"className":2668,"style":2669},[145],"height:0.8778em;vertical-align:-0.1944em;",[72,2671,2673,2676],{"className":2672},[150],[72,2674,2174],{"className":2675,"style":2223},[150,151],[72,2677,2679],{"className":2678},[516],[72,2680,2682,2702],{"className":2681},[386,387],[72,2683,2685,2699],{"className":2684},[391],[72,2686,2688],{"className":2687,"style":526},[395],[72,2689,2690,2693],{"style":2239},[72,2691],{"className":2692,"style":533},[403],[72,2694,2696],{"className":2695},[537,538,539,540],[72,2697,297],{"className":2698},[150,540],[72,2700,428],{"className":2701},[427],[72,2703,2705],{"className":2704},[391],[72,2706,2708],{"className":2707,"style":553},[395],[72,2709],{},[72,2711,308],{"className":2712},[574],[72,2714],{"className":2715,"style":492},[155],[72,2717,2719,2722],{"className":2718},[150],[72,2720,2174],{"className":2721,"style":2223},[150,151],[72,2723,2725],{"className":2724},[516],[72,2726,2728,2748],{"className":2727},[386,387],[72,2729,2731,2745],{"className":2730},[391],[72,2732,2734],{"className":2733,"style":526},[395],[72,2735,2736,2739],{"style":2239},[72,2737],{"className":2738,"style":533},[403],[72,2740,2742],{"className":2741},[537,538,539,540],[72,2743,348],{"className":2744},[150,540],[72,2746,428],{"className":2747},[427],[72,2749,2751],{"className":2750},[391],[72,2752,2754],{"className":2753,"style":553},[395],[72,2755],{},[72,2757,308],{"className":2758},[574],[72,2760],{"className":2761,"style":492},[155],[72,2763,2765,2768],{"className":2764},[150],[72,2766,2174],{"className":2767,"style":2223},[150,151],[72,2769,2771],{"className":2770},[516],[72,2772,2774,2794],{"className":2773},[386,387],[72,2775,2777,2791],{"className":2776},[391],[72,2778,2780],{"className":2779,"style":526},[395],[72,2781,2782,2785],{"style":2239},[72,2783],{"className":2784,"style":533},[403],[72,2786,2788],{"className":2787},[537,538,539,540],[72,2789,2656],{"className":2790},[150,540],[72,2792,428],{"className":2793},[427],[72,2795,2797],{"className":2796},[391],[72,2798,2800],{"className":2799,"style":553},[395],[72,2801],{},[72,2803,2805,2848],{"className":2804},[75],[72,2806,2808],{"className":2807},[79],[81,2809,2810],{"xmlns":83},[85,2811,2812,2845],{},[88,2813,2814,2816,2818,2820,2822,2828,2830,2832,2834,2836,2838,2840,2842],{},[295,2815,2656],{},[95,2817,2417],{},[91,2819,1916],{},[95,2821,2417],{},[286,2823,2824,2826],{},[91,2825,1916],{},[2043,2827,2045],{},[95,2829,273],{},[295,2831,2656],{},[95,2833,2417],{},[295,2835,1921],{},[95,2837,2417],{},[295,2839,2050],{},[95,2841,273],{},[295,2843,2844],{},"20,582,400",[129,2846,2847],{"encoding":131},"3 \\times d \\times d_{\\text{ff}} = 3 \\times 1{,}600 \\times 4{,}288 = 20{,}582{,}400",[72,2849,2851,2869,2888,2949,2967,2994,3021],{"className":2850,"ariaHidden":137},[136],[72,2852,2854,2857,2860,2863,2866],{"className":2853},[141],[72,2855],{"className":2856,"style":2454},[145],[72,2858,2656],{"className":2859},[150],[72,2861],{"className":2862,"style":189},[155],[72,2864,2417],{"className":2865},[193],[72,2867],{"className":2868,"style":189},[155],[72,2870,2872,2876,2879,2882,2885],{"className":2871},[141],[72,2873],{"className":2874,"style":2875},[145],"height:0.7778em;vertical-align:-0.0833em;",[72,2877,1916],{"className":2878},[150,151],[72,2880],{"className":2881,"style":189},[155],[72,2883,2417],{"className":2884},[193],[72,2886],{"className":2887,"style":189},[155],[72,2889,2891,2894,2940,2943,2946],{"className":2890},[141],[72,2892],{"className":2893,"style":2063},[145],[72,2895,2897,2900],{"className":2896},[150],[72,2898,1916],{"className":2899},[150,151],[72,2901,2903],{"className":2902},[516],[72,2904,2906,2932],{"className":2905},[386,387],[72,2907,2909,2929],{"className":2908},[391],[72,2910,2912],{"className":2911,"style":2082},[395],[72,2913,2914,2917],{"style":2085},[72,2915],{"className":2916,"style":533},[403],[72,2918,2920],{"className":2919},[537,538,539,540],[72,2921,2923],{"className":2922},[150,540],[72,2924,2926],{"className":2925},[150,2098,540],[72,2927,2045],{"className":2928},[150,540],[72,2930,428],{"className":2931},[427],[72,2933,2935],{"className":2934},[391],[72,2936,2938],{"className":2937,"style":553},[395],[72,2939],{},[72,2941],{"className":2942,"style":156},[155],[72,2944,273],{"className":2945},[160],[72,2947],{"className":2948,"style":156},[155],[72,2950,2952,2955,2958,2961,2964],{"className":2951},[141],[72,2953],{"className":2954,"style":2454},[145],[72,2956,2656],{"className":2957},[150],[72,2959],{"className":2960,"style":189},[155],[72,2962,2417],{"className":2963},[193],[72,2965],{"className":2966,"style":189},[155],[72,2968,2970,2973,2976,2982,2985,2988,2991],{"className":2969},[141],[72,2971],{"className":2972,"style":1759},[145],[72,2974,297],{"className":2975},[150],[72,2977,2979],{"className":2978},[150],[72,2980,308],{"className":2981},[574],[72,2983,1965],{"className":2984},[150],[72,2986],{"className":2987,"style":189},[155],[72,2989,2417],{"className":2990},[193],[72,2992],{"className":2993,"style":189},[155],[72,2995,2997,3000,3003,3009,3012,3015,3018],{"className":2996},[141],[72,2998],{"className":2999,"style":1759},[145],[72,3001,2131],{"className":3002},[150],[72,3004,3006],{"className":3005},[150],[72,3007,308],{"className":3008},[574],[72,3010,2141],{"className":3011},[150],[72,3013],{"className":3014,"style":156},[155],[72,3016,273],{"className":3017},[160],[72,3019],{"className":3020,"style":156},[155],[72,3022,3024,3027,3031,3037,3041,3047],{"className":3023},[141],[72,3025],{"className":3026,"style":1759},[145],[72,3028,3030],{"className":3029},[150],"20",[72,3032,3034],{"className":3033},[150],[72,3035,308],{"className":3036},[574],[72,3038,3040],{"className":3039},[150],"582",[72,3042,3044],{"className":3043},[150],[72,3045,308],{"className":3046},[574],[72,3048,3050],{"className":3049},[150],"400",[1587,3052,3053,3054,1593,3057,3060,3061],{},"两个 RMSNorm 缩放向量：",[14,3055,3056],{},"ln1",[14,3058,3059],{},"ln2"," 共 ",[72,3062,3064,3087],{"className":3063},[75],[72,3065,3067],{"className":3066},[79],[81,3068,3069],{"xmlns":83},[85,3070,3071,3084],{},[88,3072,3073,3075,3077,3079,3081],{},[295,3074,348],{},[95,3076,2417],{},[91,3078,1916],{},[95,3080,273],{},[295,3082,3083],{},"3,200",[129,3085,3086],{"encoding":131},"2 \\times d = 3{,}200",[72,3088,3090,3108,3126],{"className":3089,"ariaHidden":137},[136],[72,3091,3093,3096,3099,3102,3105],{"className":3092},[141],[72,3094],{"className":3095,"style":2454},[145],[72,3097,348],{"className":3098},[150],[72,3100],{"className":3101,"style":189},[155],[72,3103,2417],{"className":3104},[193],[72,3106],{"className":3107,"style":189},[155],[72,3109,3111,3114,3117,3120,3123],{"className":3110},[141],[72,3112],{"className":3113,"style":1934},[145],[72,3115,1916],{"className":3116},[150,151],[72,3118],{"className":3119,"style":156},[155],[72,3121,273],{"className":3122},[160],[72,3124],{"className":3125,"style":156},[155],[72,3127,3129,3132,3135,3141],{"className":3128},[141],[72,3130],{"className":3131,"style":1759},[145],[72,3133,2656],{"className":3134},[150],[72,3136,3138],{"className":3137},[150],[72,3139,308],{"className":3140},[574],[72,3142,3144],{"className":3143},[150],"200",[1587,3146,3147,3148,3198,3199,3321],{},"单层合计：",[72,3149,3151,3166],{"className":3150},[75],[72,3152,3154],{"className":3153},[79],[81,3155,3156],{"xmlns":83},[85,3157,3158,3163],{},[88,3159,3160],{},[295,3161,3162],{},"30,825,600",[129,3164,3165],{"encoding":131},"30{,}825{,}600",[72,3167,3169],{"className":3168,"ariaHidden":137},[136],[72,3170,3172,3175,3179,3185,3189,3195],{"className":3171},[141],[72,3173],{"className":3174,"style":1759},[145],[72,3176,3178],{"className":3177},[150],"30",[72,3180,3182],{"className":3181},[150],[72,3183,308],{"className":3184},[574],[72,3186,3188],{"className":3187},[150],"825",[72,3190,3192],{"className":3191},[150],[72,3193,308],{"className":3194},[574],[72,3196,1965],{"className":3197},[150]," 个参数；48 层共 ",[72,3200,3202,3225],{"className":3201},[75],[72,3203,3205],{"className":3204},[79],[81,3206,3207],{"xmlns":83},[85,3208,3209,3222],{},[88,3210,3211,3213,3215,3217,3219],{},[295,3212,1864],{},[95,3214,2417],{},[295,3216,3162],{},[95,3218,273],{},[295,3220,3221],{},"1,479,628,800",[129,3223,3224],{"encoding":131},"48 \\times 30{,}825{,}600 = 1{,}479{,}628{,}800",[72,3226,3228,3246,3282],{"className":3227,"ariaHidden":137},[136],[72,3229,3231,3234,3237,3240,3243],{"className":3230},[141],[72,3232],{"className":3233,"style":2454},[145],[72,3235,1864],{"className":3236},[150],[72,3238],{"className":3239,"style":189},[155],[72,3241,2417],{"className":3242},[193],[72,3244],{"className":3245,"style":189},[155],[72,3247,3249,3252,3255,3261,3264,3270,3273,3276,3279],{"className":3248},[141],[72,3250],{"className":3251,"style":1759},[145],[72,3253,3178],{"className":3254},[150],[72,3256,3258],{"className":3257},[150],[72,3259,308],{"className":3260},[574],[72,3262,3188],{"className":3263},[150],[72,3265,3267],{"className":3266},[150],[72,3268,308],{"className":3269},[574],[72,3271,1965],{"className":3272},[150],[72,3274],{"className":3275,"style":156},[155],[72,3277,273],{"className":3278},[160],[72,3280],{"className":3281,"style":156},[155],[72,3283,3285,3288,3291,3297,3301,3307,3311,3317],{"className":3284},[141],[72,3286],{"className":3287,"style":1759},[145],[72,3289,297],{"className":3290},[150],[72,3292,3294],{"className":3293},[150],[72,3295,308],{"className":3296},[574],[72,3298,3300],{"className":3299},[150],"479",[72,3302,3304],{"className":3303},[150],[72,3305,308],{"className":3306},[574],[72,3308,3310],{"className":3309},[150],"628",[72,3312,3314],{"className":3313},[150],[72,3315,308],{"className":3316},[574],[72,3318,3320],{"className":3319},[150],"800"," 个参数。",[10,3323,3324],{},"加上词表层与最终归一化：",[1700,3326,3327,3570],{},[1587,3328,3329,1654,3331,3382,3383,1654,3385,3435,3436],{},[14,3330,1513],{},[72,3332,3334,3352],{"className":3333},[75],[72,3335,3337],{"className":3336},[79],[81,3338,3339],{"xmlns":83},[85,3340,3341,3349],{},[88,3342,3343,3345,3347],{},[91,3344,1722],{},[95,3346,2417],{},[91,3348,1916],{},[129,3350,3351],{"encoding":131},"V \\times d",[72,3353,3355,3373],{"className":3354,"ariaHidden":137},[136],[72,3356,3358,3361,3364,3367,3370],{"className":3357},[141],[72,3359],{"className":3360,"style":849},[145],[72,3362,1722],{"className":3363,"style":189},[150,151],[72,3365],{"className":3366,"style":189},[155],[72,3368,2417],{"className":3369},[193],[72,3371],{"className":3372,"style":189},[155],[72,3374,3376,3379],{"className":3375},[141],[72,3377],{"className":3378,"style":1934},[145],[72,3380,1916],{"className":3381},[150,151],"）与 ",[14,3384,881],{},[72,3386,3388,3405],{"className":3387},[75],[72,3389,3391],{"className":3390},[79],[81,3392,3393],{"xmlns":83},[85,3394,3395,3403],{},[88,3396,3397,3399,3401],{},[91,3398,1722],{},[95,3400,2417],{},[91,3402,1916],{},[129,3404,3351],{"encoding":131},[72,3406,3408,3426],{"className":3407,"ariaHidden":137},[136],[72,3409,3411,3414,3417,3420,3423],{"className":3410},[141],[72,3412],{"className":3413,"style":849},[145],[72,3415,1722],{"className":3416,"style":189},[150,151],[72,3418],{"className":3419,"style":189},[155],[72,3421,2417],{"className":3422},[193],[72,3424],{"className":3425,"style":189},[155],[72,3427,3429,3432],{"className":3428},[141],[72,3430],{"className":3431,"style":1934},[145],[72,3433,1916],{"className":3434},[150,151],"）：",[72,3437,3439,3466],{"className":3438},[75],[72,3440,3442],{"className":3441},[79],[81,3443,3444],{"xmlns":83},[85,3445,3446,3463],{},[88,3447,3448,3450,3452,3454,3456,3458,3460],{},[295,3449,348],{},[95,3451,2417],{},[295,3453,1727],{},[95,3455,2417],{},[295,3457,1921],{},[95,3459,273],{},[295,3461,3462],{},"160,822,400",[129,3464,3465],{"encoding":131},"2 \\times 50{,}257 \\times 1{,}600 = 160{,}822{,}400",[72,3467,3469,3487,3514,3541],{"className":3468,"ariaHidden":137},[136],[72,3470,3472,3475,3478,3481,3484],{"className":3471},[141],[72,3473],{"className":3474,"style":2454},[145],[72,3476,348],{"className":3477},[150],[72,3479],{"className":3480,"style":189},[155],[72,3482,2417],{"className":3483},[193],[72,3485],{"className":3486,"style":189},[155],[72,3488,3490,3493,3496,3502,3505,3508,3511],{"className":3489},[141],[72,3491],{"className":3492,"style":1759},[145],[72,3494,1763],{"className":3495},[150],[72,3497,3499],{"className":3498},[150],[72,3500,308],{"className":3501},[574],[72,3503,1773],{"className":3504},[150],[72,3506],{"className":3507,"style":189},[155],[72,3509,2417],{"className":3510},[193],[72,3512],{"className":3513,"style":189},[155],[72,3515,3517,3520,3523,3529,3532,3535,3538],{"className":3516},[141],[72,3518],{"className":3519,"style":1759},[145],[72,3521,297],{"className":3522},[150],[72,3524,3526],{"className":3525},[150],[72,3527,308],{"className":3528},[574],[72,3530,1965],{"className":3531},[150],[72,3533],{"className":3534,"style":156},[155],[72,3536,273],{"className":3537},[160],[72,3539],{"className":3540,"style":156},[155],[72,3542,3544,3547,3551,3557,3561,3567],{"className":3543},[141],[72,3545],{"className":3546,"style":1759},[145],[72,3548,3550],{"className":3549},[150],"160",[72,3552,3554],{"className":3553},[150],[72,3555,308],{"className":3556},[574],[72,3558,3560],{"className":3559},[150],"822",[72,3562,3564],{"className":3563},[150],[72,3565,308],{"className":3566},[574],[72,3568,3050],{"className":3569},[150],[1587,3571,3572,1060,3574],{},[14,3573,887],{},[72,3575,3577,3591],{"className":3576},[75],[72,3578,3580],{"className":3579},[79],[81,3581,3582],{"xmlns":83},[85,3583,3584,3588],{},[88,3585,3586],{},[295,3587,1921],{},[129,3589,3590],{"encoding":131},"1{,}600",[72,3592,3594],{"className":3593,"ariaHidden":137},[136],[72,3595,3597,3600,3603,3609],{"className":3596},[141],[72,3598],{"className":3599,"style":1759},[145],[72,3601,297],{"className":3602},[150],[72,3604,3606],{"className":3605},[150],[72,3607,308],{"className":3608},[574],[72,3610,1965],{"className":3611},[150],[10,3613,3614,1060],{},[36,3615,3616],{},"总可训练参数量",[72,3618,3620],{"className":3619},[232],[72,3621,3623,3671],{"className":3622},[75],[72,3624,3626],{"className":3625},[79],[81,3627,3628],{"xmlns":83,"display":241},[85,3629,3630,3668],{},[88,3631,3632,3640,3642,3644,3646,3648,3650,3652,3654,3657,3660,3663,3666],{},[286,3633,3634,3637],{},[91,3635,3636],{},"N",[2043,3638,3639],{},"params",[95,3641,273],{},[295,3643,3221],{},[95,3645,113],{},[295,3647,3462],{},[95,3649,113],{},[295,3651,1921],{},[95,3653,273],{},[295,3655,3656],{},"1,640,452,800",[95,3658,3659],{},"≈",[295,3661,3662],{},"1.640",[2043,3664,3665],{}," B",[91,3667,359],{"mathvariant":100},[129,3669,3670],{"encoding":131},"N_{\\text{params}} = 1{,}479{,}628{,}800 + 160{,}822{,}400 + 1{,}600 = 1{,}640{,}452{,}800 \\approx 1.640\\text{ B}.",[72,3672,3674,3738,3783,3819,3846,3893],{"className":3673,"ariaHidden":137},[136],[72,3675,3677,3680,3729,3732,3735],{"className":3676},[141],[72,3678],{"className":3679,"style":2216},[145],[72,3681,3683,3687],{"className":3682},[150],[72,3684,3636],{"className":3685,"style":3686},[150,151],"margin-right:0.109em;",[72,3688,3690],{"className":3689},[516],[72,3691,3693,3721],{"className":3692},[386,387],[72,3694,3696,3718],{"className":3695},[391],[72,3697,3700],{"className":3698,"style":3699},[395],"height:0.1514em;",[72,3701,3703,3706],{"style":3702},"top:-2.55em;margin-left:-0.109em;margin-right:0.05em;",[72,3704],{"className":3705,"style":533},[403],[72,3707,3709],{"className":3708},[537,538,539,540],[72,3710,3712],{"className":3711},[150,540],[72,3713,3715],{"className":3714},[150,2098,540],[72,3716,3639],{"className":3717},[150,540],[72,3719,428],{"className":3720},[427],[72,3722,3724],{"className":3723},[391],[72,3725,3727],{"className":3726,"style":2258},[395],[72,3728],{},[72,3730],{"className":3731,"style":156},[155],[72,3733,273],{"className":3734},[160],[72,3736],{"className":3737,"style":156},[155],[72,3739,3741,3744,3747,3753,3756,3762,3765,3771,3774,3777,3780],{"className":3740},[141],[72,3742],{"className":3743,"style":1759},[145],[72,3745,297],{"className":3746},[150],[72,3748,3750],{"className":3749},[150],[72,3751,308],{"className":3752},[574],[72,3754,3300],{"className":3755},[150],[72,3757,3759],{"className":3758},[150],[72,3760,308],{"className":3761},[574],[72,3763,3310],{"className":3764},[150],[72,3766,3768],{"className":3767},[150],[72,3769,308],{"className":3770},[574],[72,3772,3320],{"className":3773},[150],[72,3775],{"className":3776,"style":189},[155],[72,3778,113],{"className":3779},[193],[72,3781],{"className":3782,"style":189},[155],[72,3784,3786,3789,3792,3798,3801,3807,3810,3813,3816],{"className":3785},[141],[72,3787],{"className":3788,"style":1759},[145],[72,3790,3550],{"className":3791},[150],[72,3793,3795],{"className":3794},[150],[72,3796,308],{"className":3797},[574],[72,3799,3560],{"className":3800},[150],[72,3802,3804],{"className":3803},[150],[72,3805,308],{"className":3806},[574],[72,3808,3050],{"className":3809},[150],[72,3811],{"className":3812,"style":189},[155],[72,3814,113],{"className":3815},[193],[72,3817],{"className":3818,"style":189},[155],[72,3820,3822,3825,3828,3834,3837,3840,3843],{"className":3821},[141],[72,3823],{"className":3824,"style":1759},[145],[72,3826,297],{"className":3827},[150],[72,3829,3831],{"className":3830},[150],[72,3832,308],{"className":3833},[574],[72,3835,1965],{"className":3836},[150],[72,3838],{"className":3839,"style":156},[155],[72,3841,273],{"className":3842},[160],[72,3844],{"className":3845,"style":156},[155],[72,3847,3849,3852,3855,3861,3865,3871,3875,3881,3884,3887,3890],{"className":3848},[141],[72,3850],{"className":3851,"style":1759},[145],[72,3853,297],{"className":3854},[150],[72,3856,3858],{"className":3857},[150],[72,3859,308],{"className":3860},[574],[72,3862,3864],{"className":3863},[150],"640",[72,3866,3868],{"className":3867},[150],[72,3869,308],{"className":3870},[574],[72,3872,3874],{"className":3873},[150],"452",[72,3876,3878],{"className":3877},[150],[72,3879,308],{"className":3880},[574],[72,3882,3320],{"className":3883},[150],[72,3885],{"className":3886,"style":156},[155],[72,3888,3659],{"className":3889},[160],[72,3891],{"className":3892,"style":156},[155],[72,3894,3896,3899,3902,3908],{"className":3895},[141],[72,3897],{"className":3898,"style":1740},[145],[72,3900,3662],{"className":3901},[150],[72,3903,3905],{"className":3904},[150,2098],[72,3906,3665],{"className":3907},[150],[72,3909,359],{"className":3910},[150],[10,3912,3913,3914,4016,4017,4056],{},"按单精度浮点数（FP32，每参数 4 字节）存储，仅加载模型权重需要 ",[72,3915,3917,3943],{"className":3916},[75],[72,3918,3920],{"className":3919},[79],[81,3921,3922],{"xmlns":83},[85,3923,3924,3940],{},[88,3925,3926,3929,3932,3934,3937],{},[295,3927,3928],{},"6,561,811,200",[2043,3930,3931],{}," bytes",[95,3933,3659],{},[295,3935,3936],{},"6.56",[2043,3938,3939],{}," GB",[129,3941,3942],{"encoding":131},"6{,}561{,}811{,}200\\text{ bytes} \\approx 6.56\\text{ GB}",[72,3944,3946,4001],{"className":3945,"ariaHidden":137},[136],[72,3947,3949,3953,3957,3963,3967,3973,3977,3983,3986,3992,3995,3998],{"className":3948},[141],[72,3950],{"className":3951,"style":3952},[145],"height:0.8889em;vertical-align:-0.1944em;",[72,3954,3956],{"className":3955},[150],"6",[72,3958,3960],{"className":3959},[150],[72,3961,308],{"className":3962},[574],[72,3964,3966],{"className":3965},[150],"561",[72,3968,3970],{"className":3969},[150],[72,3971,308],{"className":3972},[574],[72,3974,3976],{"className":3975},[150],"811",[72,3978,3980],{"className":3979},[150],[72,3981,308],{"className":3982},[574],[72,3984,3144],{"className":3985},[150],[72,3987,3989],{"className":3988},[150,2098],[72,3990,3931],{"className":3991},[150],[72,3993],{"className":3994,"style":156},[155],[72,3996,3659],{"className":3997},[160],[72,3999],{"className":4000,"style":156},[155],[72,4002,4004,4007,4010],{"className":4003},[141],[72,4005],{"className":4006,"style":1740},[145],[72,4008,3936],{"className":4009},[150],[72,4011,4013],{"className":4012},[150,2098],[72,4014,3939],{"className":4015},[150],"（或 ",[72,4018,4020,4038],{"className":4019},[75],[72,4021,4023],{"className":4022},[79],[81,4024,4025],{"xmlns":83},[85,4026,4027,4035],{},[88,4028,4029,4032],{},[295,4030,4031],{},"6.11",[2043,4033,4034],{}," GiB",[129,4036,4037],{"encoding":131},"6.11\\text{ GiB}",[72,4039,4041],{"className":4040,"ariaHidden":137},[136],[72,4042,4044,4047,4050],{"className":4043},[141],[72,4045],{"className":4046,"style":1740},[145],[72,4048,4031],{"className":4049},[150],[72,4051,4053],{"className":4052},[150,2098],[72,4054,4034],{"className":4055},[150],"）。",[2143,4058,4060,4061,4114],{"id":4059},"_2-单样本前向传播矩阵乘法-flops-拆解b1b-1b1","2. 单样本前向传播矩阵乘法 FLOPs 拆解（",[72,4062,4064,4083],{"className":4063},[75],[72,4065,4067],{"className":4066},[79],[81,4068,4069],{"xmlns":83},[85,4070,4071,4080],{},[88,4072,4073,4076,4078],{},[91,4074,4075],{},"B",[95,4077,273],{},[295,4079,297],{},[129,4081,4082],{"encoding":131},"B = 1",[72,4084,4086,4105],{"className":4085,"ariaHidden":137},[136],[72,4087,4089,4092,4096,4099,4102],{"className":4088},[141],[72,4090],{"className":4091,"style":1740},[145],[72,4093,4075],{"className":4094,"style":4095},[150,151],"margin-right:0.0502em;",[72,4097],{"className":4098,"style":156},[155],[72,4100,273],{"className":4101},[160],[72,4103],{"className":4104,"style":156},[155],[72,4106,4108,4111],{"className":4107},[141],[72,4109],{"className":4110,"style":1895},[145],[72,4112,297],{"className":4113},[150],"）",[10,4116,4117,4118,4224,4225,4266],{},"利用矩阵乘法规则 ",[72,4119,4121,4158],{"className":4120},[75],[72,4122,4124],{"className":4123},[79],[81,4125,4126],{"xmlns":83},[85,4127,4128,4155],{},[88,4129,4130,4132,4135,4137,4140,4142,4145,4147,4149,4151,4153],{},[95,4131,108],{"stretchy":107},[91,4133,4134],{},"m",[95,4136,2417],{},[91,4138,4139],{},"n",[95,4141,125],{"stretchy":107},[91,4143,4144],{"mathvariant":100},"@",[95,4146,108],{"stretchy":107},[91,4148,4139],{},[95,4150,2417],{},[91,4152,10],{},[95,4154,125],{"stretchy":107},[129,4156,4157],{"encoding":131},"(m \\times n) @ (n \\times p)",[72,4159,4161,4182,4212],{"className":4160,"ariaHidden":137},[136],[72,4162,4164,4167,4170,4173,4176,4179],{"className":4163},[141],[72,4165],{"className":4166,"style":170},[145],[72,4168,108],{"className":4169},[182],[72,4171,4134],{"className":4172},[150,151],[72,4174],{"className":4175,"style":189},[155],[72,4177,2417],{"className":4178},[193],[72,4180],{"className":4181,"style":189},[155],[72,4183,4185,4188,4191,4194,4197,4200,4203,4206,4209],{"className":4184},[141],[72,4186],{"className":4187,"style":170},[145],[72,4189,4139],{"className":4190},[150,151],[72,4192,125],{"className":4193},[218],[72,4195,4144],{"className":4196},[150],[72,4198,108],{"className":4199},[182],[72,4201,4139],{"className":4202},[150,151],[72,4204],{"className":4205,"style":189},[155],[72,4207,2417],{"className":4208},[193],[72,4210],{"className":4211,"style":189},[155],[72,4213,4215,4218,4221],{"className":4214},[141],[72,4216],{"className":4217,"style":170},[145],[72,4219,10],{"className":4220},[150,151],[72,4222,125],{"className":4223},[218]," 需要 ",[72,4226,4228,4248],{"className":4227},[75],[72,4229,4231],{"className":4230},[79],[81,4232,4233],{"xmlns":83},[85,4234,4235,4245],{},[88,4236,4237,4239,4241,4243],{},[295,4238,348],{},[91,4240,4134],{},[91,4242,4139],{},[91,4244,10],{},[129,4246,4247],{"encoding":131},"2mnp",[72,4249,4251],{"className":4250,"ariaHidden":137},[136],[72,4252,4254,4257,4260,4263],{"className":4253},[141],[72,4255],{"className":4256,"style":1759},[145],[72,4258,348],{"className":4259},[150],[72,4261,295],{"className":4262},[150,151],[72,4264,10],{"className":4265},[150,151]," FLOPs：",[923,4268,4269,4341],{},[926,4270,4271],{},[929,4272,4273,4276,4279,4282,4338],{},[932,4274,4275],{},"模块位置",[932,4277,4278],{},"矩阵乘法运算",[932,4280,4281],{},"单层 FLOPs 公式",[932,4283,4285,4286,4114],{"align":4284},"right","48 层总 FLOPs（",[72,4287,4289,4308],{"className":4288},[75],[72,4290,4292],{"className":4291},[79],[81,4293,4294],{"xmlns":83},[85,4295,4296,4305],{},[88,4297,4298,4300,4302],{},[91,4299,1791],{},[95,4301,273],{},[295,4303,4304],{},"1024",[129,4306,4307],{"encoding":131},"S=1024",[72,4309,4311,4329],{"className":4310,"ariaHidden":137},[136],[72,4312,4314,4317,4320,4323,4326],{"className":4313},[141],[72,4315],{"className":4316,"style":1740},[145],[72,4318,1791],{"className":4319,"style":1812},[150,151],[72,4321],{"className":4322,"style":156},[155],[72,4324,273],{"className":4325},[160],[72,4327],{"className":4328,"style":156},[155],[72,4330,4332,4335],{"className":4331},[141],[72,4333],{"className":4334,"style":1895},[145],[72,4336,4304],{"className":4337},[150],[932,4339,4340],{"align":4284},"占总 FLOPs 比例",[945,4342,4343,4709,5197,5739,5991],{},[929,4344,4345,4416,4543,4615,4706],{},[950,4346,4347,4348,125],{},"Attention 投影 (",[72,4349,4351,4377],{"className":4350},[75],[72,4352,4354],{"className":4353},[79],[81,4355,4356],{"xmlns":83},[85,4357,4358,4374],{},[88,4359,4360,4362,4364,4366,4368,4370,4372],{},[91,4361,2177],{},[95,4363,308],{"separator":137},[91,4365,2186],{},[95,4367,308],{"separator":137},[91,4369,1722],{},[95,4371,308],{"separator":137},[91,4373,2203],{},[129,4375,4376],{"encoding":131},"Q, K, V, O",[72,4378,4380],{"className":4379,"ariaHidden":137},[136],[72,4381,4383,4386,4389,4392,4395,4398,4401,4404,4407,4410,4413],{"className":4382},[141],[72,4384],{"className":4385,"style":2669},[145],[72,4387,2177],{"className":4388},[150,151],[72,4390,308],{"className":4391},[574],[72,4393],{"className":4394,"style":492},[155],[72,4396,2186],{"className":4397,"style":2296},[150,151],[72,4399,308],{"className":4400},[574],[72,4402],{"className":4403,"style":492},[155],[72,4405,1722],{"className":4406,"style":189},[150,151],[72,4408,308],{"className":4409},[574],[72,4411],{"className":4412,"style":492},[155],[72,4414,2203],{"className":4415,"style":2389},[150,151],[950,4417,4418],{},[72,4419,4421,4459],{"className":4420},[75],[72,4422,4424],{"className":4423},[79],[81,4425,4426],{"xmlns":83},[85,4427,4428,4456],{},[88,4429,4430,4432,4434,4436,4438,4440,4442,4444,4446,4448,4450,4452,4454],{},[295,4431,2131],{},[95,4433,2417],{},[95,4435,108],{"stretchy":107},[91,4437,1791],{},[95,4439,2417],{},[91,4441,1916],{},[95,4443,125],{"stretchy":107},[91,4445,4144],{"mathvariant":100},[95,4447,108],{"stretchy":107},[91,4449,1916],{},[95,4451,2417],{},[91,4453,1916],{},[95,4455,125],{"stretchy":107},[129,4457,4458],{"encoding":131},"4 \\times (S \\times d) @ (d \\times d)",[72,4460,4462,4480,4501,4531],{"className":4461,"ariaHidden":137},[136],[72,4463,4465,4468,4471,4474,4477],{"className":4464},[141],[72,4466],{"className":4467,"style":2454},[145],[72,4469,2131],{"className":4470},[150],[72,4472],{"className":4473,"style":189},[155],[72,4475,2417],{"className":4476},[193],[72,4478],{"className":4479,"style":189},[155],[72,4481,4483,4486,4489,4492,4495,4498],{"className":4482},[141],[72,4484],{"className":4485,"style":170},[145],[72,4487,108],{"className":4488},[182],[72,4490,1791],{"className":4491,"style":1812},[150,151],[72,4493],{"className":4494,"style":189},[155],[72,4496,2417],{"className":4497},[193],[72,4499],{"className":4500,"style":189},[155],[72,4502,4504,4507,4510,4513,4516,4519,4522,4525,4528],{"className":4503},[141],[72,4505],{"className":4506,"style":170},[145],[72,4508,1916],{"className":4509},[150,151],[72,4511,125],{"className":4512},[218],[72,4514,4144],{"className":4515},[150],[72,4517,108],{"className":4518},[182],[72,4520,1916],{"className":4521},[150,151],[72,4523],{"className":4524,"style":189},[155],[72,4526,2417],{"className":4527},[193],[72,4529],{"className":4530,"style":189},[155],[72,4532,4534,4537,4540],{"className":4533},[141],[72,4535],{"className":4536,"style":170},[145],[72,4538,1916],{"className":4539},[150,151],[72,4541,125],{"className":4542},[218],[950,4544,4545],{},[72,4546,4548,4571],{"className":4547},[75],[72,4549,4551],{"className":4550},[79],[81,4552,4553],{"xmlns":83},[85,4554,4555,4568],{},[88,4556,4557,4560,4562],{},[295,4558,4559],{},"8",[91,4561,1791],{},[2419,4563,4564,4566],{},[91,4565,1916],{},[295,4567,348],{},[129,4569,4570],{"encoding":131},"8 S d^2",[72,4572,4574],{"className":4573,"ariaHidden":137},[136],[72,4575,4577,4580,4583,4586],{"className":4576},[141],[72,4578],{"className":4579,"style":2473},[145],[72,4581,4559],{"className":4582},[150],[72,4584,1791],{"className":4585,"style":1812},[150,151],[72,4587,4589,4592],{"className":4588},[150],[72,4590,1916],{"className":4591},[150,151],[72,4593,4595],{"className":4594},[516],[72,4596,4598],{"className":4597},[386],[72,4599,4601],{"className":4600},[391],[72,4602,4604],{"className":4603,"style":2473},[395],[72,4605,4606,4609],{"style":2494},[72,4607],{"className":4608,"style":533},[403],[72,4610,4612],{"className":4611},[537,538,539,540],[72,4613,348],{"className":4614},[150,540],[950,4616,4617],{"align":4284},[72,4618,4620,4644],{"className":4619},[75],[72,4621,4623],{"className":4622},[79],[81,4624,4625],{"xmlns":83},[85,4626,4627,4641],{},[88,4628,4629,4632,4634],{},[295,4630,4631],{},"1.0066",[95,4633,2417],{},[2419,4635,4636,4638],{},[295,4637,2598],{},[295,4639,4640],{},"12",[129,4642,4643],{"encoding":131},"1.0066 \\times 10^{12}",[72,4645,4647,4665],{"className":4646,"ariaHidden":137},[136],[72,4648,4650,4653,4656,4659,4662],{"className":4649},[141],[72,4651],{"className":4652,"style":2454},[145],[72,4654,4631],{"className":4655},[150],[72,4657],{"className":4658,"style":189},[155],[72,4660,2417],{"className":4661},[193],[72,4663],{"className":4664,"style":189},[155],[72,4666,4668,4671,4674],{"className":4667},[141],[72,4669],{"className":4670,"style":2473},[145],[72,4672,297],{"className":4673},[150],[72,4675,4677,4680],{"className":4676},[150],[72,4678,259],{"className":4679},[150],[72,4681,4683],{"className":4682},[516],[72,4684,4686],{"className":4685},[386],[72,4687,4689],{"className":4688},[391],[72,4690,4692],{"className":4691,"style":2473},[395],[72,4693,4694,4697],{"style":2494},[72,4695],{"className":4696,"style":533},[403],[72,4698,4700],{"className":4699},[537,538,539,540],[72,4701,4703],{"className":4702},[150,540],[72,4704,4640],{"className":4705},[150,540],[950,4707,4708],{"align":4284},"22.30%",[929,4710,4711,4800,5033,5104,5194],{},[950,4712,4713,4714,125],{},"Attention 分数与加权 (",[72,4715,4717,4745],{"className":4716},[75],[72,4718,4720],{"className":4719},[79],[81,4721,4722],{"xmlns":83},[85,4723,4724,4742],{},[88,4725,4726,4728,4735,4737,4740],{},[91,4727,2177],{},[2419,4729,4730,4732],{},[91,4731,2186],{},[91,4733,4734],{"mathvariant":100},"⊤",[95,4736,308],{"separator":137},[91,4738,4739],{},"P",[91,4741,1722],{},[129,4743,4744],{"encoding":131},"QK^\\top, PV",[72,4746,4748],{"className":4747,"ariaHidden":137},[136],[72,4749,4751,4755,4758,4788,4791,4794,4797],{"className":4750},[141],[72,4752],{"className":4753,"style":4754},[145],"height:1.0435em;vertical-align:-0.1944em;",[72,4756,2177],{"className":4757},[150,151],[72,4759,4761,4764],{"className":4760},[150],[72,4762,2186],{"className":4763,"style":2296},[150,151],[72,4765,4767],{"className":4766},[516],[72,4768,4770],{"className":4769},[386],[72,4771,4773],{"className":4772},[391],[72,4774,4777],{"className":4775,"style":4776},[395],"height:0.8491em;",[72,4778,4779,4782],{"style":2494},[72,4780],{"className":4781,"style":533},[403],[72,4783,4785],{"className":4784},[537,538,539,540],[72,4786,4734],{"className":4787},[150,540],[72,4789,308],{"className":4790},[574],[72,4792],{"className":4793,"style":492},[155],[72,4795,4739],{"className":4796,"style":2223},[150,151],[72,4798,1722],{"className":4799,"style":189},[150,151],[950,4801,4802],{},[72,4803,4805,4856],{"className":4804},[75],[72,4806,4808],{"className":4807},[79],[81,4809,4810],{"xmlns":83},[85,4811,4812,4853],{},[88,4813,4814,4816,4818,4820,4822,4824,4826,4828,4835,4837,4839,4841,4847,4849,4851],{},[295,4815,348],{},[95,4817,2417],{},[91,4819,1983],{},[95,4821,2417],{},[95,4823,108],{"stretchy":107},[91,4825,1791],{},[95,4827,2417],{},[286,4829,4830,4832],{},[91,4831,1916],{},[91,4833,4834],{},"k",[95,4836,125],{"stretchy":107},[91,4838,4144],{"mathvariant":100},[95,4840,108],{"stretchy":107},[286,4842,4843,4845],{},[91,4844,1916],{},[91,4846,4834],{},[95,4848,2417],{},[91,4850,1791],{},[95,4852,125],{"stretchy":107},[129,4854,4855],{"encoding":131},"2 \\times H \\times (S \\times d_k) @ (d_k \\times S)",[72,4857,4859,4877,4895,4916,5021],{"className":4858,"ariaHidden":137},[136],[72,4860,4862,4865,4868,4871,4874],{"className":4861},[141],[72,4863],{"className":4864,"style":2454},[145],[72,4866,348],{"className":4867},[150],[72,4869],{"className":4870,"style":189},[155],[72,4872,2417],{"className":4873},[193],[72,4875],{"className":4876,"style":189},[155],[72,4878,4880,4883,4886,4889,4892],{"className":4879},[141],[72,4881],{"className":4882,"style":849},[145],[72,4884,1983],{"className":4885,"style":2004},[150,151],[72,4887],{"className":4888,"style":189},[155],[72,4890,2417],{"className":4891},[193],[72,4893],{"className":4894,"style":189},[155],[72,4896,4898,4901,4904,4907,4910,4913],{"className":4897},[141],[72,4899],{"className":4900,"style":170},[145],[72,4902,108],{"className":4903},[182],[72,4905,1791],{"className":4906,"style":1812},[150,151],[72,4908],{"className":4909,"style":189},[155],[72,4911,2417],{"className":4912},[193],[72,4914],{"className":4915,"style":189},[155],[72,4917,4919,4922,4963,4966,4969,4972,5012,5015,5018],{"className":4918},[141],[72,4920],{"className":4921,"style":170},[145],[72,4923,4925,4928],{"className":4924},[150],[72,4926,1916],{"className":4927},[150,151],[72,4929,4931],{"className":4930},[516],[72,4932,4934,4955],{"className":4933},[386,387],[72,4935,4937,4952],{"className":4936},[391],[72,4938,4940],{"className":4939,"style":2082},[395],[72,4941,4942,4945],{"style":2085},[72,4943],{"className":4944,"style":533},[403],[72,4946,4948],{"className":4947},[537,538,539,540],[72,4949,4834],{"className":4950,"style":4951},[150,151,540],"margin-right:0.0315em;",[72,4953,428],{"className":4954},[427],[72,4956,4958],{"className":4957},[391],[72,4959,4961],{"className":4960,"style":553},[395],[72,4962],{},[72,4964,125],{"className":4965},[218],[72,4967,4144],{"className":4968},[150],[72,4970,108],{"className":4971},[182],[72,4973,4975,4978],{"className":4974},[150],[72,4976,1916],{"className":4977},[150,151],[72,4979,4981],{"className":4980},[516],[72,4982,4984,5004],{"className":4983},[386,387],[72,4985,4987,5001],{"className":4986},[391],[72,4988,4990],{"className":4989,"style":2082},[395],[72,4991,4992,4995],{"style":2085},[72,4993],{"className":4994,"style":533},[403],[72,4996,4998],{"className":4997},[537,538,539,540],[72,4999,4834],{"className":5000,"style":4951},[150,151,540],[72,5002,428],{"className":5003},[427],[72,5005,5007],{"className":5006},[391],[72,5008,5010],{"className":5009,"style":553},[395],[72,5011],{},[72,5013],{"className":5014,"style":189},[155],[72,5016,2417],{"className":5017},[193],[72,5019],{"className":5020,"style":189},[155],[72,5022,5024,5027,5030],{"className":5023},[141],[72,5025],{"className":5026,"style":170},[145],[72,5028,1791],{"className":5029,"style":1812},[150,151],[72,5031,125],{"className":5032},[218],[950,5034,5035],{},[72,5036,5038,5060],{"className":5037},[75],[72,5039,5041],{"className":5040},[79],[81,5042,5043],{"xmlns":83},[85,5044,5045,5057],{},[88,5046,5047,5049,5055],{},[295,5048,2131],{},[2419,5050,5051,5053],{},[91,5052,1791],{},[295,5054,348],{},[91,5056,1916],{},[129,5058,5059],{"encoding":131},"4 S^2 d",[72,5061,5063],{"className":5062,"ariaHidden":137},[136],[72,5064,5066,5069,5072,5101],{"className":5065},[141],[72,5067],{"className":5068,"style":2473},[145],[72,5070,2131],{"className":5071},[150],[72,5073,5075,5078],{"className":5074},[150],[72,5076,1791],{"className":5077,"style":1812},[150,151],[72,5079,5081],{"className":5080},[516],[72,5082,5084],{"className":5083},[386],[72,5085,5087],{"className":5086},[391],[72,5088,5090],{"className":5089,"style":2473},[395],[72,5091,5092,5095],{"style":2494},[72,5093],{"className":5094,"style":533},[403],[72,5096,5098],{"className":5097},[537,538,539,540],[72,5099,348],{"className":5100},[150,540],[72,5102,1916],{"className":5103},[150,151],[950,5105,5106],{"align":4284},[72,5107,5109,5132],{"className":5108},[75],[72,5110,5112],{"className":5111},[79],[81,5113,5114],{"xmlns":83},[85,5115,5116,5129],{},[88,5117,5118,5121,5123],{},[295,5119,5120],{},"0.3221",[95,5122,2417],{},[2419,5124,5125,5127],{},[295,5126,2598],{},[295,5128,4640],{},[129,5130,5131],{"encoding":131},"0.3221 \\times 10^{12}",[72,5133,5135,5153],{"className":5134,"ariaHidden":137},[136],[72,5136,5138,5141,5144,5147,5150],{"className":5137},[141],[72,5139],{"className":5140,"style":2454},[145],[72,5142,5120],{"className":5143},[150],[72,5145],{"className":5146,"style":189},[155],[72,5148,2417],{"className":5149},[193],[72,5151],{"className":5152,"style":189},[155],[72,5154,5156,5159,5162],{"className":5155},[141],[72,5157],{"className":5158,"style":2473},[145],[72,5160,297],{"className":5161},[150],[72,5163,5165,5168],{"className":5164},[150],[72,5166,259],{"className":5167},[150],[72,5169,5171],{"className":5170},[516],[72,5172,5174],{"className":5173},[386],[72,5175,5177],{"className":5176},[391],[72,5178,5180],{"className":5179,"style":2473},[395],[72,5181,5182,5185],{"style":2494},[72,5183],{"className":5184,"style":533},[403],[72,5186,5188],{"className":5187},[537,538,539,540],[72,5189,5191],{"className":5190},[150,540],[72,5192,4640],{"className":5193},[150,540],[950,5195,5196],{"align":4284},"7.14%",[929,5198,5199,5379,5553,5646,5736],{},[950,5200,5201,5202,125],{},"SwiGLU FFN (",[72,5203,5205,5238],{"className":5204},[75],[72,5206,5208],{"className":5207},[79],[81,5209,5210],{"xmlns":83},[85,5211,5212,5236],{},[88,5213,5214,5220,5222,5228,5230],{},[286,5215,5216,5218],{},[91,5217,2174],{},[295,5219,297],{},[95,5221,308],{"separator":137},[286,5223,5224,5226],{},[91,5225,2174],{},[295,5227,348],{},[95,5229,308],{"separator":137},[286,5231,5232,5234],{},[91,5233,2174],{},[295,5235,2656],{},[129,5237,2659],{"encoding":131},[72,5239,5241],{"className":5240,"ariaHidden":137},[136],[72,5242,5244,5247,5287,5290,5293,5333,5336,5339],{"className":5243},[141],[72,5245],{"className":5246,"style":2669},[145],[72,5248,5250,5253],{"className":5249},[150],[72,5251,2174],{"className":5252,"style":2223},[150,151],[72,5254,5256],{"className":5255},[516],[72,5257,5259,5279],{"className":5258},[386,387],[72,5260,5262,5276],{"className":5261},[391],[72,5263,5265],{"className":5264,"style":526},[395],[72,5266,5267,5270],{"style":2239},[72,5268],{"className":5269,"style":533},[403],[72,5271,5273],{"className":5272},[537,538,539,540],[72,5274,297],{"className":5275},[150,540],[72,5277,428],{"className":5278},[427],[72,5280,5282],{"className":5281},[391],[72,5283,5285],{"className":5284,"style":553},[395],[72,5286],{},[72,5288,308],{"className":5289},[574],[72,5291],{"className":5292,"style":492},[155],[72,5294,5296,5299],{"className":5295},[150],[72,5297,2174],{"className":5298,"style":2223},[150,151],[72,5300,5302],{"className":5301},[516],[72,5303,5305,5325],{"className":5304},[386,387],[72,5306,5308,5322],{"className":5307},[391],[72,5309,5311],{"className":5310,"style":526},[395],[72,5312,5313,5316],{"style":2239},[72,5314],{"className":5315,"style":533},[403],[72,5317,5319],{"className":5318},[537,538,539,540],[72,5320,348],{"className":5321},[150,540],[72,5323,428],{"className":5324},[427],[72,5326,5328],{"className":5327},[391],[72,5329,5331],{"className":5330,"style":553},[395],[72,5332],{},[72,5334,308],{"className":5335},[574],[72,5337],{"className":5338,"style":492},[155],[72,5340,5342,5345],{"className":5341},[150],[72,5343,2174],{"className":5344,"style":2223},[150,151],[72,5346,5348],{"className":5347},[516],[72,5349,5351,5371],{"className":5350},[386,387],[72,5352,5354,5368],{"className":5353},[391],[72,5355,5357],{"className":5356,"style":526},[395],[72,5358,5359,5362],{"style":2239},[72,5360],{"className":5361,"style":533},[403],[72,5363,5365],{"className":5364},[537,538,539,540],[72,5366,2656],{"className":5367},[150,540],[72,5369,428],{"className":5370},[427],[72,5372,5374],{"className":5373},[391],[72,5375,5377],{"className":5376,"style":553},[395],[72,5378],{},[950,5380,5381],{},[72,5382,5384,5426],{"className":5383},[75],[72,5385,5387],{"className":5386},[79],[81,5388,5389],{"xmlns":83},[85,5390,5391,5423],{},[88,5392,5393,5395,5397,5399,5401,5403,5405,5407,5409,5411,5413,5415,5421],{},[295,5394,2656],{},[95,5396,2417],{},[95,5398,108],{"stretchy":107},[91,5400,1791],{},[95,5402,2417],{},[91,5404,1916],{},[95,5406,125],{"stretchy":107},[91,5408,4144],{"mathvariant":100},[95,5410,108],{"stretchy":107},[91,5412,1916],{},[95,5414,2417],{},[286,5416,5417,5419],{},[91,5418,1916],{},[2043,5420,2045],{},[95,5422,125],{"stretchy":107},[129,5424,5425],{"encoding":131},"3 \\times (S \\times d) @ (d \\times d_{\\text{ff}})",[72,5427,5429,5447,5468,5498],{"className":5428,"ariaHidden":137},[136],[72,5430,5432,5435,5438,5441,5444],{"className":5431},[141],[72,5433],{"className":5434,"style":2454},[145],[72,5436,2656],{"className":5437},[150],[72,5439],{"className":5440,"style":189},[155],[72,5442,2417],{"className":5443},[193],[72,5445],{"className":5446,"style":189},[155],[72,5448,5450,5453,5456,5459,5462,5465],{"className":5449},[141],[72,5451],{"className":5452,"style":170},[145],[72,5454,108],{"className":5455},[182],[72,5457,1791],{"className":5458,"style":1812},[150,151],[72,5460],{"className":5461,"style":189},[155],[72,5463,2417],{"className":5464},[193],[72,5466],{"className":5467,"style":189},[155],[72,5469,5471,5474,5477,5480,5483,5486,5489,5492,5495],{"className":5470},[141],[72,5472],{"className":5473,"style":170},[145],[72,5475,1916],{"className":5476},[150,151],[72,5478,125],{"className":5479},[218],[72,5481,4144],{"className":5482},[150],[72,5484,108],{"className":5485},[182],[72,5487,1916],{"className":5488},[150,151],[72,5490],{"className":5491,"style":189},[155],[72,5493,2417],{"className":5494},[193],[72,5496],{"className":5497,"style":189},[155],[72,5499,5501,5504,5550],{"className":5500},[141],[72,5502],{"className":5503,"style":170},[145],[72,5505,5507,5510],{"className":5506},[150],[72,5508,1916],{"className":5509},[150,151],[72,5511,5513],{"className":5512},[516],[72,5514,5516,5542],{"className":5515},[386,387],[72,5517,5519,5539],{"className":5518},[391],[72,5520,5522],{"className":5521,"style":2082},[395],[72,5523,5524,5527],{"style":2085},[72,5525],{"className":5526,"style":533},[403],[72,5528,5530],{"className":5529},[537,538,539,540],[72,5531,5533],{"className":5532},[150,540],[72,5534,5536],{"className":5535},[150,2098,540],[72,5537,2045],{"className":5538},[150,540],[72,5540,428],{"className":5541},[427],[72,5543,5545],{"className":5544},[391],[72,5546,5548],{"className":5547,"style":553},[395],[72,5549],{},[72,5551,125],{"className":5552},[218],[950,5554,5555],{},[72,5556,5558,5582],{"className":5557},[75],[72,5559,5561],{"className":5560},[79],[81,5562,5563],{"xmlns":83},[85,5564,5565,5579],{},[88,5566,5567,5569,5571,5573],{},[295,5568,3956],{},[91,5570,1791],{},[91,5572,1916],{},[286,5574,5575,5577],{},[91,5576,1916],{},[2043,5578,2045],{},[129,5580,5581],{"encoding":131},"6 S d d_{\\text{ff}}",[72,5583,5585],{"className":5584,"ariaHidden":137},[136],[72,5586,5588,5591,5594,5597,5600],{"className":5587},[141],[72,5589],{"className":5590,"style":2063},[145],[72,5592,3956],{"className":5593},[150],[72,5595,1791],{"className":5596,"style":1812},[150,151],[72,5598,1916],{"className":5599},[150,151],[72,5601,5603,5606],{"className":5602},[150],[72,5604,1916],{"className":5605},[150,151],[72,5607,5609],{"className":5608},[516],[72,5610,5612,5638],{"className":5611},[386,387],[72,5613,5615,5635],{"className":5614},[391],[72,5616,5618],{"className":5617,"style":2082},[395],[72,5619,5620,5623],{"style":2085},[72,5621],{"className":5622,"style":533},[403],[72,5624,5626],{"className":5625},[537,538,539,540],[72,5627,5629],{"className":5628},[150,540],[72,5630,5632],{"className":5631},[150,2098,540],[72,5633,2045],{"className":5634},[150,540],[72,5636,428],{"className":5637},[427],[72,5639,5641],{"className":5640},[391],[72,5642,5644],{"className":5643,"style":553},[395],[72,5645],{},[950,5647,5648],{"align":4284},[72,5649,5651,5674],{"className":5650},[75],[72,5652,5654],{"className":5653},[79],[81,5655,5656],{"xmlns":83},[85,5657,5658,5671],{},[88,5659,5660,5663,5665],{},[295,5661,5662],{},"3.0351",[95,5664,2417],{},[2419,5666,5667,5669],{},[295,5668,2598],{},[295,5670,4640],{},[129,5672,5673],{"encoding":131},"3.0351 \\times 10^{12}",[72,5675,5677,5695],{"className":5676,"ariaHidden":137},[136],[72,5678,5680,5683,5686,5689,5692],{"className":5679},[141],[72,5681],{"className":5682,"style":2454},[145],[72,5684,5662],{"className":5685},[150],[72,5687],{"className":5688,"style":189},[155],[72,5690,2417],{"className":5691},[193],[72,5693],{"className":5694,"style":189},[155],[72,5696,5698,5701,5704],{"className":5697},[141],[72,5699],{"className":5700,"style":2473},[145],[72,5702,297],{"className":5703},[150],[72,5705,5707,5710],{"className":5706},[150],[72,5708,259],{"className":5709},[150],[72,5711,5713],{"className":5712},[516],[72,5714,5716],{"className":5715},[386],[72,5717,5719],{"className":5718},[391],[72,5720,5722],{"className":5721,"style":2473},[395],[72,5723,5724,5727],{"style":2494},[72,5725],{"className":5726,"style":533},[403],[72,5728,5730],{"className":5729},[537,538,539,540],[72,5731,5733],{"className":5732},[150,540],[72,5734,4640],{"className":5735},[150,540],[950,5737,5738],{"align":4284},"67.25%",[929,5740,5741,5746,5851,5898,5988],{},[950,5742,5743,5744],{},"输出层 ",[14,5745,881],{},[950,5747,5748],{},[72,5749,5751,5785],{"className":5750},[75],[72,5752,5754],{"className":5753},[79],[81,5755,5756],{"xmlns":83},[85,5757,5758,5782],{},[88,5759,5760,5762,5764,5766,5768,5770,5772,5774,5776,5778,5780],{},[95,5761,108],{"stretchy":107},[91,5763,1791],{},[95,5765,2417],{},[91,5767,1916],{},[95,5769,125],{"stretchy":107},[91,5771,4144],{"mathvariant":100},[95,5773,108],{"stretchy":107},[91,5775,1916],{},[95,5777,2417],{},[91,5779,1722],{},[95,5781,125],{"stretchy":107},[129,5783,5784],{"encoding":131},"(S \\times d) @ (d \\times V)",[72,5786,5788,5809,5839],{"className":5787,"ariaHidden":137},[136],[72,5789,5791,5794,5797,5800,5803,5806],{"className":5790},[141],[72,5792],{"className":5793,"style":170},[145],[72,5795,108],{"className":5796},[182],[72,5798,1791],{"className":5799,"style":1812},[150,151],[72,5801],{"className":5802,"style":189},[155],[72,5804,2417],{"className":5805},[193],[72,5807],{"className":5808,"style":189},[155],[72,5810,5812,5815,5818,5821,5824,5827,5830,5833,5836],{"className":5811},[141],[72,5813],{"className":5814,"style":170},[145],[72,5816,1916],{"className":5817},[150,151],[72,5819,125],{"className":5820},[218],[72,5822,4144],{"className":5823},[150],[72,5825,108],{"className":5826},[182],[72,5828,1916],{"className":5829},[150,151],[72,5831],{"className":5832,"style":189},[155],[72,5834,2417],{"className":5835},[193],[72,5837],{"className":5838,"style":189},[155],[72,5840,5842,5845,5848],{"className":5841},[141],[72,5843],{"className":5844,"style":170},[145],[72,5846,1722],{"className":5847,"style":189},[150,151],[72,5849,125],{"className":5850},[218],[950,5852,5853,5854],{},"整网一次 ",[72,5855,5857,5877],{"className":5856},[75],[72,5858,5860],{"className":5859},[79],[81,5861,5862],{"xmlns":83},[85,5863,5864,5874],{},[88,5865,5866,5868,5870,5872],{},[295,5867,348],{},[91,5869,1791],{},[91,5871,1916],{},[91,5873,1722],{},[129,5875,5876],{"encoding":131},"2 S d V",[72,5878,5880],{"className":5879,"ariaHidden":137},[136],[72,5881,5883,5886,5889,5892,5895],{"className":5882},[141],[72,5884],{"className":5885,"style":1934},[145],[72,5887,348],{"className":5888},[150],[72,5890,1791],{"className":5891,"style":1812},[150,151],[72,5893,1916],{"className":5894},[150,151],[72,5896,1722],{"className":5897,"style":189},[150,151],[950,5899,5900],{"align":4284},[72,5901,5903,5926],{"className":5902},[75],[72,5904,5906],{"className":5905},[79],[81,5907,5908],{"xmlns":83},[85,5909,5910,5923],{},[88,5911,5912,5915,5917],{},[295,5913,5914],{},"0.1492",[95,5916,2417],{},[2419,5918,5919,5921],{},[295,5920,2598],{},[295,5922,4640],{},[129,5924,5925],{"encoding":131},"0.1492 \\times 10^{12}",[72,5927,5929,5947],{"className":5928,"ariaHidden":137},[136],[72,5930,5932,5935,5938,5941,5944],{"className":5931},[141],[72,5933],{"className":5934,"style":2454},[145],[72,5936,5914],{"className":5937},[150],[72,5939],{"className":5940,"style":189},[155],[72,5942,2417],{"className":5943},[193],[72,5945],{"className":5946,"style":189},[155],[72,5948,5950,5953,5956],{"className":5949},[141],[72,5951],{"className":5952,"style":2473},[145],[72,5954,297],{"className":5955},[150],[72,5957,5959,5962],{"className":5958},[150],[72,5960,259],{"className":5961},[150],[72,5963,5965],{"className":5964},[516],[72,5966,5968],{"className":5967},[386],[72,5969,5971],{"className":5970},[391],[72,5972,5974],{"className":5973,"style":2473},[395],[72,5975,5976,5979],{"style":2494},[72,5977],{"className":5978,"style":533},[403],[72,5980,5982],{"className":5981},[537,538,539,540],[72,5983,5985],{"className":5984},[150,540],[72,5986,4640],{"className":5987},[150,540],[950,5989,5990],{"align":4284},"3.31%",[929,5992,5993,5998,6001,6003,6096],{},[950,5994,5995],{},[36,5996,5997],{},"合计",[950,5999,6000],{},"—",[950,6002,6000],{},[950,6004,6005],{"align":4284},[36,6006,6007,6095],{},[72,6008,6010,6033],{"className":6009},[75],[72,6011,6013],{"className":6012},[79],[81,6014,6015],{"xmlns":83},[85,6016,6017,6030],{},[88,6018,6019,6022,6024],{},[295,6020,6021],{},"4.5131",[95,6023,2417],{},[2419,6025,6026,6028],{},[295,6027,2598],{},[295,6029,4640],{},[129,6031,6032],{"encoding":131},"4.5131 \\times 10^{12}",[72,6034,6036,6054],{"className":6035,"ariaHidden":137},[136],[72,6037,6039,6042,6045,6048,6051],{"className":6038},[141],[72,6040],{"className":6041,"style":2454},[145],[72,6043,6021],{"className":6044},[150],[72,6046],{"className":6047,"style":189},[155],[72,6049,2417],{"className":6050},[193],[72,6052],{"className":6053,"style":189},[155],[72,6055,6057,6060,6063],{"className":6056},[141],[72,6058],{"className":6059,"style":2473},[145],[72,6061,297],{"className":6062},[150],[72,6064,6066,6069],{"className":6065},[150],[72,6067,259],{"className":6068},[150],[72,6070,6072],{"className":6071},[516],[72,6073,6075],{"className":6074},[386],[72,6076,6078],{"className":6077},[391],[72,6079,6081],{"className":6080,"style":2473},[395],[72,6082,6083,6086],{"style":2494},[72,6084],{"className":6085,"style":533},[403],[72,6087,6089],{"className":6088},[537,538,539,540],[72,6090,6092],{"className":6091},[150,540],[72,6093,4640],{"className":6094},[150,540]," (4.51 TFLOPs)",[950,6097,6098],{"align":4284},[36,6099,6100],{},"100.00%",[10,6102,6103],{},"可以看到：",[1584,6105,6106,6173,6380],{},[1587,6107,6108,6109,6168,6169,6172],{},"在常规上下文长度 ",[72,6110,6112,6129],{"className":6111},[75],[72,6113,6115],{"className":6114},[79],[81,6116,6117],{"xmlns":83},[85,6118,6119,6127],{},[88,6120,6121,6123,6125],{},[91,6122,1791],{},[95,6124,273],{},[295,6126,1796],{},[129,6128,1799],{"encoding":131},[72,6130,6132,6150],{"className":6131,"ariaHidden":137},[136],[72,6133,6135,6138,6141,6144,6147],{"className":6134},[141],[72,6136],{"className":6137,"style":1740},[145],[72,6139,1791],{"className":6140,"style":1812},[150,151],[72,6142],{"className":6143,"style":156},[155],[72,6145,273],{"className":6146},[160],[72,6148],{"className":6149,"style":156},[155],[72,6151,6153,6156,6159,6165],{"className":6152},[141],[72,6154],{"className":6155,"style":1759},[145],[72,6157,297],{"className":6158},[150],[72,6160,6162],{"className":6161},[150],[72,6163,308],{"className":6164},[574],[72,6166,1840],{"className":6167},[150]," 下，",[36,6170,6171],{},"SwiGLU FFN 占据了 67.25% 的主导计算量","，加上 QKVO 投影的 22.30%，线性投影层合计超过 89%。",[1587,6174,6175,6176,6228,6229,6281,6282,6341,6342,6376,6377,6379],{},"随着模型规模从 GPT-2 Small（",[72,6177,6179,6198],{"className":6178},[75],[72,6180,6182],{"className":6181},[79],[81,6183,6184],{"xmlns":83},[85,6185,6186,6195],{},[88,6187,6188,6190,6192],{},[91,6189,1916],{},[95,6191,273],{},[295,6193,6194],{},"768",[129,6196,6197],{"encoding":131},"d=768",[72,6199,6201,6219],{"className":6200,"ariaHidden":137},[136],[72,6202,6204,6207,6210,6213,6216],{"className":6203},[141],[72,6205],{"className":6206,"style":1934},[145],[72,6208,1916],{"className":6209},[150,151],[72,6211],{"className":6212,"style":156},[155],[72,6214,273],{"className":6215},[160],[72,6217],{"className":6218,"style":156},[155],[72,6220,6222,6225],{"className":6221},[141],[72,6223],{"className":6224,"style":1895},[145],[72,6226,6194],{"className":6227},[150],"）增大到 GPT-2 XL（",[72,6230,6232,6251],{"className":6231},[75],[72,6233,6235],{"className":6234},[79],[81,6236,6237],{"xmlns":83},[85,6238,6239,6248],{},[88,6240,6241,6243,6245],{},[91,6242,1916],{},[95,6244,273],{},[295,6246,6247],{},"1600",[129,6249,6250],{"encoding":131},"d=1600",[72,6252,6254,6272],{"className":6253,"ariaHidden":137},[136],[72,6255,6257,6260,6263,6266,6269],{"className":6256},[141],[72,6258],{"className":6259,"style":1934},[145],[72,6261,1916],{"className":6262},[150,151],[72,6264],{"className":6265,"style":156},[155],[72,6267,273],{"className":6268},[160],[72,6270],{"className":6271,"style":156},[155],[72,6273,6275,6278],{"className":6274},[141],[72,6276],{"className":6277,"style":1895},[145],[72,6279,6247],{"className":6280},[150],"），与 ",[72,6283,6285,6303],{"className":6284},[75],[72,6286,6288],{"className":6287},[79],[81,6289,6290],{"xmlns":83},[85,6291,6292,6300],{},[88,6293,6294],{},[2419,6295,6296,6298],{},[91,6297,1916],{},[295,6299,348],{},[129,6301,6302],{"encoding":131},"d^2",[72,6304,6306],{"className":6305,"ariaHidden":137},[136],[72,6307,6309,6312],{"className":6308},[141],[72,6310],{"className":6311,"style":2473},[145],[72,6313,6315,6318],{"className":6314},[150],[72,6316,1916],{"className":6317},[150,151],[72,6319,6321],{"className":6320},[516],[72,6322,6324],{"className":6323},[386],[72,6325,6327],{"className":6326},[391],[72,6328,6330],{"className":6329,"style":2473},[395],[72,6331,6332,6335],{"style":2494},[72,6333],{"className":6334,"style":533},[403],[72,6336,6338],{"className":6337},[537,538,539,540],[72,6339,348],{"className":6340},[150,540]," 成正比的线性层（FFN + 投影）占比持续上升，而与 ",[72,6343,6345,6361],{"className":6344},[75],[72,6346,6348],{"className":6347},[79],[81,6349,6350],{"xmlns":83},[85,6351,6352,6358],{},[88,6353,6354,6356],{},[91,6355,1722],{},[91,6357,1916],{},[129,6359,6360],{"encoding":131},"V d",[72,6362,6364],{"className":6363,"ariaHidden":137},[136],[72,6365,6367,6370,6373],{"className":6366},[141],[72,6368],{"className":6369,"style":1934},[145],[72,6371,1722],{"className":6372,"style":189},[150,151],[72,6374,1916],{"className":6375},[150,151]," 成正比的 ",[14,6378,881],{}," 占比从约 19% 降至 3.31%。",[1587,6381,6382,6383,6411,6412,6453,6454,6488,6489,6548,6549,6623,6624,6659,6660,6699,6700,6738,6739,6742],{},"反过来，如果保持 GPT-2 XL 不变，把上下文长度 ",[72,6384,6386,6399],{"className":6385},[75],[72,6387,6389],{"className":6388},[79],[81,6390,6391],{"xmlns":83},[85,6392,6393,6397],{},[88,6394,6395],{},[91,6396,1791],{},[129,6398,1791],{"encoding":131},[72,6400,6402],{"className":6401,"ariaHidden":137},[136],[72,6403,6405,6408],{"className":6404},[141],[72,6406],{"className":6407,"style":1740},[145],[72,6409,1791],{"className":6410,"style":1812},[150,151]," 扩大 16 倍到 ",[72,6413,6415,6430],{"className":6414},[75],[72,6416,6418],{"className":6417},[79],[81,6419,6420],{"xmlns":83},[85,6421,6422,6427],{},[88,6423,6424],{},[295,6425,6426],{},"16,384",[129,6428,6429],{"encoding":131},"16{,}384",[72,6431,6433],{"className":6432,"ariaHidden":137},[136],[72,6434,6436,6439,6443,6449],{"className":6435},[141],[72,6437],{"className":6438,"style":1759},[145],[72,6440,6442],{"className":6441},[150],"16",[72,6444,6446],{"className":6445},[150],[72,6447,308],{"className":6448},[574],[72,6450,6452],{"className":6451},[150],"384","，线性层 FLOPs 呈线性放大 ",[72,6455,6457,6473],{"className":6456},[75],[72,6458,6460],{"className":6459},[79],[81,6461,6462],{"xmlns":83},[85,6463,6464,6470],{},[88,6465,6466,6468],{},[295,6467,6442],{},[95,6469,2417],{},[129,6471,6472],{"encoding":131},"16\\times",[72,6474,6476],{"className":6475,"ariaHidden":137},[136],[72,6477,6479,6482,6485],{"className":6478},[141],[72,6480],{"className":6481,"style":2454},[145],[72,6483,6442],{"className":6484},[150],[72,6486,2417],{"className":6487},[150],"，而与 ",[72,6490,6492,6510],{"className":6491},[75],[72,6493,6495],{"className":6494},[79],[81,6496,6497],{"xmlns":83},[85,6498,6499,6507],{},[88,6500,6501],{},[2419,6502,6503,6505],{},[91,6504,1791],{},[295,6506,348],{},[129,6508,6509],{"encoding":131},"S^2",[72,6511,6513],{"className":6512,"ariaHidden":137},[136],[72,6514,6516,6519],{"className":6515},[141],[72,6517],{"className":6518,"style":2473},[145],[72,6520,6522,6525],{"className":6521},[150],[72,6523,1791],{"className":6524,"style":1812},[150,151],[72,6526,6528],{"className":6527},[516],[72,6529,6531],{"className":6530},[386],[72,6532,6534],{"className":6533},[391],[72,6535,6537],{"className":6536,"style":2473},[395],[72,6538,6539,6542],{"style":2494},[72,6540],{"className":6541,"style":533},[403],[72,6543,6545],{"className":6544},[537,538,539,540],[72,6546,348],{"className":6547},[150,540]," 成正比的注意力矩阵乘法（",[72,6550,6552,6576],{"className":6551},[75],[72,6553,6555],{"className":6554},[79],[81,6556,6557],{"xmlns":83},[85,6558,6559,6573],{},[88,6560,6561,6563,6565,6571],{},[295,6562,2131],{},[91,6564,1859],{},[2419,6566,6567,6569],{},[91,6568,1791],{},[295,6570,348],{},[91,6572,1916],{},[129,6574,6575],{"encoding":131},"4 L S^2 d",[72,6577,6579],{"className":6578,"ariaHidden":137},[136],[72,6580,6582,6585,6588,6591,6620],{"className":6581},[141],[72,6583],{"className":6584,"style":2473},[145],[72,6586,2131],{"className":6587},[150],[72,6589,1859],{"className":6590},[150,151],[72,6592,6594,6597],{"className":6593},[150],[72,6595,1791],{"className":6596,"style":1812},[150,151],[72,6598,6600],{"className":6599},[516],[72,6601,6603],{"className":6602},[386],[72,6604,6606],{"className":6605},[391],[72,6607,6609],{"className":6608,"style":2473},[395],[72,6610,6611,6614],{"style":2494},[72,6612],{"className":6613,"style":533},[403],[72,6615,6617],{"className":6616},[537,538,539,540],[72,6618,348],{"className":6619},[150,540],[72,6621,1916],{"className":6622},[150,151],"）会暴增 ",[72,6625,6627,6644],{"className":6626},[75],[72,6628,6630],{"className":6629},[79],[81,6631,6632],{"xmlns":83},[85,6633,6634,6641],{},[88,6635,6636,6639],{},[295,6637,6638],{},"256",[95,6640,2417],{},[129,6642,6643],{"encoding":131},"256\\times",[72,6645,6647],{"className":6646,"ariaHidden":137},[136],[72,6648,6650,6653,6656],{"className":6649},[141],[72,6651],{"className":6652,"style":2454},[145],[72,6654,6638],{"className":6655},[150],[72,6657,2417],{"className":6658},[150],"，达到 ",[72,6661,6663,6681],{"className":6662},[75],[72,6664,6666],{"className":6665},[79],[81,6667,6668],{"xmlns":83},[85,6669,6670,6678],{},[88,6671,6672,6675],{},[295,6673,6674],{},"82.46",[2043,6676,6677],{}," TFLOPs",[129,6679,6680],{"encoding":131},"82.46\\text{ TFLOPs}",[72,6682,6684],{"className":6683,"ariaHidden":137},[136],[72,6685,6687,6690,6693],{"className":6686},[141],[72,6688],{"className":6689,"style":1740},[145],[72,6691,6674],{"className":6692},[150],[72,6694,6696],{"className":6695},[150,2098],[72,6697,6677],{"className":6698},[150],"，占整网总计算量（",[72,6701,6703,6720],{"className":6702},[75],[72,6704,6706],{"className":6705},[79],[81,6707,6708],{"xmlns":83},[85,6709,6710,6717],{},[88,6711,6712,6715],{},[295,6713,6714],{},"146.15",[2043,6716,6677],{},[129,6718,6719],{"encoding":131},"146.15\\text{ TFLOPs}",[72,6721,6723],{"className":6722,"ariaHidden":137},[136],[72,6724,6726,6729,6732],{"className":6725},[141],[72,6727],{"className":6728,"style":1740},[145],[72,6730,6714],{"className":6731},[150],[72,6733,6735],{"className":6734},[150,2098],[72,6736,6677],{"className":6737},[150],"）的 ",[36,6740,6741],{},"56.42%","，成为新的性能瓶颈。这也正是长上下文场景必须引入 FlashAttention 的根本原因。",[64,6744,6745],{"id":6745},"官方测试验证结果",[10,6747,6748,6749,6752],{},"运行 ",[14,6750,6751],{},"test_model.py"," 中的全部 13 项模型测试：",[1062,6754,6758],{"className":6755,"code":6757,"language":2098,"meta":1067},[6756],"language-text","tests\u002Ftest_model.py::test_transformer_block PASSED\ntests\u002Ftest_model.py::test_transformer_lm PASSED\ntests\u002Ftest_model.py::test_transformer_lm_truncated_input PASSED\n",[14,6759,6757],{"__ignoreMap":1067},[10,6761,6762,6763,6765,6766,889],{},"至此，",[14,6764,6751],{}," 全模块 13 个测试用例 100% 通过。下一篇回到文本入口，完成 ",[52,6767,6769],{"href":6768},"\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding\u002F","Byte-Level BPE 分词器训练与流式编码实现",[10,6771,6772],{},[6773,6774,6775,6776,6781],"em",{},"封面使用 ",[52,6777,6780],{"href":6778,"rel":6779},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Faigc-image-generator",[56],"aigc-image-generator"," 辅助生成，属于概念插画；准确的架构细节与 FLOPs 核算以正文为准。",[64,6783,6784],{"id":6784},"参考资料",[1700,6786,6787,6794,6801],{},[1587,6788,6789],{},[52,6790,6793],{"href":6791,"rel":6792},"https:\u002F\u002Fgithub.com\u002Fstanford-cs336\u002Fassignment1-basics",[56],"Stanford CS336 Assignment 1 官方仓库",[1587,6795,6796],{},[52,6797,6800],{"href":6798,"rel":6799},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2002.04745",[56],"Xiong et al., 2020: On Layer Normalization in the Transformer Architecture",[1587,6802,6803],{},[52,6804,6807],{"href":6805,"rel":6806},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2001.08361",[56],"Kaplan et al., 2020: Scaling Laws for Neural Language Models",[6809,6810,6811],"style",{},"html pre.shiki code .sRGqv, html code.shiki .sRGqv{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#ACAFFF}html pre.shiki code .sWURj, html code.shiki .sWURj{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#EEF0F9}html pre.shiki code .sf2cM, html code.shiki .sf2cM{--shiki-default:#F47067;--shiki-light:#D73A49;--shiki-cyber:#EEF0F9}html pre.shiki code .sq3c-, html code.shiki .sq3c-{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#00DAEF}html pre.shiki code .sA9aF, html code.shiki .sA9aF{--shiki-default:#F69D50;--shiki-default-font-style:inherit;--shiki-light:#E36209;--shiki-light-font-style:inherit;--shiki-cyber:#4BF3C8;--shiki-cyber-font-style:italic}html pre.shiki code .slEfO, html code.shiki .slEfO{--shiki-default:#F47067;--shiki-default-font-style:inherit;--shiki-light:#D73A49;--shiki-light-font-style:inherit;--shiki-cyber:#54B9FF;--shiki-cyber-font-style:italic}html pre.shiki code .svkvy, html code.shiki .svkvy{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#EEF0F9}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html.cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html pre.shiki code .sST-V, html code.shiki .sST-V{--shiki-default:#F47067;--shiki-light:#D73A49;--shiki-cyber:#54B9FF}html pre.shiki code .sS_0i, html code.shiki .sS_0i{--shiki-default:#DCBDFB;--shiki-light:#6F42C1;--shiki-cyber:#00DAEF}html pre.shiki code .sbU-9, html code.shiki .sbU-9{--shiki-default:#ADBAC7;--shiki-default-font-style:inherit;--shiki-light:#24292E;--shiki-light-font-style:inherit;--shiki-cyber:#ACAFFF;--shiki-cyber-font-style:italic}html pre.shiki code .s9NhK, html code.shiki .s9NhK{--shiki-default:#ADBAC7;--shiki-default-font-style:inherit;--shiki-light:#24292E;--shiki-light-font-style:inherit;--shiki-cyber:#FFD493;--shiki-cyber-font-style:italic}html pre.shiki code .sgENW, html code.shiki .sgENW{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#FFD493}",{"title":1067,"searchDepth":1111,"depth":1130,"links":6813},[6814,6815,6817,6819,6825,6826],{"id":66,"depth":1111,"text":67},{"id":892,"depth":1111,"text":6816},"对齐官方 state_dict 键名与 nn.ModuleList",{"id":1342,"depth":1111,"text":6818},"为什么 TransformerLM 返回未归一化的 Logits",{"id":1686,"depth":1111,"text":6820,"children":6821},"资源核算（Problem transformer_accounting）：以 GPT-2 XL 为例",[6822,6823],{"id":2145,"depth":1130,"text":2146},{"id":4059,"depth":1130,"text":6824},"2. 单样本前向传播矩阵乘法 FLOPs 拆解（B=1B = 1B=1）",{"id":6745,"depth":1111,"text":6745},{"id":6784,"depth":1111,"text":6784},[6828],"学习笔记","\u002Fimages\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm\u002Fcs336-pre-norm-transformer-block-lm-cover.webp","干净残差流主干穿过多个 Pre-Norm 子层并汇聚到词表投影层的 AI 辅助概念插画","2026-10-23","将 RMSNorm、带 RoPE 的因果多头自注意力和 SwiGLU 组装成 Pre-Norm Transformer Block，搭建完整 TransformerLM，并核算 GPT-2 XL 的参数量与前向 FLOPs。",false,"md",{"codeUrl":54,"attachmentUrl":61},true,"\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm",null,{"title":5,"description":6832},"Stanford CS336","posts\u002Fcs336-pre-norm-transformer-block-lm",[6843,6844,6845,227,6846],"CS336","PyTorch","Transformer","FLOPs","7fKm_2Q8IUDx2epmbDgoGud2pXIBpoj3EKlboXJ1_nY",[6849,6863,6879,6893,6907,6911,6925,6940,6954,6966,6981,6993,7006,7018,7029,7040,7056,7070,7081,7093,7107,7120,7134,7146,7160,7172,7187,7201,7215,7226,7239,7256,7271,7284,7296,7308,7321,7334],{"slug":6850,"path":6851,"title":6852,"date":6853,"tags":6854,"description":6858,"cover":6859,"coverAlt":6860,"excerpt":6861,"draft":6833,"hidden":6833,"published":6836,"readingTime":1177,"series":6840,"seriesOrder":1177,"codeUrl":54,"attachmentUrl":6862},"cs336-cross-entropy-adamw-training-loop","\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop","Stanford CS336 学习记录 08：数值稳定交叉熵、AdamW、余弦调度与完整训练闭环","2026-10-27",[6843,6844,6855,6856,6857],"AdamW","CrossEntropy","训练闭环","用 LogSumExp 消去交叉熵中的指数溢出，手写解耦权重衰减的 AdamW 优化器、全局 L2 梯度裁剪、Warmup 余弦退火调度、memmap 数据采样、Checkpoint 序列化与 Top-p 解码，完成 CS336 Assignment 1 全部 48 项测试。","\u002Fimages\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop\u002Fcs336-cross-entropy-adamw-training-loop-cover.webp","滑动窗口采样、余弦退火曲线、双动量优化轨迹与检查点归档构成的训练闭环 AI 辅助概念插画","作为 Stanford CS336 Assignment 1 系列的收官篇，我们已经拥有了分词器（bpe.py、tokenizer.py）和完整的语言模型架构（transformer.py）。要让模型在语料上真正训练并生成文本，还需","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-08-adamw-training-loop-20261027.zip",{"slug":6864,"path":6865,"title":6866,"date":6867,"tags":6868,"description":6875,"cover":6876,"coverAlt":6877,"excerpt":6878,"draft":6833,"hidden":6833,"published":6836,"readingTime":1177},"editcache-video-dit-inference-acceleration","\u002Fposts\u002Feditcache-video-dit-inference-acceleration","EditCache：视频 Diffusion Transformer (DiT) 局部编辑的跨步特征复用与区域重算加速","2026-10-25",[6869,6870,6871,6872,6873,6874],"扩散模型","视频生成","AI Infra","推理加速","GPU","论文解读","拆解视频扩散模型（Wan2.1-VACE）局部编辑中的三重算力冗余，通过可重入 Latent、跨步 Block 隐状态缓存复用、带 Halo 边界的局部区域重算与自适应推理起点，在单卡 H100 上实现 2.46× 模型计算加速与 1.49× 严格同步端到端加速。","\u002Fimages\u002Fposts\u002Feditcache-video-dit-inference-acceleration\u002Feditcache-dit-cover-20261025.webp","视频扩散 Transformer 时空三维体素网格中冻结背景缓存与暖橙色局部编辑重算通道的等距技术插画","在基于 Diffusion Transformer（DiT）的视频编辑与局部重绘（Video Inpainting \u002F Local Editing）任务中，用户的典型工作流并不是一次性生成最终成片，而是针对同一段源视频反复调整掩码区",{"slug":6880,"path":6881,"title":6882,"date":6867,"tags":6883,"description":6888,"cover":6889,"coverAlt":6890,"excerpt":6891,"draft":6833,"hidden":6833,"published":6836,"readingTime":1177,"series":6840,"seriesOrder":1166,"codeUrl":54,"attachmentUrl":6892},"cs336-bpe-tokenizer-training-encoding","\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding","Stanford CS336 学习记录 07：Byte-Level BPE 分词器训练、倒排索引加速与流式编码",[6843,6884,6885,6886,6887],"Tokenizer","BPE","Tiktoken","算法优化","实现 GPT-2 正则预分词、特殊 token 硬边界切分、基于 pair-to-word 倒排索引的增量 BPE 合并算法（\u003C 0.25s 通过速度基准），以及与 tiktoken 完全对齐的 Tokenizer。","\u002Fimages\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding\u002Fcs336-bpe-tokenizer-training-encoding-cover.webp","UTF-8 字节块经过正则切分闸口后沿合并树聚合成子词单元的 AI 辅助概念插画","前面六篇完成了 Transformer 语言模型本身的全部神经网络组件。要让模型真正处理自然语言文本，还需要打通入口处的 ByteLevel BPE（BytePair Encoding）分词器。这一篇记录 cs336basics\u002Fb","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-07-bpe-tokenizer-20261025.zip",{"slug":6894,"path":6895,"title":6896,"date":6897,"tags":6898,"description":6903,"cover":6904,"coverAlt":6905,"excerpt":6906,"draft":6833,"hidden":6833,"published":6836,"readingTime":1177},"in-situ-autoguidance-editable-scientific-illustration","\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration","从原位自校正引导（In-situ Autoguidance）到可编辑科研插图生成（AutoFigure-Edit）","2026-10-24",[6869,6899,6900,6901,6902,6874],"多模态","Agent","ICML","ACL","串联两项生成式模型与多模态智能体研究：如何通过推理期局部随机微扰在单模型内构造免训练扩散自引导向量（ICML 2025 Workshop 第一作者，arXiv:2510.17136），以及如何通过五阶段视觉-符号混合流水线生成原生可编辑 SVG 科研插图（ACL 2026 Demo，FigureBench 83.0% 胜率）。","\u002Fimages\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration\u002Fautoguidance-autofigure-cover-20261024.webp","左侧扩散流形自校正轨迹矢量场与右侧五阶段可编辑 SVG 科研插图分层网格相衔接的等距技术插画","生成式视觉模型正在经历从“生成高保真像素（Pixel Synthesis）”向“可控结构化生成（Controllable & Editable Generation）”的演进。在这一进程中，我作为第一作者或核心贡献者参与了两项分别处",{"slug":6908,"path":6837,"title":5,"date":6831,"tags":6909,"description":6832,"cover":6829,"coverAlt":6830,"excerpt":6910,"draft":6833,"hidden":6833,"published":6836,"readingTime":1177,"series":6840,"seriesOrder":1155,"codeUrl":54,"attachmentUrl":61},"cs336-pre-norm-transformer-block-lm",[6843,6844,6845,227,6846],"前五篇把 Linear、Embedding、RMSNorm、SwiGLU、RotaryPositionalEmbedding 和因果 MultiHeadSelfAttention 全部单独打通了。这一篇进入 Assignment 1",{"slug":6912,"path":6913,"title":6914,"date":6831,"tags":6915,"description":6921,"cover":6922,"coverAlt":6923,"excerpt":6924,"draft":6833,"hidden":6833,"published":6836,"readingTime":1247},"xgrammar-concurrent-cache-generation-fix","\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix","XGrammar 结构化解码缓存剖析：Unlimited 模式并发 Clear 字节记账竞态与代际修复（PR #1011）",[6871,6916,6917,6918,6919,6920],"C++","XGrammar","结构化解码","并发编程","源码分析","拆解 XGrammar GrammarCompiler 的双层缓存架构与 ThreadSafeLRUCache 锁分离实现，定位 Unlimited 模式下 Clear() 字节残留与在途编译（In-flight Computation）跨代污染根因，给出代际号 + 读写锁原子修复与零 sleep 确定性并发 GoogleTest 验证。","\u002Fimages\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix\u002Fxgrammar-cache-fix-cover-20261022.webp","双层磨砂玻璃流道经暖橙色代际闸门过滤后汇入晶格缓存矩阵的等距技术插画","在大语言模型服务（如 SGLang、vLLM）的结构化输出场景中，JSON Schema 或 EBNF 语法需要先编译为词表级掩码缓存（CompiledGrammar），才能在每个 Decode 步以微秒级开销生成合法 token ",{"slug":6926,"path":6927,"title":6928,"date":6929,"tags":6930,"description":6936,"cover":6937,"coverAlt":6938,"excerpt":6939,"draft":6833,"hidden":6833,"published":6836,"readingTime":1247},"multi-constraint-agent-routing-dag-eval","\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval","多约束生产级 Agent 系统实践：两阶段工具路由、DAG 局部重规划与轨迹级评测闭环","2026-10-22",[6900,6931,6932,6933,6934,6935],"LLM","工具调用","上下文工程","DPO","强化学习","围绕跨境酒旅等强耦合多约束业务场景，拆解向量 Top-K 召回 + LLM 精排 + 跨语言实体链接与 Schema 门控（96%+ 调用合法率）、基于状态机的 DAG 局部子图重规划、Observation 冗余裁剪（节省 40% Token），以及从线上失败恢复轨迹构造 SFT\u002FDPO 偏好对的评测与训练闭环。","\u002Fimages\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval\u002Fagent-dag-routing-cover-20261020.webp","候选模块经漏斗滤网进入双分支有向无环图轨道并在受阻节点触发暖橙色局部旁路重连的等距技术插画","在玩具级 Demo 里，几个工具加上一段 ReAct 循环就能跑通大多数单步问答。但当 Agent 进入跨境酒旅、复杂交易或研发沙箱等真实业务生产环境时，单靠线性 ReAct 循环很快会撞上四堵墙： 1. 工具空间与参数耦合爆炸：候",{"slug":6941,"path":6942,"title":6943,"date":6944,"tags":6945,"description":6949,"cover":6950,"coverAlt":6951,"excerpt":6952,"draft":6833,"hidden":6833,"published":6836,"readingTime":6953},"llm-mrd-distillation-auditable-review-agents","\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents","多视角推理蒸馏（LLM-MRD）与可审计生成式评审系统（DeepReviewer 2.0）深度拆解","2026-10-21",[6946,6899,6947,6900,6948,6874],"知识蒸馏","Qwen2.5-VL","Post-training","对比大模型落地的两条互补路径：如何用 Qwen2.5-VL 离线多视角分析结合 MLP 残差校准与温度缩放 KL 蒸馏，将推理能力压入 358M 学生模型实现 17.2 ms 独立推理（LLM-MRD，DASFAA 2026 Oral）；以及如何通过主张-证据-风险台账、PASA 可比性门控与导出预算校验构建可审计的同行评审 Agent（DeepReviewer 2.0）。","\u002Fimages\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents\u002Fmrd-review-distill-cover-20261016.webp","三块半透明几何棱镜发出平行光束穿过中央校准环汇聚至右侧高密度晶体模块的等距技术插画","把大语言模型或视觉语言模型（VLM）的深度推理能力用到实际系统中，通常会面临一个两难选择： 如果场景受限于毫秒级在线延迟（如高吞吐多模态内容审核与虚假信息检测）：直接在线调用 7B～72B 的多模态大模型做长链条思维链（CoT）生成",18,{"slug":6955,"path":6956,"title":6957,"date":6944,"tags":6958,"description":6961,"cover":6962,"coverAlt":6963,"excerpt":6964,"draft":6833,"hidden":6833,"published":6836,"readingTime":1166,"series":6840,"seriesOrder":1144,"codeUrl":54,"attachmentUrl":6965},"cs336-rope-rotary-position-embedding","\u002Fposts\u002Fcs336-rope-rotary-position-embedding","Stanford CS336 学习记录 05：RoPE 旋转位置编码与带位置的多头注意力",[6843,6844,6845,6959,6960],"RoPE","Attention","从二维平面正交旋转推导 RoPE 的奇偶特征配对公式，理解为什么内积只依赖相对位置差 (i - j)，并用 register_buffer 预计算 cos\u002Fsin 缓存接入因果多头自注意力。","\u002Fimages\u002Fposts\u002Fcs336-rope-rotary-position-embedding\u002Fcs336-rope-rotary-position-embedding-cover.webp","二维特征平面中的旋转向量对与预计算余弦正弦网格的 AI 辅助概念插画","上一篇完成了不含位置编码的因果多头自注意力。如果只看那一套矩阵乘法，把输入序列前两个 token 交换顺序（并在因果 mask 允许的范围内看同一个后缀 query 对它们的点积），点积本身并不知道哪个 key 离当前 query ","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-05-rope-mha-20261021.zip",{"slug":6967,"path":6968,"title":6969,"date":6970,"tags":6971,"description":6977,"cover":6978,"coverAlt":6979,"excerpt":6980,"draft":6833,"hidden":6833,"published":6836,"readingTime":1177},"llm-gateway-semantic-cache-dynamic-batching","\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching","高并发 LLM 推理网关架构实践：两级精确\u002F语义缓存、50ms 有界动态批处理与尾部延迟治理","2026-10-19",[6871,6972,6973,6974,6975,6976],"LLM 推理","分布式系统","Redis","FastAPI","后端","深度拆解生产级大模型推理网关 LLM-Gateway 的核心系统设计：基于 SHA-256 与 Redis 向量余弦相似度的两级精确\u002F语义缓存、50ms 有界等待动态批处理调度器（Dynamic Batching）、令牌桶多租户限流与 Prometheus 全链路可观测性，将命中请求 P99 延迟从 5,307 ms 压降至 17 ms。","\u002Fimages\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching\u002Fllm-gateway-cover-20261019.webp","高并发请求流依次穿过精确哈希滤网、向量球面语义缓存层与 50ms 动态批处理时间窗拼装器的等距系统架构插画","在将大语言模型接入高并发业务系统时，如果让上游微服务直接裸连底层推理引擎（vLLM \u002F SGLang 或外部模型 API），系统很快会暴露出三个典型的生产痛点： 1. 高频重复与近义查询消耗巨量算力：大量用户请求在语义上高度等价（例",{"slug":6982,"path":6983,"title":6984,"date":6970,"tags":6985,"description":6987,"cover":6988,"coverAlt":6989,"excerpt":6990,"draft":6833,"hidden":6833,"published":6836,"readingTime":1247,"series":6840,"seriesOrder":1136,"codeUrl":6991,"attachmentUrl":6992},"cs336-multihead-attention-shapes","\u002Fposts\u002Fcs336-multihead-attention-shapes","Stanford CS336 学习记录 04：多头注意力怎样拆头、计算和合头",[6843,6844,6845,6960,6986],"张量形状","沿着 (2, 5, 64) 的输入追踪 QKV 投影、reshape、transpose、共享因果 mask 和合头，理解每个头仍处理完整序列，以及最后为什么还需要输出 Linear。","\u002Fimages\u002Fposts\u002Fcs336-multihead-attention-shapes\u002Fcs336-multihead-attention-shapes-cover.webp","宽向量束分成多条并行特征轨道后重新合并的 AI 辅助概念插画","上一篇完成了缩放点积注意力。那个函数接收已经准备好的 Q、K、V，计算匹配分数，应用 mask，再用权重汇总 V。这一篇继续把它放进一个带参数的多头自注意力模块。 这次最需要看清的是“拆头”。代码里只是几行 reshape 和 tr","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002F31363223498fafd1528c550a51efbe0aed231736","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-04-multihead-attention-20261019.zip",{"slug":6994,"path":6995,"title":6996,"date":6997,"tags":6998,"description":7002,"cover":7003,"coverAlt":7004,"excerpt":7005,"draft":6833,"hidden":6833,"published":6836,"readingTime":1188},"qwen25-sft-dpo-alignment-tax-ablation","\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation","大模型后训练偏好对齐实录：Qwen2.5 SFT 多维消融、动态参考 DPO 与对齐税（Alignment Tax）量化分析","2026-10-18",[6948,6934,6999,7000,6935,7001],"SFT","LoRA","Qwen2.5","基于 Qwen2.5 完整复现 Instruct SFT → Bradley-Terry DPO 两阶段后训练管线：通过 3 随机种子消融 LoRA Rank、学习率与数据规模（GSM8K +12.73pp），实现免双倍权重的动态 Adapter 切换 DPO 训练，并定量揭示偏好对齐对确定性数学推理的对齐税效应。","\u002Fimages\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation\u002Fqwen25-sft-dpo-cover-20261018.webp","左侧监督微调轨迹流经中央 Bradley-Terry 偏好分流天平分离出正负样本通道并连接右侧对齐税监控仪表盘的等距技术插画","在大语言模型后训练（PostTraining）管线中，监督微调（SFT）负责教会模型任务格式与领域推理模式，而直接偏好优化（DPO, Direct Preference Optimization）则利用成对偏好数据 $(x, yw,",{"slug":7007,"path":7008,"title":7009,"date":6997,"tags":7010,"description":7012,"cover":7013,"coverAlt":7014,"excerpt":7015,"draft":6833,"hidden":6833,"published":6836,"readingTime":1210,"series":6840,"seriesOrder":1130,"codeUrl":7016,"attachmentUrl":7017},"cs336-softmax-attention-mask","\u002Fposts\u002Fcs336-softmax-attention-mask","Stanford CS336 学习记录 03：Softmax 的数值稳定与 Attention Mask",[6843,6844,6845,7011,6960],"Softmax","从减去最大值为何不改变 Softmax 开始，理清函数与有参数层的区别，再沿着 Q、K、V 的形状理解缩放点积注意力，以及 mask 为什么必须填负无穷。","\u002Fimages\u002Fposts\u002Fcs336-softmax-attention-mask\u002Fcs336-softmax-attention-mask-cover.webp","注意力分数、权重与因果遮罩的 AI 辅助概念插画","上一篇写到 SwiGLU，接下来进入 Attention。真正开始相乘之前，先做了 Softmax：它把一组分数变成总和为 1 的权重，后面用这些权重汇总不同 token 提供的信息。 这段学习里，有两个很具体的问题值得记下来。一个","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fa39eb684710817f7f389626423fcb26ca0f6d629","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-03-softmax-sdpa-20261018.zip",{"slug":7019,"path":7020,"title":7021,"date":7022,"tags":7023,"description":7024,"cover":7025,"coverAlt":7026,"excerpt":7027,"draft":6833,"hidden":6833,"published":6836,"readingTime":1188,"series":6840,"seriesOrder":1111,"codeUrl":7016,"attachmentUrl":7028},"cs336-rmsnorm-swiglu","\u002Fposts\u002Fcs336-rmsnorm-swiglu","Stanford CS336 学习记录 02：RMSNorm 的广播与 SwiGLU 的门控","2026-10-16",[6843,6844,6845,23,26],"用一个 token 向量理解均方根归一化，拆开 dim=-1 与 keepdim=True 的作用，再沿着两条分支读懂 SiLU、逐元素乘法和 SwiGLU。","\u002Fimages\u002Fposts\u002Fcs336-rmsnorm-swiglu\u002Fcs336-rmsnorm-swiglu-cover.webp","特征尺度整理和双分支门控的 AI 辅助概念插画","上一篇把 Linear 和 Embedding 的参数形状理顺了。继续往下，最先遇到的是 RMSNorm：公式不算长，但把它落到一批 token 上时，dim=1、keepdim=True 和广播必须一起理解。 然后是 SiLU 和","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-02-rmsnorm-swiglu-20261016.zip",{"slug":7030,"path":7031,"title":7032,"date":7033,"tags":7034,"description":7035,"cover":7036,"coverAlt":7037,"excerpt":7038,"draft":6833,"hidden":6833,"published":6836,"readingTime":1199,"series":6840,"seriesOrder":1073,"codeUrl":7016,"attachmentUrl":7039},"cs336-linear-embedding-shapes","\u002Fposts\u002Fcs336-linear-embedding-shapes","Stanford CS336 学习记录 01：从 Linear、Embedding 搞清张量形状","2026-10-14",[6843,6844,6845,16,20],"从 PyTorch 类、参数与 forward 的关系开始，理解 Linear 为什么保存两行三列的权重、计算时为什么要转置，再区分 Embedding 查表与线性变换。","\u002Fimages\u002Fposts\u002Fcs336-linear-embedding-shapes\u002Fcs336-linear-embedding-shapes-cover.webp","token 方块、向量表与矩阵变换的 AI 辅助概念插画","开始沿着 Stanford CS336 的 Assignment 1 一步步实现。BPE 之前已经接触过，这轮先跳过分词器，从 Transformer 的基础层开始。这里记录的是当时写代码时真正卡住的问题，以及对应测试能确认到的范围","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-01-linear-embedding-20261014.zip",{"slug":7041,"path":7042,"title":7043,"date":7044,"tags":7045,"description":7050,"cover":7051,"coverAlt":7052,"excerpt":7053,"draft":6833,"hidden":6833,"published":6836,"readingTime":1221,"series":7046,"seriesOrder":1247,"codeUrl":7054,"attachmentUrl":7055},"cmu-15-779-kv-cache-granularity-study","\u002Fposts\u002Fcmu-15-779-kv-cache-granularity-study","CMU 15-779 学习记录 14：KV 页大小、共享边界与 CPU 实现成本","2026-10-12",[7046,7047,7048,7049],"CMU 15-779","KV cache","PagedAttention","系统实验","用 210 组 CPU 配置比较连续与分页 KV cache，观察 31\u002F32-token 共享前缀的页边界，分别记录活跃槽位、池容量、复制计数和完整调用耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling\u002Fcmu-serving-cover-20261019.webp","浅色手绘分页存储抽屉与共享前缀卡片，旁边连接不同长度的请求队列","个人课程项目中的 CPU 实验记录，延续记录 11 的分页 KV 实现。已完成 210 组配置与 1470 次计时，输出均通过连续存储参考对照；本篇不代表正式作业提交、整门课程或项目全部交付完成。 同样是共享 31 个 token，","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F200faf569eccf94fb125e65110bb5d14094d454a","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fkv-cache-granularity-20261019.zip",{"slug":7057,"path":7058,"title":7059,"date":7060,"tags":7061,"description":7064,"cover":7065,"coverAlt":7066,"excerpt":7067,"draft":6833,"hidden":6833,"published":6836,"readingTime":1144,"series":7046,"seriesOrder":1227,"codeUrl":7068,"attachmentUrl":7069},"cmu-15-779-moe-routing-capacity","\u002Fposts\u002Fcmu-15-779-moe-routing-capacity","CMU 15-779 学习记录 13：MoE 路由、稀疏分发与容量溢出","2026-10-09",[7046,7062,7063],"MoE","Routing","用 CPU top-2 MoE 检查稀疏分发的输出与梯度，统计 expert-token 行数，并保留容量丢弃会改变输出的反例。","\u002Fimages\u002Fposts\u002Fcmu-15-779-moe-routing-capacity\u002Fcmu-moe-cover-20261019.webp","浅色手绘分拣器将小珠分配到四个专家工作屋，队列呈现负载不均","第 18 讲的个人配套实验。36 号脚本串行运行 CPU 专家，检查 topk 路由后的分发与合并；没有训练 router，也没有执行 GPU grouped GEMM 或跨设备 alltoall。 上一节记录低秩适配，这次转向每个","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fc4851e0cedadf13af7c421867e83c8ff57244fa0","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-parallel-serving-lora-moe-20261019.zip",{"slug":7071,"path":7072,"title":7073,"date":7074,"tags":7075,"description":7077,"cover":7078,"coverAlt":7079,"excerpt":7080,"draft":6833,"hidden":6833,"published":6836,"readingTime":1144,"series":7046,"seriesOrder":1221,"codeUrl":7068,"attachmentUrl":7069},"cmu-15-779-lora-uniform-quantization","\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization","CMU 15-779 学习记录 12：LoRA 首步梯度、权重合并与均匀 4-bit 打包","2026-10-06",[7046,7000,7076],"Quantization","用一次 CPU SGD 更新检查 LoRA 初始化、冻结基座与合并等价，再分别记录均匀 4-bit 存储字节数和量化误差。","\u002Fimages\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization\u002Fcmu-lora-cover-20261019.webp","浅色手绘的大计算板、两块细长适配板和成对打包的小方块","第 17 讲的个人配套实验，35 号脚本实际执行一次 CPU LoRA 更新，并实现均匀 4bit 打包。实验检查机制与数值误差，不评估微调任务效果，也没有实现完整 QLoRA。 这次把两个问题分开记录：低秩分支是否能合并进原权重，",{"slug":7082,"path":7083,"title":7084,"date":7085,"tags":7086,"description":7090,"cover":7051,"coverAlt":7091,"excerpt":7092,"draft":6833,"hidden":6833,"published":6836,"readingTime":1166,"series":7046,"seriesOrder":1210,"codeUrl":7068,"attachmentUrl":7069},"cmu-15-779-kv-serving-speculative-sampling","\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling","CMU 15-779 学习记录 11：分页 KV、连续批处理与推测采样","2026-10-03",[7046,7087,7088,7089],"LLM Serving","KV Cache","Speculative Decoding","在 CPU 上验证 KV 页共享和写时复制，用单位成本模型检查连续批处理，再用条件概率小模型核对推测采样的接受与补采样。","浅色手绘的共享卡片抽屉与不断补入小珠子的请求托盘","第 15、16 讲的个人配套实验。33 号实现 CPU 分页 KV 存储，37 号模拟请求准入，34 号运行小概率模型的推测采样。它们分别验证存储、调度和分布问题，没有测量真实 LLM 服务吞吐或 GPU 延迟。 记录 08已经对照",{"slug":7094,"path":7095,"title":7096,"date":7097,"tags":7098,"description":7103,"cover":7104,"coverAlt":7105,"excerpt":7106,"draft":6833,"hidden":6833,"published":6836,"readingTime":1259},"qwen3-mid-post-train-mlx","\u002Fposts\u002Fqwen3-mid-post-train-mlx","在 M2 Pro 上完成 Qwen3 两阶段训练：Mid Train、SFT 与可复核评估","2026-09-30 11:19:10",[6931,7099,7100,7000,7101,6999,7102],"Qwen3","MLX","持续预训练","Apple Silicon","用 Qwen3 0.6B Base 和 MLX 完成领域持续训练与监督微调，记录数据划分、聊天模板、损失掩码、数值稳定性修复和独立测试结果。","\u002Fimages\u002Fposts\u002Fqwen3-mid-post-train-mlx\u002Fqwen3-mlx-cover-20261018.webp","Apple Silicon 统一内存芯片底座支撑两阶段持续预训练与带精确 Loss 掩码滤网的监督微调桥接插画","在一台 M2 Pro、16 GB 内存的 Mac 上，能否把一个 Base 语言模型接续训练成输出 JSON 的运维事件分级助手？这次项目完成了领域持续训练、监督微调、检查点选择、重新加载和独立测试，并保留了权重、日志和数据哈希。 ",{"slug":7108,"path":7109,"title":7110,"date":7111,"tags":7112,"description":7116,"cover":7117,"coverAlt":7118,"excerpt":7119,"draft":6833,"hidden":6833,"published":6836,"readingTime":1155,"series":7046,"seriesOrder":1199,"codeUrl":7068,"attachmentUrl":7069},"cmu-15-779-cpu-parallelism-pipeline","\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline","CMU 15-779 学习记录 10：加权梯度、状态分片与同步流水线","2026-09-30",[7046,7113,7114,7115],"Parallel Training","Gloo","Pipeline","用双 CPU 进程 Gloo 检查不等数据分片的梯度，推导持久状态字节数，再用依赖模型比较 GPipe 与同步 1F1B 的激活存活。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline\u002Fcmu-parallel-cover-20261019.webp","浅色手绘的四阶段桌面流水线与按数量汇总的托盘","第 12 至 14 讲的个人配套实验，脚本编号 31、32 不对应官方作业。Gloo 实际运行两个 CPU 进程；张量切分、状态字节数和流水线分别按下文的数学或成本模型解释。真实 GPU 实测按学习安排跳过。 上一节检查单个计算图的",{"slug":7121,"path":7122,"title":7123,"date":7124,"tags":7125,"description":7128,"cover":7129,"coverAlt":7130,"excerpt":7131,"draft":6833,"hidden":6833,"published":6836,"readingTime":1166,"series":7046,"seriesOrder":1188,"codeUrl":7132,"attachmentUrl":7133},"cmu-15-779-cpu-compiler-optimizations","\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations","CMU 15-779 学习记录 09：CPU 编译优化、合法重写与候选搜索","2026-09-27",[7046,7126,7127,6844],"ML Compiler","TorchInductor","用 CPU 实验验证 BatchNorm 折叠、RMSNorm 重排与非线性反例，测量分块候选，并运行 TorchInductor，分别记录首调用与预热后的时间。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations\u002Fcmu-cpu-compiler-cover-20261019.webp","淡紫计算积木经过薄荷绿折叠模具变成浅蓝积木，前方摆放四种候选托盘的浅色手绘插画","编译章节的个人配套实验，编号 30 不对应官方作业。图重写、分块候选和 TorchInductor 均在 CPU 上运行；真实 GPU 实验按学习安排跳过。 上一节记录了分块 Attention 和 KV cache。这次转到编译优","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F9261bea6aedf31cdccbbab2541ca693fc97b3743","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-compiler-optimizations-20261019.zip",{"slug":7135,"path":7136,"title":7137,"date":7138,"tags":7139,"description":7141,"cover":7142,"coverAlt":7143,"excerpt":7144,"draft":6833,"hidden":6833,"published":6836,"readingTime":1221,"series":7046,"seriesOrder":1177,"codeUrl":7054,"attachmentUrl":7145},"cmu-15-779-attention-online-softmax","\u002Fposts\u002Fcmu-15-779-attention-online-softmax","CMU 15-779 学习记录 08：分块 Attention、在线 Softmax 与 KV cache","2026-09-24",[7046,6960,7011,7140,6844],"FlashAttention","在 CPU 上对照完整 Attention、在线 Softmax 分块实现与 PyTorch SDPA，检查因果掩码、浮点误差和分数块大小，理解 FlashAttention 的一部分算法基础。","\u002Fimages\u002Fposts\u002Fcmu-15-779-attention-online-softmax\u002Fcmu-online-attention-cover-20261016.webp","浅紫色卡片分批进入绿色计算托盘，旁边三个状态容器保存累积量的浅色手绘插画","第 5 讲 Transformers & Attention 的个人配套实验。28\u002F29 号是 CPU 单头前向与 KV cache；后续补入 38 号 CUDA 多头合成张量的显式 SDPA MATH\u002FFLASH 对照。没有手写 ","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcuda-triton-sdpa-20261019.zip",{"slug":7147,"path":7148,"title":7149,"date":7150,"tags":7151,"description":7156,"cover":7157,"coverAlt":7158,"excerpt":7159,"draft":6833,"hidden":6833,"published":6836,"readingTime":1199,"series":7046,"seriesOrder":1166,"codeUrl":7054,"attachmentUrl":7145},"cmu-15-779-matmul-tiling-tree-reduction","\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction","CMU 15-779 学习记录 07：矩阵分块、数据复用与树形归约","2026-09-21",[7046,7152,7153,7154,7155],"CUDA","矩阵乘法","数据复用","归约","用 CPU 模型统计矩阵分块的输入读取次数，再比较串行求和和树形归约的依赖深度，记录补零、同步与浮点加法顺序的边界。","\u002Fimages\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction\u002Fcmu-matmul-reduction-cover-20261016.webp","矩阵积木在共享托盘中复用，旁边的小珠沿树形管道逐层合并的浅色手绘插画","GPU Architecture & CUDA Programming 的个人配套实验。首次记录用 CPU 模型检查数据复用与依赖深度；后续补入单卡 GPU 原生 CUDA 矩阵验证，以及 Triton 两阶段归约和矩阵乘法实测。 ",{"slug":7161,"path":7162,"title":7163,"date":7164,"tags":7165,"description":7168,"cover":7169,"coverAlt":7170,"excerpt":7171,"draft":6833,"hidden":6833,"published":6836,"readingTime":1199,"series":7046,"seriesOrder":1155,"codeUrl":7054,"attachmentUrl":7145},"cmu-15-779-cuda-thread-indexing","\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing","CMU 15-779 学习记录 06：CUDA 线程索引与边界检查","2026-09-18",[7046,7152,6873,7166,7167],"线程索引","边界检查","用 CPU 演示 CUDA 的 block、thread 与全局下标：10 个元素如何分给 3 个线程块，为什么要跳过越界线程，以及这些验证能说明什么。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing\u002Fcmu-cuda-indexing-cover-20261015.webp","分组的浅紫色计算方块与末端边界挡板组成线程分配教具的浅色插画","GPU Architecture & CUDA Programming 的个人配套练习。首次记录为 CPU 索引演示；后续在单卡 GPU 环境补入原生 CUDA 编译、运行与 memcheck，以及 Triton 向量加法实测。 上",{"slug":7173,"path":7174,"title":7175,"date":7176,"tags":7177,"description":7181,"cover":7182,"coverAlt":7183,"excerpt":7184,"draft":6833,"hidden":6833,"published":6836,"readingTime":1199,"series":7046,"seriesOrder":1144,"codeUrl":7185,"attachmentUrl":7186},"cmu-15-779-profiler-operator-batching","\u002Fposts\u002Fcmu-15-779-profiler-operator-batching","CMU 15-779 学习记录 05：Profiler、算子调用与批量推理","2026-09-15",[7046,6844,7178,7179,7180],"Profiler","批量推理","性能分析","对照 ML Systems 基础，用 Profiler 拆开两层网络的执行过程，再固定 1000 个样本扫描 batch size，观察算子调用次数、尾批和 CPU 耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-profiler-operator-batching\u002Fcmu-pytorch-profiling-cover-20261015.webp","放大镜观察线性层与 ReLU 模块，旁边面板呈现执行片段的浅色技术教具插画","本篇对照课程官网的 Introduction to ML Systems 整理，是个人配套 CPU 实验，不是课程官方作业。脚本编号是练习顺序，不对应课程讲次。 前几篇一直在补 PyTorch 基础。到了这一步，我在学习聊天里问：现","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fcecf5110d8a95161bb5763eadfca94cd7a0cc50f","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-profiler-batching-20261015.zip",{"slug":7188,"path":7189,"title":7190,"date":7191,"tags":7192,"description":7195,"cover":7196,"coverAlt":7197,"excerpt":7198,"draft":6833,"hidden":6833,"published":6836,"readingTime":1188,"series":7046,"seriesOrder":1136,"codeUrl":7199,"attachmentUrl":7200},"cmu-15-779-pytorch-linear-relu-absolute-value","\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value","CMU 15-779 学习记录 04：多输出线性层、ReLU 与绝对值网络","2026-09-12",[7046,6844,7153,7193,7194],"ReLU","神经网络","把 nn.Linear 对应到矩阵乘法，弄清多个输出怎样计算，再逐层拆开一个绝对值网络，比较手动设置参数与训练得到的结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value\u002Fcmu-pytorch-absolute-cover-20261015.webp","双分支经过折角模块后汇入紫色 V 形曲线面板的浅色技术教具插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的批量推理与矩阵乘法，我继续把 model(x) 拆开看：一层里有多个输出时，每个数是怎么算出来的？两","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F4cc63904a3a583da4a2122c246a14b43aac38528","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-linear-relu-abs-20261015.zip",{"slug":7202,"path":7203,"title":7204,"date":7205,"tags":7206,"description":7209,"cover":7210,"coverAlt":7211,"excerpt":7212,"draft":6833,"hidden":6833,"published":6836,"readingTime":1188,"series":7046,"seriesOrder":1130,"codeUrl":7213,"attachmentUrl":7214},"cmu-15-779-pytorch-dataloader-checkpoints-batching","\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching","CMU 15-779 学习记录 03：DataLoader、模型保存与批量推理","2026-09-09",[7046,6844,7207,7179,7208],"DataLoader","模型保存","用 DataLoader 分批训练平方函数，保存并加载模型参数，再比较 1000 个样本与 1 个样本的推理耗时，最后把批量计算对应到矩阵乘法。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching\u002Fcmu-pytorch-batching-cover-20261015.webp","样本卡片按批次进入计算模块，旁边配有参数存储盒和输出矩阵面板的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的平方函数实验，这一节继续练习分批训练、保存参数和独立预测。最后做了一次计时：同一个小网络，逐个处理 ","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-batching-20261015.zip",{"slug":7216,"path":7217,"title":7218,"date":7219,"tags":7220,"description":7221,"cover":7222,"coverAlt":7223,"excerpt":7224,"draft":6833,"hidden":6833,"published":6836,"readingTime":1188,"series":7046,"seriesOrder":1111,"codeUrl":7213,"attachmentUrl":7225},"cmu-15-779-pytorch-linear-module-relu","\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu","CMU 15-779 学习记录 02：nn.Linear、张量形状与 ReLU","2026-09-06",[7046,6844,7194,7193],"从带偏置的线性回归开始，弄清 nn.Linear 和 nn.Module 的分工，再用平方函数观察线性模型的限制、ReLU 的作用和测试点的选择。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu\u002Fcmu-pytorch-relu-cover-20261015.webp","输入经过两层神经网络和 ReLU 折角模块，输出曲线拟合结果的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 上一篇里，我用一个参数拟合了 $y=2x$。这一篇继续顺着往下写，先加上偏置，再把手写参数换成 nn.Linea","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-models-20261015.zip",{"slug":7227,"path":7228,"title":7229,"date":7230,"tags":7231,"description":7234,"cover":7235,"coverAlt":7236,"excerpt":7237,"draft":6833,"hidden":6833,"published":6836,"readingTime":1199,"series":7046,"seriesOrder":1073,"codeUrl":7213,"attachmentUrl":7238},"cmu-15-779-pytorch-autograd-linear-regression","\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression","CMU 15-779 学习记录 01：PyTorch 自动求导与线性回归","2026-09-03",[7046,6844,7232,7233],"自动求导","线性回归","从自动求导和手动梯度下降开始，比较学习率，再用 SGD 训练线性回归。记录 w * 4 写错后停在 1.25 的原因和修正结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression\u002Fcmu-pytorch-autograd-cover-20261015.webp","梯度曲面上的小球沿路径下降，前侧面板展示散点与拟合直线的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 开始学习 CMU 15779 时，我会 Python，但还没用过 PyTorch。这次先在 Mac 上用 CPU","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-basics-20261013.zip",{"slug":7240,"path":7241,"title":7242,"date":7243,"tags":7244,"description":7250,"cover":7251,"coverAlt":7252,"excerpt":7253,"draft":6833,"hidden":6833,"published":6836,"readingTime":1227,"codeUrl":7254,"attachmentUrl":7255},"embodied-pickplace-lab-robustness","\u002Fposts\u002Fembodied-pickplace-lab-robustness","Pickplace Lab 项目复盘：抓取成功率之外的时延与失败分析","2026-08-29",[7245,7246,7247,6844,7248,7249],"具身智能","模仿学习","MuJoCo","机器人","鲁棒性","从 MuJoCo 抓取搬运闭环出发，记录 22,100 个评测回合。动作平滑在自建任务的 50 ms 延迟下有效，换到 Panda Lift 却没有支持同样的收益。","\u002Fimages\u002Fposts\u002Fembodied-pickplace-lab-robustness\u002Fpickplace-grasp-cover-20261016.webp","MuJoCo 实际渲染的笛卡尔夹爪抓取方块场景，桌面上标出放置目标","Pickplace Lab 从一个简单任务开始：让夹爪抓起方块，搬到指定位置，放下，再退开。先把仿真、示范采集、训练和闭环评测接起来，随后再问：干净条件下成功的策略，换一种执行条件，还能不能完成任务？ 最值得讨论的结果出现在指令延迟","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Ftree\u002Fv0.1.0","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Freleases\u002Fdownload\u002Fv0.1.0\u002Fpickplace-lab-research-v1.zip",{"slug":7257,"path":7258,"title":7259,"date":7260,"tags":7261,"description":7266,"cover":7267,"coverAlt":7268,"excerpt":7269,"draft":6833,"hidden":6833,"published":6836,"readingTime":7270},"qwen25-vl-sglang-verl-single-gpu-lab","\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab","从 Mac 到单卡 GPU：Qwen2.5-VL 部署、GRPO 与计数 LoRA 实践","2026-08-19",[6947,6899,7262,7263,7264,7265,7100],"SGLang","verl","GRPO","推理优化","在 M2 Pro 和云端单卡上部署 Qwen2.5-VL-3B，实测连续批处理、前缀缓存与 SGLang 参数调优，再用 verl 完成多模态 GRPO 更新和同条件前后评测，记录有效优化、兼容性问题与未观察到提升的结果。","\u002Fimages\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab\u002Fqvl-lab-cover-20261010.webp","笔记本与单卡计算模块通过图像和 token 光轨连接的实验台插画","这次实验从一个很具体的问题开始：M2 Pro、16 GB 统一内存，能不能在本机跑 Qwen2.5VL？跑起来之后，我又想把图文问答、推理性能测量和强化学习训练接成一条完整流程。 本机的 MLX 4bit 服务和云端的 SGLang",29,{"slug":7272,"path":7273,"title":7274,"date":7275,"tags":7276,"description":7280,"cover":7281,"coverAlt":7282,"excerpt":7283,"draft":6833,"hidden":6833,"published":6836,"readingTime":1227},"cycleresearcher-post-training","\u002Fposts\u002Fcycleresearcher-post-training","从 DeepReviewer 2.0 到 CycleResearcher：科研 Agent 如何真正做 Post-training","2026-08-07 17:33:05",[6931,6948,6900,7277,7278,7279],"SimPO","Reward Model","ICLR","区分推理时 Agent 编排与模型后训练，并拆解 CycleResearcher 的领域 SFT、生成式 Reward Model、Iterative SimPO 和评测风险。","\u002Fimages\u002Fposts\u002Fcycleresearcher-post-training\u002Fcycleresearcher-cover-20261018.webp","策略生成模块产出多份候选文稿经多维评审棱镜排序后通过偏好梯度反馈弧闭环优化的等距技术插画","看到一个系统会检索、规划、反思和调用工具，不能据此判断它做过 Posttraining。Agent 描述的是模型在运行时如何完成任务；Posttraining 描述的是预训练之后，是否继续用监督数据、偏好数据或奖励信号更新模型参数。",{"slug":7285,"path":7286,"title":7287,"date":7288,"tags":7289,"description":7291,"cover":7292,"coverAlt":7293,"excerpt":7294,"draft":6833,"hidden":6833,"published":6836,"readingTime":1247,"series":7295,"seriesOrder":1073},"cuda-fused-rmsnorm-kernel","\u002Fposts\u002Fcuda-fused-rmsnorm-kernel","从 PyTorch 到 CUDA Kernel：融合 RMSNorm 在 RTX 4090D 上加速 4.27 倍","2026-07-22 18:16:50",[7152,6871,6873,6844,6972,7290],"Kernel 优化","从零实现 PyTorch CUDA 扩展，拆解 RMSNorm 的访存瓶颈、warp 归约、向量化加载、算子融合与冷热缓存 benchmark。","\u002Fimages\u002Fposts\u002Fcuda-fused-rmsnorm-kernel\u002Fcuda-rmsnorm-cover-20261018.webp","融合残差加法与向量化 float4 访存经 Warp Shuffle 蝴蝶树归约输出归一化张量的等距硬件架构插画","为了准备 AI Infra 岗位，我做了一个尽量小而完整的 CUDA 项目：不追求堆很多算子，而是选一个真实的 LLM 推理路径，把实现、正确性、benchmark 和性能解释全部做扎实。 项目实现了两个 PyTorch CUDA ","AI Infra 实战",{"slug":7297,"path":7298,"title":7299,"date":7300,"tags":7301,"description":7303,"cover":7304,"coverAlt":7305,"excerpt":7306,"draft":6833,"hidden":6833,"published":6836,"readingTime":1177,"series":7307,"seriesOrder":1073},"agent-memory","\u002Fposts\u002Fagent-memory","Agent 上下文工程与分层记忆架构：从 Observation 剪枝、滚动摘要到向量召回闭环","2026-04-24",[6931,6900,7302,6871],"RAG","结合生产级多约束 Agent 与超 100K Token 科研智能体实践，系统拆解长程会话中的上下文膨胀与注意力稀释问题：涵盖 Observation 结构化剪枝（降低约 40% Token 开销）、滚动工作摘要、向量检索长期记忆及 Prefix Cache 友好的上下文组装。","\u002Fimages\u002Fposts\u002Fagent-memory\u002Fagent-memory-cover-20261018.webp","底层高噪数据经剪枝滤网进入中期滑动窗口与金色状态台账并由顶层向量星座按需召回的三层记忆架构插画","在构建多步工具调用（ToolUse）与长周期自主规划（Autonomous Planning）的 Agent 系统时，上下文管理（Context Engineering）往往比单纯改写 System Prompt 更决定线上稳定性。","Agent 上下文工程",{"slug":7309,"path":7310,"title":7311,"date":7312,"tags":7313,"description":7316,"cover":7317,"coverAlt":7318,"excerpt":7319,"draft":6833,"hidden":6833,"published":6836,"readingTime":1144,"series":7320,"seriesOrder":1111},"backend-stack-deep-dive","\u002Fposts\u002Fbackend-stack-deep-dive","高并发 AI 平台后端基础设施实践：FastAPI 异步网关、SQLAlchemy 2.0 Async、PostgreSQL 与 Docker 沙箱编排","2026-04-07",[6976,6973,6975,7314,7315,6871],"PostgreSQL","Docker","结合 DeepScientist 生产级 AI 科研平台实践，拆解 FastAPI ASGI 异步事件循环、SQLAlchemy 2.0 + AsyncPG 连接池调优、PostgreSQL + MinIO 存储分层与 Docker SDK 动态沙箱容器编排的职责边界与核心设计。","\u002Fimages\u002Fposts\u002Fbackend-stack-deep-dive\u002Fbackend-stack-cover-20261018.webp","ASGI 异步事件循环涡轮将非阻塞协程流分发至 PostgreSQL 连接池、MinIO 存储矩阵与动态 Docker 沙箱的等距架构插画","结合 DeepScientist 生产级 AI 科研平台（服务 7,000+ 用户）的实际工程落地，本文系统拆解支撑高并发流式推理与容器沙箱调度的后端基础设施栈——FastAPI（ASGI 异步网关 + SSE 流式推送）、SQLA","DeepScientist 工程复盘",{"slug":7322,"path":7323,"title":7324,"date":7325,"tags":7326,"description":7329,"cover":7330,"coverAlt":7331,"excerpt":7332,"draft":6833,"hidden":6833,"published":6836,"readingTime":1210,"series":7333,"seriesOrder":1073},"minicode-source-analysis","\u002Fposts\u002Fminicode-source-analysis","MiniCode 架构与工具评测深度拆解：开源终端 Agentic Coding 引擎（1k+ Stars）的 ReAct、MCP 与 30 项基准验证","2026-04-06",[6900,7327,7328,6920,6871],"TypeScript","MCP","深度剖析开源终端 AI 编程智能体 MiniCode（1k+ GitHub Stars）的核心架构：从零重型依赖的 ReAct 规划循环、声明式工具注册表与权限拦截层、MCP 协议帧自动协商，到 30 项可执行 Tool-Use 评测基准（86.7% 完成率、100% 错误恢复率）的失败分布归因与闭环优化。","\u002Fimages\u002Fposts\u002Fminicode-source-analysis\u002Fminicode-cover-20261018.webp","终端编程智能体控制台被 ReAct 状态环、Unified Diff 补丁预览门与 MCP 协议插槽环绕的等距技术插画","MiniCode 是我作为核心开发者参与构建的开源终端 Agentic Coding 引擎（目前已在 GitHub 获得 1,000+ Stars）。项目的核心设计目标是：在不引入庞大运行时框架的前提下，用约 5,000 行高内聚的","AI 项目源码导读",{"slug":7335,"path":7336,"title":7337,"date":7325,"tags":7338,"description":7339,"cover":7340,"coverAlt":7341,"excerpt":7342,"draft":6833,"hidden":6833,"published":6836,"readingTime":1199,"series":7320,"seriesOrder":1073},"deepscientist-tech-stack","\u002Fposts\u002Fdeepscientist-tech-stack","DeepScientist 架构与运行时全解析：超 100K 上下文科研智能体、MCP 沙箱与高并发流式系统",[6900,6871,7328,7315,6975,6973],"深度拆解服务 7,000+ 用户的科研自主智能体平台 DeepScientist：从一题一仓库（Quest）持久化状态机、MCP 工具栈（memory \u002F artifact \u002F bash_exec）、Docker SDK 动态容器编排，到 SSE + WebSocket 双流通信与 PostgreSQL \u002F Redis \u002F Prometheus 生产级基础设施。","\u002Fimages\u002Fposts\u002Fdeepscientist-tech-stack\u002Fdeepscientist-cover-20261018.webp","科研智能体编排核心通过 memory、artifact 与隔离容器沙箱三座协议桥连接双流工作台的等距架构插画","DeepScientist（deepscientist.cc）是我们在西湖大学自然语言处理实验室研发的自主科学发现与科研管理平台，目前累计服务 7,000+ 科研用户。与单轮对话或纯文本文献问答工具不同，科研智能体需要在数小时乃至数",1795088946145]