[{"data":1,"prerenderedAt":9812},["ShallowReactive",2],{"post-\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop":3,"archive-public-posts":9304},{"id":4,"title":5,"body":6,"categories":9282,"cover":9284,"coverAlt":9285,"date":9286,"description":9287,"draft":9288,"extension":9289,"haloPublished":9288,"haloSync":9288,"hidden":9288,"legacy":9288,"meta":9290,"navigation":9291,"path":9292,"published":9288,"readingTime":9293,"seo":9294,"series":9295,"seriesOrder":6095,"stem":9296,"tags":9297,"__hash__":9303},"posts\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop.md","Stanford CS336 学习记录 08：数值稳定交叉熵、AdamW、余弦调度与完整训练闭环",{"type":7,"value":8,"toc":9261},"minimark",[9,87,94,108,188,478,884,1026,1247,1951,1962,2175,2476,2554,2631,3052,3431,3594,3616,3620,3813,5492,5813,5822,5825,6247,6899,6903,6938,8436,8439,8476,8482,8657,8801,8808,8812,8839,8872,8882,9165,9168,9175,9182,9218,9230,9233,9257],[10,11,12,13,17,18,21,22,25,26,17,30,17,33,36,37,86],"p",{},"作为 Stanford CS336 Assignment 1 系列的收官篇，我们已经拥有了分词器（",[14,15,16],"code",{},"bpe.py","、",[14,19,20],{},"tokenizer.py","）和完整的语言模型架构（",[14,23,24],{},"transformer.py","）。要让模型在语料上真正训练并生成文本，还需要完成最后四大支柱：",[27,28,29],"strong",{},"数值稳定的交叉熵损失",[27,31,32],{},"AdamW 优化器与学习率\u002F梯度控制",[27,34,35],{},"流式数据采样与 Checkpoint 序列化","，以及 ",[27,38,39,40,85],{},"Temperature + Top-",[41,42,45,66],"span",{"className":43},[44],"katex",[41,46,49],{"className":47},[48],"katex-mathml",[50,51,53],"math",{"xmlns":52},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[54,55,56,62],"semantics",{},[57,58,59],"mrow",{},[60,61,10],"mi",{},[63,64,10],"annotation",{"encoding":65},"application\u002Fx-tex",[41,67,71],{"className":68,"ariaHidden":70},[69],"katex-html","true",[41,72,75,80],{"className":73},[74],"base",[41,76],{"className":77,"style":79},[78],"strut","height:0.625em;vertical-align:-0.1944em;",[41,81,10],{"className":82},[83,84],"mord","mathnormal"," 自回归解码","。",[10,88,89,90,93],{},"本篇完成后，",[14,91,92],{},"liangqianxing\u002Fcs336-labs"," 仓库通过了官方测试套件的全部测试。",[10,95,96,103,104],{},[97,98,102],"a",{"href":99,"rel":100},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fb5c93ad0cc73178b2b2a62e518a0900da47a0fb7",[101],"nofollow","查看代码仓库"," · ",[97,105,107],{"href":106},"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-08-adamw-training-loop-20261027.zip","下载本节代码与测试 ZIP",[109,110,112,113,154,155,187],"h2",{"id":111},"数值稳定-cross-entropy把-logloglog-和-expexpexp-消掉","数值稳定 Cross-Entropy：把 ",[41,114,116,135],{"className":115},[44],[41,117,119],{"className":118},[48],[50,120,121],{"xmlns":52},[54,122,123,132],{},[57,124,125,128],{},[60,126,127],{},"log",[129,130,131],"mo",{},"⁡",[63,133,134],{"encoding":65},"\\log",[41,136,138],{"className":137,"ariaHidden":70},[69],[41,139,141,145],{"className":140},[74],[41,142],{"className":143,"style":144},[78],"height:0.8889em;vertical-align:-0.1944em;",[41,146,149,150],{"className":147},[148],"mop","lo",[41,151,153],{"style":152},"margin-right:0.0139em;","g"," 和 ",[41,156,158,175],{"className":157},[44],[41,159,161],{"className":160},[48],[50,162,163],{"xmlns":52},[54,164,165,172],{},[57,166,167,170],{},[60,168,169],{},"exp",[129,171,131],{},[63,173,174],{"encoding":65},"\\exp",[41,176,178],{"className":177,"ariaHidden":70},[69],[41,179,181,184],{"className":180},[74],[41,182],{"className":183,"style":79},[78],[41,185,169],{"className":186},[148]," 消掉",[10,189,190,191,315,316,477],{},"给定未归一化 logits ",[41,192,194,230],{"className":193},[44],[41,195,197],{"className":196},[48],[50,198,199],{"xmlns":52},[54,200,201,227],{},[57,202,203,206,209],{},[60,204,205],{},"o",[129,207,208],{},"∈",[210,211,212,216],"msup",{},[60,213,215],{"mathvariant":214},"double-struck","R",[57,217,218,221,224],{},[60,219,220],{},"B",[129,222,223],{},"×",[60,225,226],{},"V",[63,228,229],{"encoding":65},"o \\in \\mathbb{R}^{B \\times V}",[41,231,233,255],{"className":232,"ariaHidden":70},[69],[41,234,236,240,243,248,252],{"className":235},[74],[41,237],{"className":238,"style":239},[78],"height:0.5782em;vertical-align:-0.0391em;",[41,241,205],{"className":242},[83,84],[41,244],{"className":245,"style":247},[246],"mspace","margin-right:0.2778em;",[41,249,208],{"className":250},[251],"mrel",[41,253],{"className":254,"style":247},[246],[41,256,258,262],{"className":257},[74],[41,259],{"className":260,"style":261},[78],"height:0.8413em;",[41,263,265,269],{"className":264},[83],[41,266,215],{"className":267},[83,268],"mathbb",[41,270,273],{"className":271},[272],"msupsub",[41,274,277],{"className":275},[276],"vlist-t",[41,278,281],{"className":279},[280],"vlist-r",[41,282,285],{"className":283,"style":261},[284],"vlist",[41,286,288,293],{"style":287},"top:-3.063em;margin-right:0.05em;",[41,289],{"className":290,"style":292},[291],"pstrut","height:2.7em;",[41,294,300],{"className":295},[296,297,298,299],"sizing","reset-size6","size3","mtight",[41,301,303,307,311],{"className":302},[83,299],[41,304,220],{"className":305,"style":306},[83,84,299],"margin-right:0.0502em;",[41,308,223],{"className":309},[310,299],"mbin",[41,312,226],{"className":313,"style":314},[83,84,299],"margin-right:0.2222em;"," 和真实类别下标 ",[41,317,319,367],{"className":318},[44],[41,320,322],{"className":321},[48],[50,323,324],{"xmlns":52},[54,325,326,364],{},[57,327,328,331,333,337,341,344,347,349,351,354,357],{},[60,329,330],{},"y",[129,332,208],{},[129,334,336],{"stretchy":335},"false","{",[338,339,340],"mn",{},"0",[129,342,343],{"separator":70},",",[129,345,346],{},"…",[129,348,343],{"separator":70},[60,350,226],{},[129,352,353],{},"−",[338,355,356],{},"1",[210,358,359,362],{},[129,360,361],{"stretchy":335},"}",[60,363,220],{},[63,365,366],{"encoding":65},"y \\in \\{0, \\dots, V-1\\}^B",[41,368,370,390,437],{"className":369,"ariaHidden":70},[69],[41,371,373,377,381,384,387],{"className":372},[74],[41,374],{"className":375,"style":376},[78],"height:0.7335em;vertical-align:-0.1944em;",[41,378,330],{"className":379,"style":380},[83,84],"margin-right:0.0359em;",[41,382],{"className":383,"style":247},[246],[41,385,208],{"className":386},[251],[41,388],{"className":389,"style":247},[246],[41,391,393,397,401,404,408,412,416,419,422,425,428,431,434],{"className":392},[74],[41,394],{"className":395,"style":396},[78],"height:1em;vertical-align:-0.25em;",[41,398,336],{"className":399},[400],"mopen",[41,402,340],{"className":403},[83],[41,405,343],{"className":406},[407],"mpunct",[41,409],{"className":410,"style":411},[246],"margin-right:0.1667em;",[41,413,346],{"className":414},[415],"minner",[41,417],{"className":418,"style":411},[246],[41,420,343],{"className":421},[407],[41,423],{"className":424,"style":411},[246],[41,426,226],{"className":427,"style":314},[83,84],[41,429],{"className":430,"style":314},[246],[41,432,353],{"className":433},[310],[41,435],{"className":436,"style":314},[246],[41,438,440,444,447],{"className":439},[74],[41,441],{"className":442,"style":443},[78],"height:1.0913em;vertical-align:-0.25em;",[41,445,356],{"className":446},[83],[41,448,451,454],{"className":449},[450],"mclose",[41,452,361],{"className":453},[450],[41,455,457],{"className":456},[272],[41,458,460],{"className":459},[276],[41,461,463],{"className":462},[280],[41,464,466],{"className":465,"style":261},[284],[41,467,468,471],{"style":287},[41,469],{"className":470,"style":292},[291],[41,472,474],{"className":473},[296,297,298,299],[41,475,220],{"className":476,"style":306},[83,84,299],"，单样本交叉熵损失定义为负对数似然：",[41,479,482],{"className":480},[481],"katex-display",[41,483,485,578],{"className":484},[44],[41,486,488],{"className":487},[48],[50,489,491],{"xmlns":52,"display":490},"block",[54,492,493,575],{},[57,494,495,499,502,504,506,508,511,514,516,518,520,572],{},[60,496,498],{"mathvariant":497},"normal","ℓ",[129,500,501],{"stretchy":335},"(",[60,503,205],{},[129,505,343],{"separator":70},[60,507,330],{},[129,509,510],{"stretchy":335},")",[129,512,513],{},"=",[129,515,353],{},[60,517,127],{},[129,519,131],{},[521,522,523,540],"mfrac",{},[57,524,525,527,529,531,538],{},[60,526,169],{},[129,528,131],{},[129,530,501],{"stretchy":335},[532,533,534,536],"msub",{},[60,535,205],{},[60,537,330],{},[129,539,510],{"stretchy":335},[57,541,542,558,560,562,564,570],{},[543,544,545,548,556],"munderover",{},[129,546,547],{},"∑",[57,549,550,552,554],{},[60,551,97],{},[129,553,513],{},[338,555,356],{},[60,557,226],{},[60,559,169],{},[129,561,131],{},[129,563,501],{"stretchy":335},[532,565,566,568],{},[60,567,205],{},[60,569,97],{},[129,571,510],{"stretchy":335},[60,573,574],{"mathvariant":497},".",[63,576,577],{"encoding":65},"\\ell(o, y) = -\\log \\frac{\\exp(o_y)}{\\sum_{a=1}^{V} \\exp(o_a)}.",[41,579,581,617],{"className":580,"ariaHidden":70},[69],[41,582,584,587,590,593,596,599,602,605,608,611,614],{"className":583},[74],[41,585],{"className":586,"style":396},[78],[41,588,498],{"className":589},[83],[41,591,501],{"className":592},[400],[41,594,205],{"className":595},[83,84],[41,597,343],{"className":598},[407],[41,600],{"className":601,"style":411},[246],[41,603,330],{"className":604,"style":380},[83,84],[41,606,510],{"className":607},[450],[41,609],{"className":610,"style":247},[246],[41,612,513],{"className":613},[251],[41,615],{"className":616,"style":247},[246],[41,618,620,624,627,630,635,638,881],{"className":619},[74],[41,621],{"className":622,"style":623},[78],"height:2.5979em;vertical-align:-1.1709em;",[41,625,353],{"className":626},[83],[41,628],{"className":629,"style":411},[246],[41,631,149,633],{"className":632},[148],[41,634,153],{"style":152},[41,636],{"className":637,"style":411},[246],[41,639,641,645,878],{"className":640},[83],[41,642],{"className":643},[400,644],"nulldelimiter",[41,646,648],{"className":647},[521],[41,649,652,869],{"className":650},[276,651],"vlist-t2",[41,653,655,866],{"className":654},[280],[41,656,659,796,807],{"className":657,"style":658},[284],"height:1.427em;",[41,660,662,666],{"style":661},"top:-2.1288em;",[41,663],{"className":664,"style":665},[291],"height:3em;",[41,667,669,741,744,747,750,793],{"className":668},[83],[41,670,672,678],{"className":671},[148],[41,673,547],{"className":674,"style":677},[148,675,676],"op-symbol","small-op","position:relative;top:0em;",[41,679,681],{"className":680},[272],[41,682,684,732],{"className":683},[276,651],[41,685,687,727],{"className":686},[280],[41,688,691,712],{"className":689,"style":690},[284],"height:0.9812em;",[41,692,694,697],{"style":693},"top:-2.4003em;margin-left:0em;margin-right:0.05em;",[41,695],{"className":696,"style":292},[291],[41,698,700],{"className":699},[296,297,298,299],[41,701,703,706,709],{"className":702},[83,299],[41,704,97],{"className":705},[83,84,299],[41,707,513],{"className":708},[251,299],[41,710,356],{"className":711},[83,299],[41,713,715,718],{"style":714},"top:-3.2029em;margin-right:0.05em;",[41,716],{"className":717,"style":292},[291],[41,719,721],{"className":720},[296,297,298,299],[41,722,724],{"className":723},[83,299],[41,725,226],{"className":726,"style":314},[83,84,299],[41,728,731],{"className":729},[730],"vlist-s","​",[41,733,735],{"className":734},[280],[41,736,739],{"className":737,"style":738},[284],"height:0.2997em;",[41,740],{},[41,742],{"className":743,"style":411},[246],[41,745,169],{"className":746},[148],[41,748,501],{"className":749},[400],[41,751,753,756],{"className":752},[83],[41,754,205],{"className":755},[83,84],[41,757,759],{"className":758},[272],[41,760,762,784],{"className":761},[276,651],[41,763,765,781],{"className":764},[280],[41,766,769],{"className":767,"style":768},[284],"height:0.1514em;",[41,770,772,775],{"style":771},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[41,773],{"className":774,"style":292},[291],[41,776,778],{"className":777},[296,297,298,299],[41,779,97],{"className":780},[83,84,299],[41,782,731],{"className":783},[730],[41,785,787],{"className":786},[280],[41,788,791],{"className":789,"style":790},[284],"height:0.15em;",[41,792],{},[41,794,510],{"className":795},[450],[41,797,799,802],{"style":798},"top:-3.23em;",[41,800],{"className":801,"style":665},[291],[41,803],{"className":804,"style":806},[805],"frac-line","border-bottom-width:0.04em;",[41,808,810,813],{"style":809},"top:-3.677em;",[41,811],{"className":812,"style":665},[291],[41,814,816,819,822,863],{"className":815},[83],[41,817,169],{"className":818},[148],[41,820,501],{"className":821},[400],[41,823,825,828],{"className":824},[83],[41,826,205],{"className":827},[83,84],[41,829,831],{"className":830},[272],[41,832,834,854],{"className":833},[276,651],[41,835,837,851],{"className":836},[280],[41,838,840],{"className":839,"style":768},[284],[41,841,842,845],{"style":771},[41,843],{"className":844,"style":292},[291],[41,846,848],{"className":847},[296,297,298,299],[41,849,330],{"className":850,"style":380},[83,84,299],[41,852,731],{"className":853},[730],[41,855,857],{"className":856},[280],[41,858,861],{"className":859,"style":860},[284],"height:0.2861em;",[41,862],{},[41,864,510],{"className":865},[450],[41,867,731],{"className":868},[730],[41,870,872],{"className":871},[280],[41,873,876],{"className":874,"style":875},[284],"height:1.1709em;",[41,877],{},[41,879],{"className":880},[450,644],[41,882,574],{"className":883},[83],[10,885,886,887,890,891,894,895,898,899,902,903,1014,1015,1018,1019,1022,1023,86],{},"如果先调用 ",[14,888,889],{},"softmax(o)"," 再取 ",[14,892,893],{},"-torch.log(prob)","，当 logits 被放大到 ",[14,896,897],{},"1000.0 * inputs","（对应官方测试 ",[14,900,901],{},"test_cross_entropy"," 的溢出测试分支）时，非最大类的概率 ",[41,904,906,937],{"className":905},[44],[41,907,909],{"className":908},[48],[50,910,911],{"xmlns":52},[54,912,913,934],{},[57,914,915,917,919,921,927,929,932],{},[60,916,169],{},[129,918,131],{},[129,920,501],{"stretchy":335},[532,922,923,925],{},[60,924,205],{},[60,926,330],{},[129,928,353],{},[60,930,931],{},"m",[129,933,510],{"stretchy":335},[63,935,936],{"encoding":65},"\\exp(o_y - m)",[41,938,940,1002],{"className":939,"ariaHidden":70},[69],[41,941,943,947,950,953,993,996,999],{"className":942},[74],[41,944],{"className":945,"style":946},[78],"height:1.0361em;vertical-align:-0.2861em;",[41,948,169],{"className":949},[148],[41,951,501],{"className":952},[400],[41,954,956,959],{"className":955},[83],[41,957,205],{"className":958},[83,84],[41,960,962],{"className":961},[272],[41,963,965,985],{"className":964},[276,651],[41,966,968,982],{"className":967},[280],[41,969,971],{"className":970,"style":768},[284],[41,972,973,976],{"style":771},[41,974],{"className":975,"style":292},[291],[41,977,979],{"className":978},[296,297,298,299],[41,980,330],{"className":981,"style":380},[83,84,299],[41,983,731],{"className":984},[730],[41,986,988],{"className":987},[280],[41,989,991],{"className":990,"style":860},[284],[41,992],{},[41,994],{"className":995,"style":314},[246],[41,997,353],{"className":998},[310],[41,1000],{"className":1001,"style":314},[246],[41,1003,1005,1008,1011],{"className":1004},[74],[41,1006],{"className":1007,"style":396},[78],[41,1009,931],{"className":1010},[83,84],[41,1012,510],{"className":1013},[450]," 会直接下溢成浮点数 ",[14,1016,1017],{},"0.0","，随后 ",[14,1020,1021],{},"log(0.0)"," 变成 ",[14,1024,1025],{},"-inf",[10,1027,1028,1029,1178,1179,1182,1183,1215,1216,1246],{},"正确做法是先把分式拆开，在减去行最大值 ",[41,1030,1032,1065],{"className":1031},[44],[41,1033,1035],{"className":1034},[48],[50,1036,1037],{"xmlns":52},[54,1038,1039,1062],{},[57,1040,1041,1043,1045,1056],{},[60,1042,931],{},[129,1044,513],{},[532,1046,1047,1054],{},[57,1048,1049,1052],{},[60,1050,1051],{},"max",[129,1053,131],{},[60,1055,97],{},[532,1057,1058,1060],{},[60,1059,205],{},[60,1061,97],{},[63,1063,1064],{"encoding":65},"m = \\max_a o_a",[41,1066,1068,1087],{"className":1067,"ariaHidden":70},[69],[41,1069,1071,1075,1078,1081,1084],{"className":1070},[74],[41,1072],{"className":1073,"style":1074},[78],"height:0.4306em;",[41,1076,931],{"className":1077},[83,84],[41,1079],{"className":1080,"style":247},[246],[41,1082,513],{"className":1083},[251],[41,1085],{"className":1086,"style":247},[246],[41,1088,1090,1094,1135,1138],{"className":1089},[74],[41,1091],{"className":1092,"style":1093},[78],"height:0.5806em;vertical-align:-0.15em;",[41,1095,1097,1100],{"className":1096},[148],[41,1098,1051],{"className":1099},[148],[41,1101,1103],{"className":1102},[272],[41,1104,1106,1127],{"className":1105},[276,651],[41,1107,1109,1124],{"className":1108},[280],[41,1110,1112],{"className":1111,"style":768},[284],[41,1113,1115,1118],{"style":1114},"top:-2.55em;margin-right:0.05em;",[41,1116],{"className":1117,"style":292},[291],[41,1119,1121],{"className":1120},[296,297,298,299],[41,1122,97],{"className":1123},[83,84,299],[41,1125,731],{"className":1126},[730],[41,1128,1130],{"className":1129},[280],[41,1131,1133],{"className":1132,"style":790},[284],[41,1134],{},[41,1136],{"className":1137,"style":411},[246],[41,1139,1141,1144],{"className":1140},[83],[41,1142,205],{"className":1143},[83,84],[41,1145,1147],{"className":1146},[272],[41,1148,1150,1170],{"className":1149},[276,651],[41,1151,1153,1167],{"className":1152},[280],[41,1154,1156],{"className":1155,"style":768},[284],[41,1157,1158,1161],{"style":771},[41,1159],{"className":1160,"style":292},[291],[41,1162,1164],{"className":1163},[296,297,298,299],[41,1165,97],{"className":1166},[83,84,299],[41,1168,731],{"className":1169},[730],[41,1171,1173],{"className":1172},[280],[41,1174,1176],{"className":1175,"style":790},[284],[41,1177],{}," 后直接化简为 ",[27,1180,1181],{},"LogSumExp"," 形式，把分子上的 ",[41,1184,1186,1201],{"className":1185},[44],[41,1187,1189],{"className":1188},[48],[50,1190,1191],{"xmlns":52},[54,1192,1193,1199],{},[57,1194,1195,1197],{},[60,1196,127],{},[129,1198,131],{},[63,1200,134],{"encoding":65},[41,1202,1204],{"className":1203,"ariaHidden":70},[69],[41,1205,1207,1210],{"className":1206},[74],[41,1208],{"className":1209,"style":144},[78],[41,1211,149,1213],{"className":1212},[148],[41,1214,153],{"style":152}," 与 ",[41,1217,1219,1234],{"className":1218},[44],[41,1220,1222],{"className":1221},[48],[50,1223,1224],{"xmlns":52},[54,1225,1226,1232],{},[57,1227,1228,1230],{},[60,1229,169],{},[129,1231,131],{},[63,1233,174],{"encoding":65},[41,1235,1237],{"className":1236,"ariaHidden":70},[69],[41,1238,1240,1243],{"className":1239},[74],[41,1241],{"className":1242,"style":79},[78],[41,1244,169],{"className":1245},[148]," 解析抵消：",[41,1248,1250],{"className":1249},[481],[41,1251,1253,1397],{"className":1252},[44],[41,1254,1256],{"className":1255},[48],[50,1257,1258],{"xmlns":52,"display":490},[54,1259,1260,1394],{},[57,1261,1262,1264,1266,1268,1270,1272,1274,1276,1278,1280,1282,1338,1340,1342,1344,1358,1360,1362,1364,1370,1372,1374,1376,1378,1380,1386,1388,1390,1392],{},[60,1263,498],{"mathvariant":497},[129,1265,501],{"stretchy":335},[60,1267,205],{},[129,1269,343],{"separator":70},[60,1271,330],{},[129,1273,510],{"stretchy":335},[129,1275,513],{},[129,1277,353],{},[60,1279,127],{},[129,1281,131],{},[521,1283,1284,1304],{},[57,1285,1286,1288,1290,1292,1298,1300,1302],{},[60,1287,169],{},[129,1289,131],{},[129,1291,501],{"stretchy":335},[532,1293,1294,1296],{},[60,1295,205],{},[60,1297,330],{},[129,1299,353],{},[60,1301,931],{},[129,1303,510],{"stretchy":335},[57,1305,1306,1320,1322,1324,1326,1332,1334,1336],{},[543,1307,1308,1310,1318],{},[129,1309,547],{},[57,1311,1312,1314,1316],{},[60,1313,97],{},[129,1315,513],{},[338,1317,356],{},[60,1319,226],{},[60,1321,169],{},[129,1323,131],{},[129,1325,501],{"stretchy":335},[532,1327,1328,1330],{},[60,1329,205],{},[60,1331,97],{},[129,1333,353],{},[60,1335,931],{},[129,1337,510],{"stretchy":335},[129,1339,513],{},[60,1341,127],{},[129,1343,131],{},[543,1345,1346,1348,1356],{},[129,1347,547],{},[57,1349,1350,1352,1354],{},[60,1351,97],{},[129,1353,513],{},[338,1355,356],{},[60,1357,226],{},[60,1359,169],{},[129,1361,131],{},[129,1363,501],{"stretchy":335},[532,1365,1366,1368],{},[60,1367,205],{},[60,1369,97],{},[129,1371,353],{},[60,1373,931],{},[129,1375,510],{"stretchy":335},[129,1377,353],{},[129,1379,501],{"stretchy":335},[532,1381,1382,1384],{},[60,1383,205],{},[60,1385,330],{},[129,1387,353],{},[60,1389,931],{},[129,1391,510],{"stretchy":335},[60,1393,574],{"mathvariant":497},[63,1395,1396],{"encoding":65},"\\ell(o, y)\n= -\\log \\frac{\\exp(o_y - m)}{\\sum_{a=1}^{V} \\exp(o_a - m)}\n= \\log \\sum_{a=1}^{V} \\exp(o_a - m) - (o_y - m).",[41,1398,1400,1436,1709,1857,1878,1936],{"className":1399,"ariaHidden":70},[69],[41,1401,1403,1406,1409,1412,1415,1418,1421,1424,1427,1430,1433],{"className":1402},[74],[41,1404],{"className":1405,"style":396},[78],[41,1407,498],{"className":1408},[83],[41,1410,501],{"className":1411},[400],[41,1413,205],{"className":1414},[83,84],[41,1416,343],{"className":1417},[407],[41,1419],{"className":1420,"style":411},[246],[41,1422,330],{"className":1423,"style":380},[83,84],[41,1425,510],{"className":1426},[450],[41,1428],{"className":1429,"style":247},[246],[41,1431,513],{"className":1432},[251],[41,1434],{"className":1435,"style":247},[246],[41,1437,1439,1442,1445,1448,1453,1456,1700,1703,1706],{"className":1438},[74],[41,1440],{"className":1441,"style":623},[78],[41,1443,353],{"className":1444},[83],[41,1446],{"className":1447,"style":411},[246],[41,1449,149,1451],{"className":1450},[148],[41,1452,153],{"style":152},[41,1454],{"className":1455,"style":411},[246],[41,1457,1459,1462,1697],{"className":1458},[83],[41,1460],{"className":1461},[400,644],[41,1463,1465],{"className":1464},[521],[41,1466,1468,1689],{"className":1467},[276,651],[41,1469,1471,1686],{"className":1470},[280],[41,1472,1474,1609,1617],{"className":1473,"style":658},[284],[41,1475,1476,1479],{"style":661},[41,1477],{"className":1478,"style":665},[291],[41,1480,1482,1545,1548,1551,1554,1594,1597,1600,1603,1606],{"className":1481},[83],[41,1483,1485,1488],{"className":1484},[148],[41,1486,547],{"className":1487,"style":677},[148,675,676],[41,1489,1491],{"className":1490},[272],[41,1492,1494,1537],{"className":1493},[276,651],[41,1495,1497,1534],{"className":1496},[280],[41,1498,1500,1520],{"className":1499,"style":690},[284],[41,1501,1502,1505],{"style":693},[41,1503],{"className":1504,"style":292},[291],[41,1506,1508],{"className":1507},[296,297,298,299],[41,1509,1511,1514,1517],{"className":1510},[83,299],[41,1512,97],{"className":1513},[83,84,299],[41,1515,513],{"className":1516},[251,299],[41,1518,356],{"className":1519},[83,299],[41,1521,1522,1525],{"style":714},[41,1523],{"className":1524,"style":292},[291],[41,1526,1528],{"className":1527},[296,297,298,299],[41,1529,1531],{"className":1530},[83,299],[41,1532,226],{"className":1533,"style":314},[83,84,299],[41,1535,731],{"className":1536},[730],[41,1538,1540],{"className":1539},[280],[41,1541,1543],{"className":1542,"style":738},[284],[41,1544],{},[41,1546],{"className":1547,"style":411},[246],[41,1549,169],{"className":1550},[148],[41,1552,501],{"className":1553},[400],[41,1555,1557,1560],{"className":1556},[83],[41,1558,205],{"className":1559},[83,84],[41,1561,1563],{"className":1562},[272],[41,1564,1566,1586],{"className":1565},[276,651],[41,1567,1569,1583],{"className":1568},[280],[41,1570,1572],{"className":1571,"style":768},[284],[41,1573,1574,1577],{"style":771},[41,1575],{"className":1576,"style":292},[291],[41,1578,1580],{"className":1579},[296,297,298,299],[41,1581,97],{"className":1582},[83,84,299],[41,1584,731],{"className":1585},[730],[41,1587,1589],{"className":1588},[280],[41,1590,1592],{"className":1591,"style":790},[284],[41,1593],{},[41,1595],{"className":1596,"style":314},[246],[41,1598,353],{"className":1599},[310],[41,1601],{"className":1602,"style":314},[246],[41,1604,931],{"className":1605},[83,84],[41,1607,510],{"className":1608},[450],[41,1610,1611,1614],{"style":798},[41,1612],{"className":1613,"style":665},[291],[41,1615],{"className":1616,"style":806},[805],[41,1618,1619,1622],{"style":809},[41,1620],{"className":1621,"style":665},[291],[41,1623,1625,1628,1631,1671,1674,1677,1680,1683],{"className":1624},[83],[41,1626,169],{"className":1627},[148],[41,1629,501],{"className":1630},[400],[41,1632,1634,1637],{"className":1633},[83],[41,1635,205],{"className":1636},[83,84],[41,1638,1640],{"className":1639},[272],[41,1641,1643,1663],{"className":1642},[276,651],[41,1644,1646,1660],{"className":1645},[280],[41,1647,1649],{"className":1648,"style":768},[284],[41,1650,1651,1654],{"style":771},[41,1652],{"className":1653,"style":292},[291],[41,1655,1657],{"className":1656},[296,297,298,299],[41,1658,330],{"className":1659,"style":380},[83,84,299],[41,1661,731],{"className":1662},[730],[41,1664,1666],{"className":1665},[280],[41,1667,1669],{"className":1668,"style":860},[284],[41,1670],{},[41,1672],{"className":1673,"style":314},[246],[41,1675,353],{"className":1676},[310],[41,1678],{"className":1679,"style":314},[246],[41,1681,931],{"className":1682},[83,84],[41,1684,510],{"className":1685},[450],[41,1687,731],{"className":1688},[730],[41,1690,1692],{"className":1691},[280],[41,1693,1695],{"className":1694,"style":875},[284],[41,1696],{},[41,1698],{"className":1699},[450,644],[41,1701],{"className":1702,"style":247},[246],[41,1704,513],{"className":1705},[251],[41,1707],{"className":1708,"style":247},[246],[41,1710,1712,1716,1721,1724,1799,1802,1805,1808,1848,1851,1854],{"className":1711},[74],[41,1713],{"className":1714,"style":1715},[78],"height:3.0954em;vertical-align:-1.2671em;",[41,1717,149,1719],{"className":1718},[148],[41,1720,153],{"style":152},[41,1722],{"className":1723,"style":411},[246],[41,1725,1728],{"className":1726},[148,1727],"op-limits",[41,1729,1731,1790],{"className":1730},[276,651],[41,1732,1734,1787],{"className":1733},[280],[41,1735,1738,1760,1772],{"className":1736,"style":1737},[284],"height:1.8283em;",[41,1739,1741,1745],{"style":1740},"top:-1.8829em;margin-left:0em;",[41,1742],{"className":1743,"style":1744},[291],"height:3.05em;",[41,1746,1748],{"className":1747},[296,297,298,299],[41,1749,1751,1754,1757],{"className":1750},[83,299],[41,1752,97],{"className":1753},[83,84,299],[41,1755,513],{"className":1756},[251,299],[41,1758,356],{"className":1759},[83,299],[41,1761,1763,1766],{"style":1762},"top:-3.05em;",[41,1764],{"className":1765,"style":1744},[291],[41,1767,1768],{},[41,1769,547],{"className":1770},[148,675,1771],"large-op",[41,1773,1775,1778],{"style":1774},"top:-4.3em;margin-left:0em;",[41,1776],{"className":1777,"style":1744},[291],[41,1779,1781],{"className":1780},[296,297,298,299],[41,1782,1784],{"className":1783},[83,299],[41,1785,226],{"className":1786,"style":314},[83,84,299],[41,1788,731],{"className":1789},[730],[41,1791,1793],{"className":1792},[280],[41,1794,1797],{"className":1795,"style":1796},[284],"height:1.2671em;",[41,1798],{},[41,1800],{"className":1801,"style":411},[246],[41,1803,169],{"className":1804},[148],[41,1806,501],{"className":1807},[400],[41,1809,1811,1814],{"className":1810},[83],[41,1812,205],{"className":1813},[83,84],[41,1815,1817],{"className":1816},[272],[41,1818,1820,1840],{"className":1819},[276,651],[41,1821,1823,1837],{"className":1822},[280],[41,1824,1826],{"className":1825,"style":768},[284],[41,1827,1828,1831],{"style":771},[41,1829],{"className":1830,"style":292},[291],[41,1832,1834],{"className":1833},[296,297,298,299],[41,1835,97],{"className":1836},[83,84,299],[41,1838,731],{"className":1839},[730],[41,1841,1843],{"className":1842},[280],[41,1844,1846],{"className":1845,"style":790},[284],[41,1847],{},[41,1849],{"className":1850,"style":314},[246],[41,1852,353],{"className":1853},[310],[41,1855],{"className":1856,"style":314},[246],[41,1858,1860,1863,1866,1869,1872,1875],{"className":1859},[74],[41,1861],{"className":1862,"style":396},[78],[41,1864,931],{"className":1865},[83,84],[41,1867,510],{"className":1868},[450],[41,1870],{"className":1871,"style":314},[246],[41,1873,353],{"className":1874},[310],[41,1876],{"className":1877,"style":314},[246],[41,1879,1881,1884,1887,1927,1930,1933],{"className":1880},[74],[41,1882],{"className":1883,"style":946},[78],[41,1885,501],{"className":1886},[400],[41,1888,1890,1893],{"className":1889},[83],[41,1891,205],{"className":1892},[83,84],[41,1894,1896],{"className":1895},[272],[41,1897,1899,1919],{"className":1898},[276,651],[41,1900,1902,1916],{"className":1901},[280],[41,1903,1905],{"className":1904,"style":768},[284],[41,1906,1907,1910],{"style":771},[41,1908],{"className":1909,"style":292},[291],[41,1911,1913],{"className":1912},[296,297,298,299],[41,1914,330],{"className":1915,"style":380},[83,84,299],[41,1917,731],{"className":1918},[730],[41,1920,1922],{"className":1921},[280],[41,1923,1925],{"className":1924,"style":860},[284],[41,1926],{},[41,1928],{"className":1929,"style":314},[246],[41,1931,353],{"className":1932},[310],[41,1934],{"className":1935,"style":314},[246],[41,1937,1939,1942,1945,1948],{"className":1938},[74],[41,1940],{"className":1941,"style":396},[78],[41,1943,931],{"className":1944},[83,84],[41,1946,510],{"className":1947},[450],[41,1949,574],{"className":1950},[83],[10,1952,1953,1954,1957,1958,1961],{},"在 ",[14,1955,1956],{},"cs336_basics\u002Fnn_utils.py"," 中，用 ",[14,1959,1960],{},"torch.gather"," 取出目标类别对应的平移后 logit，只需 6 行即可同时支持任意前导 batch 维度与极端大数值输入：",[1963,1964,1969],"pre",{"className":1965,"code":1966,"language":1967,"meta":1968,"style":1968},"language-python shiki shiki-themes github-dark-dimmed github-light houston","def cross_entropy(inputs: torch.Tensor, targets: torch.Tensor) -> torch.Tensor:\n    max_logits = torch.amax(inputs, dim=-1, keepdim=True)\n    shifted = inputs - max_logits\n    log_sum_exp = torch.log(torch.sum(torch.exp(shifted), dim=-1))\n    target_logits = torch.gather(shifted, dim=-1, index=targets.long().unsqueeze(-1)).squeeze(-1)\n    return (log_sum_exp - target_logits).mean()\n","python","",[14,1970,1971,1999,2042,2059,2093,2154],{"__ignoreMap":1968},[41,1972,1975,1979,1983,1986,1990,1993,1996],{"class":1973,"line":1974},"line",1,[41,1976,1978],{"class":1977},"sST-V","def",[41,1980,1982],{"class":1981},"sS_0i"," cross_entropy",[41,1984,501],{"class":1985},"sWURj",[41,1987,1989],{"class":1988},"s9NhK","inputs",[41,1991,1992],{"class":1985},": torch.Tensor, ",[41,1994,1995],{"class":1988},"targets",[41,1997,1998],{"class":1985},": torch.Tensor) -> torch.Tensor:\n",[41,2000,2002,2005,2008,2011,2015,2018,2022,2025,2028,2031,2034,2036,2039],{"class":1973,"line":2001},2,[41,2003,2004],{"class":1985},"    max_logits ",[41,2006,513],{"class":2007},"sf2cM",[41,2009,2010],{"class":1985}," torch.",[41,2012,2014],{"class":2013},"sq3c-","amax",[41,2016,2017],{"class":1985},"(inputs, ",[41,2019,2021],{"class":2020},"sA9aF","dim",[41,2023,2024],{"class":2007},"=-",[41,2026,356],{"class":2027},"sgENW",[41,2029,2030],{"class":1985},", ",[41,2032,2033],{"class":2020},"keepdim",[41,2035,513],{"class":2007},[41,2037,2038],{"class":2027},"True",[41,2040,2041],{"class":1985},")\n",[41,2043,2045,2048,2050,2053,2056],{"class":1973,"line":2044},3,[41,2046,2047],{"class":1985},"    shifted ",[41,2049,513],{"class":2007},[41,2051,2052],{"class":1985}," inputs ",[41,2054,2055],{"class":2007},"-",[41,2057,2058],{"class":1985}," max_logits\n",[41,2060,2062,2065,2067,2069,2071,2074,2077,2079,2081,2084,2086,2088,2090],{"class":1973,"line":2061},4,[41,2063,2064],{"class":1985},"    log_sum_exp ",[41,2066,513],{"class":2007},[41,2068,2010],{"class":1985},[41,2070,127],{"class":2013},[41,2072,2073],{"class":1985},"(torch.",[41,2075,2076],{"class":2013},"sum",[41,2078,2073],{"class":1985},[41,2080,169],{"class":2013},[41,2082,2083],{"class":1985},"(shifted), ",[41,2085,2021],{"class":2020},[41,2087,2024],{"class":2007},[41,2089,356],{"class":2027},[41,2091,2092],{"class":1985},"))\n",[41,2094,2096,2099,2101,2103,2106,2109,2111,2113,2115,2117,2120,2122,2125,2128,2131,2134,2136,2138,2140,2143,2146,2148,2150,2152],{"class":1973,"line":2095},5,[41,2097,2098],{"class":1985},"    target_logits ",[41,2100,513],{"class":2007},[41,2102,2010],{"class":1985},[41,2104,2105],{"class":2013},"gather",[41,2107,2108],{"class":1985},"(shifted, ",[41,2110,2021],{"class":2020},[41,2112,2024],{"class":2007},[41,2114,356],{"class":2027},[41,2116,2030],{"class":1985},[41,2118,2119],{"class":2020},"index",[41,2121,513],{"class":2007},[41,2123,2124],{"class":1985},"targets.",[41,2126,2127],{"class":2013},"long",[41,2129,2130],{"class":1985},"().",[41,2132,2133],{"class":2013},"unsqueeze",[41,2135,501],{"class":1985},[41,2137,2055],{"class":2007},[41,2139,356],{"class":2027},[41,2141,2142],{"class":1985},")).",[41,2144,2145],{"class":2013},"squeeze",[41,2147,501],{"class":1985},[41,2149,2055],{"class":2007},[41,2151,356],{"class":2027},[41,2153,2041],{"class":1985},[41,2155,2157,2161,2164,2166,2169,2172],{"class":1973,"line":2156},6,[41,2158,2160],{"class":2159},"slEfO","    return",[41,2162,2163],{"class":1985}," (log_sum_exp ",[41,2165,2055],{"class":2007},[41,2167,2168],{"class":1985}," target_logits).",[41,2170,2171],{"class":2013},"mean",[41,2173,2174],{"class":1985},"()\n",[10,2176,2177,2178,86],{},"语言模型的困惑度（Perplexity）则是序列平均交叉熵的指数：",[41,2179,2181,2238],{"className":2180},[44],[41,2182,2184],{"className":2183},[48],[50,2185,2186],{"xmlns":52},[54,2187,2188,2235],{},[57,2189,2190,2193,2195,2197,2199,2201],{},[60,2191,2192],{"mathvariant":497},"PPL",[129,2194,131],{},[129,2196,513],{},[60,2198,169],{},[129,2200,131],{},[57,2202,2203,2205,2211,2227,2233],{},[129,2204,501],{"fence":70},[521,2206,2207,2209],{},[338,2208,356],{},[60,2210,931],{},[2212,2213,2214,2216,2225],"msubsup",{},[129,2215,547],{},[57,2217,2218,2221,2223],{},[60,2219,2220],{},"i",[129,2222,513],{},[338,2224,356],{},[60,2226,931],{},[532,2228,2229,2231],{},[60,2230,498],{"mathvariant":497},[60,2232,2220],{},[129,2234,510],{"fence":70},[63,2236,2237],{"encoding":65},"\\operatorname{PPL} = \\exp\\left(\\frac{1}{m}\\sum_{i=1}^m \\ell_i\\right)",[41,2239,2241,2264],{"className":2240,"ariaHidden":70},[69],[41,2242,2244,2248,2255,2258,2261],{"className":2243},[74],[41,2245],{"className":2246,"style":2247},[78],"height:0.6833em;",[41,2249,2251],{"className":2250},[148],[41,2252,2192],{"className":2253},[83,2254],"mathrm",[41,2256],{"className":2257,"style":247},[246],[41,2259,513],{"className":2260},[251],[41,2262],{"className":2263,"style":247},[246],[41,2265,2267,2271,2274,2277],{"className":2266},[74],[41,2268],{"className":2269,"style":2270},[78],"height:1.2em;vertical-align:-0.35em;",[41,2272,169],{"className":2273},[148],[41,2275],{"className":2276,"style":411},[246],[41,2278,2280,2290,2362,2365,2426,2429,2470],{"className":2279},[415],[41,2281,2285],{"className":2282,"style":2284},[400,2283],"delimcenter","top:0em;",[41,2286,501],{"className":2287},[2288,2289],"delimsizing","size1",[41,2291,2293,2296,2359],{"className":2292},[83],[41,2294],{"className":2295},[400,644],[41,2297,2299],{"className":2298},[521],[41,2300,2302,2350],{"className":2301},[276,651],[41,2303,2305,2347],{"className":2304},[280],[41,2306,2309,2324,2332],{"className":2307,"style":2308},[284],"height:0.8451em;",[41,2310,2312,2315],{"style":2311},"top:-2.655em;",[41,2313],{"className":2314,"style":665},[291],[41,2316,2318],{"className":2317},[296,297,298,299],[41,2319,2321],{"className":2320},[83,299],[41,2322,931],{"className":2323},[83,84,299],[41,2325,2326,2329],{"style":798},[41,2327],{"className":2328,"style":665},[291],[41,2330],{"className":2331,"style":806},[805],[41,2333,2335,2338],{"style":2334},"top:-3.394em;",[41,2336],{"className":2337,"style":665},[291],[41,2339,2341],{"className":2340},[296,297,298,299],[41,2342,2344],{"className":2343},[83,299],[41,2345,356],{"className":2346},[83,299],[41,2348,731],{"className":2349},[730],[41,2351,2353],{"className":2352},[280],[41,2354,2357],{"className":2355,"style":2356},[284],"height:0.345em;",[41,2358],{},[41,2360],{"className":2361},[450,644],[41,2363],{"className":2364,"style":411},[246],[41,2366,2368,2371],{"className":2367},[148],[41,2369,547],{"className":2370,"style":677},[148,675,676],[41,2372,2374],{"className":2373},[272],[41,2375,2377,2418],{"className":2376},[276,651],[41,2378,2380,2415],{"className":2379},[280],[41,2381,2384,2404],{"className":2382,"style":2383},[284],"height:0.8043em;",[41,2385,2386,2389],{"style":693},[41,2387],{"className":2388,"style":292},[291],[41,2390,2392],{"className":2391},[296,297,298,299],[41,2393,2395,2398,2401],{"className":2394},[83,299],[41,2396,2220],{"className":2397},[83,84,299],[41,2399,513],{"className":2400},[251,299],[41,2402,356],{"className":2403},[83,299],[41,2405,2406,2409],{"style":714},[41,2407],{"className":2408,"style":292},[291],[41,2410,2412],{"className":2411},[296,297,298,299],[41,2413,931],{"className":2414},[83,84,299],[41,2416,731],{"className":2417},[730],[41,2419,2421],{"className":2420},[280],[41,2422,2424],{"className":2423,"style":738},[284],[41,2425],{},[41,2427],{"className":2428,"style":411},[246],[41,2430,2432,2435],{"className":2431},[83],[41,2433,498],{"className":2434},[83],[41,2436,2438],{"className":2437},[272],[41,2439,2441,2462],{"className":2440},[276,651],[41,2442,2444,2459],{"className":2443},[280],[41,2445,2448],{"className":2446,"style":2447},[284],"height:0.3117em;",[41,2449,2450,2453],{"style":771},[41,2451],{"className":2452,"style":292},[291],[41,2454,2456],{"className":2455},[296,297,298,299],[41,2457,2220],{"className":2458},[83,84,299],[41,2460,731],{"className":2461},[730],[41,2463,2465],{"className":2464},[280],[41,2466,2468],{"className":2467,"style":790},[284],[41,2469],{},[41,2471,2473],{"className":2472,"style":2284},[450,2283],[41,2474,510],{"className":2475},[2288,2289],[109,2477,2479,2480,2553],{"id":2478},"全局-ℓ2ell_2ℓ2-梯度裁剪gradient-clipping","全局 ",[41,2481,2483,2502],{"className":2482},[44],[41,2484,2486],{"className":2485},[48],[50,2487,2488],{"xmlns":52},[54,2489,2490,2499],{},[57,2491,2492],{},[532,2493,2494,2496],{},[60,2495,498],{"mathvariant":497},[338,2497,2498],{},"2",[63,2500,2501],{"encoding":65},"\\ell_2",[41,2503,2505],{"className":2504,"ariaHidden":70},[69],[41,2506,2508,2512],{"className":2507},[74],[41,2509],{"className":2510,"style":2511},[78],"height:0.8444em;vertical-align:-0.15em;",[41,2513,2515,2518],{"className":2514},[83],[41,2516,498],{"className":2517},[83],[41,2519,2521],{"className":2520},[272],[41,2522,2524,2545],{"className":2523},[276,651],[41,2525,2527,2542],{"className":2526},[280],[41,2528,2531],{"className":2529,"style":2530},[284],"height:0.3011em;",[41,2532,2533,2536],{"style":771},[41,2534],{"className":2535,"style":292},[291],[41,2537,2539],{"className":2538},[296,297,298,299],[41,2540,2498],{"className":2541},[83,299],[41,2543,731],{"className":2544},[730],[41,2546,2548],{"className":2547},[280],[41,2549,2551],{"className":2550,"style":790},[284],[41,2552],{}," 梯度裁剪（Gradient Clipping）",[10,2555,2556,2557,2630],{},"训练过程中遇到异常样本可能产生幅度极高的梯度，导致参数更新一步飞出稳定区域。全局梯度裁剪不是按单个参数矩阵分别裁剪，而是先计算",[27,2558,2559,2560,2629],{},"所有可训练参数梯度的联合 ",[41,2561,2563,2580],{"className":2562},[44],[41,2564,2566],{"className":2565},[48],[50,2567,2568],{"xmlns":52},[54,2569,2570,2578],{},[57,2571,2572],{},[532,2573,2574,2576],{},[60,2575,498],{"mathvariant":497},[338,2577,2498],{},[63,2579,2501],{"encoding":65},[41,2581,2583],{"className":2582,"ariaHidden":70},[69],[41,2584,2586,2589],{"className":2585},[74],[41,2587],{"className":2588,"style":2511},[78],[41,2590,2592,2595],{"className":2591},[83],[41,2593,498],{"className":2594},[83],[41,2596,2598],{"className":2597},[272],[41,2599,2601,2621],{"className":2600},[276,651],[41,2602,2604,2618],{"className":2603},[280],[41,2605,2607],{"className":2606,"style":2530},[284],[41,2608,2609,2612],{"style":771},[41,2610],{"className":2611,"style":292},[291],[41,2613,2615],{"className":2614},[296,297,298,299],[41,2616,2498],{"className":2617},[83,299],[41,2619,731],{"className":2620},[730],[41,2622,2624],{"className":2623},[280],[41,2625,2627],{"className":2626,"style":790},[284],[41,2628],{}," 范数","：",[41,2632,2634],{"className":2633},[481],[41,2635,2637,2718],{"className":2636},[44],[41,2638,2640],{"className":2639},[48],[50,2641,2642],{"xmlns":52,"display":490},[54,2643,2644,2715],{},[57,2645,2646,2649,2651,2657,2659,2713],{},[60,2647,2648],{"mathvariant":497},"∥",[60,2650,153],{},[532,2652,2653,2655],{},[60,2654,2648],{"mathvariant":497},[338,2656,2498],{},[129,2658,513],{},[2660,2661,2662],"msqrt",{},[57,2663,2664,2697,2699,2701,2703,2705],{},[2665,2666,2667,2669],"munder",{},[129,2668,547],{},[57,2670,2671,2673,2675,2678,2680,2684,2686,2688,2691,2694],{},[60,2672,10],{},[129,2674,208],{},[60,2676,2677],{"mathvariant":497},"Θ",[129,2679,343],{"separator":70},[2681,2682,2683],"mtext",{},"  ",[60,2685,10],{},[60,2687,574],{"mathvariant":497},[2681,2689,2690],{},"grad",[129,2692,2693],{"mathvariant":497},"≠",[2681,2695,2696],{},"None",[60,2698,2648],{"mathvariant":497},[60,2700,10],{},[60,2702,574],{"mathvariant":497},[2681,2704,2690],{},[2212,2706,2707,2709,2711],{},[60,2708,2648],{"mathvariant":497},[338,2710,2498],{},[338,2712,2498],{},[60,2714,574],{"mathvariant":497},[63,2716,2717],{"encoding":65},"\\|g\\|_2 = \\sqrt{\\sum_{p \\in \\Theta,\\; p.\\text{grad} \\neq \\text{None}} \\|p.\\text{grad}\\|_2^2}.",[41,2719,2721,2782],{"className":2720,"ariaHidden":70},[69],[41,2722,2724,2727,2730,2733,2773,2776,2779],{"className":2723},[74],[41,2725],{"className":2726,"style":396},[78],[41,2728,2648],{"className":2729},[83],[41,2731,153],{"className":2732,"style":380},[83,84],[41,2734,2736,2739],{"className":2735},[83],[41,2737,2648],{"className":2738},[83],[41,2740,2742],{"className":2741},[272],[41,2743,2745,2765],{"className":2744},[276,651],[41,2746,2748,2762],{"className":2747},[280],[41,2749,2751],{"className":2750,"style":2530},[284],[41,2752,2753,2756],{"style":771},[41,2754],{"className":2755,"style":292},[291],[41,2757,2759],{"className":2758},[296,297,298,299],[41,2760,2498],{"className":2761},[83,299],[41,2763,731],{"className":2764},[730],[41,2766,2768],{"className":2767},[280],[41,2769,2771],{"className":2770,"style":790},[284],[41,2772],{},[41,2774],{"className":2775,"style":247},[246],[41,2777,513],{"className":2778},[251],[41,2780],{"className":2781,"style":247},[246],[41,2783,2785,2789,3049],{"className":2784},[74],[41,2786],{"className":2787,"style":2788},[78],"height:3.04em;vertical-align:-1.6002em;",[41,2790,2793],{"className":2791},[83,2792],"sqrt",[41,2794,2796,3040],{"className":2795},[276,651],[41,2797,2799,3037],{"className":2798},[280],[41,2800,2803,3014],{"className":2801,"style":2802},[284],"height:1.4398em;",[41,2804,2808,2812],{"className":2805,"style":2807},[2806],"svg-align","top:-5em;",[41,2809],{"className":2810,"style":2811},[291],"height:5em;",[41,2813,2816,2941,2944,2947,2950,2953,2959],{"className":2814,"style":2815},[83],"padding-left:1em;",[41,2817,2819],{"className":2818},[148,1727],[41,2820,2822,2932],{"className":2821},[276,651],[41,2823,2825,2929],{"className":2824},[280],[41,2826,2829,2919],{"className":2827,"style":2828},[284],"height:1.05em;",[41,2830,2832,2835],{"style":2831},"top:-1.8479em;margin-left:0em;",[41,2833],{"className":2834,"style":1744},[291],[41,2836,2838],{"className":2837},[296,297,298,299],[41,2839,2841,2844,2847,2850,2853,2857,2860,2863,2870,2913],{"className":2840},[83,299],[41,2842,10],{"className":2843},[83,84,299],[41,2845,208],{"className":2846},[251,299],[41,2848,2677],{"className":2849},[83,299],[41,2851,343],{"className":2852},[407,299],[41,2854],{"className":2855,"style":2856},[246,299],"margin-right:0.3253em;",[41,2858,10],{"className":2859},[83,84,299],[41,2861,574],{"className":2862},[83,299],[41,2864,2867],{"className":2865},[83,2866,299],"text",[41,2868,2690],{"className":2869},[83,299],[41,2871,2873,2906,2910],{"className":2872},[251,299],[41,2874,2876],{"className":2875},[251,299],[41,2877,2880],{"className":2878},[83,2879,299],"vbox",[41,2881,2884],{"className":2882},[2883,299],"thinbox",[41,2885,2888,2891,2902],{"className":2886},[2887,299],"rlap",[41,2889],{"className":2890,"style":144},[78],[41,2892,2895],{"className":2893},[2894],"inner",[41,2896,2898],{"className":2897},[83,299],[41,2899,2901],{"className":2900},[251,299],"",[41,2903],{"className":2904},[2905],"fix",[41,2907],{"className":2908},[246,2909,299],"nobreak",[41,2911,513],{"className":2912},[251,299],[41,2914,2916],{"className":2915},[83,2866,299],[41,2917,2696],{"className":2918},[83,299],[41,2920,2921,2924],{"style":1762},[41,2922],{"className":2923,"style":1744},[291],[41,2925,2926],{},[41,2927,547],{"className":2928},[148,675,1771],[41,2930,731],{"className":2931},[730],[41,2933,2935],{"className":2934},[280],[41,2936,2939],{"className":2937,"style":2938},[284],"height:1.4382em;",[41,2940],{},[41,2942],{"className":2943,"style":411},[246],[41,2945,2648],{"className":2946},[83],[41,2948,10],{"className":2949},[83,84],[41,2951,574],{"className":2952},[83],[41,2954,2956],{"className":2955},[83,2866],[41,2957,2690],{"className":2958},[83],[41,2960,2962,2965],{"className":2961},[83],[41,2963,2648],{"className":2964},[83],[41,2966,2968],{"className":2967},[272],[41,2969,2971,3005],{"className":2970},[276,651],[41,2972,2974,3002],{"className":2973},[280],[41,2975,2978,2990],{"className":2976,"style":2977},[284],"height:0.7959em;",[41,2979,2981,2984],{"style":2980},"top:-2.4337em;margin-left:0em;margin-right:0.05em;",[41,2982],{"className":2983,"style":292},[291],[41,2985,2987],{"className":2986},[296,297,298,299],[41,2988,2498],{"className":2989},[83,299],[41,2991,2993,2996],{"style":2992},"top:-3.0448em;margin-right:0.05em;",[41,2994],{"className":2995,"style":292},[291],[41,2997,2999],{"className":2998},[296,297,298,299],[41,3000,2498],{"className":3001},[83,299],[41,3003,731],{"className":3004},[730],[41,3006,3008],{"className":3007},[280],[41,3009,3012],{"className":3010,"style":3011},[284],"height:0.2663em;",[41,3013],{},[41,3015,3017,3020],{"style":3016},"top:-3.3998em;",[41,3018],{"className":3019,"style":2811},[291],[41,3021,3025],{"className":3022,"style":3024},[3023],"hide-tail","min-width:1.02em;height:3.08em;",[3026,3027,3033],"svg",{"xmlns":3028,"width":3029,"height":3030,"viewBox":3031,"preserveAspectRatio":3032},"http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","400em","3.08em","0 0 400000 3240","xMinYMin slice",[3034,3035],"path",{"d":3036},"M473,2793\nc339.3,-1799.3,509.3,-2700,510,-2702 l0 -0\nc3.3,-7.3,9.3,-11,18,-11 H400000v40H1017.7\ns-90.5,478,-276.2,1466c-185.7,988,-279.5,1483,-281.5,1485c-2,6,-10,9,-24,9\nc-8,0,-12,-0.7,-12,-2c0,-1.3,-5.3,-32,-16,-92c-50.7,-293.3,-119.7,-693.3,-207,-1200\nc0,-1.3,-5.3,8.7,-16,30c-10.7,21.3,-21.3,42.7,-32,64s-16,33,-16,33s-26,-26,-26,-26\ns76,-153,76,-153s77,-151,77,-151c0.7,0.7,35.7,202,105,604c67.3,400.7,102,602.7,104,\n606zM1001 80h400000v40H1017.7z",[41,3038,731],{"className":3039},[730],[41,3041,3043],{"className":3042},[280],[41,3044,3047],{"className":3045,"style":3046},[284],"height:1.6002em;",[41,3048],{},[41,3050,574],{"className":3051},[83],[10,3053,3054,3055,3160,3161,3332,3333,3430],{},"当 ",[41,3056,3058,3086],{"className":3057},[44],[41,3059,3061],{"className":3060},[48],[50,3062,3063],{"xmlns":52},[54,3064,3065,3083],{},[57,3066,3067,3069,3071,3077,3080],{},[60,3068,2648],{"mathvariant":497},[60,3070,153],{},[532,3072,3073,3075],{},[60,3074,2648],{"mathvariant":497},[338,3076,2498],{},[129,3078,3079],{},">",[60,3081,3082],{},"M",[63,3084,3085],{"encoding":65},"\\|g\\|_2 > M",[41,3087,3089,3150],{"className":3088,"ariaHidden":70},[69],[41,3090,3092,3095,3098,3101,3141,3144,3147],{"className":3091},[74],[41,3093],{"className":3094,"style":396},[78],[41,3096,2648],{"className":3097},[83],[41,3099,153],{"className":3100,"style":380},[83,84],[41,3102,3104,3107],{"className":3103},[83],[41,3105,2648],{"className":3106},[83],[41,3108,3110],{"className":3109},[272],[41,3111,3113,3133],{"className":3112},[276,651],[41,3114,3116,3130],{"className":3115},[280],[41,3117,3119],{"className":3118,"style":2530},[284],[41,3120,3121,3124],{"style":771},[41,3122],{"className":3123,"style":292},[291],[41,3125,3127],{"className":3126},[296,297,298,299],[41,3128,2498],{"className":3129},[83,299],[41,3131,731],{"className":3132},[730],[41,3134,3136],{"className":3135},[280],[41,3137,3139],{"className":3138,"style":790},[284],[41,3140],{},[41,3142],{"className":3143,"style":247},[246],[41,3145,3079],{"className":3146},[251],[41,3148],{"className":3149,"style":247},[246],[41,3151,3153,3156],{"className":3152},[74],[41,3154],{"className":3155,"style":2247},[78],[41,3157,3082],{"className":3158,"style":3159},[83,84],"margin-right:0.109em;"," 时，将每个参数的梯度原地乘以缩放系数 ",[41,3162,3164,3198],{"className":3163},[44],[41,3165,3167],{"className":3166},[48],[50,3168,3169],{"xmlns":52},[54,3170,3171,3195],{},[57,3172,3173],{},[521,3174,3175,3177],{},[60,3176,3082],{},[57,3178,3179,3181,3183,3189,3192],{},[60,3180,2648],{"mathvariant":497},[60,3182,153],{},[532,3184,3185,3187],{},[60,3186,2648],{"mathvariant":497},[338,3188,2498],{},[129,3190,3191],{},"+",[60,3193,3194],{},"ε",[63,3196,3197],{"encoding":65},"\\frac{M}{\\|g\\|_2 + \\varepsilon}",[41,3199,3201],{"className":3200,"ariaHidden":70},[69],[41,3202,3204,3208],{"className":3203},[74],[41,3205],{"className":3206,"style":3207},[78],"height:1.3923em;vertical-align:-0.52em;",[41,3209,3211,3214,3329],{"className":3210},[83],[41,3212],{"className":3213},[400,644],[41,3215,3217],{"className":3216},[521],[41,3218,3220,3320],{"className":3219},[276,651],[41,3221,3223,3317],{"className":3222},[280],[41,3224,3227,3295,3303],{"className":3225,"style":3226},[284],"height:0.8723em;",[41,3228,3229,3232],{"style":2311},[41,3230],{"className":3231,"style":665},[291],[41,3233,3235],{"className":3234},[296,297,298,299],[41,3236,3238,3241,3244,3289,3292],{"className":3237},[83,299],[41,3239,2648],{"className":3240},[83,299],[41,3242,153],{"className":3243,"style":380},[83,84,299],[41,3245,3247,3250],{"className":3246},[83,299],[41,3248,2648],{"className":3249},[83,299],[41,3251,3253],{"className":3252},[272],[41,3254,3256,3280],{"className":3255},[276,651],[41,3257,3259,3277],{"className":3258},[280],[41,3260,3263],{"className":3261,"style":3262},[284],"height:0.3173em;",[41,3264,3266,3270],{"style":3265},"top:-2.357em;margin-left:0em;margin-right:0.0714em;",[41,3267],{"className":3268,"style":3269},[291],"height:2.5em;",[41,3271,3274],{"className":3272},[296,3273,2289,299],"reset-size3",[41,3275,2498],{"className":3276},[83,299],[41,3278,731],{"className":3279},[730],[41,3281,3283],{"className":3282},[280],[41,3284,3287],{"className":3285,"style":3286},[284],"height:0.143em;",[41,3288],{},[41,3290,3191],{"className":3291},[310,299],[41,3293,3194],{"className":3294},[83,84,299],[41,3296,3297,3300],{"style":798},[41,3298],{"className":3299,"style":665},[291],[41,3301],{"className":3302,"style":806},[805],[41,3304,3305,3308],{"style":2334},[41,3306],{"className":3307,"style":665},[291],[41,3309,3311],{"className":3310},[296,297,298,299],[41,3312,3314],{"className":3313},[83,299],[41,3315,3082],{"className":3316,"style":3159},[83,84,299],[41,3318,731],{"className":3319},[730],[41,3321,3323],{"className":3322},[280],[41,3324,3327],{"className":3325,"style":3326},[284],"height:0.52em;",[41,3328],{},[41,3330],{"className":3331},[450,644],"（其中 ",[41,3334,3336,3364],{"className":3335},[44],[41,3337,3339],{"className":3338},[48],[50,3340,3341],{"xmlns":52},[54,3342,3343,3361],{},[57,3344,3345,3347,3349],{},[60,3346,3194],{},[129,3348,513],{},[210,3350,3351,3354],{},[338,3352,3353],{},"10",[57,3355,3356,3358],{},[129,3357,353],{},[338,3359,3360],{},"6",[63,3362,3363],{"encoding":65},"\\varepsilon = 10^{-6}",[41,3365,3367,3385],{"className":3366,"ariaHidden":70},[69],[41,3368,3370,3373,3376,3379,3382],{"className":3369},[74],[41,3371],{"className":3372,"style":1074},[78],[41,3374,3194],{"className":3375},[83,84],[41,3377],{"className":3378,"style":247},[246],[41,3380,513],{"className":3381},[251],[41,3383],{"className":3384,"style":247},[246],[41,3386,3388,3392,3395],{"className":3387},[74],[41,3389],{"className":3390,"style":3391},[78],"height:0.8141em;",[41,3393,356],{"className":3394},[83],[41,3396,3398,3401],{"className":3397},[83],[41,3399,340],{"className":3400},[83],[41,3402,3404],{"className":3403},[272],[41,3405,3407],{"className":3406},[276],[41,3408,3410],{"className":3409},[280],[41,3411,3413],{"className":3412,"style":3391},[284],[41,3414,3415,3418],{"style":287},[41,3416],{"className":3417,"style":292},[291],[41,3419,3421],{"className":3420},[296,297,298,299],[41,3422,3424,3427],{"className":3423},[83,299],[41,3425,353],{"className":3426},[83,299],[41,3428,3360],{"className":3429},[83,299],"）：",[1963,3432,3434],{"className":1965,"code":3433,"language":1967,"meta":1968,"style":1968},"grads = [p.grad for p in parameters if p.grad is not None]\ntotal_norm = torch.sqrt(sum(torch.sum(g.detach().to(torch.float32) ** 2) for g in grads))\nif total_norm > max_l2_norm:\n    scale = max_l2_norm \u002F (total_norm + eps)\n    for g in grads:\n        g.detach().mul_(scale.to(g.dtype))\n",[14,3435,3436,3476,3529,3541,3562,3574],{"__ignoreMap":1968},[41,3437,3438,3441,3443,3446,3449,3452,3455,3458,3461,3464,3467,3470,3473],{"class":1973,"line":1974},[41,3439,3440],{"class":1985},"grads ",[41,3442,513],{"class":2007},[41,3444,3445],{"class":1985}," [p.grad ",[41,3447,3448],{"class":2159},"for",[41,3450,3451],{"class":1985}," p ",[41,3453,3454],{"class":2159},"in",[41,3456,3457],{"class":1985}," parameters ",[41,3459,3460],{"class":2159},"if",[41,3462,3463],{"class":1985}," p.grad ",[41,3465,3466],{"class":1977},"is",[41,3468,3469],{"class":1977}," not",[41,3471,3472],{"class":2027}," None",[41,3474,3475],{"class":1985},"]\n",[41,3477,3478,3481,3483,3485,3487,3489,3492,3494,3496,3499,3502,3504,3507,3510,3513,3516,3519,3521,3524,3526],{"class":1973,"line":2001},[41,3479,3480],{"class":1985},"total_norm ",[41,3482,513],{"class":2007},[41,3484,2010],{"class":1985},[41,3486,2792],{"class":2013},[41,3488,501],{"class":1985},[41,3490,2076],{"class":3491},"svkvy",[41,3493,2073],{"class":1985},[41,3495,2076],{"class":2013},[41,3497,3498],{"class":1985},"(g.",[41,3500,3501],{"class":2013},"detach",[41,3503,2130],{"class":1985},[41,3505,3506],{"class":2013},"to",[41,3508,3509],{"class":1985},"(torch.float32) ",[41,3511,3512],{"class":2007},"**",[41,3514,3515],{"class":2027}," 2",[41,3517,3518],{"class":1985},") ",[41,3520,3448],{"class":2159},[41,3522,3523],{"class":1985}," g ",[41,3525,3454],{"class":2159},[41,3527,3528],{"class":1985}," grads))\n",[41,3530,3531,3533,3536,3538],{"class":1973,"line":2044},[41,3532,3460],{"class":2159},[41,3534,3535],{"class":1985}," total_norm ",[41,3537,3079],{"class":2007},[41,3539,3540],{"class":1985}," max_l2_norm:\n",[41,3542,3543,3546,3548,3551,3554,3557,3559],{"class":1973,"line":2061},[41,3544,3545],{"class":1985},"    scale ",[41,3547,513],{"class":2007},[41,3549,3550],{"class":1985}," max_l2_norm ",[41,3552,3553],{"class":2007},"\u002F",[41,3555,3556],{"class":1985}," (total_norm ",[41,3558,3191],{"class":2007},[41,3560,3561],{"class":1985}," eps)\n",[41,3563,3564,3567,3569,3571],{"class":1973,"line":2095},[41,3565,3566],{"class":2159},"    for",[41,3568,3523],{"class":1985},[41,3570,3454],{"class":2159},[41,3572,3573],{"class":1985}," grads:\n",[41,3575,3576,3579,3581,3583,3586,3589,3591],{"class":1973,"line":2156},[41,3577,3578],{"class":1985},"        g.",[41,3580,3501],{"class":2013},[41,3582,2130],{"class":1985},[41,3584,3585],{"class":2013},"mul_",[41,3587,3588],{"class":1985},"(scale.",[41,3590,3506],{"class":2013},[41,3592,3593],{"class":1985},"(g.dtype))\n",[10,3595,3596,3597,3600,3601,3604,3605,3608,3609,3611,3612,3615],{},"注意官方测试 ",[14,3598,3599],{},"test_gradient_clipping"," 会特意把最后一个参数设为 ",[14,3602,3603],{},"requires_grad_(False)","，它的 ",[14,3606,3607],{},"p.grad"," 为 ",[14,3610,2696],{},"，必须先过滤掉 ",[14,3613,3614],{},"p.grad is not None"," 才能汇总范数。",[109,3617,3619],{"id":3618},"解耦权重衰减的-adamw-优化器与训练显存核算","解耦权重衰减的 AdamW 优化器与训练显存核算",[10,3621,3622,3623,3721,3722,3754,3755,3783,3784,2630],{},"按照 Loshchilov & Hutter (2019) 与讲义算法 1，在第 ",[41,3624,3626,3658],{"className":3625},[44],[41,3627,3629],{"className":3628},[48],[50,3630,3631],{"xmlns":52},[54,3632,3633,3655],{},[57,3634,3635,3638,3640,3642,3644,3646,3648,3650,3652],{},[60,3636,3637],{},"t",[129,3639,513],{},[338,3641,356],{},[129,3643,343],{"separator":70},[338,3645,2498],{},[129,3647,343],{"separator":70},[129,3649,346],{},[129,3651,343],{"separator":70},[60,3653,3654],{},"T",[63,3656,3657],{"encoding":65},"t = 1, 2, \\dots, T",[41,3659,3661,3680],{"className":3660,"ariaHidden":70},[69],[41,3662,3664,3668,3671,3674,3677],{"className":3663},[74],[41,3665],{"className":3666,"style":3667},[78],"height:0.6151em;",[41,3669,3637],{"className":3670},[83,84],[41,3672],{"className":3673,"style":247},[246],[41,3675,513],{"className":3676},[251],[41,3678],{"className":3679,"style":247},[246],[41,3681,3683,3687,3690,3693,3696,3699,3702,3705,3708,3711,3714,3717],{"className":3682},[74],[41,3684],{"className":3685,"style":3686},[78],"height:0.8778em;vertical-align:-0.1944em;",[41,3688,356],{"className":3689},[83],[41,3691,343],{"className":3692},[407],[41,3694],{"className":3695,"style":411},[246],[41,3697,2498],{"className":3698},[83],[41,3700,343],{"className":3701},[407],[41,3703],{"className":3704,"style":411},[246],[41,3706,346],{"className":3707},[415],[41,3709],{"className":3710,"style":411},[246],[41,3712,343],{"className":3713},[407],[41,3715],{"className":3716,"style":411},[246],[41,3718,3654],{"className":3719,"style":3720},[83,84],"margin-right:0.1389em;"," 步，AdamW 对每个参数 ",[41,3723,3725,3740],{"className":3724},[44],[41,3726,3728],{"className":3727},[48],[50,3729,3730],{"xmlns":52},[54,3731,3732,3737],{},[57,3733,3734],{},[60,3735,3736],{},"θ",[63,3738,3739],{"encoding":65},"\\theta",[41,3741,3743],{"className":3742,"ariaHidden":70},[69],[41,3744,3746,3750],{"className":3745},[74],[41,3747],{"className":3748,"style":3749},[78],"height:0.6944em;",[41,3751,3736],{"className":3752,"style":3753},[83,84],"margin-right:0.0278em;"," 维护一阶动量 ",[41,3756,3758,3771],{"className":3757},[44],[41,3759,3761],{"className":3760},[48],[50,3762,3763],{"xmlns":52},[54,3764,3765,3769],{},[57,3766,3767],{},[60,3768,931],{},[63,3770,931],{"encoding":65},[41,3772,3774],{"className":3773,"ariaHidden":70},[69],[41,3775,3777,3780],{"className":3776},[74],[41,3778],{"className":3779,"style":1074},[78],[41,3781,931],{"className":3782},[83,84]," 和二阶动量 ",[41,3785,3787,3801],{"className":3786},[44],[41,3788,3790],{"className":3789},[48],[50,3791,3792],{"xmlns":52},[54,3793,3794,3799],{},[57,3795,3796],{},[60,3797,3798],{},"v",[63,3800,3798],{"encoding":65},[41,3802,3804],{"className":3803,"ariaHidden":70},[69],[41,3805,3807,3810],{"className":3806},[74],[41,3808],{"className":3809,"style":1074},[78],[41,3811,3798],{"className":3812,"style":380},[83,84],[41,3814,3816],{"className":3815},[481],[41,3817,3819,4113],{"className":3818},[44],[41,3820,3822],{"className":3821},[48],[50,3823,3824],{"xmlns":52,"display":490},[54,3825,3826,4110],{},[3827,3828,3832,3900,3966,4025],"mtable",{"rowspacing":3829,"columnalign":3830,"columnspacing":3831},"0.25em","right left","0em",[3833,3834,3835,3847],"mtr",{},[3836,3837,3838],"mtd",{},[3839,3840,3841],"mstyle",{"scriptlevel":340,"displaystyle":70},[532,3842,3843,3845],{},[60,3844,931],{},[60,3846,3637],{},[3836,3848,3849],{},[3839,3850,3851],{"scriptlevel":340,"displaystyle":70},[57,3852,3853,3855,3857,3864,3876,3878,3880,3882,3884,3890,3892,3898],{},[57,3854],{},[129,3856,513],{},[532,3858,3859,3862],{},[60,3860,3861],{},"β",[338,3863,356],{},[532,3865,3866,3868],{},[60,3867,931],{},[57,3869,3870,3872,3874],{},[60,3871,3637],{},[129,3873,353],{},[338,3875,356],{},[129,3877,3191],{},[129,3879,501],{"stretchy":335},[338,3881,356],{},[129,3883,353],{},[532,3885,3886,3888],{},[60,3887,3861],{},[338,3889,356],{},[129,3891,510],{"stretchy":335},[532,3893,3894,3896],{},[60,3895,153],{},[60,3897,3637],{},[129,3899,343],{"separator":70},[3833,3901,3902,3912],{},[3836,3903,3904],{},[3839,3905,3906],{"scriptlevel":340,"displaystyle":70},[532,3907,3908,3910],{},[60,3909,3798],{},[60,3911,3637],{},[3836,3913,3914],{},[3839,3915,3916],{"scriptlevel":340,"displaystyle":70},[57,3917,3918,3920,3922,3928,3940,3942,3944,3946,3948,3954,3956,3964],{},[57,3919],{},[129,3921,513],{},[532,3923,3924,3926],{},[60,3925,3861],{},[338,3927,2498],{},[532,3929,3930,3932],{},[60,3931,3798],{},[57,3933,3934,3936,3938],{},[60,3935,3637],{},[129,3937,353],{},[338,3939,356],{},[129,3941,3191],{},[129,3943,501],{"stretchy":335},[338,3945,356],{},[129,3947,353],{},[532,3949,3950,3952],{},[60,3951,3861],{},[338,3953,2498],{},[129,3955,510],{"stretchy":335},[2212,3957,3958,3960,3962],{},[60,3959,153],{},[60,3961,3637],{},[338,3963,2498],{},[129,3965,343],{"separator":70},[3833,3967,3968,3979],{},[3836,3969,3970],{},[3839,3971,3972],{"scriptlevel":340,"displaystyle":70},[532,3973,3974,3977],{},[60,3975,3976],{},"α",[60,3978,3637],{},[3836,3980,3981],{},[3839,3982,3983],{"scriptlevel":340,"displaystyle":70},[57,3984,3985,3987,3989,3991,4023],{},[57,3986],{},[129,3988,513],{},[60,3990,3976],{},[521,3992,3993,4009],{},[2660,3994,3995],{},[57,3996,3997,3999,4001],{},[338,3998,356],{},[129,4000,353],{},[2212,4002,4003,4005,4007],{},[60,4004,3861],{},[338,4006,2498],{},[60,4008,3637],{},[57,4010,4011,4013,4015],{},[338,4012,356],{},[129,4014,353],{},[2212,4016,4017,4019,4021],{},[60,4018,3861],{},[338,4020,356],{},[60,4022,3637],{},[129,4024,343],{"separator":70},[3833,4026,4027,4037],{},[3836,4028,4029],{},[3839,4030,4031],{"scriptlevel":340,"displaystyle":70},[532,4032,4033,4035],{},[60,4034,3736],{},[60,4036,3637],{},[3836,4038,4039],{},[3839,4040,4041],{"scriptlevel":340,"displaystyle":70},[57,4042,4043,4045,4047,4059,4061,4067,4089,4091,4093,4096,4108],{},[57,4044],{},[129,4046,513],{},[532,4048,4049,4051],{},[60,4050,3736],{},[57,4052,4053,4055,4057],{},[60,4054,3637],{},[129,4056,353],{},[338,4058,356],{},[129,4060,353],{},[532,4062,4063,4065],{},[60,4064,3976],{},[60,4066,3637],{},[521,4068,4069,4075],{},[532,4070,4071,4073],{},[60,4072,931],{},[60,4074,3637],{},[57,4076,4077,4085,4087],{},[2660,4078,4079],{},[532,4080,4081,4083],{},[60,4082,3798],{},[60,4084,3637],{},[129,4086,3191],{},[60,4088,3194],{},[129,4090,353],{},[60,4092,3976],{},[60,4094,4095],{},"λ",[532,4097,4098,4100],{},[60,4099,3736],{},[57,4101,4102,4104,4106],{},[60,4103,3637],{},[129,4105,353],{},[338,4107,356],{},[60,4109,574],{"mathvariant":497},[63,4111,4112],{"encoding":65},"\\begin{aligned}\nm_t &= \\beta_1 m_{t-1} + (1 - \\beta_1) g_t, \\\\\nv_t &= \\beta_2 v_{t-1} + (1 - \\beta_2) g_t^2, \\\\\n\\alpha_t &= \\alpha \\frac{\\sqrt{1 - \\beta_2^t}}{1 - \\beta_1^t}, \\\\\n\\theta_t &= \\theta_{t-1} - \\alpha_t \\frac{m_t}{\\sqrt{v_t} + \\varepsilon} - \\alpha \\lambda \\theta_{t-1}.\n\\end{aligned}",[41,4114,4116],{"className":4115,"ariaHidden":70},[69],[41,4117,4119,4123],{"className":4118},[74],[41,4120],{"className":4121,"style":4122},[78],"height:8.0147em;vertical-align:-3.7573em;",[41,4124,4126],{"className":4125},[83],[41,4127,4129,4357],{"className":4128},[3827],[41,4130,4133],{"className":4131},[4132],"col-align-r",[41,4134,4136,4348],{"className":4135},[276,651],[41,4137,4139,4345],{"className":4138},[280],[41,4140,4143,4194,4244,4295],{"className":4141,"style":4142},[284],"height:4.2573em;",[41,4144,4146,4150],{"style":4145},"top:-7.0473em;",[41,4147],{"className":4148,"style":4149},[291],"height:3.63em;",[41,4151,4153],{"className":4152},[83],[41,4154,4156,4159],{"className":4155},[83],[41,4157,931],{"className":4158},[83,84],[41,4160,4162],{"className":4161},[272],[41,4163,4165,4186],{"className":4164},[276,651],[41,4166,4168,4183],{"className":4167},[280],[41,4169,4172],{"className":4170,"style":4171},[284],"height:0.2806em;",[41,4173,4174,4177],{"style":771},[41,4175],{"className":4176,"style":292},[291],[41,4178,4180],{"className":4179},[296,297,298,299],[41,4181,3637],{"className":4182},[83,84,299],[41,4184,731],{"className":4185},[730],[41,4187,4189],{"className":4188},[280],[41,4190,4192],{"className":4191,"style":790},[284],[41,4193],{},[41,4195,4197,4200],{"style":4196},"top:-5.5232em;",[41,4198],{"className":4199,"style":4149},[291],[41,4201,4203],{"className":4202},[83],[41,4204,4206,4209],{"className":4205},[83],[41,4207,3798],{"className":4208,"style":380},[83,84],[41,4210,4212],{"className":4211},[272],[41,4213,4215,4236],{"className":4214},[276,651],[41,4216,4218,4233],{"className":4217},[280],[41,4219,4221],{"className":4220,"style":4171},[284],[41,4222,4224,4227],{"style":4223},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[41,4225],{"className":4226,"style":292},[291],[41,4228,4230],{"className":4229},[296,297,298,299],[41,4231,3637],{"className":4232},[83,84,299],[41,4234,731],{"className":4235},[730],[41,4237,4239],{"className":4238},[280],[41,4240,4242],{"className":4241,"style":790},[284],[41,4243],{},[41,4245,4247,4250],{"style":4246},"top:-3.2332em;",[41,4248],{"className":4249,"style":4149},[291],[41,4251,4253],{"className":4252},[83],[41,4254,4256,4260],{"className":4255},[83],[41,4257,3976],{"className":4258,"style":4259},[83,84],"margin-right:0.0037em;",[41,4261,4263],{"className":4262},[272],[41,4264,4266,4287],{"className":4265},[276,651],[41,4267,4269,4284],{"className":4268},[280],[41,4270,4272],{"className":4271,"style":4171},[284],[41,4273,4275,4278],{"style":4274},"top:-2.55em;margin-left:-0.0037em;margin-right:0.05em;",[41,4276],{"className":4277,"style":292},[291],[41,4279,4281],{"className":4280},[296,297,298,299],[41,4282,3637],{"className":4283},[83,84,299],[41,4285,731],{"className":4286},[730],[41,4288,4290],{"className":4289},[280],[41,4291,4293],{"className":4292,"style":790},[284],[41,4294],{},[41,4296,4298,4301],{"style":4297},"top:-0.8734em;",[41,4299],{"className":4300,"style":4149},[291],[41,4302,4304],{"className":4303},[83],[41,4305,4307,4310],{"className":4306},[83],[41,4308,3736],{"className":4309,"style":3753},[83,84],[41,4311,4313],{"className":4312},[272],[41,4314,4316,4337],{"className":4315},[276,651],[41,4317,4319,4334],{"className":4318},[280],[41,4320,4322],{"className":4321,"style":4171},[284],[41,4323,4325,4328],{"style":4324},"top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;",[41,4326],{"className":4327,"style":292},[291],[41,4329,4331],{"className":4330},[296,297,298,299],[41,4332,3637],{"className":4333},[83,84,299],[41,4335,731],{"className":4336},[730],[41,4338,4340],{"className":4339},[280],[41,4341,4343],{"className":4342,"style":790},[284],[41,4344],{},[41,4346,731],{"className":4347},[730],[41,4349,4351],{"className":4350},[280],[41,4352,4355],{"className":4353,"style":4354},[284],"height:3.7573em;",[41,4356],{},[41,4358,4361],{"className":4359},[4360],"col-align-l",[41,4362,4364,5484],{"className":4363},[276,651],[41,4365,4367,5481],{"className":4366},[280],[41,4368,4370,4592,4826,5093],{"className":4369,"style":4142},[284],[41,4371,4372,4375],{"style":4145},[41,4373],{"className":4374,"style":4149},[291],[41,4376,4378,4381,4384,4387,4390,4432,4482,4485,4488,4491,4494,4497,4500,4503,4506,4546,4549,4589],{"className":4377},[83],[41,4379],{"className":4380},[83],[41,4382],{"className":4383,"style":247},[246],[41,4385,513],{"className":4386},[251],[41,4388],{"className":4389,"style":247},[246],[41,4391,4393,4397],{"className":4392},[83],[41,4394,3861],{"className":4395,"style":4396},[83,84],"margin-right:0.0528em;",[41,4398,4400],{"className":4399},[272],[41,4401,4403,4424],{"className":4402},[276,651],[41,4404,4406,4421],{"className":4405},[280],[41,4407,4409],{"className":4408,"style":2530},[284],[41,4410,4412,4415],{"style":4411},"top:-2.55em;margin-left:-0.0528em;margin-right:0.05em;",[41,4413],{"className":4414,"style":292},[291],[41,4416,4418],{"className":4417},[296,297,298,299],[41,4419,356],{"className":4420},[83,299],[41,4422,731],{"className":4423},[730],[41,4425,4427],{"className":4426},[280],[41,4428,4430],{"className":4429,"style":790},[284],[41,4431],{},[41,4433,4435,4438],{"className":4434},[83],[41,4436,931],{"className":4437},[83,84],[41,4439,4441],{"className":4440},[272],[41,4442,4444,4473],{"className":4443},[276,651],[41,4445,4447,4470],{"className":4446},[280],[41,4448,4450],{"className":4449,"style":2530},[284],[41,4451,4452,4455],{"style":771},[41,4453],{"className":4454,"style":292},[291],[41,4456,4458],{"className":4457},[296,297,298,299],[41,4459,4461,4464,4467],{"className":4460},[83,299],[41,4462,3637],{"className":4463},[83,84,299],[41,4465,353],{"className":4466},[310,299],[41,4468,356],{"className":4469},[83,299],[41,4471,731],{"className":4472},[730],[41,4474,4476],{"className":4475},[280],[41,4477,4480],{"className":4478,"style":4479},[284],"height:0.2083em;",[41,4481],{},[41,4483],{"className":4484,"style":314},[246],[41,4486,3191],{"className":4487},[310],[41,4489],{"className":4490,"style":314},[246],[41,4492,501],{"className":4493},[400],[41,4495,356],{"className":4496},[83],[41,4498],{"className":4499,"style":314},[246],[41,4501,353],{"className":4502},[310],[41,4504],{"className":4505,"style":314},[246],[41,4507,4509,4512],{"className":4508},[83],[41,4510,3861],{"className":4511,"style":4396},[83,84],[41,4513,4515],{"className":4514},[272],[41,4516,4518,4538],{"className":4517},[276,651],[41,4519,4521,4535],{"className":4520},[280],[41,4522,4524],{"className":4523,"style":2530},[284],[41,4525,4526,4529],{"style":4411},[41,4527],{"className":4528,"style":292},[291],[41,4530,4532],{"className":4531},[296,297,298,299],[41,4533,356],{"className":4534},[83,299],[41,4536,731],{"className":4537},[730],[41,4539,4541],{"className":4540},[280],[41,4542,4544],{"className":4543,"style":790},[284],[41,4545],{},[41,4547,510],{"className":4548},[450],[41,4550,4552,4555],{"className":4551},[83],[41,4553,153],{"className":4554,"style":380},[83,84],[41,4556,4558],{"className":4557},[272],[41,4559,4561,4581],{"className":4560},[276,651],[41,4562,4564,4578],{"className":4563},[280],[41,4565,4567],{"className":4566,"style":4171},[284],[41,4568,4569,4572],{"style":4223},[41,4570],{"className":4571,"style":292},[291],[41,4573,4575],{"className":4574},[296,297,298,299],[41,4576,3637],{"className":4577},[83,84,299],[41,4579,731],{"className":4580},[730],[41,4582,4584],{"className":4583},[280],[41,4585,4587],{"className":4586,"style":790},[284],[41,4588],{},[41,4590,343],{"className":4591},[407],[41,4593,4594,4597],{"style":4196},[41,4595],{"className":4596,"style":4149},[291],[41,4598,4600,4603,4606,4609,4612,4652,4701,4704,4707,4710,4713,4716,4719,4722,4725,4765,4768,4823],{"className":4599},[83],[41,4601],{"className":4602},[83],[41,4604],{"className":4605,"style":247},[246],[41,4607,513],{"className":4608},[251],[41,4610],{"className":4611,"style":247},[246],[41,4613,4615,4618],{"className":4614},[83],[41,4616,3861],{"className":4617,"style":4396},[83,84],[41,4619,4621],{"className":4620},[272],[41,4622,4624,4644],{"className":4623},[276,651],[41,4625,4627,4641],{"className":4626},[280],[41,4628,4630],{"className":4629,"style":2530},[284],[41,4631,4632,4635],{"style":4411},[41,4633],{"className":4634,"style":292},[291],[41,4636,4638],{"className":4637},[296,297,298,299],[41,4639,2498],{"className":4640},[83,299],[41,4642,731],{"className":4643},[730],[41,4645,4647],{"className":4646},[280],[41,4648,4650],{"className":4649,"style":790},[284],[41,4651],{},[41,4653,4655,4658],{"className":4654},[83],[41,4656,3798],{"className":4657,"style":380},[83,84],[41,4659,4661],{"className":4660},[272],[41,4662,4664,4693],{"className":4663},[276,651],[41,4665,4667,4690],{"className":4666},[280],[41,4668,4670],{"className":4669,"style":2530},[284],[41,4671,4672,4675],{"style":4223},[41,4673],{"className":4674,"style":292},[291],[41,4676,4678],{"className":4677},[296,297,298,299],[41,4679,4681,4684,4687],{"className":4680},[83,299],[41,4682,3637],{"className":4683},[83,84,299],[41,4685,353],{"className":4686},[310,299],[41,4688,356],{"className":4689},[83,299],[41,4691,731],{"className":4692},[730],[41,4694,4696],{"className":4695},[280],[41,4697,4699],{"className":4698,"style":4479},[284],[41,4700],{},[41,4702],{"className":4703,"style":314},[246],[41,4705,3191],{"className":4706},[310],[41,4708],{"className":4709,"style":314},[246],[41,4711,501],{"className":4712},[400],[41,4714,356],{"className":4715},[83],[41,4717],{"className":4718,"style":314},[246],[41,4720,353],{"className":4721},[310],[41,4723],{"className":4724,"style":314},[246],[41,4726,4728,4731],{"className":4727},[83],[41,4729,3861],{"className":4730,"style":4396},[83,84],[41,4732,4734],{"className":4733},[272],[41,4735,4737,4757],{"className":4736},[276,651],[41,4738,4740,4754],{"className":4739},[280],[41,4741,4743],{"className":4742,"style":2530},[284],[41,4744,4745,4748],{"style":4411},[41,4746],{"className":4747,"style":292},[291],[41,4749,4751],{"className":4750},[296,297,298,299],[41,4752,2498],{"className":4753},[83,299],[41,4755,731],{"className":4756},[730],[41,4758,4760],{"className":4759},[280],[41,4761,4763],{"className":4762,"style":790},[284],[41,4764],{},[41,4766,510],{"className":4767},[450],[41,4769,4771,4774],{"className":4770},[83],[41,4772,153],{"className":4773,"style":380},[83,84],[41,4775,4777],{"className":4776},[272],[41,4778,4780,4814],{"className":4779},[276,651],[41,4781,4783,4811],{"className":4782},[280],[41,4784,4787,4799],{"className":4785,"style":4786},[284],"height:0.8641em;",[41,4788,4790,4793],{"style":4789},"top:-2.453em;margin-left:-0.0359em;margin-right:0.05em;",[41,4791],{"className":4792,"style":292},[291],[41,4794,4796],{"className":4795},[296,297,298,299],[41,4797,3637],{"className":4798},[83,84,299],[41,4800,4802,4805],{"style":4801},"top:-3.113em;margin-right:0.05em;",[41,4803],{"className":4804,"style":292},[291],[41,4806,4808],{"className":4807},[296,297,298,299],[41,4809,2498],{"className":4810},[83,299],[41,4812,731],{"className":4813},[730],[41,4815,4817],{"className":4816},[280],[41,4818,4821],{"className":4819,"style":4820},[284],"height:0.247em;",[41,4822],{},[41,4824,343],{"className":4825},[407],[41,4827,4828,4831],{"style":4246},[41,4829],{"className":4830,"style":4149},[291],[41,4832,4834,4837,4840,4843,4846,4849,5090],{"className":4833},[83],[41,4835],{"className":4836},[83],[41,4838],{"className":4839,"style":247},[246],[41,4841,513],{"className":4842},[251],[41,4844],{"className":4845,"style":247},[246],[41,4847,3976],{"className":4848,"style":4259},[83,84],[41,4850,4852,4855,5087],{"className":4851},[83],[41,4853],{"className":4854},[400,644],[41,4856,4858],{"className":4857},[521],[41,4859,4861,5078],{"className":4860},[276,651],[41,4862,4864,5075],{"className":4863},[280],[41,4865,4868,4942,4950],{"className":4866,"style":4867},[284],"height:1.63em;",[41,4869,4871,4874],{"style":4870},"top:-2.314em;",[41,4872],{"className":4873,"style":665},[291],[41,4875,4877,4880,4883,4886,4889],{"className":4876},[83],[41,4878,356],{"className":4879},[83],[41,4881],{"className":4882,"style":314},[246],[41,4884,353],{"className":4885},[310],[41,4887],{"className":4888,"style":314},[246],[41,4890,4892,4895],{"className":4891},[83],[41,4893,3861],{"className":4894,"style":4396},[83,84],[41,4896,4898],{"className":4897},[272],[41,4899,4901,4934],{"className":4900},[276,651],[41,4902,4904,4931],{"className":4903},[280],[41,4905,4908,4920],{"className":4906,"style":4907},[284],"height:0.7754em;",[41,4909,4911,4914],{"style":4910},"top:-2.4337em;margin-left:-0.0528em;margin-right:0.05em;",[41,4912],{"className":4913,"style":292},[291],[41,4915,4917],{"className":4916},[296,297,298,299],[41,4918,356],{"className":4919},[83,299],[41,4921,4922,4925],{"style":2992},[41,4923],{"className":4924,"style":292},[291],[41,4926,4928],{"className":4927},[296,297,298,299],[41,4929,3637],{"className":4930},[83,84,299],[41,4932,731],{"className":4933},[730],[41,4935,4937],{"className":4936},[280],[41,4938,4940],{"className":4939,"style":3011},[284],[41,4941],{},[41,4943,4944,4947],{"style":798},[41,4945],{"className":4946,"style":665},[291],[41,4948],{"className":4949,"style":806},[805],[41,4951,4953,4956],{"style":4952},"top:-3.6905em;",[41,4954],{"className":4955,"style":665},[291],[41,4957,4959],{"className":4958},[83],[41,4960,4962],{"className":4961},[83,2792],[41,4963,4965,5066],{"className":4964},[276,651],[41,4966,4968,5063],{"className":4967},[280],[41,4969,4972,5046],{"className":4970,"style":4971},[284],"height:0.9395em;",[41,4973,4976,4980],{"className":4974,"style":4975},[2806],"top:-3.2em;",[41,4977],{"className":4978,"style":4979},[291],"height:3.2em;",[41,4981,4983,4986,4989,4992,4995],{"className":4982,"style":2815},[83],[41,4984,356],{"className":4985},[83],[41,4987],{"className":4988,"style":314},[246],[41,4990,353],{"className":4991},[310],[41,4993],{"className":4994,"style":314},[246],[41,4996,4998,5001],{"className":4997},[83],[41,4999,3861],{"className":5000,"style":4396},[83,84],[41,5002,5004],{"className":5003},[272],[41,5005,5007,5038],{"className":5006},[276,651],[41,5008,5010,5035],{"className":5009},[280],[41,5011,5013,5024],{"className":5012,"style":4907},[284],[41,5014,5015,5018],{"style":4910},[41,5016],{"className":5017,"style":292},[291],[41,5019,5021],{"className":5020},[296,297,298,299],[41,5022,2498],{"className":5023},[83,299],[41,5025,5026,5029],{"style":2992},[41,5027],{"className":5028,"style":292},[291],[41,5030,5032],{"className":5031},[296,297,298,299],[41,5033,3637],{"className":5034},[83,84,299],[41,5036,731],{"className":5037},[730],[41,5039,5041],{"className":5040},[280],[41,5042,5044],{"className":5043,"style":3011},[284],[41,5045],{},[41,5047,5049,5052],{"style":5048},"top:-2.8995em;",[41,5050],{"className":5051,"style":4979},[291],[41,5053,5056],{"className":5054,"style":5055},[3023],"min-width:1.02em;height:1.28em;",[3026,5057,5060],{"xmlns":3028,"width":3029,"height":5058,"viewBox":5059,"preserveAspectRatio":3032},"1.28em","0 0 400000 1296",[3034,5061],{"d":5062},"M263,681c0.7,0,18,39.7,52,119\nc34,79.3,68.167,158.7,102.5,238c34.3,79.3,51.8,119.3,52.5,120\nc340,-704.7,510.7,-1060.3,512,-1067\nl0 -0\nc4.7,-7.3,11,-11,19,-11\nH40000v40H1012.3\ns-271.3,567,-271.3,567c-38.7,80.7,-84,175,-136,283c-52,108,-89.167,185.3,-111.5,232\nc-22.3,46.7,-33.8,70.3,-34.5,71c-4.7,4.7,-12.3,7,-23,7s-12,-1,-12,-1\ns-109,-253,-109,-253c-72.7,-168,-109.3,-252,-110,-252c-10.7,8,-22,16.7,-34,26\nc-22,17.3,-33.3,26,-34,26s-26,-26,-26,-26s76,-59,76,-59s76,-60,76,-60z\nM1001 80h400000v40h-400000z",[41,5064,731],{"className":5065},[730],[41,5067,5069],{"className":5068},[280],[41,5070,5073],{"className":5071,"style":5072},[284],"height:0.3005em;",[41,5074],{},[41,5076,731],{"className":5077},[730],[41,5079,5081],{"className":5080},[280],[41,5082,5085],{"className":5083,"style":5084},[284],"height:0.9523em;",[41,5086],{},[41,5088],{"className":5089},[450,644],[41,5091,343],{"className":5092},[407],[41,5094,5095,5098],{"style":4297},[41,5096],{"className":5097,"style":4149},[291],[41,5099,5101,5104,5107,5110,5113,5162,5165,5168,5171,5211,5414,5417,5420,5423,5426,5429,5478],{"className":5100},[83],[41,5102],{"className":5103},[83],[41,5105],{"className":5106,"style":247},[246],[41,5108,513],{"className":5109},[251],[41,5111],{"className":5112,"style":247},[246],[41,5114,5116,5119],{"className":5115},[83],[41,5117,3736],{"className":5118,"style":3753},[83,84],[41,5120,5122],{"className":5121},[272],[41,5123,5125,5154],{"className":5124},[276,651],[41,5126,5128,5151],{"className":5127},[280],[41,5129,5131],{"className":5130,"style":2530},[284],[41,5132,5133,5136],{"style":4324},[41,5134],{"className":5135,"style":292},[291],[41,5137,5139],{"className":5138},[296,297,298,299],[41,5140,5142,5145,5148],{"className":5141},[83,299],[41,5143,3637],{"className":5144},[83,84,299],[41,5146,353],{"className":5147},[310,299],[41,5149,356],{"className":5150},[83,299],[41,5152,731],{"className":5153},[730],[41,5155,5157],{"className":5156},[280],[41,5158,5160],{"className":5159,"style":4479},[284],[41,5161],{},[41,5163],{"className":5164,"style":314},[246],[41,5166,353],{"className":5167},[310],[41,5169],{"className":5170,"style":314},[246],[41,5172,5174,5177],{"className":5173},[83],[41,5175,3976],{"className":5176,"style":4259},[83,84],[41,5178,5180],{"className":5179},[272],[41,5181,5183,5203],{"className":5182},[276,651],[41,5184,5186,5200],{"className":5185},[280],[41,5187,5189],{"className":5188,"style":4171},[284],[41,5190,5191,5194],{"style":4274},[41,5192],{"className":5193,"style":292},[291],[41,5195,5197],{"className":5196},[296,297,298,299],[41,5198,3637],{"className":5199},[83,84,299],[41,5201,731],{"className":5202},[730],[41,5204,5206],{"className":5205},[280],[41,5207,5209],{"className":5208,"style":790},[284],[41,5210],{},[41,5212,5214,5217,5411],{"className":5213},[83],[41,5215],{"className":5216},[400,644],[41,5218,5220],{"className":5219},[521],[41,5221,5223,5402],{"className":5222},[276,651],[41,5224,5226,5399],{"className":5225},[280],[41,5227,5230,5343,5351],{"className":5228,"style":5229},[284],"height:1.1076em;",[41,5231,5232,5235],{"style":4870},[41,5233],{"className":5234,"style":665},[291],[41,5236,5238,5331,5334,5337,5340],{"className":5237},[83],[41,5239,5241],{"className":5240},[83,2792],[41,5242,5244,5322],{"className":5243},[276,651],[41,5245,5247,5319],{"className":5246},[280],[41,5248,5251,5302],{"className":5249,"style":5250},[284],"height:0.7253em;",[41,5252,5255,5258],{"className":5253,"style":5254},[2806],"top:-3em;",[41,5256],{"className":5257,"style":665},[291],[41,5259,5262],{"className":5260,"style":5261},[83],"padding-left:0.833em;",[41,5263,5265,5268],{"className":5264},[83],[41,5266,3798],{"className":5267,"style":380},[83,84],[41,5269,5271],{"className":5270},[272],[41,5272,5274,5294],{"className":5273},[276,651],[41,5275,5277,5291],{"className":5276},[280],[41,5278,5280],{"className":5279,"style":4171},[284],[41,5281,5282,5285],{"style":4223},[41,5283],{"className":5284,"style":292},[291],[41,5286,5288],{"className":5287},[296,297,298,299],[41,5289,3637],{"className":5290},[83,84,299],[41,5292,731],{"className":5293},[730],[41,5295,5297],{"className":5296},[280],[41,5298,5300],{"className":5299,"style":790},[284],[41,5301],{},[41,5303,5305,5308],{"style":5304},"top:-2.6853em;",[41,5306],{"className":5307,"style":665},[291],[41,5309,5312],{"className":5310,"style":5311},[3023],"min-width:0.853em;height:1.08em;",[3026,5313,5316],{"xmlns":3028,"width":3029,"height":5314,"viewBox":5315,"preserveAspectRatio":3032},"1.08em","0 0 400000 1080",[3034,5317],{"d":5318},"M95,702\nc-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14\nc0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54\nc44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10\ns173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429\nc69,-144,104.5,-217.7,106.5,-221\nl0 -0\nc5.3,-9.3,12,-14,20,-14\nH400000v40H845.2724\ns-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7\nc-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z\nM834 80h400000v40h-400000z",[41,5320,731],{"className":5321},[730],[41,5323,5325],{"className":5324},[280],[41,5326,5329],{"className":5327,"style":5328},[284],"height:0.3147em;",[41,5330],{},[41,5332],{"className":5333,"style":314},[246],[41,5335,3191],{"className":5336},[310],[41,5338],{"className":5339,"style":314},[246],[41,5341,3194],{"className":5342},[83,84],[41,5344,5345,5348],{"style":798},[41,5346],{"className":5347,"style":665},[291],[41,5349],{"className":5350,"style":806},[805],[41,5352,5353,5356],{"style":809},[41,5354],{"className":5355,"style":665},[291],[41,5357,5359],{"className":5358},[83],[41,5360,5362,5365],{"className":5361},[83],[41,5363,931],{"className":5364},[83,84],[41,5366,5368],{"className":5367},[272],[41,5369,5371,5391],{"className":5370},[276,651],[41,5372,5374,5388],{"className":5373},[280],[41,5375,5377],{"className":5376,"style":4171},[284],[41,5378,5379,5382],{"style":771},[41,5380],{"className":5381,"style":292},[291],[41,5383,5385],{"className":5384},[296,297,298,299],[41,5386,3637],{"className":5387},[83,84,299],[41,5389,731],{"className":5390},[730],[41,5392,5394],{"className":5393},[280],[41,5395,5397],{"className":5396,"style":790},[284],[41,5398],{},[41,5400,731],{"className":5401},[730],[41,5403,5405],{"className":5404},[280],[41,5406,5409],{"className":5407,"style":5408},[284],"height:1.0007em;",[41,5410],{},[41,5412],{"className":5413},[450,644],[41,5415],{"className":5416,"style":314},[246],[41,5418,353],{"className":5419},[310],[41,5421],{"className":5422,"style":314},[246],[41,5424,3976],{"className":5425,"style":4259},[83,84],[41,5427,4095],{"className":5428},[83,84],[41,5430,5432,5435],{"className":5431},[83],[41,5433,3736],{"className":5434,"style":3753},[83,84],[41,5436,5438],{"className":5437},[272],[41,5439,5441,5470],{"className":5440},[276,651],[41,5442,5444,5467],{"className":5443},[280],[41,5445,5447],{"className":5446,"style":2530},[284],[41,5448,5449,5452],{"style":4324},[41,5450],{"className":5451,"style":292},[291],[41,5453,5455],{"className":5454},[296,297,298,299],[41,5456,5458,5461,5464],{"className":5457},[83,299],[41,5459,3637],{"className":5460},[83,84,299],[41,5462,353],{"className":5463},[310,299],[41,5465,356],{"className":5466},[83,299],[41,5468,731],{"className":5469},[730],[41,5471,5473],{"className":5472},[280],[41,5474,5476],{"className":5475,"style":4479},[284],[41,5477],{},[41,5479,574],{"className":5480},[83],[41,5482,731],{"className":5483},[730],[41,5485,5487],{"className":5486},[280],[41,5488,5490],{"className":5489,"style":4354},[284],[41,5491],{},[10,5493,5494,5495,5792,5793,5796,5797,5800,5801,5804,5805,5808,5809,5812],{},"这里最关键的区别在于：",[27,5496,5497,5498,5543,5544,5573,5574,5603,5604,5674,5675],{},"权重衰减项 ",[41,5499,5501,5521],{"className":5500},[44],[41,5502,5504],{"className":5503},[48],[50,5505,5506],{"xmlns":52},[54,5507,5508,5518],{},[57,5509,5510,5512,5514,5516],{},[129,5511,353],{},[60,5513,3976],{},[60,5515,4095],{},[60,5517,3736],{},[63,5519,5520],{"encoding":65},"-\\alpha \\lambda \\theta",[41,5522,5524],{"className":5523,"ariaHidden":70},[69],[41,5525,5527,5531,5534,5537,5540],{"className":5526},[74],[41,5528],{"className":5529,"style":5530},[78],"height:0.7778em;vertical-align:-0.0833em;",[41,5532,353],{"className":5533},[83],[41,5535,3976],{"className":5536,"style":4259},[83,84],[41,5538,4095],{"className":5539},[83,84],[41,5541,3736],{"className":5542,"style":3753},[83,84]," 直接使用原始学习率 ",[41,5545,5547,5561],{"className":5546},[44],[41,5548,5550],{"className":5549},[48],[50,5551,5552],{"xmlns":52},[54,5553,5554,5558],{},[57,5555,5556],{},[60,5557,3976],{},[63,5559,5560],{"encoding":65},"\\alpha",[41,5562,5564],{"className":5563,"ariaHidden":70},[69],[41,5565,5567,5570],{"className":5566},[74],[41,5568],{"className":5569,"style":1074},[78],[41,5571,3976],{"className":5572,"style":4259},[83,84]," 与解耦的权重衰减率 ",[41,5575,5577,5591],{"className":5576},[44],[41,5578,5580],{"className":5579},[48],[50,5581,5582],{"xmlns":52},[54,5583,5584,5588],{},[57,5585,5586],{},[60,5587,4095],{},[63,5589,5590],{"encoding":65},"\\lambda",[41,5592,5594],{"className":5593,"ariaHidden":70},[69],[41,5595,5597,5600],{"className":5596},[74],[41,5598],{"className":5599,"style":3749},[78],[41,5601,4095],{"className":5602},[83,84],"，不混入梯度 ",[41,5605,5607,5625],{"className":5606},[44],[41,5608,5610],{"className":5609},[48],[50,5611,5612],{"xmlns":52},[54,5613,5614,5622],{},[57,5615,5616],{},[532,5617,5618,5620],{},[60,5619,153],{},[60,5621,3637],{},[63,5623,5624],{"encoding":65},"g_t",[41,5626,5628],{"className":5627,"ariaHidden":70},[69],[41,5629,5631,5634],{"className":5630},[74],[41,5632],{"className":5633,"style":79},[78],[41,5635,5637,5640],{"className":5636},[83],[41,5638,153],{"className":5639,"style":380},[83,84],[41,5641,5643],{"className":5642},[272],[41,5644,5646,5666],{"className":5645},[276,651],[41,5647,5649,5663],{"className":5648},[280],[41,5650,5652],{"className":5651,"style":4171},[284],[41,5653,5654,5657],{"style":4223},[41,5655],{"className":5656,"style":292},[291],[41,5658,5660],{"className":5659},[296,297,298,299],[41,5661,3637],{"className":5662},[83,84,299],[41,5664,731],{"className":5665},[730],[41,5667,5669],{"className":5668},[280],[41,5670,5672],{"className":5671,"style":790},[284],[41,5673],{}," 和二阶动量分母 ",[41,5676,5678,5698],{"className":5677},[44],[41,5679,5681],{"className":5680},[48],[50,5682,5683],{"xmlns":52},[54,5684,5685,5695],{},[57,5686,5687],{},[2660,5688,5689],{},[532,5690,5691,5693],{},[60,5692,3798],{},[60,5694,3637],{},[63,5696,5697],{"encoding":65},"\\sqrt{v_t}",[41,5699,5701],{"className":5700,"ariaHidden":70},[69],[41,5702,5704,5708],{"className":5703},[74],[41,5705],{"className":5706,"style":5707},[78],"height:1.04em;vertical-align:-0.3147em;",[41,5709,5711],{"className":5710},[83,2792],[41,5712,5714,5784],{"className":5713},[276,651],[41,5715,5717,5781],{"className":5716},[280],[41,5718,5720,5769],{"className":5719,"style":5250},[284],[41,5721,5723,5726],{"className":5722,"style":5254},[2806],[41,5724],{"className":5725,"style":665},[291],[41,5727,5729],{"className":5728,"style":5261},[83],[41,5730,5732,5735],{"className":5731},[83],[41,5733,3798],{"className":5734,"style":380},[83,84],[41,5736,5738],{"className":5737},[272],[41,5739,5741,5761],{"className":5740},[276,651],[41,5742,5744,5758],{"className":5743},[280],[41,5745,5747],{"className":5746,"style":4171},[284],[41,5748,5749,5752],{"style":4223},[41,5750],{"className":5751,"style":292},[291],[41,5753,5755],{"className":5754},[296,297,298,299],[41,5756,3637],{"className":5757},[83,84,299],[41,5759,731],{"className":5760},[730],[41,5762,5764],{"className":5763},[280],[41,5765,5767],{"className":5766,"style":790},[284],[41,5768],{},[41,5770,5771,5774],{"style":5304},[41,5772],{"className":5773,"style":665},[291],[41,5775,5777],{"className":5776,"style":5311},[3023],[3026,5778,5779],{"xmlns":3028,"width":3029,"height":5314,"viewBox":5315,"preserveAspectRatio":3032},[3034,5780],{"d":5318},[41,5782,731],{"className":5783},[730],[41,5785,5787],{"className":5786},[280],[41,5788,5790],{"className":5789,"style":5328},[284],[41,5791],{},"。状态字典 ",[14,5794,5795],{},"self.state[p]"," 中保存步数 ",[14,5798,5799],{},"step","、一阶矩 ",[14,5802,5803],{},"exp_avg"," 和二阶矩 ",[14,5806,5807],{},"exp_avg_sq","，从而天然支持 ",[14,5810,5811],{},"optimizer.state_dict()"," 序列化。",[5814,5815,5817,5818,5821],"h3",{"id":5816},"adamw-训练峰值显存核算problem-adamw_accounting","AdamW 训练峰值显存核算（Problem ",[14,5819,5820],{},"adamw_accounting","）",[10,5823,5824],{},"假设全部使用 FP32（每元素 4 字节），训练时的显存由四部分组成：",[5826,5827,5828,5921,6006],"ol",{},[5829,5830,5831,2630,5834,5920],"li",{},[27,5832,5833],{},"模型参数（Parameters）",[41,5835,5837,5860],{"className":5836},[44],[41,5838,5840],{"className":5839},[48],[50,5841,5842],{"xmlns":52},[54,5843,5844,5857],{},[57,5845,5846,5849],{},[338,5847,5848],{},"4",[532,5850,5851,5854],{},[60,5852,5853],{},"N",[2681,5855,5856],{},"params",[63,5858,5859],{"encoding":65},"4 N_{\\text{params}}",[41,5861,5863],{"className":5862,"ariaHidden":70},[69],[41,5864,5866,5870,5873],{"className":5865},[74],[41,5867],{"className":5868,"style":5869},[78],"height:0.9694em;vertical-align:-0.2861em;",[41,5871,5848],{"className":5872},[83],[41,5874,5876,5879],{"className":5875},[83],[41,5877,5853],{"className":5878,"style":3159},[83,84],[41,5880,5882],{"className":5881},[272],[41,5883,5885,5912],{"className":5884},[276,651],[41,5886,5888,5909],{"className":5887},[280],[41,5889,5891],{"className":5890,"style":768},[284],[41,5892,5894,5897],{"style":5893},"top:-2.55em;margin-left:-0.109em;margin-right:0.05em;",[41,5895],{"className":5896,"style":292},[291],[41,5898,5900],{"className":5899},[296,297,298,299],[41,5901,5903],{"className":5902},[83,299],[41,5904,5906],{"className":5905},[83,2866,299],[41,5907,5856],{"className":5908},[83,299],[41,5910,731],{"className":5911},[730],[41,5913,5915],{"className":5914},[280],[41,5916,5918],{"className":5917,"style":860},[284],[41,5919],{}," 字节；",[5829,5922,5923,2630,5926,5920],{},[27,5924,5925],{},"参数梯度（Gradients）",[41,5927,5929,5948],{"className":5928},[44],[41,5930,5932],{"className":5931},[48],[50,5933,5934],{"xmlns":52},[54,5935,5936,5946],{},[57,5937,5938,5940],{},[338,5939,5848],{},[532,5941,5942,5944],{},[60,5943,5853],{},[2681,5945,5856],{},[63,5947,5859],{"encoding":65},[41,5949,5951],{"className":5950,"ariaHidden":70},[69],[41,5952,5954,5957,5960],{"className":5953},[74],[41,5955],{"className":5956,"style":5869},[78],[41,5958,5848],{"className":5959},[83],[41,5961,5963,5966],{"className":5962},[83],[41,5964,5853],{"className":5965,"style":3159},[83,84],[41,5967,5969],{"className":5968},[272],[41,5970,5972,5998],{"className":5971},[276,651],[41,5973,5975,5995],{"className":5974},[280],[41,5976,5978],{"className":5977,"style":768},[284],[41,5979,5980,5983],{"style":5893},[41,5981],{"className":5982,"style":292},[291],[41,5984,5986],{"className":5985},[296,297,298,299],[41,5987,5989],{"className":5988},[83,299],[41,5990,5992],{"className":5991},[83,2866,299],[41,5993,5856],{"className":5994},[83,299],[41,5996,731],{"className":5997},[730],[41,5999,6001],{"className":6000},[280],[41,6002,6004],{"className":6003,"style":860},[284],[41,6005],{},[5829,6007,6008,2630,6067,6246],{},[27,6009,6010,6011,1215,6039,5821],{},"AdamW 优化器状态（Optimizer State，",[41,6012,6014,6027],{"className":6013},[44],[41,6015,6017],{"className":6016},[48],[50,6018,6019],{"xmlns":52},[54,6020,6021,6025],{},[57,6022,6023],{},[60,6024,931],{},[63,6026,931],{"encoding":65},[41,6028,6030],{"className":6029,"ariaHidden":70},[69],[41,6031,6033,6036],{"className":6032},[74],[41,6034],{"className":6035,"style":1074},[78],[41,6037,931],{"className":6038},[83,84],[41,6040,6042,6055],{"className":6041},[44],[41,6043,6045],{"className":6044},[48],[50,6046,6047],{"xmlns":52},[54,6048,6049,6053],{},[57,6050,6051],{},[60,6052,3798],{},[63,6054,3798],{"encoding":65},[41,6056,6058],{"className":6057,"ariaHidden":70},[69],[41,6059,6061,6064],{"className":6060},[74],[41,6062],{"className":6063,"style":1074},[78],[41,6065,3798],{"className":6066,"style":380},[83,84],[41,6068,6070,6105],{"className":6069},[44],[41,6071,6073],{"className":6072},[48],[50,6074,6075],{"xmlns":52},[54,6076,6077,6102],{},[57,6078,6079,6081,6083,6085,6091,6093,6096],{},[338,6080,2498],{},[129,6082,223],{},[338,6084,5848],{},[532,6086,6087,6089],{},[60,6088,5853],{},[2681,6090,5856],{},[129,6092,513],{},[338,6094,6095],{},"8",[532,6097,6098,6100],{},[60,6099,5853],{},[2681,6101,5856],{},[63,6103,6104],{"encoding":65},"2 \\times 4 N_{\\text{params}} = 8 N_{\\text{params}}",[41,6106,6108,6127,6191],{"className":6107,"ariaHidden":70},[69],[41,6109,6111,6115,6118,6121,6124],{"className":6110},[74],[41,6112],{"className":6113,"style":6114},[78],"height:0.7278em;vertical-align:-0.0833em;",[41,6116,2498],{"className":6117},[83],[41,6119],{"className":6120,"style":314},[246],[41,6122,223],{"className":6123},[310],[41,6125],{"className":6126,"style":314},[246],[41,6128,6130,6133,6136,6182,6185,6188],{"className":6129},[74],[41,6131],{"className":6132,"style":5869},[78],[41,6134,5848],{"className":6135},[83],[41,6137,6139,6142],{"className":6138},[83],[41,6140,5853],{"className":6141,"style":3159},[83,84],[41,6143,6145],{"className":6144},[272],[41,6146,6148,6174],{"className":6147},[276,651],[41,6149,6151,6171],{"className":6150},[280],[41,6152,6154],{"className":6153,"style":768},[284],[41,6155,6156,6159],{"style":5893},[41,6157],{"className":6158,"style":292},[291],[41,6160,6162],{"className":6161},[296,297,298,299],[41,6163,6165],{"className":6164},[83,299],[41,6166,6168],{"className":6167},[83,2866,299],[41,6169,5856],{"className":6170},[83,299],[41,6172,731],{"className":6173},[730],[41,6175,6177],{"className":6176},[280],[41,6178,6180],{"className":6179,"style":860},[284],[41,6181],{},[41,6183],{"className":6184,"style":247},[246],[41,6186,513],{"className":6187},[251],[41,6189],{"className":6190,"style":247},[246],[41,6192,6194,6197,6200],{"className":6193},[74],[41,6195],{"className":6196,"style":5869},[78],[41,6198,6095],{"className":6199},[83],[41,6201,6203,6206],{"className":6202},[83],[41,6204,5853],{"className":6205,"style":3159},[83,84],[41,6207,6209],{"className":6208},[272],[41,6210,6212,6238],{"className":6211},[276,651],[41,6213,6215,6235],{"className":6214},[280],[41,6216,6218],{"className":6217,"style":768},[284],[41,6219,6220,6223],{"style":5893},[41,6221],{"className":6222,"style":292},[291],[41,6224,6226],{"className":6225},[296,297,298,299],[41,6227,6229],{"className":6228},[83,299],[41,6230,6232],{"className":6231},[83,2866,299],[41,6233,5856],{"className":6234},[83,299],[41,6236,731],{"className":6237},[730],[41,6239,6241],{"className":6240},[280],[41,6242,6244],{"className":6243,"style":860},[284],[41,6245],{}," 字节。",[10,6248,6249,6250,6332,6333,6489,6490,6652,6653,6726,6727,6730,6731,6788,6789,6842,6843,6895,6896,86],{},"仅这三项静态状态就高达 ",[41,6251,6253,6274],{"className":6252},[44],[41,6254,6256],{"className":6255},[48],[50,6257,6258],{"xmlns":52},[54,6259,6260,6271],{},[57,6261,6262,6265],{},[338,6263,6264],{},"16",[532,6266,6267,6269],{},[60,6268,5853],{},[2681,6270,5856],{},[63,6272,6273],{"encoding":65},"16 N_{\\text{params}}",[41,6275,6277],{"className":6276,"ariaHidden":70},[69],[41,6278,6280,6283,6286],{"className":6279},[74],[41,6281],{"className":6282,"style":5869},[78],[41,6284,6264],{"className":6285},[83],[41,6287,6289,6292],{"className":6288},[83],[41,6290,5853],{"className":6291,"style":3159},[83,84],[41,6293,6295],{"className":6294},[272],[41,6296,6298,6324],{"className":6297},[276,651],[41,6299,6301,6321],{"className":6300},[280],[41,6302,6304],{"className":6303,"style":768},[284],[41,6305,6306,6309],{"style":5893},[41,6307],{"className":6308,"style":292},[291],[41,6310,6312],{"className":6311},[296,297,298,299],[41,6313,6315],{"className":6314},[83,299],[41,6316,6318],{"className":6317},[83,2866,299],[41,6319,5856],{"className":6320},[83,299],[41,6322,731],{"className":6323},[730],[41,6325,6327],{"className":6326},[280],[41,6328,6330],{"className":6329,"style":860},[284],[41,6331],{}," 字节。对于上一篇核算的 GPT-2 XL（",[41,6334,6336,6369],{"className":6335},[44],[41,6337,6339],{"className":6338},[48],[50,6340,6341],{"xmlns":52},[54,6342,6343,6366],{},[57,6344,6345,6351,6354,6357,6359],{},[532,6346,6347,6349],{},[60,6348,5853],{},[2681,6350,5856],{},[129,6352,6353],{},"≈",[338,6355,6356],{},"1.640",[129,6358,223],{},[210,6360,6361,6363],{},[338,6362,3353],{},[338,6364,6365],{},"9",[63,6367,6368],{"encoding":65},"N_{\\text{params}} \\approx 1.640 \\times 10^9",[41,6370,6372,6433,6451],{"className":6371,"ariaHidden":70},[69],[41,6373,6375,6378,6424,6427,6430],{"className":6374},[74],[41,6376],{"className":6377,"style":5869},[78],[41,6379,6381,6384],{"className":6380},[83],[41,6382,5853],{"className":6383,"style":3159},[83,84],[41,6385,6387],{"className":6386},[272],[41,6388,6390,6416],{"className":6389},[276,651],[41,6391,6393,6413],{"className":6392},[280],[41,6394,6396],{"className":6395,"style":768},[284],[41,6397,6398,6401],{"style":5893},[41,6399],{"className":6400,"style":292},[291],[41,6402,6404],{"className":6403},[296,297,298,299],[41,6405,6407],{"className":6406},[83,299],[41,6408,6410],{"className":6409},[83,2866,299],[41,6411,5856],{"className":6412},[83,299],[41,6414,731],{"className":6415},[730],[41,6417,6419],{"className":6418},[280],[41,6420,6422],{"className":6421,"style":860},[284],[41,6423],{},[41,6425],{"className":6426,"style":247},[246],[41,6428,6353],{"className":6429},[251],[41,6431],{"className":6432,"style":247},[246],[41,6434,6436,6439,6442,6445,6448],{"className":6435},[74],[41,6437],{"className":6438,"style":6114},[78],[41,6440,6356],{"className":6441},[83],[41,6443],{"className":6444,"style":314},[246],[41,6446,223],{"className":6447},[310],[41,6449],{"className":6450,"style":314},[246],[41,6452,6454,6457,6460],{"className":6453},[74],[41,6455],{"className":6456,"style":3391},[78],[41,6458,356],{"className":6459},[83],[41,6461,6463,6466],{"className":6462},[83],[41,6464,340],{"className":6465},[83],[41,6467,6469],{"className":6468},[272],[41,6470,6472],{"className":6471},[276],[41,6473,6475],{"className":6474},[280],[41,6476,6478],{"className":6477,"style":3391},[284],[41,6479,6480,6483],{"style":287},[41,6481],{"className":6482,"style":292},[291],[41,6484,6486],{"className":6485},[296,297,298,299],[41,6487,6365],{"className":6488},[83,299],"），静态显存基数 ",[41,6491,6493,6533],{"className":6492},[44],[41,6494,6496],{"className":6495},[48],[50,6497,6498],{"xmlns":52},[54,6499,6500,6530],{},[57,6501,6502,6505,6507,6509,6511,6514,6516,6522,6524,6527],{},[60,6503,6504],{},"b",[129,6506,513],{},[338,6508,6264],{},[129,6510,223],{},[338,6512,6513],{},"1.64045",[129,6515,223],{},[210,6517,6518,6520],{},[338,6519,3353],{},[338,6521,6365],{},[129,6523,6353],{},[338,6525,6526],{},"26.25",[2681,6528,6529],{}," GB",[63,6531,6532],{"encoding":65},"b = 16 \\times 1.64045 \\times 10^9 \\approx 26.25\\text{ GB}",[41,6534,6536,6554,6572,6590,6637],{"className":6535,"ariaHidden":70},[69],[41,6537,6539,6542,6545,6548,6551],{"className":6538},[74],[41,6540],{"className":6541,"style":3749},[78],[41,6543,6504],{"className":6544},[83,84],[41,6546],{"className":6547,"style":247},[246],[41,6549,513],{"className":6550},[251],[41,6552],{"className":6553,"style":247},[246],[41,6555,6557,6560,6563,6566,6569],{"className":6556},[74],[41,6558],{"className":6559,"style":6114},[78],[41,6561,6264],{"className":6562},[83],[41,6564],{"className":6565,"style":314},[246],[41,6567,223],{"className":6568},[310],[41,6570],{"className":6571,"style":314},[246],[41,6573,6575,6578,6581,6584,6587],{"className":6574},[74],[41,6576],{"className":6577,"style":6114},[78],[41,6579,6513],{"className":6580},[83],[41,6582],{"className":6583,"style":314},[246],[41,6585,223],{"className":6586},[310],[41,6588],{"className":6589,"style":314},[246],[41,6591,6593,6596,6599,6628,6631,6634],{"className":6592},[74],[41,6594],{"className":6595,"style":3391},[78],[41,6597,356],{"className":6598},[83],[41,6600,6602,6605],{"className":6601},[83],[41,6603,340],{"className":6604},[83],[41,6606,6608],{"className":6607},[272],[41,6609,6611],{"className":6610},[276],[41,6612,6614],{"className":6613},[280],[41,6615,6617],{"className":6616,"style":3391},[284],[41,6618,6619,6622],{"style":287},[41,6620],{"className":6621,"style":292},[291],[41,6623,6625],{"className":6624},[296,297,298,299],[41,6626,6365],{"className":6627},[83,299],[41,6629],{"className":6630,"style":247},[246],[41,6632,6353],{"className":6633},[251],[41,6635],{"className":6636,"style":247},[246],[41,6638,6640,6643,6646],{"className":6639},[74],[41,6641],{"className":6642,"style":2247},[78],[41,6644,6526],{"className":6645},[83],[41,6647,6649],{"className":6648},[83,2866],[41,6650,6529],{"className":6651},[83],"。再加上每层前向激活值（RMSNorm、QKV、注意力分数矩阵 ",[41,6654,6656,6680],{"className":6655},[44],[41,6657,6659],{"className":6658},[48],[50,6660,6661],{"xmlns":52},[54,6662,6663,6677],{},[57,6664,6665,6667,6670],{},[60,6666,220],{},[60,6668,6669],{},"H",[210,6671,6672,6675],{},[60,6673,6674],{},"S",[338,6676,2498],{},[63,6678,6679],{"encoding":65},"B H S^2",[41,6681,6683],{"className":6682,"ariaHidden":70},[69],[41,6684,6686,6689,6692,6696],{"className":6685},[74],[41,6687],{"className":6688,"style":3391},[78],[41,6690,220],{"className":6691,"style":306},[83,84],[41,6693,6669],{"className":6694,"style":6695},[83,84],"margin-right:0.0813em;",[41,6697,6699,6703],{"className":6698},[83],[41,6700,6674],{"className":6701,"style":6702},[83,84],"margin-right:0.0576em;",[41,6704,6706],{"className":6705},[272],[41,6707,6709],{"className":6708},[276],[41,6710,6712],{"className":6711},[280],[41,6713,6715],{"className":6714,"style":3391},[284],[41,6716,6717,6720],{"style":287},[41,6718],{"className":6719,"style":292},[291],[41,6721,6723],{"className":6722},[296,297,298,299],[41,6724,2498],{"className":6725},[83,299],"、注意力权重、SwiGLU 中间激活等）随 ",[14,6728,6729],{},"batch_size"," 线性增长的斜率项 ",[41,6732,6734,6753],{"className":6733},[44],[41,6735,6737],{"className":6736},[48],[50,6738,6739],{"xmlns":52},[54,6740,6741,6750],{},[57,6742,6743,6745,6748],{},[60,6744,97],{},[129,6746,6747],{},"⋅",[2681,6749,6729],{},[63,6751,6752],{"encoding":65},"a \\cdot \\text{batch\\_size}",[41,6754,6756,6775],{"className":6755,"ariaHidden":70},[69],[41,6757,6759,6763,6766,6769,6772],{"className":6758},[74],[41,6760],{"className":6761,"style":6762},[78],"height:0.4445em;",[41,6764,97],{"className":6765},[83,84],[41,6767],{"className":6768,"style":314},[246],[41,6770,6747],{"className":6771},[310],[41,6773],{"className":6774,"style":314},[246],[41,6776,6778,6782],{"className":6777},[74],[41,6779],{"className":6780,"style":6781},[78],"height:1.0044em;vertical-align:-0.31em;",[41,6783,6785],{"className":6784},[83,2866],[41,6786,6729],{"className":6787},[83],"（在 ",[41,6790,6792,6811],{"className":6791},[44],[41,6793,6795],{"className":6794},[48],[50,6796,6797],{"xmlns":52},[54,6798,6799,6808],{},[57,6800,6801,6803,6805],{},[60,6802,6674],{},[129,6804,513],{},[338,6806,6807],{},"1024",[63,6809,6810],{"encoding":65},"S=1024",[41,6812,6814,6832],{"className":6813,"ariaHidden":70},[69],[41,6815,6817,6820,6823,6826,6829],{"className":6816},[74],[41,6818],{"className":6819,"style":2247},[78],[41,6821,6674],{"className":6822,"style":6702},[83,84],[41,6824],{"className":6825,"style":247},[246],[41,6827,513],{"className":6828},[251],[41,6830],{"className":6831,"style":247},[246],[41,6833,6835,6839],{"className":6834},[74],[41,6836],{"className":6837,"style":6838},[78],"height:0.6444em;",[41,6840,6807],{"className":6841},[83]," 下约为 ",[41,6844,6846,6868],{"className":6845},[44],[41,6847,6849],{"className":6848},[48],[50,6850,6851],{"xmlns":52},[54,6852,6853,6865],{},[57,6854,6855,6858,6860,6862],{},[338,6856,6857],{},"14.9",[2681,6859,6529],{},[60,6861,3553],{"mathvariant":497},[2681,6863,6864],{},"sample",[63,6866,6867],{"encoding":65},"14.9\\text{ GB} \u002F \\text{sample}",[41,6869,6871],{"className":6870,"ariaHidden":70},[69],[41,6872,6874,6877,6880,6886,6889],{"className":6873},[74],[41,6875],{"className":6876,"style":396},[78],[41,6878,6857],{"className":6879},[83],[41,6881,6883],{"className":6882},[83,2866],[41,6884,6529],{"className":6885},[83],[41,6887,3553],{"className":6888},[83],[41,6890,6892],{"className":6891},[83,2866],[41,6893,6864],{"className":6894},[83],"），在单张 80GB H100 GPU 上不启用激活重计算（Activation Checkpointing）或混合精度时，最大只能塞下 ",[14,6897,6898],{},"batch_size = 3",[109,6900,6902],{"id":6901},"warmup-余弦退火学习率调度","Warmup + 余弦退火学习率调度",[10,6904,6905,6908,6909,6937],{},[14,6906,6907],{},"get_lr_cosine_schedule"," 将训练步 ",[41,6910,6912,6925],{"className":6911},[44],[41,6913,6915],{"className":6914},[48],[50,6916,6917],{"xmlns":52},[54,6918,6919,6923],{},[57,6920,6921],{},[60,6922,3637],{},[63,6924,3637],{"encoding":65},[41,6926,6928],{"className":6927,"ariaHidden":70},[69],[41,6929,6931,6934],{"className":6930},[74],[41,6932],{"className":6933,"style":3667},[78],[41,6935,3637],{"className":6936},[83,84]," 划分为三个阶段：",[41,6939,6941],{"className":6940},[481],[41,6942,6944,7203],{"className":6943},[44],[41,6945,6947],{"className":6946},[48],[50,6948,6949],{"xmlns":52,"display":490},[54,6950,6951,7200],{},[57,6952,6953,6959,6961],{},[532,6954,6955,6957],{},[60,6956,3976],{},[60,6958,3637],{},[129,6960,513],{},[57,6962,6963,6965],{},[129,6964,336],{"fence":70},[3827,6966,6970,7023,7161],{"rowspacing":6967,"columnalign":6968,"columnspacing":6969},"0.36em","left left","1em",[3833,6971,6972,7001],{},[3836,6973,6974],{},[3839,6975,6976],{"scriptlevel":340,"displaystyle":335},[57,6977,6978,6991,6999],{},[3839,6979,6980],{"scriptlevel":340,"displaystyle":70},[521,6981,6982,6984],{},[60,6983,3637],{},[532,6985,6986,6988],{},[60,6987,3654],{},[60,6989,6990],{},"w",[532,6992,6993,6995,6997],{},[60,6994,3976],{},[60,6996,1051],{},[129,6998,131],{},[129,7000,343],{"separator":70},[3836,7002,7003],{},[3839,7004,7005],{"scriptlevel":340,"displaystyle":335},[57,7006,7007,7009,7012,7018,7020],{},[60,7008,3637],{},[129,7010,7011],{},"\u003C",[532,7013,7014,7016],{},[60,7015,3654],{},[60,7017,6990],{},[246,7019],{"width":6969},[2681,7021,7022],{},"（线性预热）",[3833,7024,7025,7131],{},[3836,7026,7027],{},[3839,7028,7029],{"scriptlevel":340,"displaystyle":335},[57,7030,7031,7040,7042,7050,7107,7109,7117,7119,7127,7129],{},[532,7032,7033,7035,7038],{},[60,7034,3976],{},[60,7036,7037],{},"min",[129,7039,131],{},[129,7041,3191],{},[3839,7043,7044],{"scriptlevel":340,"displaystyle":70},[521,7045,7046,7048],{},[338,7047,356],{},[338,7049,2498],{},[57,7051,7052,7054,7056,7058,7061,7063,7105],{},[129,7053,501],{"fence":70},[338,7055,356],{},[129,7057,3191],{},[60,7059,7060],{},"cos",[129,7062,131],{},[57,7064,7065,7067,7100,7103],{},[129,7066,501],{"fence":70},[3839,7068,7069],{"scriptlevel":340,"displaystyle":70},[521,7070,7071,7083],{},[57,7072,7073,7075,7077],{},[60,7074,3637],{},[129,7076,353],{},[532,7078,7079,7081],{},[60,7080,3654],{},[60,7082,6990],{},[57,7084,7085,7092,7094],{},[532,7086,7087,7089],{},[60,7088,3654],{},[60,7090,7091],{},"c",[129,7093,353],{},[532,7095,7096,7098],{},[60,7097,3654],{},[60,7099,6990],{},[60,7101,7102],{},"π",[129,7104,510],{"fence":70},[129,7106,510],{"fence":70},[129,7108,501],{"stretchy":335},[532,7110,7111,7113,7115],{},[60,7112,3976],{},[60,7114,1051],{},[129,7116,131],{},[129,7118,353],{},[532,7120,7121,7123,7125],{},[60,7122,3976],{},[60,7124,7037],{},[129,7126,131],{},[129,7128,510],{"stretchy":335},[129,7130,343],{"separator":70},[3836,7132,7133],{},[3839,7134,7135],{"scriptlevel":340,"displaystyle":335},[57,7136,7137,7143,7146,7148,7150,7156,7158],{},[532,7138,7139,7141],{},[60,7140,3654],{},[60,7142,6990],{},[129,7144,7145],{},"≤",[60,7147,3637],{},[129,7149,7145],{},[532,7151,7152,7154],{},[60,7153,3654],{},[60,7155,7091],{},[246,7157],{"width":6969},[2681,7159,7160],{},"（余弦退火）",[3833,7162,7163,7179],{},[3836,7164,7165],{},[3839,7166,7167],{"scriptlevel":340,"displaystyle":335},[57,7168,7169,7177],{},[532,7170,7171,7173,7175],{},[60,7172,3976],{},[60,7174,7037],{},[129,7176,131],{},[129,7178,343],{"separator":70},[3836,7180,7181],{},[3839,7182,7183],{"scriptlevel":340,"displaystyle":335},[57,7184,7185,7187,7189,7195,7197],{},[60,7186,3637],{},[129,7188,3079],{},[532,7190,7191,7193],{},[60,7192,3654],{},[60,7194,7091],{},[246,7196],{"width":6969},[2681,7198,7199],{},"（尾部保持）",[63,7201,7202],{"encoding":65},"\\alpha_t =\n\\begin{cases}\n\\dfrac{t}{T_w} \\alpha_{\\max}, & t \u003C T_w \\quad\\text{（线性预热）} \\\\[6pt]\n\\alpha_{\\min} + \\dfrac{1}{2}\\left(1 + \\cos\\left(\\dfrac{t - T_w}{T_c - T_w}\\pi\\right)\\right)(\\alpha_{\\max} - \\alpha_{\\min}), & T_w \\le t \\le T_c \\quad\\text{（余弦退火）} \\\\[6pt]\n\\alpha_{\\min}, & t > T_c \\quad\\text{（尾部保持）}\n\\end{cases}",[41,7204,7206,7261],{"className":7205,"ariaHidden":70},[69],[41,7207,7209,7212,7252,7255,7258],{"className":7208},[74],[41,7210],{"className":7211,"style":1093},[78],[41,7213,7215,7218],{"className":7214},[83],[41,7216,3976],{"className":7217,"style":4259},[83,84],[41,7219,7221],{"className":7220},[272],[41,7222,7224,7244],{"className":7223},[276,651],[41,7225,7227,7241],{"className":7226},[280],[41,7228,7230],{"className":7229,"style":4171},[284],[41,7231,7232,7235],{"style":4274},[41,7233],{"className":7234,"style":292},[291],[41,7236,7238],{"className":7237},[296,297,298,299],[41,7239,3637],{"className":7240},[83,84,299],[41,7242,731],{"className":7243},[730],[41,7245,7247],{"className":7246},[280],[41,7248,7250],{"className":7249,"style":790},[284],[41,7251],{},[41,7253],{"className":7254,"style":247},[246],[41,7256,513],{"className":7257},[251],[41,7259],{"className":7260,"style":247},[246],[41,7262,7264,7268],{"className":7263},[74],[41,7265],{"className":7266,"style":7267},[78],"height:6.2461em;vertical-align:-2.873em;",[41,7269,7271,7370,8433],{"className":7270},[415],[41,7272,7274],{"className":7273},[400],[41,7275,7278],{"className":7276},[2288,7277],"mult",[41,7279,7281,7361],{"className":7280},[276,651],[41,7282,7284,7358],{"className":7283},[280],[41,7285,7288,7303,7322,7334,7346],{"className":7286,"style":7287},[284],"height:3.25em;",[41,7289,7291,7295],{"style":7290},"top:-1.366em;",[41,7292],{"className":7293,"style":7294},[291],"height:3.216em;",[41,7296,7300],{"className":7297},[7298,7299],"delimsizinginner","delim-size4",[41,7301,7302],{},"⎩",[41,7304,7306,7309],{"style":7305},"top:-1.358em;",[41,7307],{"className":7308,"style":7294},[291],[41,7310,7312],{"style":7311},"height:1.216em;width:0.8889em;",[3026,7313,7319],{"xmlns":3028,"width":7314,"height":7315,"style":7316,"viewBox":7317,"preserveAspectRatio":7318},"0.8889em","1.216em","width:0.8889em","0 0 888.89 1216","xMinYMin",[3034,7320],{"d":7321},"M384 0 H504 V1216 H384z M384 0 H504 V1216 H384z",[41,7323,7325,7328],{"style":7324},"top:-3.216em;",[41,7326],{"className":7327,"style":7294},[291],[41,7329,7331],{"className":7330},[7298,7299],[41,7332,7333],{},"⎨",[41,7335,7337,7340],{"style":7336},"top:-4.358em;",[41,7338],{"className":7339,"style":7294},[291],[41,7341,7342],{"style":7311},[3026,7343,7344],{"xmlns":3028,"width":7314,"height":7315,"style":7316,"viewBox":7317,"preserveAspectRatio":7318},[3034,7345],{"d":7321},[41,7347,7349,7352],{"style":7348},"top:-5.566em;",[41,7350],{"className":7351,"style":7294},[291],[41,7353,7355],{"className":7354},[7298,7299],[41,7356,7357],{},"⎧",[41,7359,731],{"className":7360},[730],[41,7362,7364],{"className":7363},[280],[41,7365,7368],{"className":7366,"style":7367},[284],"height:2.75em;",[41,7369],{},[41,7371,7373],{"className":7372},[83],[41,7374,7376,8147,8152],{"className":7375},[3827],[41,7377,7379],{"className":7378},[4360],[41,7380,7382,8138],{"className":7381},[276,651],[41,7383,7385,8135],{"className":7384},[280],[41,7386,7389,7558,8071],{"className":7387,"style":7388},[284],"height:3.373em;",[41,7390,7392,7396],{"style":7391},"top:-5.531em;",[41,7393],{"className":7394,"style":7395},[291],"height:3.45em;",[41,7397,7399,7502,7555],{"className":7398},[83],[41,7400,7402,7405,7499],{"className":7401},[83],[41,7403],{"className":7404},[400,644],[41,7406,7408],{"className":7407},[521],[41,7409,7411,7490],{"className":7410},[276,651],[41,7412,7414,7487],{"className":7413},[280],[41,7415,7418,7468,7476],{"className":7416,"style":7417},[284],"height:1.2921em;",[41,7419,7420,7423],{"style":4870},[41,7421],{"className":7422,"style":665},[291],[41,7424,7426],{"className":7425},[83],[41,7427,7429,7432],{"className":7428},[83],[41,7430,3654],{"className":7431,"style":3720},[83,84],[41,7433,7435],{"className":7434},[272],[41,7436,7438,7460],{"className":7437},[276,651],[41,7439,7441,7457],{"className":7440},[280],[41,7442,7444],{"className":7443,"style":768},[284],[41,7445,7447,7450],{"style":7446},"top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;",[41,7448],{"className":7449,"style":292},[291],[41,7451,7453],{"className":7452},[296,297,298,299],[41,7454,6990],{"className":7455,"style":7456},[83,84,299],"margin-right:0.0269em;",[41,7458,731],{"className":7459},[730],[41,7461,7463],{"className":7462},[280],[41,7464,7466],{"className":7465,"style":790},[284],[41,7467],{},[41,7469,7470,7473],{"style":798},[41,7471],{"className":7472,"style":665},[291],[41,7474],{"className":7475,"style":806},[805],[41,7477,7478,7481],{"style":809},[41,7479],{"className":7480,"style":665},[291],[41,7482,7484],{"className":7483},[83],[41,7485,3637],{"className":7486},[83,84],[41,7488,731],{"className":7489},[730],[41,7491,7493],{"className":7492},[280],[41,7494,7497],{"className":7495,"style":7496},[284],"height:0.836em;",[41,7498],{},[41,7500],{"className":7501},[450,644],[41,7503,7505,7508],{"className":7504},[83],[41,7506,3976],{"className":7507,"style":4259},[83,84],[41,7509,7511],{"className":7510},[272],[41,7512,7514,7547],{"className":7513},[276,651],[41,7515,7517,7544],{"className":7516},[280],[41,7518,7520],{"className":7519,"style":768},[284],[41,7521,7522,7525],{"style":4274},[41,7523],{"className":7524,"style":292},[291],[41,7526,7528],{"className":7527},[296,297,298,299],[41,7529,7531],{"className":7530},[83,299],[41,7532,7534,7537,7540],{"className":7533},[148,299],[41,7535,931],{"className":7536},[299],[41,7538,97],{"className":7539},[299],[41,7541,7543],{"className":7542},[299],"x",[41,7545,731],{"className":7546},[730],[41,7548,7550],{"className":7549},[280],[41,7551,7553],{"className":7552,"style":790},[284],[41,7554],{},[41,7556,343],{"className":7557},[407],[41,7559,7561,7564],{"style":7560},"top:-3.049em;",[41,7562],{"className":7563,"style":7395},[291],[41,7565,7567,7621,7624,7627,7630,7694,7697,7946,7949,7952,8004,8007,8010,8013,8065,8068],{"className":7566},[83],[41,7568,7570,7573],{"className":7569},[83],[41,7571,3976],{"className":7572,"style":4259},[83,84],[41,7574,7576],{"className":7575},[272],[41,7577,7579,7613],{"className":7578},[276,651],[41,7580,7582,7610],{"className":7581},[280],[41,7583,7586],{"className":7584,"style":7585},[284],"height:0.3175em;",[41,7587,7588,7591],{"style":4274},[41,7589],{"className":7590,"style":292},[291],[41,7592,7594],{"className":7593},[296,297,298,299],[41,7595,7597],{"className":7596},[83,299],[41,7598,7600,7603,7606],{"className":7599},[148,299],[41,7601,931],{"className":7602},[299],[41,7604,2220],{"className":7605},[299],[41,7607,7609],{"className":7608},[299],"n",[41,7611,731],{"className":7612},[730],[41,7614,7616],{"className":7615},[280],[41,7617,7619],{"className":7618,"style":790},[284],[41,7620],{},[41,7622],{"className":7623,"style":314},[246],[41,7625,3191],{"className":7626},[310],[41,7628],{"className":7629,"style":314},[246],[41,7631,7633,7636,7691],{"className":7632},[83],[41,7634],{"className":7635},[400,644],[41,7637,7639],{"className":7638},[521],[41,7640,7642,7682],{"className":7641},[276,651],[41,7643,7645,7679],{"className":7644},[280],[41,7646,7649,7660,7668],{"className":7647,"style":7648},[284],"height:1.3214em;",[41,7650,7651,7654],{"style":4870},[41,7652],{"className":7653,"style":665},[291],[41,7655,7657],{"className":7656},[83],[41,7658,2498],{"className":7659},[83],[41,7661,7662,7665],{"style":798},[41,7663],{"className":7664,"style":665},[291],[41,7666],{"className":7667,"style":806},[805],[41,7669,7670,7673],{"style":809},[41,7671],{"className":7672,"style":665},[291],[41,7674,7676],{"className":7675},[83],[41,7677,356],{"className":7678},[83],[41,7680,731],{"className":7681},[730],[41,7683,7685],{"className":7684},[280],[41,7686,7689],{"className":7687,"style":7688},[284],"height:0.686em;",[41,7690],{},[41,7692],{"className":7693},[450,644],[41,7695],{"className":7696,"style":411},[246],[41,7698,7700,7706,7709,7712,7715,7718,7721,7724,7940],{"className":7699},[415],[41,7701,7703],{"className":7702,"style":2284},[400,2283],[41,7704,501],{"className":7705},[2288,298],[41,7707,356],{"className":7708},[83],[41,7710],{"className":7711,"style":314},[246],[41,7713,3191],{"className":7714},[310],[41,7716],{"className":7717,"style":314},[246],[41,7719,7060],{"className":7720},[148],[41,7722],{"className":7723,"style":411},[246],[41,7725,7727,7733,7931,7934],{"className":7726},[415],[41,7728,7730],{"className":7729,"style":2284},[400,2283],[41,7731,501],{"className":7732},[2288,298],[41,7734,7736,7739,7928],{"className":7735},[83],[41,7737],{"className":7738},[400,644],[41,7740,7742],{"className":7741},[521],[41,7743,7745,7920],{"className":7744},[276,651],[41,7746,7748,7917],{"className":7747},[280],[41,7749,7752,7849,7857],{"className":7750,"style":7751},[284],"height:1.3603em;",[41,7753,7754,7757],{"style":4870},[41,7755],{"className":7756,"style":665},[291],[41,7758,7760,7800,7803,7806,7809],{"className":7759},[83],[41,7761,7763,7766],{"className":7762},[83],[41,7764,3654],{"className":7765,"style":3720},[83,84],[41,7767,7769],{"className":7768},[272],[41,7770,7772,7792],{"className":7771},[276,651],[41,7773,7775,7789],{"className":7774},[280],[41,7776,7778],{"className":7777,"style":768},[284],[41,7779,7780,7783],{"style":7446},[41,7781],{"className":7782,"style":292},[291],[41,7784,7786],{"className":7785},[296,297,298,299],[41,7787,7091],{"className":7788},[83,84,299],[41,7790,731],{"className":7791},[730],[41,7793,7795],{"className":7794},[280],[41,7796,7798],{"className":7797,"style":790},[284],[41,7799],{},[41,7801],{"className":7802,"style":314},[246],[41,7804,353],{"className":7805},[310],[41,7807],{"className":7808,"style":314},[246],[41,7810,7812,7815],{"className":7811},[83],[41,7813,3654],{"className":7814,"style":3720},[83,84],[41,7816,7818],{"className":7817},[272],[41,7819,7821,7841],{"className":7820},[276,651],[41,7822,7824,7838],{"className":7823},[280],[41,7825,7827],{"className":7826,"style":768},[284],[41,7828,7829,7832],{"style":7446},[41,7830],{"className":7831,"style":292},[291],[41,7833,7835],{"className":7834},[296,297,298,299],[41,7836,6990],{"className":7837,"style":7456},[83,84,299],[41,7839,731],{"className":7840},[730],[41,7842,7844],{"className":7843},[280],[41,7845,7847],{"className":7846,"style":790},[284],[41,7848],{},[41,7850,7851,7854],{"style":798},[41,7852],{"className":7853,"style":665},[291],[41,7855],{"className":7856,"style":806},[805],[41,7858,7859,7862],{"style":809},[41,7860],{"className":7861,"style":665},[291],[41,7863,7865,7868,7871,7874,7877],{"className":7864},[83],[41,7866,3637],{"className":7867},[83,84],[41,7869],{"className":7870,"style":314},[246],[41,7872,353],{"className":7873},[310],[41,7875],{"className":7876,"style":314},[246],[41,7878,7880,7883],{"className":7879},[83],[41,7881,3654],{"className":7882,"style":3720},[83,84],[41,7884,7886],{"className":7885},[272],[41,7887,7889,7909],{"className":7888},[276,651],[41,7890,7892,7906],{"className":7891},[280],[41,7893,7895],{"className":7894,"style":768},[284],[41,7896,7897,7900],{"style":7446},[41,7898],{"className":7899,"style":292},[291],[41,7901,7903],{"className":7902},[296,297,298,299],[41,7904,6990],{"className":7905,"style":7456},[83,84,299],[41,7907,731],{"className":7908},[730],[41,7910,7912],{"className":7911},[280],[41,7913,7915],{"className":7914,"style":790},[284],[41,7916],{},[41,7918,731],{"className":7919},[730],[41,7921,7923],{"className":7922},[280],[41,7924,7926],{"className":7925,"style":7496},[284],[41,7927],{},[41,7929],{"className":7930},[450,644],[41,7932,7102],{"className":7933,"style":380},[83,84],[41,7935,7937],{"className":7936,"style":2284},[450,2283],[41,7938,510],{"className":7939},[2288,298],[41,7941,7943],{"className":7942,"style":2284},[450,2283],[41,7944,510],{"className":7945},[2288,298],[41,7947],{"className":7948,"style":411},[246],[41,7950,501],{"className":7951},[400],[41,7953,7955,7958],{"className":7954},[83],[41,7956,3976],{"className":7957,"style":4259},[83,84],[41,7959,7961],{"className":7960},[272],[41,7962,7964,7996],{"className":7963},[276,651],[41,7965,7967,7993],{"className":7966},[280],[41,7968,7970],{"className":7969,"style":768},[284],[41,7971,7972,7975],{"style":4274},[41,7973],{"className":7974,"style":292},[291],[41,7976,7978],{"className":7977},[296,297,298,299],[41,7979,7981],{"className":7980},[83,299],[41,7982,7984,7987,7990],{"className":7983},[148,299],[41,7985,931],{"className":7986},[299],[41,7988,97],{"className":7989},[299],[41,7991,7543],{"className":7992},[299],[41,7994,731],{"className":7995},[730],[41,7997,7999],{"className":7998},[280],[41,8000,8002],{"className":8001,"style":790},[284],[41,8003],{},[41,8005],{"className":8006,"style":314},[246],[41,8008,353],{"className":8009},[310],[41,8011],{"className":8012,"style":314},[246],[41,8014,8016,8019],{"className":8015},[83],[41,8017,3976],{"className":8018,"style":4259},[83,84],[41,8020,8022],{"className":8021},[272],[41,8023,8025,8057],{"className":8024},[276,651],[41,8026,8028,8054],{"className":8027},[280],[41,8029,8031],{"className":8030,"style":7585},[284],[41,8032,8033,8036],{"style":4274},[41,8034],{"className":8035,"style":292},[291],[41,8037,8039],{"className":8038},[296,297,298,299],[41,8040,8042],{"className":8041},[83,299],[41,8043,8045,8048,8051],{"className":8044},[148,299],[41,8046,931],{"className":8047},[299],[41,8049,2220],{"className":8050},[299],[41,8052,7609],{"className":8053},[299],[41,8055,731],{"className":8056},[730],[41,8058,8060],{"className":8059},[280],[41,8061,8063],{"className":8062,"style":790},[284],[41,8064],{},[41,8066,510],{"className":8067},[450],[41,8069,343],{"className":8070},[407],[41,8072,8074,8077],{"style":8073},"top:-1.009em;",[41,8075],{"className":8076,"style":7395},[291],[41,8078,8080,8132],{"className":8079},[83],[41,8081,8083,8086],{"className":8082},[83],[41,8084,3976],{"className":8085,"style":4259},[83,84],[41,8087,8089],{"className":8088},[272],[41,8090,8092,8124],{"className":8091},[276,651],[41,8093,8095,8121],{"className":8094},[280],[41,8096,8098],{"className":8097,"style":7585},[284],[41,8099,8100,8103],{"style":4274},[41,8101],{"className":8102,"style":292},[291],[41,8104,8106],{"className":8105},[296,297,298,299],[41,8107,8109],{"className":8108},[83,299],[41,8110,8112,8115,8118],{"className":8111},[148,299],[41,8113,931],{"className":8114},[299],[41,8116,2220],{"className":8117},[299],[41,8119,7609],{"className":8120},[299],[41,8122,731],{"className":8123},[730],[41,8125,8127],{"className":8126},[280],[41,8128,8130],{"className":8129,"style":790},[284],[41,8131],{},[41,8133,343],{"className":8134},[407],[41,8136,731],{"className":8137},[730],[41,8139,8141],{"className":8140},[280],[41,8142,8145],{"className":8143,"style":8144},[284],"height:2.873em;",[41,8146],{},[41,8148],{"className":8149,"style":8151},[8150],"arraycolsep","width:1em;",[41,8153,8155],{"className":8154},[4360],[41,8156,8158,8425],{"className":8157},[276,651],[41,8159,8161,8422],{"className":8160},[280],[41,8162,8164,8235,8353],{"className":8163,"style":7388},[284],[41,8165,8166,8169],{"style":7391},[41,8167],{"className":8168,"style":7395},[291],[41,8170,8172,8175,8178,8181,8184,8224,8228],{"className":8171},[83],[41,8173,3637],{"className":8174},[83,84],[41,8176],{"className":8177,"style":247},[246],[41,8179,7011],{"className":8180},[251],[41,8182],{"className":8183,"style":247},[246],[41,8185,8187,8190],{"className":8186},[83],[41,8188,3654],{"className":8189,"style":3720},[83,84],[41,8191,8193],{"className":8192},[272],[41,8194,8196,8216],{"className":8195},[276,651],[41,8197,8199,8213],{"className":8198},[280],[41,8200,8202],{"className":8201,"style":768},[284],[41,8203,8204,8207],{"style":7446},[41,8205],{"className":8206,"style":292},[291],[41,8208,8210],{"className":8209},[296,297,298,299],[41,8211,6990],{"className":8212,"style":7456},[83,84,299],[41,8214,731],{"className":8215},[730],[41,8217,8219],{"className":8218},[280],[41,8220,8222],{"className":8221,"style":790},[284],[41,8223],{},[41,8225],{"className":8226,"style":8227},[246],"margin-right:1em;",[41,8229,8231],{"className":8230},[83,2866],[41,8232,7022],{"className":8233},[83,8234],"cjk_fallback",[41,8236,8237,8240],{"style":7560},[41,8238],{"className":8239,"style":7395},[291],[41,8241,8243,8283,8286,8289,8292,8295,8298,8301,8304,8344,8347],{"className":8242},[83],[41,8244,8246,8249],{"className":8245},[83],[41,8247,3654],{"className":8248,"style":3720},[83,84],[41,8250,8252],{"className":8251},[272],[41,8253,8255,8275],{"className":8254},[276,651],[41,8256,8258,8272],{"className":8257},[280],[41,8259,8261],{"className":8260,"style":768},[284],[41,8262,8263,8266],{"style":7446},[41,8264],{"className":8265,"style":292},[291],[41,8267,8269],{"className":8268},[296,297,298,299],[41,8270,6990],{"className":8271,"style":7456},[83,84,299],[41,8273,731],{"className":8274},[730],[41,8276,8278],{"className":8277},[280],[41,8279,8281],{"className":8280,"style":790},[284],[41,8282],{},[41,8284],{"className":8285,"style":247},[246],[41,8287,7145],{"className":8288},[251],[41,8290],{"className":8291,"style":247},[246],[41,8293,3637],{"className":8294},[83,84],[41,8296],{"className":8297,"style":247},[246],[41,8299,7145],{"className":8300},[251],[41,8302],{"className":8303,"style":247},[246],[41,8305,8307,8310],{"className":8306},[83],[41,8308,3654],{"className":8309,"style":3720},[83,84],[41,8311,8313],{"className":8312},[272],[41,8314,8316,8336],{"className":8315},[276,651],[41,8317,8319,8333],{"className":8318},[280],[41,8320,8322],{"className":8321,"style":768},[284],[41,8323,8324,8327],{"style":7446},[41,8325],{"className":8326,"style":292},[291],[41,8328,8330],{"className":8329},[296,297,298,299],[41,8331,7091],{"className":8332},[83,84,299],[41,8334,731],{"className":8335},[730],[41,8337,8339],{"className":8338},[280],[41,8340,8342],{"className":8341,"style":790},[284],[41,8343],{},[41,8345],{"className":8346,"style":8227},[246],[41,8348,8350],{"className":8349},[83,2866],[41,8351,7160],{"className":8352},[83,8234],[41,8354,8355,8358],{"style":8073},[41,8356],{"className":8357,"style":7395},[291],[41,8359,8361,8364,8367,8370,8373,8413,8416],{"className":8360},[83],[41,8362,3637],{"className":8363},[83,84],[41,8365],{"className":8366,"style":247},[246],[41,8368,3079],{"className":8369},[251],[41,8371],{"className":8372,"style":247},[246],[41,8374,8376,8379],{"className":8375},[83],[41,8377,3654],{"className":8378,"style":3720},[83,84],[41,8380,8382],{"className":8381},[272],[41,8383,8385,8405],{"className":8384},[276,651],[41,8386,8388,8402],{"className":8387},[280],[41,8389,8391],{"className":8390,"style":768},[284],[41,8392,8393,8396],{"style":7446},[41,8394],{"className":8395,"style":292},[291],[41,8397,8399],{"className":8398},[296,297,298,299],[41,8400,7091],{"className":8401},[83,84,299],[41,8403,731],{"className":8404},[730],[41,8406,8408],{"className":8407},[280],[41,8409,8411],{"className":8410,"style":790},[284],[41,8412],{},[41,8414],{"className":8415,"style":8227},[246],[41,8417,8419],{"className":8418},[83,2866],[41,8420,7199],{"className":8421},[83,8234],[41,8423,731],{"className":8424},[730],[41,8426,8428],{"className":8427},[280],[41,8429,8431],{"className":8430,"style":8144},[284],[41,8432],{},[41,8434],{"className":8435},[450,644],[10,8437,8438],{},"线性预热避免了训练最初几步二阶动量尚未稳定时过大的步长冲毁初始表示，随后的余弦退火让模型在后期平滑收敛到高曲率极小值区域。",[109,8440,8442,8443,8446,8447,8475],{"id":8441},"随机窗口采样-get_batchcheckpoint-与-top-ppp-解码","随机窗口采样 ",[14,8444,8445],{},"get_batch","、Checkpoint 与 Top-",[41,8448,8450,8463],{"className":8449},[44],[41,8451,8453],{"className":8452},[48],[50,8454,8455],{"xmlns":52},[54,8456,8457,8461],{},[57,8458,8459],{},[60,8460,10],{},[63,8462,10],{"encoding":65},[41,8464,8466],{"className":8465,"ariaHidden":70},[69],[41,8467,8469,8472],{"className":8468},[74],[41,8470],{"className":8471,"style":79},[78],[41,8473,10],{"className":8474},[83,84]," 解码",[5814,8477,8479,8480],{"id":8478},"_1-向量化滑动窗口采样-get_batch","1. 向量化滑动窗口采样 ",[14,8481,8445],{},[10,8483,8484,8485,8513,8514,8517,8518,8521,8522,8550,8551,8650,8651,8653,8654,3430],{},"给定长度为 ",[41,8486,8488,8501],{"className":8487},[44],[41,8489,8491],{"className":8490},[48],[50,8492,8493],{"xmlns":52},[54,8494,8495,8499],{},[57,8496,8497],{},[60,8498,5853],{},[63,8500,5853],{"encoding":65},[41,8502,8504],{"className":8503,"ariaHidden":70},[69],[41,8505,8507,8510],{"className":8506},[74],[41,8508],{"className":8509,"style":2247},[78],[41,8511,5853],{"className":8512,"style":3159},[83,84]," 的一维 token 数组 ",[14,8515,8516],{},"dataset"," 和上下文长度 ",[14,8519,8520],{},"context_length"," ",[41,8523,8525,8538],{"className":8524},[44],[41,8526,8528],{"className":8527},[48],[50,8529,8530],{"xmlns":52},[54,8531,8532,8536],{},[57,8533,8534],{},[60,8535,931],{},[63,8537,931],{"encoding":65},[41,8539,8541],{"className":8540,"ariaHidden":70},[69],[41,8542,8544,8547],{"className":8543},[74],[41,8545],{"className":8546,"style":1074},[78],[41,8548,931],{"className":8549},[83,84],"，合法的起点下标范围是 ",[41,8552,8554,8586],{"className":8553},[44],[41,8555,8557],{"className":8556},[48],[50,8558,8559],{"xmlns":52},[54,8560,8561,8583],{},[57,8562,8563,8566,8568,8570,8572,8574,8576,8578,8580],{},[129,8564,8565],{"stretchy":335},"[",[338,8567,340],{},[129,8569,343],{"separator":70},[60,8571,5853],{},[129,8573,353],{},[60,8575,931],{},[129,8577,353],{},[338,8579,356],{},[129,8581,8582],{"stretchy":335},"]",[63,8584,8585],{"encoding":65},"[0, N - m - 1]",[41,8587,8589,8619,8638],{"className":8588,"ariaHidden":70},[69],[41,8590,8592,8595,8598,8601,8604,8607,8610,8613,8616],{"className":8591},[74],[41,8593],{"className":8594,"style":396},[78],[41,8596,8565],{"className":8597},[400],[41,8599,340],{"className":8600},[83],[41,8602,343],{"className":8603},[407],[41,8605],{"className":8606,"style":411},[246],[41,8608,5853],{"className":8609,"style":3159},[83,84],[41,8611],{"className":8612,"style":314},[246],[41,8614,353],{"className":8615},[310],[41,8617],{"className":8618,"style":314},[246],[41,8620,8622,8626,8629,8632,8635],{"className":8621},[74],[41,8623],{"className":8624,"style":8625},[78],"height:0.6667em;vertical-align:-0.0833em;",[41,8627,931],{"className":8628},[83,84],[41,8630],{"className":8631,"style":314},[246],[41,8633,353],{"className":8634},[310],[41,8636],{"className":8637,"style":314},[246],[41,8639,8641,8644,8647],{"className":8640},[74],[41,8642],{"className":8643,"style":396},[78],[41,8645,356],{"className":8646},[83],[41,8648,8582],{"className":8649},[450],"（因为标签 ",[14,8652,330],{}," 需要向右偏移 1 位取到 ",[14,8655,8656],{},"start + m",[1963,8658,8660],{"className":1965,"code":8659,"language":1967,"meta":1968,"style":1968},"start_indices = np.random.randint(0, len(dataset) - context_length, size=(batch_size,))\noffsets = np.arange(context_length)\nwindow_indices = start_indices[:, None] + offsets[None, :]\nx = torch.as_tensor(dataset[window_indices], dtype=torch.long).to(device)\ny = torch.as_tensor(dataset[window_indices + 1], dtype=torch.long).to(device)\n",[14,8661,8662,8700,8716,8741,8769],{"__ignoreMap":1968},[41,8663,8664,8667,8669,8672,8675,8677,8679,8681,8684,8687,8689,8692,8695,8697],{"class":1973,"line":1974},[41,8665,8666],{"class":1985},"start_indices ",[41,8668,513],{"class":2007},[41,8670,8671],{"class":1985}," np.random.",[41,8673,8674],{"class":2013},"randint",[41,8676,501],{"class":1985},[41,8678,340],{"class":2027},[41,8680,2030],{"class":1985},[41,8682,8683],{"class":3491},"len",[41,8685,8686],{"class":1985},"(dataset) ",[41,8688,2055],{"class":2007},[41,8690,8691],{"class":1985}," context_length, ",[41,8693,8694],{"class":2020},"size",[41,8696,513],{"class":2007},[41,8698,8699],{"class":1985},"(batch_size,))\n",[41,8701,8702,8705,8707,8710,8713],{"class":1973,"line":2001},[41,8703,8704],{"class":1985},"offsets ",[41,8706,513],{"class":2007},[41,8708,8709],{"class":1985}," np.",[41,8711,8712],{"class":2013},"arange",[41,8714,8715],{"class":1985},"(context_length)\n",[41,8717,8718,8721,8723,8726,8728,8731,8733,8736,8738],{"class":1973,"line":2044},[41,8719,8720],{"class":1985},"window_indices ",[41,8722,513],{"class":2007},[41,8724,8725],{"class":1985}," start_indices[:, ",[41,8727,2696],{"class":2027},[41,8729,8730],{"class":1985},"] ",[41,8732,3191],{"class":2007},[41,8734,8735],{"class":1985}," offsets[",[41,8737,2696],{"class":2027},[41,8739,8740],{"class":1985},", :]\n",[41,8742,8743,8746,8748,8750,8753,8756,8759,8761,8764,8766],{"class":1973,"line":2061},[41,8744,8745],{"class":1985},"x ",[41,8747,513],{"class":2007},[41,8749,2010],{"class":1985},[41,8751,8752],{"class":2013},"as_tensor",[41,8754,8755],{"class":1985},"(dataset[window_indices], ",[41,8757,8758],{"class":2020},"dtype",[41,8760,513],{"class":2007},[41,8762,8763],{"class":1985},"torch.long).",[41,8765,3506],{"class":2013},[41,8767,8768],{"class":1985},"(device)\n",[41,8770,8771,8774,8776,8778,8780,8783,8785,8788,8791,8793,8795,8797,8799],{"class":1973,"line":2095},[41,8772,8773],{"class":1985},"y ",[41,8775,513],{"class":2007},[41,8777,2010],{"class":1985},[41,8779,8752],{"class":2013},[41,8781,8782],{"class":1985},"(dataset[window_indices ",[41,8784,3191],{"class":2007},[41,8786,8787],{"class":2027}," 1",[41,8789,8790],{"class":1985},"], ",[41,8792,8758],{"class":2020},[41,8794,513],{"class":2007},[41,8796,8763],{"class":1985},[41,8798,3506],{"class":2013},[41,8800,8768],{"class":1985},[10,8802,8803,8804,8807],{},"配合 ",[14,8805,8806],{},"np.memmap(path, dtype=np.uint16, mode=\"r\")","，即使面对数十 GB 的 OpenWebText 词元文件，操作系统也只会按页调入当前 batch 采样的切片，实现恒定内存开销的数据加载。",[5814,8809,8811],{"id":8810},"_2-checkpoint-保存与恢复","2. Checkpoint 保存与恢复",[10,8813,8814,8817,8818,17,8821,8823,8824,8827,8828,8831,8832,8834,8835,8838],{},[14,8815,8816],{},"save_checkpoint"," 将 ",[14,8819,8820],{},"model.state_dict()",[14,8822,5811],{}," 和当前迭代步数 ",[14,8825,8826],{},"iteration"," 打包为字典写入磁盘或二进制流；",[14,8829,8830],{},"load_checkpoint"," 恢复模型与优化器动量状态并返回 ",[14,8833,8826],{},"，通过了官方 ",[14,8836,8837],{},"test_checkpointing"," 对所有权重与动量张量的精确比对。",[5814,8840,8842,8843,8871],{"id":8841},"_3-temperature-缩放与-top-pppnucleus采样解码","3. Temperature 缩放与 Top-",[41,8844,8846,8859],{"className":8845},[44],[41,8847,8849],{"className":8848},[48],[50,8850,8851],{"xmlns":52},[54,8852,8853,8857],{},[57,8854,8855],{},[60,8856,10],{},[63,8858,10],{"encoding":65},[41,8860,8862],{"className":8861,"ariaHidden":70},[69],[41,8863,8865,8868],{"className":8864},[74],[41,8866],{"className":8867,"style":79},[78],[41,8869,10],{"className":8870},[83,84],"（Nucleus）采样解码",[10,8873,1953,8874,8877,8878,8881],{},[14,8875,8876],{},"cs336_basics\u002Fdecoding.py"," 中，",[14,8879,8880],{},"generate"," 接收前缀 prompt，每步取最后一个位置的 logits：",[5826,8883,8884,8949],{},[5829,8885,8886,8887,8918,8919,8948],{},"先除以温度 ",[41,8888,8890,8905],{"className":8889},[44],[41,8891,8893],{"className":8892},[48],[50,8894,8895],{"xmlns":52},[54,8896,8897,8902],{},[57,8898,8899],{},[60,8900,8901],{},"τ",[63,8903,8904],{"encoding":65},"\\tau",[41,8906,8908],{"className":8907,"ariaHidden":70},[69],[41,8909,8911,8914],{"className":8910},[74],[41,8912],{"className":8913,"style":1074},[78],[41,8915,8901],{"className":8916,"style":8917},[83,84],"margin-right:0.1132em;"," 并做 Softmax 得到概率分布 ",[41,8920,8922,8936],{"className":8921},[44],[41,8923,8925],{"className":8924},[48],[50,8926,8927],{"xmlns":52},[54,8928,8929,8934],{},[57,8930,8931],{},[60,8932,8933],{},"q",[63,8935,8933],{"encoding":65},[41,8937,8939],{"className":8938,"ariaHidden":70},[69],[41,8940,8942,8945],{"className":8941},[74],[41,8943],{"className":8944,"style":79},[78],[41,8946,8933],{"className":8947,"style":380},[83,84],"；",[5829,8950,8951,8952,9033,9034,9127,9128,9156,9157,9160,9161,9164],{},"将概率按从大到小排序得到 ",[41,8953,8955,8975],{"className":8954},[44],[41,8956,8958],{"className":8957},[48],[50,8959,8960],{"xmlns":52},[54,8961,8962,8972],{},[57,8963,8964],{},[8965,8966,8967,8969],"mover",{"accent":70},[60,8968,8933],{},[129,8970,8971],{},"~",[63,8973,8974],{"encoding":65},"\\tilde{q}",[41,8976,8978],{"className":8977,"ariaHidden":70},[69],[41,8979,8981,8985],{"className":8980},[74],[41,8982],{"className":8983,"style":8984},[78],"height:0.8623em;vertical-align:-0.1944em;",[41,8986,8989],{"className":8987},[83,8988],"accent",[41,8990,8992,9024],{"className":8991},[276,651],[41,8993,8995,9021],{"className":8994},[280],[41,8996,8999,9007],{"className":8997,"style":8998},[284],"height:0.6679em;",[41,9000,9001,9004],{"style":5254},[41,9002],{"className":9003,"style":665},[291],[41,9005,8933],{"className":9006,"style":380},[83,84],[41,9008,9010,9013],{"style":9009},"top:-3.35em;",[41,9011],{"className":9012,"style":665},[291],[41,9014,9018],{"className":9015,"style":9017},[9016],"accent-body","left:-0.1667em;",[41,9019,8971],{"className":9020},[83],[41,9022,731],{"className":9023},[730],[41,9025,9027],{"className":9026},[280],[41,9028,9031],{"className":9029,"style":9030},[284],"height:0.1944em;",[41,9032],{},"，计算累积和 ",[41,9035,9037,9064],{"className":9036},[44],[41,9038,9040],{"className":9039},[48],[50,9041,9042],{"xmlns":52},[54,9043,9044,9061],{},[57,9045,9046,9049,9051,9053,9059],{},[60,9047,9048],{"mathvariant":497},"cumsum",[129,9050,131],{},[129,9052,501],{"stretchy":335},[8965,9054,9055,9057],{"accent":70},[60,9056,8933],{},[129,9058,8971],{},[129,9060,510],{"stretchy":335},[63,9062,9063],{"encoding":65},"\\operatorname{cumsum}(\\tilde{q})",[41,9065,9067],{"className":9066,"ariaHidden":70},[69],[41,9068,9070,9073,9079,9082,9124],{"className":9069},[74],[41,9071],{"className":9072,"style":396},[78],[41,9074,9076],{"className":9075},[148],[41,9077,9048],{"className":9078},[83,2254],[41,9080,501],{"className":9081},[400],[41,9083,9085],{"className":9084},[83,8988],[41,9086,9088,9116],{"className":9087},[276,651],[41,9089,9091,9113],{"className":9090},[280],[41,9092,9094,9102],{"className":9093,"style":8998},[284],[41,9095,9096,9099],{"style":5254},[41,9097],{"className":9098,"style":665},[291],[41,9100,8933],{"className":9101,"style":380},[83,84],[41,9103,9104,9107],{"style":9009},[41,9105],{"className":9106,"style":665},[291],[41,9108,9110],{"className":9109,"style":9017},[9016],[41,9111,8971],{"className":9112},[83],[41,9114,731],{"className":9115},[730],[41,9117,9119],{"className":9118},[280],[41,9120,9122],{"className":9121,"style":9030},[284],[41,9123],{},[41,9125,510],{"className":9126},[450],"，把进入当前 token 之前累积概率已经达到阈值 ",[41,9129,9131,9144],{"className":9130},[44],[41,9132,9134],{"className":9133},[48],[50,9135,9136],{"xmlns":52},[54,9137,9138,9142],{},[57,9139,9140],{},[60,9141,10],{},[63,9143,10],{"encoding":65},[41,9145,9147],{"className":9146,"ariaHidden":70},[69],[41,9148,9150,9153],{"className":9149},[74],[41,9151],{"className":9152,"style":79},[78],[41,9154,10],{"className":9155},[83,84]," 的尾部低概率词置零（",[14,9158,9159],{},"cumulative_probs - sorted_probs >= top_p","），重新归一化后调用 ",[14,9162,9163],{},"torch.multinomial"," 采样。",[109,9166,9167],{"id":9167},"全量测试套件与最小训练闭环验证",[10,9169,9170,9171,9174],{},"在仓库根目录执行 ",[14,9172,9173],{},"bash scripts\u002Fcheck_completed.sh","，运行官方全部 7 个测试文件：",[1963,9176,9180],{"className":9177,"code":9179,"language":2866,"meta":1968},[9178],"language-text","tests\u002Ftest_data.py::test_get_batch PASSED\ntests\u002Ftest_model.py (13 passed)\ntests\u002Ftest_nn_utils.py (3 passed)\ntests\u002Ftest_optimizer.py (2 passed)\ntests\u002Ftest_serialization.py::test_checkpointing PASSED\ntests\u002Ftest_tokenizer.py (23 passed, 2 skipped)\ntests\u002Ftest_train_bpe.py (3 passed)\n======================== 46 passed, 2 skipped in 4.66s =========================\n",[14,9181,9179],{"__ignoreMap":1968},[10,9183,9184,9185,9188,9189,9217],{},"同时执行 ",[14,9186,9187],{},"python -m cs336_basics.train"," 完成从合成词元加载、Warmup 余弦学习率调度、前向\u002F反向传播、梯度裁剪、AdamW 更新、验证集困惑度评估到 Top-",[41,9190,9192,9205],{"className":9191},[44],[41,9193,9195],{"className":9194},[48],[50,9196,9197],{"xmlns":52},[54,9198,9199,9203],{},[57,9200,9201],{},[60,9202,10],{},[63,9204,10],{"encoding":65},[41,9206,9208],{"className":9207,"ariaHidden":70},[69],[41,9209,9211,9214],{"className":9210},[74],[41,9212],{"className":9213,"style":79},[78],[41,9215,10],{"className":9216},[83,84]," 文本生成的完整端到端闭环验证。",[10,9219,9220],{},[9221,9222,9223,9224,9229],"em",{},"封面使用 ",[97,9225,9228],{"href":9226,"rel":9227},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Faigc-image-generator",[101],"aigc-image-generator"," 辅助生成，属于概念插画；准确的公式、代码与测试结果以正文为准。",[109,9231,9232],{"id":9232},"参考资料",[9234,9235,9236,9243,9250],"ul",{},[5829,9237,9238],{},[97,9239,9242],{"href":9240,"rel":9241},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1711.05101",[101],"Loshchilov & Hutter, 2019: Decoupled Weight Decay Regularization (AdamW)",[5829,9244,9245],{},[97,9246,9249],{"href":9247,"rel":9248},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1904.09751",[101],"Holtzman et al., 2020: The Curious Case of Neural Text Degeneration (Top-p Sampling)",[5829,9251,9252],{},[97,9253,9256],{"href":9254,"rel":9255},"https:\u002F\u002Fgithub.com\u002Fstanford-cs336\u002Fassignment1-basics",[101],"Stanford CS336 Assignment 1 官方仓库",[9258,9259,9260],"style",{},"html pre.shiki code .sST-V, html code.shiki .sST-V{--shiki-default:#F47067;--shiki-light:#D73A49;--shiki-cyber:#54B9FF}html pre.shiki code .sS_0i, html code.shiki .sS_0i{--shiki-default:#DCBDFB;--shiki-light:#6F42C1;--shiki-cyber:#00DAEF}html pre.shiki code .sWURj, html code.shiki .sWURj{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#EEF0F9}html pre.shiki code .s9NhK, html code.shiki .s9NhK{--shiki-default:#ADBAC7;--shiki-default-font-style:inherit;--shiki-light:#24292E;--shiki-light-font-style:inherit;--shiki-cyber:#FFD493;--shiki-cyber-font-style:italic}html pre.shiki code .sf2cM, html code.shiki .sf2cM{--shiki-default:#F47067;--shiki-light:#D73A49;--shiki-cyber:#EEF0F9}html pre.shiki code .sq3c-, html code.shiki .sq3c-{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#00DAEF}html pre.shiki code .sA9aF, html code.shiki .sA9aF{--shiki-default:#F69D50;--shiki-default-font-style:inherit;--shiki-light:#E36209;--shiki-light-font-style:inherit;--shiki-cyber:#4BF3C8;--shiki-cyber-font-style:italic}html pre.shiki code .sgENW, html code.shiki .sgENW{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#FFD493}html pre.shiki code .slEfO, html code.shiki .slEfO{--shiki-default:#F47067;--shiki-default-font-style:inherit;--shiki-light:#D73A49;--shiki-light-font-style:inherit;--shiki-cyber:#54B9FF;--shiki-cyber-font-style:italic}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html.cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html pre.shiki code .svkvy, html code.shiki .svkvy{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#EEF0F9}",{"title":1968,"searchDepth":2001,"depth":2044,"links":9262},[9263,9265,9267,9271,9272,9280,9281],{"id":111,"depth":2001,"text":9264},"数值稳定 Cross-Entropy：把 log⁡\\loglog 和 exp⁡\\expexp 消掉",{"id":2478,"depth":2001,"text":9266},"全局 ℓ2\\ell_2ℓ2​ 梯度裁剪（Gradient Clipping）",{"id":3618,"depth":2001,"text":3619,"children":9268},[9269],{"id":5816,"depth":2044,"text":9270},"AdamW 训练峰值显存核算（Problem adamw_accounting）",{"id":6901,"depth":2001,"text":6902},{"id":8441,"depth":2001,"text":9273,"children":9274},"随机窗口采样 get_batch、Checkpoint 与 Top-ppp 解码",[9275,9277,9278],{"id":8478,"depth":2044,"text":9276},"1. 向量化滑动窗口采样 get_batch",{"id":8810,"depth":2044,"text":8811},{"id":8841,"depth":2044,"text":9279},"3. Temperature 缩放与 Top-ppp（Nucleus）采样解码",{"id":9167,"depth":2001,"text":9167},{"id":9232,"depth":2001,"text":9232},[9283],"学习笔记","\u002Fimages\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop\u002Fcs336-cross-entropy-adamw-training-loop-cover.webp","滑动窗口采样、余弦退火曲线、双动量优化轨迹与检查点归档构成的训练闭环 AI 辅助概念插画","2026-10-27","用 LogSumExp 消去交叉熵中的指数溢出，手写解耦权重衰减的 AdamW 优化器、全局 L2 梯度裁剪、Warmup 余弦退火调度、memmap 数据采样、Checkpoint 序列化与 Top-p 解码，完成 CS336 Assignment 1 全部 48 项测试。",false,"md",{"codeUrl":99,"attachmentUrl":106},true,"\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop",null,{"title":5,"description":9287},"Stanford CS336","posts\u002Fcs336-cross-entropy-adamw-training-loop",[9298,9299,9300,9301,9302],"CS336","PyTorch","AdamW","CrossEntropy","训练闭环","0cVoVXMLBnDAfbXgv16KaGN4jgc7_TPOTlUfuyBafKw",[9305,9310,9326,9341,9355,9369,9384,9399,9413,9425,9440,9452,9466,9479,9492,9506,9523,9538,9549,9561,9576,9589,9603,9615,9629,9641,9656,9670,9684,9695,9708,9725,9740,9753,9765,9777,9790,9803],{"slug":9306,"path":9292,"title":5,"date":9286,"tags":9307,"description":9287,"cover":9284,"coverAlt":9285,"excerpt":9308,"draft":9288,"hidden":9288,"published":9291,"readingTime":9309,"series":9295,"seriesOrder":9309,"codeUrl":99,"attachmentUrl":106},"cs336-cross-entropy-adamw-training-loop",[9298,9299,9300,9301,9302],"作为 Stanford CS336 Assignment 1 系列的收官篇，我们已经拥有了分词器（bpe.py、tokenizer.py）和完整的语言模型架构（transformer.py）。要让模型在语料上真正训练并生成文本，还需",8,{"slug":9311,"path":9312,"title":9313,"date":9314,"tags":9315,"description":9322,"cover":9323,"coverAlt":9324,"excerpt":9325,"draft":9288,"hidden":9288,"published":9291,"readingTime":9309},"editcache-video-dit-inference-acceleration","\u002Fposts\u002Feditcache-video-dit-inference-acceleration","EditCache：视频 Diffusion Transformer (DiT) 局部编辑的跨步特征复用与区域重算加速","2026-10-25",[9316,9317,9318,9319,9320,9321],"扩散模型","视频生成","AI Infra","推理加速","GPU","论文解读","拆解视频扩散模型（Wan2.1-VACE）局部编辑中的三重算力冗余，通过可重入 Latent、跨步 Block 隐状态缓存复用、带 Halo 边界的局部区域重算与自适应推理起点，在单卡 H100 上实现 2.46× 模型计算加速与 1.49× 严格同步端到端加速。","\u002Fimages\u002Fposts\u002Feditcache-video-dit-inference-acceleration\u002Feditcache-dit-cover-20261025.webp","视频扩散 Transformer 时空三维体素网格中冻结背景缓存与暖橙色局部编辑重算通道的等距技术插画","在基于 Diffusion Transformer（DiT）的视频编辑与局部重绘（Video Inpainting \u002F Local Editing）任务中，用户的典型工作流并不是一次性生成最终成片，而是针对同一段源视频反复调整掩码区",{"slug":9327,"path":9328,"title":9329,"date":9314,"tags":9330,"description":9335,"cover":9336,"coverAlt":9337,"excerpt":9338,"draft":9288,"hidden":9288,"published":9291,"readingTime":9309,"series":9295,"seriesOrder":9339,"codeUrl":99,"attachmentUrl":9340},"cs336-bpe-tokenizer-training-encoding","\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding","Stanford CS336 学习记录 07：Byte-Level BPE 分词器训练、倒排索引加速与流式编码",[9298,9331,9332,9333,9334],"Tokenizer","BPE","Tiktoken","算法优化","实现 GPT-2 正则预分词、特殊 token 硬边界切分、基于 pair-to-word 倒排索引的增量 BPE 合并算法（\u003C 0.25s 通过速度基准），以及与 tiktoken 完全对齐的 Tokenizer。","\u002Fimages\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding\u002Fcs336-bpe-tokenizer-training-encoding-cover.webp","UTF-8 字节块经过正则切分闸口后沿合并树聚合成子词单元的 AI 辅助概念插画","前面六篇完成了 Transformer 语言模型本身的全部神经网络组件。要让模型真正处理自然语言文本，还需要打通入口处的 ByteLevel BPE（BytePair Encoding）分词器。这一篇记录 cs336basics\u002Fb",7,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-07-bpe-tokenizer-20261025.zip",{"slug":9342,"path":9343,"title":9344,"date":9345,"tags":9346,"description":9351,"cover":9352,"coverAlt":9353,"excerpt":9354,"draft":9288,"hidden":9288,"published":9291,"readingTime":9309},"in-situ-autoguidance-editable-scientific-illustration","\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration","从原位自校正引导（In-situ Autoguidance）到可编辑科研插图生成（AutoFigure-Edit）","2026-10-24",[9316,9347,9348,9349,9350,9321],"多模态","Agent","ICML","ACL","串联两项生成式模型与多模态智能体研究：如何通过推理期局部随机微扰在单模型内构造免训练扩散自引导向量（ICML 2025 Workshop 第一作者，arXiv:2510.17136），以及如何通过五阶段视觉-符号混合流水线生成原生可编辑 SVG 科研插图（ACL 2026 Demo，FigureBench 83.0% 胜率）。","\u002Fimages\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration\u002Fautoguidance-autofigure-cover-20261024.webp","左侧扩散流形自校正轨迹矢量场与右侧五阶段可编辑 SVG 科研插图分层网格相衔接的等距技术插画","生成式视觉模型正在经历从“生成高保真像素（Pixel Synthesis）”向“可控结构化生成（Controllable & Editable Generation）”的演进。在这一进程中，我作为第一作者或核心贡献者参与了两项分别处",{"slug":9356,"path":9357,"title":9358,"date":9359,"tags":9360,"description":9364,"cover":9365,"coverAlt":9366,"excerpt":9367,"draft":9288,"hidden":9288,"published":9291,"readingTime":9309,"series":9295,"seriesOrder":2156,"codeUrl":99,"attachmentUrl":9368},"cs336-pre-norm-transformer-block-lm","\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm","Stanford CS336 学习记录 06：Pre-Norm Transformer Block 与完整语言模型装配","2026-10-23",[9298,9299,9361,9362,9363],"Transformer","Pre-Norm","FLOPs","将 RMSNorm、带 RoPE 的因果多头自注意力和 SwiGLU 组装成 Pre-Norm Transformer Block，搭建完整 TransformerLM，并核算 GPT-2 XL 的参数量与前向 FLOPs。","\u002Fimages\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm\u002Fcs336-pre-norm-transformer-block-lm-cover.webp","干净残差流主干穿过多个 Pre-Norm 子层并汇聚到词表投影层的 AI 辅助概念插画","前五篇把 Linear、Embedding、RMSNorm、SwiGLU、RotaryPositionalEmbedding 和因果 MultiHeadSelfAttention 全部单独打通了。这一篇进入 Assignment 1","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-06-transformer-block-lm-20261023.zip",{"slug":9370,"path":9371,"title":9372,"date":9359,"tags":9373,"description":9379,"cover":9380,"coverAlt":9381,"excerpt":9382,"draft":9288,"hidden":9288,"published":9291,"readingTime":9383},"xgrammar-concurrent-cache-generation-fix","\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix","XGrammar 结构化解码缓存剖析：Unlimited 模式并发 Clear 字节记账竞态与代际修复（PR #1011）",[9318,9374,9375,9376,9377,9378],"C++","XGrammar","结构化解码","并发编程","源码分析","拆解 XGrammar GrammarCompiler 的双层缓存架构与 ThreadSafeLRUCache 锁分离实现，定位 Unlimited 模式下 Clear() 字节残留与在途编译（In-flight Computation）跨代污染根因，给出代际号 + 读写锁原子修复与零 sleep 确定性并发 GoogleTest 验证。","\u002Fimages\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix\u002Fxgrammar-cache-fix-cover-20261022.webp","双层磨砂玻璃流道经暖橙色代际闸门过滤后汇入晶格缓存矩阵的等距技术插画","在大语言模型服务（如 SGLang、vLLM）的结构化输出场景中，JSON Schema 或 EBNF 语法需要先编译为词表级掩码缓存（CompiledGrammar），才能在每个 Decode 步以微秒级开销生成合法 token ",14,{"slug":9385,"path":9386,"title":9387,"date":9388,"tags":9389,"description":9395,"cover":9396,"coverAlt":9397,"excerpt":9398,"draft":9288,"hidden":9288,"published":9291,"readingTime":9383},"multi-constraint-agent-routing-dag-eval","\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval","多约束生产级 Agent 系统实践：两阶段工具路由、DAG 局部重规划与轨迹级评测闭环","2026-10-22",[9348,9390,9391,9392,9393,9394],"LLM","工具调用","上下文工程","DPO","强化学习","围绕跨境酒旅等强耦合多约束业务场景，拆解向量 Top-K 召回 + LLM 精排 + 跨语言实体链接与 Schema 门控（96%+ 调用合法率）、基于状态机的 DAG 局部子图重规划、Observation 冗余裁剪（节省 40% Token），以及从线上失败恢复轨迹构造 SFT\u002FDPO 偏好对的评测与训练闭环。","\u002Fimages\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval\u002Fagent-dag-routing-cover-20261020.webp","候选模块经漏斗滤网进入双分支有向无环图轨道并在受阻节点触发暖橙色局部旁路重连的等距技术插画","在玩具级 Demo 里，几个工具加上一段 ReAct 循环就能跑通大多数单步问答。但当 Agent 进入跨境酒旅、复杂交易或研发沙箱等真实业务生产环境时，单靠线性 ReAct 循环很快会撞上四堵墙： 1. 工具空间与参数耦合爆炸：候",{"slug":9400,"path":9401,"title":9402,"date":9403,"tags":9404,"description":9408,"cover":9409,"coverAlt":9410,"excerpt":9411,"draft":9288,"hidden":9288,"published":9291,"readingTime":9412},"llm-mrd-distillation-auditable-review-agents","\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents","多视角推理蒸馏（LLM-MRD）与可审计生成式评审系统（DeepReviewer 2.0）深度拆解","2026-10-21",[9405,9347,9406,9348,9407,9321],"知识蒸馏","Qwen2.5-VL","Post-training","对比大模型落地的两条互补路径：如何用 Qwen2.5-VL 离线多视角分析结合 MLP 残差校准与温度缩放 KL 蒸馏，将推理能力压入 358M 学生模型实现 17.2 ms 独立推理（LLM-MRD，DASFAA 2026 Oral）；以及如何通过主张-证据-风险台账、PASA 可比性门控与导出预算校验构建可审计的同行评审 Agent（DeepReviewer 2.0）。","\u002Fimages\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents\u002Fmrd-review-distill-cover-20261016.webp","三块半透明几何棱镜发出平行光束穿过中央校准环汇聚至右侧高密度晶体模块的等距技术插画","把大语言模型或视觉语言模型（VLM）的深度推理能力用到实际系统中，通常会面临一个两难选择： 如果场景受限于毫秒级在线延迟（如高吞吐多模态内容审核与虚假信息检测）：直接在线调用 7B～72B 的多模态大模型做长链条思维链（CoT）生成",18,{"slug":9414,"path":9415,"title":9416,"date":9403,"tags":9417,"description":9420,"cover":9421,"coverAlt":9422,"excerpt":9423,"draft":9288,"hidden":9288,"published":9291,"readingTime":9339,"series":9295,"seriesOrder":2095,"codeUrl":99,"attachmentUrl":9424},"cs336-rope-rotary-position-embedding","\u002Fposts\u002Fcs336-rope-rotary-position-embedding","Stanford CS336 学习记录 05：RoPE 旋转位置编码与带位置的多头注意力",[9298,9299,9361,9418,9419],"RoPE","Attention","从二维平面正交旋转推导 RoPE 的奇偶特征配对公式，理解为什么内积只依赖相对位置差 (i - j)，并用 register_buffer 预计算 cos\u002Fsin 缓存接入因果多头自注意力。","\u002Fimages\u002Fposts\u002Fcs336-rope-rotary-position-embedding\u002Fcs336-rope-rotary-position-embedding-cover.webp","二维特征平面中的旋转向量对与预计算余弦正弦网格的 AI 辅助概念插画","上一篇完成了不含位置编码的因果多头自注意力。如果只看那一套矩阵乘法，把输入序列前两个 token 交换顺序（并在因果 mask 允许的范围内看同一个后缀 query 对它们的点积），点积本身并不知道哪个 key 离当前 query ","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-05-rope-mha-20261021.zip",{"slug":9426,"path":9427,"title":9428,"date":9429,"tags":9430,"description":9436,"cover":9437,"coverAlt":9438,"excerpt":9439,"draft":9288,"hidden":9288,"published":9291,"readingTime":9309},"llm-gateway-semantic-cache-dynamic-batching","\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching","高并发 LLM 推理网关架构实践：两级精确\u002F语义缓存、50ms 有界动态批处理与尾部延迟治理","2026-10-19",[9318,9431,9432,9433,9434,9435],"LLM 推理","分布式系统","Redis","FastAPI","后端","深度拆解生产级大模型推理网关 LLM-Gateway 的核心系统设计：基于 SHA-256 与 Redis 向量余弦相似度的两级精确\u002F语义缓存、50ms 有界等待动态批处理调度器（Dynamic Batching）、令牌桶多租户限流与 Prometheus 全链路可观测性，将命中请求 P99 延迟从 5,307 ms 压降至 17 ms。","\u002Fimages\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching\u002Fllm-gateway-cover-20261019.webp","高并发请求流依次穿过精确哈希滤网、向量球面语义缓存层与 50ms 动态批处理时间窗拼装器的等距系统架构插画","在将大语言模型接入高并发业务系统时，如果让上游微服务直接裸连底层推理引擎（vLLM \u002F SGLang 或外部模型 API），系统很快会暴露出三个典型的生产痛点： 1. 高频重复与近义查询消耗巨量算力：大量用户请求在语义上高度等价（例",{"slug":9441,"path":9442,"title":9443,"date":9429,"tags":9444,"description":9446,"cover":9447,"coverAlt":9448,"excerpt":9449,"draft":9288,"hidden":9288,"published":9291,"readingTime":9383,"series":9295,"seriesOrder":2061,"codeUrl":9450,"attachmentUrl":9451},"cs336-multihead-attention-shapes","\u002Fposts\u002Fcs336-multihead-attention-shapes","Stanford CS336 学习记录 04：多头注意力怎样拆头、计算和合头",[9298,9299,9361,9419,9445],"张量形状","沿着 (2, 5, 64) 的输入追踪 QKV 投影、reshape、transpose、共享因果 mask 和合头，理解每个头仍处理完整序列，以及最后为什么还需要输出 Linear。","\u002Fimages\u002Fposts\u002Fcs336-multihead-attention-shapes\u002Fcs336-multihead-attention-shapes-cover.webp","宽向量束分成多条并行特征轨道后重新合并的 AI 辅助概念插画","上一篇完成了缩放点积注意力。那个函数接收已经准备好的 Q、K、V，计算匹配分数，应用 mask，再用权重汇总 V。这一篇继续把它放进一个带参数的多头自注意力模块。 这次最需要看清的是“拆头”。代码里只是几行 reshape 和 tr","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002F31363223498fafd1528c550a51efbe0aed231736","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-04-multihead-attention-20261019.zip",{"slug":9453,"path":9454,"title":9455,"date":9456,"tags":9457,"description":9461,"cover":9462,"coverAlt":9463,"excerpt":9464,"draft":9288,"hidden":9288,"published":9291,"readingTime":9465},"qwen25-sft-dpo-alignment-tax-ablation","\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation","大模型后训练偏好对齐实录：Qwen2.5 SFT 多维消融、动态参考 DPO 与对齐税（Alignment Tax）量化分析","2026-10-18",[9407,9393,9458,9459,9394,9460],"SFT","LoRA","Qwen2.5","基于 Qwen2.5 完整复现 Instruct SFT → Bradley-Terry DPO 两阶段后训练管线：通过 3 随机种子消融 LoRA Rank、学习率与数据规模（GSM8K +12.73pp），实现免双倍权重的动态 Adapter 切换 DPO 训练，并定量揭示偏好对齐对确定性数学推理的对齐税效应。","\u002Fimages\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation\u002Fqwen25-sft-dpo-cover-20261018.webp","左侧监督微调轨迹流经中央 Bradley-Terry 偏好分流天平分离出正负样本通道并连接右侧对齐税监控仪表盘的等距技术插画","在大语言模型后训练（PostTraining）管线中，监督微调（SFT）负责教会模型任务格式与领域推理模式，而直接偏好优化（DPO, Direct Preference Optimization）则利用成对偏好数据 $(x, yw,",9,{"slug":9467,"path":9468,"title":9469,"date":9456,"tags":9470,"description":9472,"cover":9473,"coverAlt":9474,"excerpt":9475,"draft":9288,"hidden":9288,"published":9291,"readingTime":9476,"series":9295,"seriesOrder":2044,"codeUrl":9477,"attachmentUrl":9478},"cs336-softmax-attention-mask","\u002Fposts\u002Fcs336-softmax-attention-mask","Stanford CS336 学习记录 03：Softmax 的数值稳定与 Attention Mask",[9298,9299,9361,9471,9419],"Softmax","从减去最大值为何不改变 Softmax 开始，理清函数与有参数层的区别，再沿着 Q、K、V 的形状理解缩放点积注意力，以及 mask 为什么必须填负无穷。","\u002Fimages\u002Fposts\u002Fcs336-softmax-attention-mask\u002Fcs336-softmax-attention-mask-cover.webp","注意力分数、权重与因果遮罩的 AI 辅助概念插画","上一篇写到 SwiGLU，接下来进入 Attention。真正开始相乘之前，先做了 Softmax：它把一组分数变成总和为 1 的权重，后面用这些权重汇总不同 token 提供的信息。 这段学习里，有两个很具体的问题值得记下来。一个",11,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fa39eb684710817f7f389626423fcb26ca0f6d629","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-03-softmax-sdpa-20261018.zip",{"slug":9480,"path":9481,"title":9482,"date":9483,"tags":9484,"description":9487,"cover":9488,"coverAlt":9489,"excerpt":9490,"draft":9288,"hidden":9288,"published":9291,"readingTime":9465,"series":9295,"seriesOrder":2001,"codeUrl":9477,"attachmentUrl":9491},"cs336-rmsnorm-swiglu","\u002Fposts\u002Fcs336-rmsnorm-swiglu","Stanford CS336 学习记录 02：RMSNorm 的广播与 SwiGLU 的门控","2026-10-16",[9298,9299,9361,9485,9486],"RMSNorm","SwiGLU","用一个 token 向量理解均方根归一化，拆开 dim=-1 与 keepdim=True 的作用，再沿着两条分支读懂 SiLU、逐元素乘法和 SwiGLU。","\u002Fimages\u002Fposts\u002Fcs336-rmsnorm-swiglu\u002Fcs336-rmsnorm-swiglu-cover.webp","特征尺度整理和双分支门控的 AI 辅助概念插画","上一篇把 Linear 和 Embedding 的参数形状理顺了。继续往下，最先遇到的是 RMSNorm：公式不算长，但把它落到一批 token 上时，dim=1、keepdim=True 和广播必须一起理解。 然后是 SiLU 和","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-02-rmsnorm-swiglu-20261016.zip",{"slug":9493,"path":9494,"title":9495,"date":9496,"tags":9497,"description":9500,"cover":9501,"coverAlt":9502,"excerpt":9503,"draft":9288,"hidden":9288,"published":9291,"readingTime":9504,"series":9295,"seriesOrder":1974,"codeUrl":9477,"attachmentUrl":9505},"cs336-linear-embedding-shapes","\u002Fposts\u002Fcs336-linear-embedding-shapes","Stanford CS336 学习记录 01：从 Linear、Embedding 搞清张量形状","2026-10-14",[9298,9299,9361,9498,9499],"Linear","Embedding","从 PyTorch 类、参数与 forward 的关系开始，理解 Linear 为什么保存两行三列的权重、计算时为什么要转置，再区分 Embedding 查表与线性变换。","\u002Fimages\u002Fposts\u002Fcs336-linear-embedding-shapes\u002Fcs336-linear-embedding-shapes-cover.webp","token 方块、向量表与矩阵变换的 AI 辅助概念插画","开始沿着 Stanford CS336 的 Assignment 1 一步步实现。BPE 之前已经接触过，这轮先跳过分词器，从 Transformer 的基础层开始。这里记录的是当时写代码时真正卡住的问题，以及对应测试能确认到的范围",10,"\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-01-linear-embedding-20261014.zip",{"slug":9507,"path":9508,"title":9509,"date":9510,"tags":9511,"description":9516,"cover":9517,"coverAlt":9518,"excerpt":9519,"draft":9288,"hidden":9288,"published":9291,"readingTime":9520,"series":9512,"seriesOrder":9383,"codeUrl":9521,"attachmentUrl":9522},"cmu-15-779-kv-cache-granularity-study","\u002Fposts\u002Fcmu-15-779-kv-cache-granularity-study","CMU 15-779 学习记录 14：KV 页大小、共享边界与 CPU 实现成本","2026-10-12",[9512,9513,9514,9515],"CMU 15-779","KV cache","PagedAttention","系统实验","用 210 组 CPU 配置比较连续与分页 KV cache，观察 31\u002F32-token 共享前缀的页边界，分别记录活跃槽位、池容量、复制计数和完整调用耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling\u002Fcmu-serving-cover-20261019.webp","浅色手绘分页存储抽屉与共享前缀卡片，旁边连接不同长度的请求队列","个人课程项目中的 CPU 实验记录，延续记录 11 的分页 KV 实现。已完成 210 组配置与 1470 次计时，输出均通过连续存储参考对照；本篇不代表正式作业提交、整门课程或项目全部交付完成。 同样是共享 31 个 token，",12,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F200faf569eccf94fb125e65110bb5d14094d454a","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fkv-cache-granularity-20261019.zip",{"slug":9524,"path":9525,"title":9526,"date":9527,"tags":9528,"description":9531,"cover":9532,"coverAlt":9533,"excerpt":9534,"draft":9288,"hidden":9288,"published":9291,"readingTime":2095,"series":9512,"seriesOrder":9535,"codeUrl":9536,"attachmentUrl":9537},"cmu-15-779-moe-routing-capacity","\u002Fposts\u002Fcmu-15-779-moe-routing-capacity","CMU 15-779 学习记录 13：MoE 路由、稀疏分发与容量溢出","2026-10-09",[9512,9529,9530],"MoE","Routing","用 CPU top-2 MoE 检查稀疏分发的输出与梯度，统计 expert-token 行数，并保留容量丢弃会改变输出的反例。","\u002Fimages\u002Fposts\u002Fcmu-15-779-moe-routing-capacity\u002Fcmu-moe-cover-20261019.webp","浅色手绘分拣器将小珠分配到四个专家工作屋，队列呈现负载不均","第 18 讲的个人配套实验。36 号脚本串行运行 CPU 专家，检查 topk 路由后的分发与合并；没有训练 router，也没有执行 GPU grouped GEMM 或跨设备 alltoall。 上一节记录低秩适配，这次转向每个",13,"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fc4851e0cedadf13af7c421867e83c8ff57244fa0","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-parallel-serving-lora-moe-20261019.zip",{"slug":9539,"path":9540,"title":9541,"date":9542,"tags":9543,"description":9545,"cover":9546,"coverAlt":9547,"excerpt":9548,"draft":9288,"hidden":9288,"published":9291,"readingTime":2095,"series":9512,"seriesOrder":9520,"codeUrl":9536,"attachmentUrl":9537},"cmu-15-779-lora-uniform-quantization","\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization","CMU 15-779 学习记录 12：LoRA 首步梯度、权重合并与均匀 4-bit 打包","2026-10-06",[9512,9459,9544],"Quantization","用一次 CPU SGD 更新检查 LoRA 初始化、冻结基座与合并等价，再分别记录均匀 4-bit 存储字节数和量化误差。","\u002Fimages\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization\u002Fcmu-lora-cover-20261019.webp","浅色手绘的大计算板、两块细长适配板和成对打包的小方块","第 17 讲的个人配套实验，35 号脚本实际执行一次 CPU LoRA 更新，并实现均匀 4bit 打包。实验检查机制与数值误差，不评估微调任务效果，也没有实现完整 QLoRA。 这次把两个问题分开记录：低秩分支是否能合并进原权重，",{"slug":9550,"path":9551,"title":9552,"date":9553,"tags":9554,"description":9558,"cover":9517,"coverAlt":9559,"excerpt":9560,"draft":9288,"hidden":9288,"published":9291,"readingTime":9339,"series":9512,"seriesOrder":9476,"codeUrl":9536,"attachmentUrl":9537},"cmu-15-779-kv-serving-speculative-sampling","\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling","CMU 15-779 学习记录 11：分页 KV、连续批处理与推测采样","2026-10-03",[9512,9555,9556,9557],"LLM Serving","KV Cache","Speculative Decoding","在 CPU 上验证 KV 页共享和写时复制，用单位成本模型检查连续批处理，再用条件概率小模型核对推测采样的接受与补采样。","浅色手绘的共享卡片抽屉与不断补入小珠子的请求托盘","第 15、16 讲的个人配套实验。33 号实现 CPU 分页 KV 存储，37 号模拟请求准入，34 号运行小概率模型的推测采样。它们分别验证存储、调度和分布问题，没有测量真实 LLM 服务吞吐或 GPU 延迟。 记录 08已经对照",{"slug":9562,"path":9563,"title":9564,"date":9565,"tags":9566,"description":9571,"cover":9572,"coverAlt":9573,"excerpt":9574,"draft":9288,"hidden":9288,"published":9291,"readingTime":9575},"qwen3-mid-post-train-mlx","\u002Fposts\u002Fqwen3-mid-post-train-mlx","在 M2 Pro 上完成 Qwen3 两阶段训练：Mid Train、SFT 与可复核评估","2026-09-30 11:19:10",[9390,9567,9568,9459,9569,9458,9570],"Qwen3","MLX","持续预训练","Apple Silicon","用 Qwen3 0.6B Base 和 MLX 完成领域持续训练与监督微调，记录数据划分、聊天模板、损失掩码、数值稳定性修复和独立测试结果。","\u002Fimages\u002Fposts\u002Fqwen3-mid-post-train-mlx\u002Fqwen3-mlx-cover-20261018.webp","Apple Silicon 统一内存芯片底座支撑两阶段持续预训练与带精确 Loss 掩码滤网的监督微调桥接插画","在一台 M2 Pro、16 GB 内存的 Mac 上，能否把一个 Base 语言模型接续训练成输出 JSON 的运维事件分级助手？这次项目完成了领域持续训练、监督微调、检查点选择、重新加载和独立测试，并保留了权重、日志和数据哈希。 ",16,{"slug":9577,"path":9578,"title":9579,"date":9580,"tags":9581,"description":9585,"cover":9586,"coverAlt":9587,"excerpt":9588,"draft":9288,"hidden":9288,"published":9291,"readingTime":2156,"series":9512,"seriesOrder":9504,"codeUrl":9536,"attachmentUrl":9537},"cmu-15-779-cpu-parallelism-pipeline","\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline","CMU 15-779 学习记录 10：加权梯度、状态分片与同步流水线","2026-09-30",[9512,9582,9583,9584],"Parallel Training","Gloo","Pipeline","用双 CPU 进程 Gloo 检查不等数据分片的梯度，推导持久状态字节数，再用依赖模型比较 GPipe 与同步 1F1B 的激活存活。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline\u002Fcmu-parallel-cover-20261019.webp","浅色手绘的四阶段桌面流水线与按数量汇总的托盘","第 12 至 14 讲的个人配套实验，脚本编号 31、32 不对应官方作业。Gloo 实际运行两个 CPU 进程；张量切分、状态字节数和流水线分别按下文的数学或成本模型解释。真实 GPU 实测按学习安排跳过。 上一节检查单个计算图的",{"slug":9590,"path":9591,"title":9592,"date":9593,"tags":9594,"description":9597,"cover":9598,"coverAlt":9599,"excerpt":9600,"draft":9288,"hidden":9288,"published":9291,"readingTime":9339,"series":9512,"seriesOrder":9465,"codeUrl":9601,"attachmentUrl":9602},"cmu-15-779-cpu-compiler-optimizations","\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations","CMU 15-779 学习记录 09：CPU 编译优化、合法重写与候选搜索","2026-09-27",[9512,9595,9596,9299],"ML Compiler","TorchInductor","用 CPU 实验验证 BatchNorm 折叠、RMSNorm 重排与非线性反例，测量分块候选，并运行 TorchInductor，分别记录首调用与预热后的时间。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations\u002Fcmu-cpu-compiler-cover-20261019.webp","淡紫计算积木经过薄荷绿折叠模具变成浅蓝积木，前方摆放四种候选托盘的浅色手绘插画","编译章节的个人配套实验，编号 30 不对应官方作业。图重写、分块候选和 TorchInductor 均在 CPU 上运行；真实 GPU 实验按学习安排跳过。 上一节记录了分块 Attention 和 KV cache。这次转到编译优","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F9261bea6aedf31cdccbbab2541ca693fc97b3743","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-compiler-optimizations-20261019.zip",{"slug":9604,"path":9605,"title":9606,"date":9607,"tags":9608,"description":9610,"cover":9611,"coverAlt":9612,"excerpt":9613,"draft":9288,"hidden":9288,"published":9291,"readingTime":9520,"series":9512,"seriesOrder":9309,"codeUrl":9521,"attachmentUrl":9614},"cmu-15-779-attention-online-softmax","\u002Fposts\u002Fcmu-15-779-attention-online-softmax","CMU 15-779 学习记录 08：分块 Attention、在线 Softmax 与 KV cache","2026-09-24",[9512,9419,9471,9609,9299],"FlashAttention","在 CPU 上对照完整 Attention、在线 Softmax 分块实现与 PyTorch SDPA，检查因果掩码、浮点误差和分数块大小，理解 FlashAttention 的一部分算法基础。","\u002Fimages\u002Fposts\u002Fcmu-15-779-attention-online-softmax\u002Fcmu-online-attention-cover-20261016.webp","浅紫色卡片分批进入绿色计算托盘，旁边三个状态容器保存累积量的浅色手绘插画","第 5 讲 Transformers & Attention 的个人配套实验。28\u002F29 号是 CPU 单头前向与 KV cache；后续补入 38 号 CUDA 多头合成张量的显式 SDPA MATH\u002FFLASH 对照。没有手写 ","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcuda-triton-sdpa-20261019.zip",{"slug":9616,"path":9617,"title":9618,"date":9619,"tags":9620,"description":9625,"cover":9626,"coverAlt":9627,"excerpt":9628,"draft":9288,"hidden":9288,"published":9291,"readingTime":9504,"series":9512,"seriesOrder":9339,"codeUrl":9521,"attachmentUrl":9614},"cmu-15-779-matmul-tiling-tree-reduction","\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction","CMU 15-779 学习记录 07：矩阵分块、数据复用与树形归约","2026-09-21",[9512,9621,9622,9623,9624],"CUDA","矩阵乘法","数据复用","归约","用 CPU 模型统计矩阵分块的输入读取次数，再比较串行求和和树形归约的依赖深度，记录补零、同步与浮点加法顺序的边界。","\u002Fimages\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction\u002Fcmu-matmul-reduction-cover-20261016.webp","矩阵积木在共享托盘中复用，旁边的小珠沿树形管道逐层合并的浅色手绘插画","GPU Architecture & CUDA Programming 的个人配套实验。首次记录用 CPU 模型检查数据复用与依赖深度；后续补入单卡 GPU 原生 CUDA 矩阵验证，以及 Triton 两阶段归约和矩阵乘法实测。 ",{"slug":9630,"path":9631,"title":9632,"date":9633,"tags":9634,"description":9637,"cover":9638,"coverAlt":9639,"excerpt":9640,"draft":9288,"hidden":9288,"published":9291,"readingTime":9504,"series":9512,"seriesOrder":2156,"codeUrl":9521,"attachmentUrl":9614},"cmu-15-779-cuda-thread-indexing","\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing","CMU 15-779 学习记录 06：CUDA 线程索引与边界检查","2026-09-18",[9512,9621,9320,9635,9636],"线程索引","边界检查","用 CPU 演示 CUDA 的 block、thread 与全局下标：10 个元素如何分给 3 个线程块，为什么要跳过越界线程，以及这些验证能说明什么。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing\u002Fcmu-cuda-indexing-cover-20261015.webp","分组的浅紫色计算方块与末端边界挡板组成线程分配教具的浅色插画","GPU Architecture & CUDA Programming 的个人配套练习。首次记录为 CPU 索引演示；后续在单卡 GPU 环境补入原生 CUDA 编译、运行与 memcheck，以及 Triton 向量加法实测。 上",{"slug":9642,"path":9643,"title":9644,"date":9645,"tags":9646,"description":9650,"cover":9651,"coverAlt":9652,"excerpt":9653,"draft":9288,"hidden":9288,"published":9291,"readingTime":9504,"series":9512,"seriesOrder":2095,"codeUrl":9654,"attachmentUrl":9655},"cmu-15-779-profiler-operator-batching","\u002Fposts\u002Fcmu-15-779-profiler-operator-batching","CMU 15-779 学习记录 05：Profiler、算子调用与批量推理","2026-09-15",[9512,9299,9647,9648,9649],"Profiler","批量推理","性能分析","对照 ML Systems 基础，用 Profiler 拆开两层网络的执行过程，再固定 1000 个样本扫描 batch size，观察算子调用次数、尾批和 CPU 耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-profiler-operator-batching\u002Fcmu-pytorch-profiling-cover-20261015.webp","放大镜观察线性层与 ReLU 模块，旁边面板呈现执行片段的浅色技术教具插画","本篇对照课程官网的 Introduction to ML Systems 整理，是个人配套 CPU 实验，不是课程官方作业。脚本编号是练习顺序，不对应课程讲次。 前几篇一直在补 PyTorch 基础。到了这一步，我在学习聊天里问：现","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fcecf5110d8a95161bb5763eadfca94cd7a0cc50f","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-profiler-batching-20261015.zip",{"slug":9657,"path":9658,"title":9659,"date":9660,"tags":9661,"description":9664,"cover":9665,"coverAlt":9666,"excerpt":9667,"draft":9288,"hidden":9288,"published":9291,"readingTime":9465,"series":9512,"seriesOrder":2061,"codeUrl":9668,"attachmentUrl":9669},"cmu-15-779-pytorch-linear-relu-absolute-value","\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value","CMU 15-779 学习记录 04：多输出线性层、ReLU 与绝对值网络","2026-09-12",[9512,9299,9622,9662,9663],"ReLU","神经网络","把 nn.Linear 对应到矩阵乘法，弄清多个输出怎样计算，再逐层拆开一个绝对值网络，比较手动设置参数与训练得到的结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value\u002Fcmu-pytorch-absolute-cover-20261015.webp","双分支经过折角模块后汇入紫色 V 形曲线面板的浅色技术教具插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的批量推理与矩阵乘法，我继续把 model(x) 拆开看：一层里有多个输出时，每个数是怎么算出来的？两","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F4cc63904a3a583da4a2122c246a14b43aac38528","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-linear-relu-abs-20261015.zip",{"slug":9671,"path":9672,"title":9673,"date":9674,"tags":9675,"description":9678,"cover":9679,"coverAlt":9680,"excerpt":9681,"draft":9288,"hidden":9288,"published":9291,"readingTime":9465,"series":9512,"seriesOrder":2044,"codeUrl":9682,"attachmentUrl":9683},"cmu-15-779-pytorch-dataloader-checkpoints-batching","\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching","CMU 15-779 学习记录 03：DataLoader、模型保存与批量推理","2026-09-09",[9512,9299,9676,9648,9677],"DataLoader","模型保存","用 DataLoader 分批训练平方函数，保存并加载模型参数，再比较 1000 个样本与 1 个样本的推理耗时，最后把批量计算对应到矩阵乘法。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching\u002Fcmu-pytorch-batching-cover-20261015.webp","样本卡片按批次进入计算模块，旁边配有参数存储盒和输出矩阵面板的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的平方函数实验，这一节继续练习分批训练、保存参数和独立预测。最后做了一次计时：同一个小网络，逐个处理 ","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-batching-20261015.zip",{"slug":9685,"path":9686,"title":9687,"date":9688,"tags":9689,"description":9690,"cover":9691,"coverAlt":9692,"excerpt":9693,"draft":9288,"hidden":9288,"published":9291,"readingTime":9465,"series":9512,"seriesOrder":2001,"codeUrl":9682,"attachmentUrl":9694},"cmu-15-779-pytorch-linear-module-relu","\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu","CMU 15-779 学习记录 02：nn.Linear、张量形状与 ReLU","2026-09-06",[9512,9299,9663,9662],"从带偏置的线性回归开始，弄清 nn.Linear 和 nn.Module 的分工，再用平方函数观察线性模型的限制、ReLU 的作用和测试点的选择。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu\u002Fcmu-pytorch-relu-cover-20261015.webp","输入经过两层神经网络和 ReLU 折角模块，输出曲线拟合结果的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 上一篇里，我用一个参数拟合了 $y=2x$。这一篇继续顺着往下写，先加上偏置，再把手写参数换成 nn.Linea","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-models-20261015.zip",{"slug":9696,"path":9697,"title":9698,"date":9699,"tags":9700,"description":9703,"cover":9704,"coverAlt":9705,"excerpt":9706,"draft":9288,"hidden":9288,"published":9291,"readingTime":9504,"series":9512,"seriesOrder":1974,"codeUrl":9682,"attachmentUrl":9707},"cmu-15-779-pytorch-autograd-linear-regression","\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression","CMU 15-779 学习记录 01：PyTorch 自动求导与线性回归","2026-09-03",[9512,9299,9701,9702],"自动求导","线性回归","从自动求导和手动梯度下降开始，比较学习率，再用 SGD 训练线性回归。记录 w * 4 写错后停在 1.25 的原因和修正结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression\u002Fcmu-pytorch-autograd-cover-20261015.webp","梯度曲面上的小球沿路径下降，前侧面板展示散点与拟合直线的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 开始学习 CMU 15779 时，我会 Python，但还没用过 PyTorch。这次先在 Mac 上用 CPU","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-basics-20261013.zip",{"slug":9709,"path":9710,"title":9711,"date":9712,"tags":9713,"description":9719,"cover":9720,"coverAlt":9721,"excerpt":9722,"draft":9288,"hidden":9288,"published":9291,"readingTime":9535,"codeUrl":9723,"attachmentUrl":9724},"embodied-pickplace-lab-robustness","\u002Fposts\u002Fembodied-pickplace-lab-robustness","Pickplace Lab 项目复盘：抓取成功率之外的时延与失败分析","2026-08-29",[9714,9715,9716,9299,9717,9718],"具身智能","模仿学习","MuJoCo","机器人","鲁棒性","从 MuJoCo 抓取搬运闭环出发，记录 22,100 个评测回合。动作平滑在自建任务的 50 ms 延迟下有效，换到 Panda Lift 却没有支持同样的收益。","\u002Fimages\u002Fposts\u002Fembodied-pickplace-lab-robustness\u002Fpickplace-grasp-cover-20261016.webp","MuJoCo 实际渲染的笛卡尔夹爪抓取方块场景，桌面上标出放置目标","Pickplace Lab 从一个简单任务开始：让夹爪抓起方块，搬到指定位置，放下，再退开。先把仿真、示范采集、训练和闭环评测接起来，随后再问：干净条件下成功的策略，换一种执行条件，还能不能完成任务？ 最值得讨论的结果出现在指令延迟","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Ftree\u002Fv0.1.0","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Freleases\u002Fdownload\u002Fv0.1.0\u002Fpickplace-lab-research-v1.zip",{"slug":9726,"path":9727,"title":9728,"date":9729,"tags":9730,"description":9735,"cover":9736,"coverAlt":9737,"excerpt":9738,"draft":9288,"hidden":9288,"published":9291,"readingTime":9739},"qwen25-vl-sglang-verl-single-gpu-lab","\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab","从 Mac 到单卡 GPU：Qwen2.5-VL 部署、GRPO 与计数 LoRA 实践","2026-08-19",[9406,9347,9731,9732,9733,9734,9568],"SGLang","verl","GRPO","推理优化","在 M2 Pro 和云端单卡上部署 Qwen2.5-VL-3B，实测连续批处理、前缀缓存与 SGLang 参数调优，再用 verl 完成多模态 GRPO 更新和同条件前后评测，记录有效优化、兼容性问题与未观察到提升的结果。","\u002Fimages\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab\u002Fqvl-lab-cover-20261010.webp","笔记本与单卡计算模块通过图像和 token 光轨连接的实验台插画","这次实验从一个很具体的问题开始：M2 Pro、16 GB 统一内存，能不能在本机跑 Qwen2.5VL？跑起来之后，我又想把图文问答、推理性能测量和强化学习训练接成一条完整流程。 本机的 MLX 4bit 服务和云端的 SGLang",29,{"slug":9741,"path":9742,"title":9743,"date":9744,"tags":9745,"description":9749,"cover":9750,"coverAlt":9751,"excerpt":9752,"draft":9288,"hidden":9288,"published":9291,"readingTime":9535},"cycleresearcher-post-training","\u002Fposts\u002Fcycleresearcher-post-training","从 DeepReviewer 2.0 到 CycleResearcher：科研 Agent 如何真正做 Post-training","2026-08-07 17:33:05",[9390,9407,9348,9746,9747,9748],"SimPO","Reward Model","ICLR","区分推理时 Agent 编排与模型后训练，并拆解 CycleResearcher 的领域 SFT、生成式 Reward Model、Iterative SimPO 和评测风险。","\u002Fimages\u002Fposts\u002Fcycleresearcher-post-training\u002Fcycleresearcher-cover-20261018.webp","策略生成模块产出多份候选文稿经多维评审棱镜排序后通过偏好梯度反馈弧闭环优化的等距技术插画","看到一个系统会检索、规划、反思和调用工具，不能据此判断它做过 Posttraining。Agent 描述的是模型在运行时如何完成任务；Posttraining 描述的是预训练之后，是否继续用监督数据、偏好数据或奖励信号更新模型参数。",{"slug":9754,"path":9755,"title":9756,"date":9757,"tags":9758,"description":9760,"cover":9761,"coverAlt":9762,"excerpt":9763,"draft":9288,"hidden":9288,"published":9291,"readingTime":9383,"series":9764,"seriesOrder":1974},"cuda-fused-rmsnorm-kernel","\u002Fposts\u002Fcuda-fused-rmsnorm-kernel","从 PyTorch 到 CUDA Kernel：融合 RMSNorm 在 RTX 4090D 上加速 4.27 倍","2026-07-22 18:16:50",[9621,9318,9320,9299,9431,9759],"Kernel 优化","从零实现 PyTorch CUDA 扩展，拆解 RMSNorm 的访存瓶颈、warp 归约、向量化加载、算子融合与冷热缓存 benchmark。","\u002Fimages\u002Fposts\u002Fcuda-fused-rmsnorm-kernel\u002Fcuda-rmsnorm-cover-20261018.webp","融合残差加法与向量化 float4 访存经 Warp Shuffle 蝴蝶树归约输出归一化张量的等距硬件架构插画","为了准备 AI Infra 岗位，我做了一个尽量小而完整的 CUDA 项目：不追求堆很多算子，而是选一个真实的 LLM 推理路径，把实现、正确性、benchmark 和性能解释全部做扎实。 项目实现了两个 PyTorch CUDA ","AI Infra 实战",{"slug":9766,"path":9767,"title":9768,"date":9769,"tags":9770,"description":9772,"cover":9773,"coverAlt":9774,"excerpt":9775,"draft":9288,"hidden":9288,"published":9291,"readingTime":9309,"series":9776,"seriesOrder":1974},"agent-memory","\u002Fposts\u002Fagent-memory","Agent 上下文工程与分层记忆架构：从 Observation 剪枝、滚动摘要到向量召回闭环","2026-04-24",[9390,9348,9771,9318],"RAG","结合生产级多约束 Agent 与超 100K Token 科研智能体实践，系统拆解长程会话中的上下文膨胀与注意力稀释问题：涵盖 Observation 结构化剪枝（降低约 40% Token 开销）、滚动工作摘要、向量检索长期记忆及 Prefix Cache 友好的上下文组装。","\u002Fimages\u002Fposts\u002Fagent-memory\u002Fagent-memory-cover-20261018.webp","底层高噪数据经剪枝滤网进入中期滑动窗口与金色状态台账并由顶层向量星座按需召回的三层记忆架构插画","在构建多步工具调用（ToolUse）与长周期自主规划（Autonomous Planning）的 Agent 系统时，上下文管理（Context Engineering）往往比单纯改写 System Prompt 更决定线上稳定性。","Agent 上下文工程",{"slug":9778,"path":9779,"title":9780,"date":9781,"tags":9782,"description":9785,"cover":9786,"coverAlt":9787,"excerpt":9788,"draft":9288,"hidden":9288,"published":9291,"readingTime":2095,"series":9789,"seriesOrder":2001},"backend-stack-deep-dive","\u002Fposts\u002Fbackend-stack-deep-dive","高并发 AI 平台后端基础设施实践：FastAPI 异步网关、SQLAlchemy 2.0 Async、PostgreSQL 与 Docker 沙箱编排","2026-04-07",[9435,9432,9434,9783,9784,9318],"PostgreSQL","Docker","结合 DeepScientist 生产级 AI 科研平台实践，拆解 FastAPI ASGI 异步事件循环、SQLAlchemy 2.0 + AsyncPG 连接池调优、PostgreSQL + MinIO 存储分层与 Docker SDK 动态沙箱容器编排的职责边界与核心设计。","\u002Fimages\u002Fposts\u002Fbackend-stack-deep-dive\u002Fbackend-stack-cover-20261018.webp","ASGI 异步事件循环涡轮将非阻塞协程流分发至 PostgreSQL 连接池、MinIO 存储矩阵与动态 Docker 沙箱的等距架构插画","结合 DeepScientist 生产级 AI 科研平台（服务 7,000+ 用户）的实际工程落地，本文系统拆解支撑高并发流式推理与容器沙箱调度的后端基础设施栈——FastAPI（ASGI 异步网关 + SSE 流式推送）、SQLA","DeepScientist 工程复盘",{"slug":9791,"path":9792,"title":9793,"date":9794,"tags":9795,"description":9798,"cover":9799,"coverAlt":9800,"excerpt":9801,"draft":9288,"hidden":9288,"published":9291,"readingTime":9476,"series":9802,"seriesOrder":1974},"minicode-source-analysis","\u002Fposts\u002Fminicode-source-analysis","MiniCode 架构与工具评测深度拆解：开源终端 Agentic Coding 引擎（1k+ Stars）的 ReAct、MCP 与 30 项基准验证","2026-04-06",[9348,9796,9797,9378,9318],"TypeScript","MCP","深度剖析开源终端 AI 编程智能体 MiniCode（1k+ GitHub Stars）的核心架构：从零重型依赖的 ReAct 规划循环、声明式工具注册表与权限拦截层、MCP 协议帧自动协商，到 30 项可执行 Tool-Use 评测基准（86.7% 完成率、100% 错误恢复率）的失败分布归因与闭环优化。","\u002Fimages\u002Fposts\u002Fminicode-source-analysis\u002Fminicode-cover-20261018.webp","终端编程智能体控制台被 ReAct 状态环、Unified Diff 补丁预览门与 MCP 协议插槽环绕的等距技术插画","MiniCode 是我作为核心开发者参与构建的开源终端 Agentic Coding 引擎（目前已在 GitHub 获得 1,000+ Stars）。项目的核心设计目标是：在不引入庞大运行时框架的前提下，用约 5,000 行高内聚的","AI 项目源码导读",{"slug":9804,"path":9805,"title":9806,"date":9794,"tags":9807,"description":9808,"cover":9809,"coverAlt":9810,"excerpt":9811,"draft":9288,"hidden":9288,"published":9291,"readingTime":9504,"series":9789,"seriesOrder":1974},"deepscientist-tech-stack","\u002Fposts\u002Fdeepscientist-tech-stack","DeepScientist 架构与运行时全解析：超 100K 上下文科研智能体、MCP 沙箱与高并发流式系统",[9348,9318,9797,9784,9434,9432],"深度拆解服务 7,000+ 用户的科研自主智能体平台 DeepScientist：从一题一仓库（Quest）持久化状态机、MCP 工具栈（memory \u002F artifact \u002F bash_exec）、Docker SDK 动态容器编排，到 SSE + WebSocket 双流通信与 PostgreSQL \u002F Redis \u002F Prometheus 生产级基础设施。","\u002Fimages\u002Fposts\u002Fdeepscientist-tech-stack\u002Fdeepscientist-cover-20261018.webp","科研智能体编排核心通过 memory、artifact 与隔离容器沙箱三座协议桥连接双流工作台的等距架构插画","DeepScientist（deepscientist.cc）是我们在西湖大学自然语言处理实验室研发的自主科学发现与科研管理平台，目前累计服务 7,000+ 科研用户。与单轮对话或纯文本文献问答工具不同，科研智能体需要在数小时乃至数",1795088946193]