[{"data":1,"prerenderedAt":5845},["ShallowReactive",2],{"post-\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression":3,"archive-public-posts":5345},{"id":4,"title":5,"body":6,"categories":5326,"cover":5328,"coverAlt":5329,"date":5330,"description":5331,"draft":5332,"extension":5333,"haloPublished":5332,"haloSync":5332,"hidden":5332,"legacy":5332,"meta":5334,"navigation":82,"path":5335,"published":5332,"readingTime":5336,"seo":5337,"series":5338,"seriesOrder":2799,"stem":5339,"tags":5340,"__hash__":5344},"posts\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression.md","CMU 15-779 学习记录 01：PyTorch 自动求导与线性回归",{"type":7,"value":8,"toc":5312},"minimark",[9,16,19,22,48,52,55,219,450,453,489,562,779,1204,1264,1511,1533,1536,1570,1860,1863,2056,2059,2067,2106,2308,2427,2432,2439,2549,2552,2716,2727,2731,2734,2764,2770,3090,3093,3167,3214,3249,3252,3279,3290,3308,3318,3325,3338,3344,3347,3424,3427,3430,3436,3674,3677,4062,4069,4076,4515,4518,4524,4562,4575,4583,4586,4605,4645,4652,4907,4910,4922,4925,5039,5046,5049,5059,5121,5124,5186,5308],[10,11,12],"blockquote",{},[13,14,15],"p",{},"本篇整理的是学习 CMU 15-779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。",[13,17,18],{},"开始学习 CMU 15-779 时，我会 Python，但还没用过 PyTorch。这次先在 Mac 上用 CPU 做基础练习：计算导数、手动更新参数，再用 SGD 从一组数据里学出线性关系。",[13,20,21],{},"这篇记录整理的是课程入门阶段的第一组练习。我把运行结果和中间写错的地方一起留下，方便以后回看。",[13,23,24,25,29,30,37,38,42,43,47],{},"实验使用 Python 3.11.15、PyTorch 2.14.1，张量采用默认的 ",[26,27,28],"code",{},"float32","，运行设备是 CPU。三个脚本和依赖文件可以在 ",[31,32,36],"a",{"href":33,"rel":34},"https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs",[35],"nofollow","GitHub 代码仓库","查看，也可以下载",[31,39,41],{"href":40},"\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-basics-20261013.zip","实验代码 ZIP","。参考阅读是 ",[31,44,46],{"href":45},"\u002Flearning\u002Fcmu-15-779\u002Fmaterials\u002Fpytorch-paper-zh\u002F","PyTorch 中文译稿","的第 4.3 节「自动微分」。",[49,50,51],"h2",{"id":51},"自动求导算的是哪个导数",[13,53,54],{},"最开始的例子很短：",[56,57,62],"pre",{"className":58,"code":59,"language":60,"meta":61,"style":61},"language-python shiki shiki-themes github-dark-dimmed github-light houston","import torch\n\nx = torch.tensor(2.0, requires_grad=True)\ny = x * x\n\nprint(\"y =\", y.item())\nprint(\"求导前 =\", x.grad)\n\ny.backward()\n\nprint(\"求导后 =\", x.grad.item())\n","python","",[26,63,64,77,84,123,140,145,167,180,185,197,202],{"__ignoreMap":61},[65,66,69,73],"span",{"class":67,"line":68},"line",1,[65,70,72],{"class":71},"slEfO","import",[65,74,76],{"class":75},"sWURj"," torch\n",[65,78,80],{"class":67,"line":79},2,[65,81,83],{"emptyLinePlaceholder":82},true,"\n",[65,85,87,90,94,97,101,104,108,111,115,117,120],{"class":67,"line":86},3,[65,88,89],{"class":75},"x ",[65,91,93],{"class":92},"sf2cM","=",[65,95,96],{"class":75}," torch.",[65,98,100],{"class":99},"sq3c-","tensor",[65,102,103],{"class":75},"(",[65,105,107],{"class":106},"sgENW","2.0",[65,109,110],{"class":75},", ",[65,112,114],{"class":113},"sA9aF","requires_grad",[65,116,93],{"class":92},[65,118,119],{"class":106},"True",[65,121,122],{"class":75},")\n",[65,124,126,129,131,134,137],{"class":67,"line":125},4,[65,127,128],{"class":75},"y ",[65,130,93],{"class":92},[65,132,133],{"class":75}," x ",[65,135,136],{"class":92},"*",[65,138,139],{"class":75}," x\n",[65,141,143],{"class":67,"line":142},5,[65,144,83],{"emptyLinePlaceholder":82},[65,146,148,152,154,158,161,164],{"class":67,"line":147},6,[65,149,151],{"class":150},"svkvy","print",[65,153,103],{"class":75},[65,155,157],{"class":156},"sWqY2","\"y =\"",[65,159,160],{"class":75},", y.",[65,162,163],{"class":99},"item",[65,165,166],{"class":75},"())\n",[65,168,170,172,174,177],{"class":67,"line":169},7,[65,171,151],{"class":150},[65,173,103],{"class":75},[65,175,176],{"class":156},"\"求导前 =\"",[65,178,179],{"class":75},", x.grad)\n",[65,181,183],{"class":67,"line":182},8,[65,184,83],{"emptyLinePlaceholder":82},[65,186,188,191,194],{"class":67,"line":187},9,[65,189,190],{"class":75},"y.",[65,192,193],{"class":99},"backward",[65,195,196],{"class":75},"()\n",[65,198,200],{"class":67,"line":199},10,[65,201,83],{"emptyLinePlaceholder":82},[65,203,205,207,209,212,215,217],{"class":67,"line":204},11,[65,206,151],{"class":150},[65,208,103],{"class":75},[65,210,211],{"class":156},"\"求导后 =\"",[65,213,214],{"class":75},", x.grad.",[65,216,163],{"class":99},[65,218,166],{"class":75},[13,220,221,222,341,342,377,378,430,431,434,435,438,439,441,442,445,446,449],{},"这里 ",[65,223,226,262],{"className":224},[225],"katex",[65,227,230],{"className":228},[229],"katex-mathml",[231,232,234],"math",{"xmlns":233},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[235,236,237,257],"semantics",{},[238,239,240,244,247],"mrow",{},[241,242,243],"mi",{},"y",[245,246,93],"mo",{},[248,249,250,253],"msup",{},[241,251,252],{},"x",[254,255,256],"mn",{},"2",[258,259,261],"annotation",{"encoding":260},"application\u002Fx-tex","y=x^2",[65,263,267,294],{"className":264,"ariaHidden":266},[265],"katex-html","true",[65,268,271,276,282,287,291],{"className":269},[270],"base",[65,272],{"className":273,"style":275},[274],"strut","height:0.625em;vertical-align:-0.1944em;",[65,277,243],{"className":278,"style":281},[279,280],"mord","mathnormal","margin-right:0.0359em;",[65,283],{"className":284,"style":286},[285],"mspace","margin-right:0.2778em;",[65,288,93],{"className":289},[290],"mrel",[65,292],{"className":293,"style":286},[285],[65,295,297,301],{"className":296},[270],[65,298],{"className":299,"style":300},[274],"height:0.8141em;",[65,302,304,307],{"className":303},[279],[65,305,252],{"className":306},[279,280],[65,308,311],{"className":309},[310],"msupsub",[65,312,315],{"className":313},[314],"vlist-t",[65,316,319],{"className":317},[318],"vlist-r",[65,320,323],{"className":321,"style":300},[322],"vlist",[65,324,326,331],{"style":325},"top:-3.063em;margin-right:0.05em;",[65,327],{"className":328,"style":330},[329],"pstrut","height:2.7em;",[65,332,338],{"className":333},[334,335,336,337],"sizing","reset-size6","size3","mtight",[65,339,256],{"className":340},[279,337],"，导数是 ",[65,343,345,361],{"className":344},[225],[65,346,348],{"className":347},[229],[231,349,350],{"xmlns":233},[235,351,352,358],{},[238,353,354,356],{},[254,355,256],{},[241,357,252],{},[258,359,360],{"encoding":260},"2x",[65,362,364],{"className":363,"ariaHidden":266},[265],[65,365,367,371,374],{"className":366},[270],[65,368],{"className":369,"style":370},[274],"height:0.6444em;",[65,372,256],{"className":373},[279],[65,375,252],{"className":376},[279,280],"。在 ",[65,379,381,399],{"className":380},[225],[65,382,384],{"className":383},[229],[231,385,386],{"xmlns":233},[235,387,388,396],{},[238,389,390,392,394],{},[241,391,252],{},[245,393,93],{},[254,395,256],{},[258,397,398],{"encoding":260},"x=2",[65,400,402,421],{"className":401,"ariaHidden":266},[265],[65,403,405,409,412,415,418],{"className":404},[270],[65,406],{"className":407,"style":408},[274],"height:0.4306em;",[65,410,252],{"className":411},[279,280],[65,413],{"className":414,"style":286},[285],[65,416,93],{"className":417},[290],[65,419],{"className":420,"style":286},[285],[65,422,424,427],{"className":423},[270],[65,425],{"className":426,"style":370},[274],[65,428,256],{"className":429},[279]," 时，函数值和导数都等于 4，但它们的含义不同。",[26,432,433],{},"requires_grad=True"," 让 PyTorch 记录后续运算的求导关系；调用 ",[26,436,437],{},"backward()"," 后，关于 ",[26,440,252],{}," 的梯度存入 ",[26,443,444],{},"x.grad","。",[26,447,448],{},".item()"," 用来取出单元素张量中的数值。",[13,451,452],{},"我接着把计算改成了：",[56,454,456],{"className":58,"code":455,"language":60,"meta":61,"style":61},"z = y * y\nz.backward()\nprint(x.grad)\n",[26,457,458,473,482],{"__ignoreMap":61},[65,459,460,463,465,468,470],{"class":67,"line":68},[65,461,462],{"class":75},"z ",[65,464,93],{"class":92},[65,466,467],{"class":75}," y ",[65,469,136],{"class":92},[65,471,472],{"class":75}," y\n",[65,474,475,478,480],{"class":67,"line":79},[65,476,477],{"class":75},"z.",[65,479,193],{"class":99},[65,481,196],{"class":75},[65,483,484,486],{"class":67,"line":86},[65,485,151],{"class":150},[65,487,488],{"class":75},"(x.grad)\n",[13,490,491,492,494,495,497,498,501,502,532,533,561],{},"这一段接在前面创建 ",[26,493,252],{},"、",[26,496,243],{}," 的代码后面，替换原来的 ",[26,499,500],{},"y.backward()","。此时求的是 ",[65,503,505,519],{"className":504},[225],[65,506,508],{"className":507},[229],[231,509,510],{"xmlns":233},[235,511,512,517],{},[238,513,514],{},[241,515,516],{},"z",[258,518,516],{"encoding":260},[65,520,522],{"className":521,"ariaHidden":266},[265],[65,523,525,528],{"className":524},[270],[65,526],{"className":527,"style":408},[274],[65,529,516],{"className":530,"style":531},[279,280],"margin-right:0.044em;"," 关于 ",[65,534,536,549],{"className":535},[225],[65,537,539],{"className":538},[229],[231,540,541],{"xmlns":233},[235,542,543,547],{},[238,544,545],{},[241,546,252],{},[258,548,252],{"encoding":260},[65,550,552],{"className":551,"ariaHidden":266},[265],[65,553,555,558],{"className":554},[270],[65,556],{"className":557,"style":408},[274],[65,559,252],{"className":560},[279,280]," 的导数：",[65,563,566],{"className":564},[565],"katex-display",[65,567,569,617],{"className":568},[225],[65,570,572],{"className":571},[229],[231,573,575],{"xmlns":233,"display":574},"block",[235,576,577,614],{},[238,578,579,581,583,589,592,595,597,599,605,607],{},[241,580,243],{},[245,582,93],{},[248,584,585,587],{},[241,586,252],{},[254,588,256],{},[245,590,591],{"separator":266},",",[285,593],{"width":594},"2em",[241,596,516],{},[245,598,93],{},[248,600,601,603],{},[241,602,243],{},[254,604,256],{},[245,606,93],{},[248,608,609,611],{},[241,610,252],{},[254,612,613],{},"4",[258,615,616],{"encoding":260},"y=x^2,\\qquad z=y^2=x^4",[65,618,620,638,700,744],{"className":619,"ariaHidden":266},[265],[65,621,623,626,629,632,635],{"className":622},[270],[65,624],{"className":625,"style":275},[274],[65,627,243],{"className":628,"style":281},[279,280],[65,630],{"className":631,"style":286},[285],[65,633,93],{"className":634},[290],[65,636],{"className":637,"style":286},[285],[65,639,641,645,676,680,684,688,691,694,697],{"className":640},[270],[65,642],{"className":643,"style":644},[274],"height:1.0585em;vertical-align:-0.1944em;",[65,646,648,651],{"className":647},[279],[65,649,252],{"className":650},[279,280],[65,652,654],{"className":653},[310],[65,655,657],{"className":656},[314],[65,658,660],{"className":659},[318],[65,661,664],{"className":662,"style":663},[322],"height:0.8641em;",[65,665,667,670],{"style":666},"top:-3.113em;margin-right:0.05em;",[65,668],{"className":669,"style":330},[329],[65,671,673],{"className":672},[334,335,336,337],[65,674,256],{"className":675},[279,337],[65,677,591],{"className":678},[679],"mpunct",[65,681],{"className":682,"style":683},[285],"margin-right:2em;",[65,685],{"className":686,"style":687},[285],"margin-right:0.1667em;",[65,689,516],{"className":690,"style":531},[279,280],[65,692],{"className":693,"style":286},[285],[65,695,93],{"className":696},[290],[65,698],{"className":699,"style":286},[285],[65,701,703,706,735,738,741],{"className":702},[270],[65,704],{"className":705,"style":644},[274],[65,707,709,712],{"className":708},[279],[65,710,243],{"className":711,"style":281},[279,280],[65,713,715],{"className":714},[310],[65,716,718],{"className":717},[314],[65,719,721],{"className":720},[318],[65,722,724],{"className":723,"style":663},[322],[65,725,726,729],{"style":666},[65,727],{"className":728,"style":330},[329],[65,730,732],{"className":731},[334,335,336,337],[65,733,256],{"className":734},[279,337],[65,736],{"className":737,"style":286},[285],[65,739,93],{"className":740},[290],[65,742],{"className":743,"style":286},[285],[65,745,747,750],{"className":746},[270],[65,748],{"className":749,"style":663},[274],[65,751,753,756],{"className":752},[279],[65,754,252],{"className":755},[279,280],[65,757,759],{"className":758},[310],[65,760,762],{"className":761},[314],[65,763,765],{"className":764},[318],[65,766,768],{"className":767,"style":663},[322],[65,769,770,773],{"style":666},[65,771],{"className":772,"style":330},[329],[65,774,776],{"className":775},[334,335,336,337],[65,777,613],{"className":778},[279,337],[65,780,782],{"className":781},[565],[65,783,785,867],{"className":784},[225],[65,786,788],{"className":787},[229],[231,789,790],{"xmlns":233,"display":574},[235,791,792,864],{},[238,793,794,810,812,826,840,842,844,846,849,851,853,855,857],{},[795,796,797,804],"mfrac",{},[238,798,799,802],{},[241,800,801],{},"d",[241,803,516],{},[238,805,806,808],{},[241,807,801],{},[241,809,252],{},[245,811,93],{},[795,813,814,820],{},[238,815,816,818],{},[241,817,801],{},[241,819,516],{},[238,821,822,824],{},[241,823,801],{},[241,825,243],{},[795,827,828,834],{},[238,829,830,832],{},[241,831,801],{},[241,833,243],{},[238,835,836,838],{},[241,837,801],{},[241,839,252],{},[245,841,93],{},[254,843,256],{},[241,845,243],{},[245,847,848],{},"⋅",[254,850,256],{},[241,852,252],{},[245,854,93],{},[254,856,613],{},[248,858,859,861],{},[241,860,252],{},[254,862,863],{},"3",[258,865,866],{"encoding":260},"\\frac{dz}{dx}\n=\\frac{dz}{dy}\\frac{dy}{dx}\n=2y\\cdot 2x\n=4x^3",[65,868,870,968,1121,1145,1166],{"className":869,"ariaHidden":266},[265],[65,871,873,877,959,962,965],{"className":872},[270],[65,874],{"className":875,"style":876},[274],"height:2.0574em;vertical-align:-0.686em;",[65,878,880,885,955],{"className":879},[279],[65,881],{"className":882},[883,884],"mopen","nulldelimiter",[65,886,888],{"className":887},[795],[65,889,892,946],{"className":890},[314,891],"vlist-t2",[65,893,895,941],{"className":894},[318],[65,896,899,915,926],{"className":897,"style":898},[322],"height:1.3714em;",[65,900,902,906],{"style":901},"top:-2.314em;",[65,903],{"className":904,"style":905},[329],"height:3em;",[65,907,909,912],{"className":908},[279],[65,910,801],{"className":911},[279,280],[65,913,252],{"className":914},[279,280],[65,916,918,921],{"style":917},"top:-3.23em;",[65,919],{"className":920,"style":905},[329],[65,922],{"className":923,"style":925},[924],"frac-line","border-bottom-width:0.04em;",[65,927,929,932],{"style":928},"top:-3.677em;",[65,930],{"className":931,"style":905},[329],[65,933,935,938],{"className":934},[279],[65,936,801],{"className":937},[279,280],[65,939,516],{"className":940,"style":531},[279,280],[65,942,945],{"className":943},[944],"vlist-s","​",[65,947,949],{"className":948},[318],[65,950,953],{"className":951,"style":952},[322],"height:0.686em;",[65,954],{},[65,956],{"className":957},[958,884],"mclose",[65,960],{"className":961,"style":286},[285],[65,963,93],{"className":964},[290],[65,966],{"className":967,"style":286},[285],[65,969,971,975,1044,1112,1115,1118],{"className":970},[270],[65,972],{"className":973,"style":974},[274],"height:2.2519em;vertical-align:-0.8804em;",[65,976,978,981,1041],{"className":977},[279],[65,979],{"className":980},[883,884],[65,982,984],{"className":983},[795],[65,985,987,1032],{"className":986},[314,891],[65,988,990,1029],{"className":989},[318],[65,991,993,1007,1015],{"className":992,"style":898},[322],[65,994,995,998],{"style":901},[65,996],{"className":997,"style":905},[329],[65,999,1001,1004],{"className":1000},[279],[65,1002,801],{"className":1003},[279,280],[65,1005,243],{"className":1006,"style":281},[279,280],[65,1008,1009,1012],{"style":917},[65,1010],{"className":1011,"style":905},[329],[65,1013],{"className":1014,"style":925},[924],[65,1016,1017,1020],{"style":928},[65,1018],{"className":1019,"style":905},[329],[65,1021,1023,1026],{"className":1022},[279],[65,1024,801],{"className":1025},[279,280],[65,1027,516],{"className":1028,"style":531},[279,280],[65,1030,945],{"className":1031},[944],[65,1033,1035],{"className":1034},[318],[65,1036,1039],{"className":1037,"style":1038},[322],"height:0.8804em;",[65,1040],{},[65,1042],{"className":1043},[958,884],[65,1045,1047,1050,1109],{"className":1046},[279],[65,1048],{"className":1049},[883,884],[65,1051,1053],{"className":1052},[795],[65,1054,1056,1101],{"className":1055},[314,891],[65,1057,1059,1098],{"className":1058},[318],[65,1060,1062,1076,1084],{"className":1061,"style":898},[322],[65,1063,1064,1067],{"style":901},[65,1065],{"className":1066,"style":905},[329],[65,1068,1070,1073],{"className":1069},[279],[65,1071,801],{"className":1072},[279,280],[65,1074,252],{"className":1075},[279,280],[65,1077,1078,1081],{"style":917},[65,1079],{"className":1080,"style":905},[329],[65,1082],{"className":1083,"style":925},[924],[65,1085,1086,1089],{"style":928},[65,1087],{"className":1088,"style":905},[329],[65,1090,1092,1095],{"className":1091},[279],[65,1093,801],{"className":1094},[279,280],[65,1096,243],{"className":1097,"style":281},[279,280],[65,1099,945],{"className":1100},[944],[65,1102,1104],{"className":1103},[318],[65,1105,1107],{"className":1106,"style":952},[322],[65,1108],{},[65,1110],{"className":1111},[958,884],[65,1113],{"className":1114,"style":286},[285],[65,1116,93],{"className":1117},[290],[65,1119],{"className":1120,"style":286},[285],[65,1122,1124,1128,1131,1134,1138,1142],{"className":1123},[270],[65,1125],{"className":1126,"style":1127},[274],"height:0.8389em;vertical-align:-0.1944em;",[65,1129,256],{"className":1130},[279],[65,1132,243],{"className":1133,"style":281},[279,280],[65,1135],{"className":1136,"style":1137},[285],"margin-right:0.2222em;",[65,1139,848],{"className":1140},[1141],"mbin",[65,1143],{"className":1144,"style":1137},[285],[65,1146,1148,1151,1154,1157,1160,1163],{"className":1147},[270],[65,1149],{"className":1150,"style":370},[274],[65,1152,256],{"className":1153},[279],[65,1155,252],{"className":1156},[279,280],[65,1158],{"className":1159,"style":286},[285],[65,1161,93],{"className":1162},[290],[65,1164],{"className":1165,"style":286},[285],[65,1167,1169,1172,1175],{"className":1168},[270],[65,1170],{"className":1171,"style":663},[274],[65,1173,613],{"className":1174},[279],[65,1176,1178,1181],{"className":1177},[279],[65,1179,252],{"className":1180},[279,280],[65,1182,1184],{"className":1183},[310],[65,1185,1187],{"className":1186},[314],[65,1188,1190],{"className":1189},[318],[65,1191,1193],{"className":1192,"style":663},[322],[65,1194,1195,1198],{"style":666},[65,1196],{"className":1197,"style":330},[329],[65,1199,1201],{"className":1200},[334,335,336,337],[65,1202,863],{"className":1203},[279,337],[13,1205,1206,1207,1257,1258,1260,1261,1263],{},"所以在 ",[65,1208,1210,1227],{"className":1209},[225],[65,1211,1213],{"className":1212},[229],[231,1214,1215],{"xmlns":233},[235,1216,1217,1225],{},[238,1218,1219,1221,1223],{},[241,1220,252],{},[245,1222,93],{},[254,1224,256],{},[258,1226,398],{"encoding":260},[65,1228,1230,1248],{"className":1229,"ariaHidden":266},[265],[65,1231,1233,1236,1239,1242,1245],{"className":1232},[270],[65,1234],{"className":1235,"style":408},[274],[65,1237,252],{"className":1238},[279,280],[65,1240],{"className":1241,"style":286},[285],[65,1243,93],{"className":1244},[290],[65,1246],{"className":1247,"style":286},[285],[65,1249,1251,1254],{"className":1250},[270],[65,1252],{"className":1253,"style":370},[274],[65,1255,256],{"className":1256},[279]," 时，",[26,1259,444],{}," 对应 32。我原来的注释写成了「输出 y 的梯度函数」，应当改为「z 关于 x 的导数，在 x=2 处的值」。",[26,1262,444],{}," 保存的是这个位置的梯度数值。",[13,1265,1266,1267,1269,1270,1321,1322,1403,1404,1510],{},"当时我还问：把 ",[26,1268,252],{}," 声明为 2 后，是不是只能在这个位置求导？可以换位置，但需要重新创建输入并重新计算表达式。例如，令 ",[65,1271,1273,1291],{"className":1272},[225],[65,1274,1276],{"className":1275},[229],[231,1277,1278],{"xmlns":233},[235,1279,1280,1288],{},[238,1281,1282,1284,1286],{},[241,1283,252],{},[245,1285,93],{},[254,1287,863],{},[258,1289,1290],{"encoding":260},"x=3",[65,1292,1294,1312],{"className":1293,"ariaHidden":266},[265],[65,1295,1297,1300,1303,1306,1309],{"className":1296},[270],[65,1298],{"className":1299,"style":408},[274],[65,1301,252],{"className":1302},[279,280],[65,1304],{"className":1305,"style":286},[285],[65,1307,93],{"className":1308},[290],[65,1310],{"className":1311,"style":286},[285],[65,1313,1315,1318],{"className":1314},[270],[65,1316],{"className":1317,"style":370},[274],[65,1319,863],{"className":1320},[279]," 后重新计算 ",[65,1323,1325,1347],{"className":1324},[225],[65,1326,1328],{"className":1327},[229],[231,1329,1330],{"xmlns":233},[235,1331,1332,1344],{},[238,1333,1334,1336,1338],{},[241,1335,516],{},[245,1337,93],{},[248,1339,1340,1342],{},[241,1341,252],{},[254,1343,613],{},[258,1345,1346],{"encoding":260},"z=x^4",[65,1348,1350,1368],{"className":1349,"ariaHidden":266},[265],[65,1351,1353,1356,1359,1362,1365],{"className":1352},[270],[65,1354],{"className":1355,"style":408},[274],[65,1357,516],{"className":1358,"style":531},[279,280],[65,1360],{"className":1361,"style":286},[285],[65,1363,93],{"className":1364},[290],[65,1366],{"className":1367,"style":286},[285],[65,1369,1371,1374],{"className":1370},[270],[65,1372],{"className":1373,"style":300},[274],[65,1375,1377,1380],{"className":1376},[279],[65,1378,252],{"className":1379},[279,280],[65,1381,1383],{"className":1382},[310],[65,1384,1386],{"className":1385},[314],[65,1387,1389],{"className":1388},[318],[65,1390,1392],{"className":1391,"style":300},[322],[65,1393,1394,1397],{"style":325},[65,1395],{"className":1396,"style":330},[329],[65,1398,1400],{"className":1399},[334,335,336,337],[65,1401,613],{"className":1402},[279,337],"，这个位置的导数就是 ",[65,1405,1407,1435],{"className":1406},[225],[65,1408,1410],{"className":1409},[229],[231,1411,1412],{"xmlns":233},[235,1413,1414,1432],{},[238,1415,1416,1418,1421,1427,1429],{},[254,1417,613],{},[245,1419,1420],{},"×",[248,1422,1423,1425],{},[254,1424,863],{},[254,1426,863],{},[245,1428,93],{},[254,1430,1431],{},"108",[258,1433,1434],{"encoding":260},"4\\times 3^3=108",[65,1436,1438,1457,1501],{"className":1437,"ariaHidden":266},[265],[65,1439,1441,1445,1448,1451,1454],{"className":1440},[270],[65,1442],{"className":1443,"style":1444},[274],"height:0.7278em;vertical-align:-0.0833em;",[65,1446,613],{"className":1447},[279],[65,1449],{"className":1450,"style":1137},[285],[65,1452,1420],{"className":1453},[1141],[65,1455],{"className":1456,"style":1137},[285],[65,1458,1460,1463,1492,1495,1498],{"className":1459},[270],[65,1461],{"className":1462,"style":300},[274],[65,1464,1466,1469],{"className":1465},[279],[65,1467,863],{"className":1468},[279],[65,1470,1472],{"className":1471},[310],[65,1473,1475],{"className":1474},[314],[65,1476,1478],{"className":1477},[318],[65,1479,1481],{"className":1480,"style":300},[322],[65,1482,1483,1486],{"style":325},[65,1484],{"className":1485,"style":330},[329],[65,1487,1489],{"className":1488},[334,335,336,337],[65,1490,863],{"className":1491},[279,337],[65,1493],{"className":1494,"style":286},[285],[65,1496,93],{"className":1497},[290],[65,1499],{"className":1500,"style":286},[285],[65,1502,1504,1507],{"className":1503},[270],[65,1505],{"className":1506,"style":370},[274],[65,1508,1431],{"className":1509},[279],"。旧的前向计算对应旧输入，不能直接用它代表新位置。",[13,1512,1513,1514,1516,1517,1520,1521,1524,1525,1528,1529,1532],{},"参数名称也容易写错：正确写法是 ",[26,1515,114],{},"，带一个 ",[26,1518,1519],{},"s","；",[26,1522,1523],{},"require_grad"," 和 ",[26,1526,1527],{},"request_grad"," 都不能作为 ",[26,1530,1531],{},"torch.tensor()"," 的这个参数。",[49,1534,1535],{"id":1535},"用梯度把参数推向目标",[13,1537,1538,1539,1569],{},"下一步是让参数 ",[65,1540,1542,1556],{"className":1541},[225],[65,1543,1545],{"className":1544},[229],[231,1546,1547],{"xmlns":233},[235,1548,1549,1554],{},[238,1550,1551],{},[241,1552,1553],{},"w",[258,1555,1553],{"encoding":260},[65,1557,1559],{"className":1558,"ariaHidden":266},[265],[65,1560,1562,1565],{"className":1561},[270],[65,1563],{"className":1564,"style":408},[274],[65,1566,1553],{"className":1567,"style":1568},[279,280],"margin-right:0.0269em;"," 从 0 开始接近 3。损失函数定义为：",[65,1571,1573],{"className":1572},[565],[65,1574,1576,1648],{"className":1575},[225],[65,1577,1579],{"className":1578},[229],[231,1580,1581],{"xmlns":233,"display":574},[235,1582,1583,1645],{},[238,1584,1585,1588,1591,1593,1596,1598,1600,1602,1605,1607,1613,1615,1617,1631,1633,1635,1637,1639,1641,1643],{},[241,1586,1587],{},"L",[245,1589,103],{"stretchy":1590},"false",[241,1592,1553],{},[245,1594,1595],{"stretchy":1590},")",[245,1597,93],{},[245,1599,103],{"stretchy":1590},[241,1601,1553],{},[245,1603,1604],{},"−",[254,1606,863],{},[248,1608,1609,1611],{},[245,1610,1595],{"stretchy":1590},[254,1612,256],{},[245,1614,591],{"separator":266},[285,1616],{"width":594},[795,1618,1619,1625],{},[238,1620,1621,1623],{},[241,1622,801],{},[241,1624,1587],{},[238,1626,1627,1629],{},[241,1628,801],{},[241,1630,1553],{},[245,1632,93],{},[254,1634,256],{},[245,1636,103],{"stretchy":1590},[241,1638,1553],{},[245,1640,1604],{},[254,1642,863],{},[245,1644,1595],{"stretchy":1590},[258,1646,1647],{"encoding":260},"L(w)=(w-3)^2,\\qquad \\frac{dL}{dw}=2(w-3)",[65,1649,1651,1679,1700,1824,1848],{"className":1650,"ariaHidden":266},[265],[65,1652,1654,1658,1661,1664,1667,1670,1673,1676],{"className":1653},[270],[65,1655],{"className":1656,"style":1657},[274],"height:1em;vertical-align:-0.25em;",[65,1659,1587],{"className":1660},[279,280],[65,1662,103],{"className":1663},[883],[65,1665,1553],{"className":1666,"style":1568},[279,280],[65,1668,1595],{"className":1669},[958],[65,1671],{"className":1672,"style":286},[285],[65,1674,93],{"className":1675},[290],[65,1677],{"className":1678,"style":286},[285],[65,1680,1682,1685,1688,1691,1694,1697],{"className":1681},[270],[65,1683],{"className":1684,"style":1657},[274],[65,1686,103],{"className":1687},[883],[65,1689,1553],{"className":1690,"style":1568},[279,280],[65,1692],{"className":1693,"style":1137},[285],[65,1695,1604],{"className":1696},[1141],[65,1698],{"className":1699,"style":1137},[285],[65,1701,1703,1706,1709,1738,1741,1744,1747,1815,1818,1821],{"className":1702},[270],[65,1704],{"className":1705,"style":876},[274],[65,1707,863],{"className":1708},[279],[65,1710,1712,1715],{"className":1711},[958],[65,1713,1595],{"className":1714},[958],[65,1716,1718],{"className":1717},[310],[65,1719,1721],{"className":1720},[314],[65,1722,1724],{"className":1723},[318],[65,1725,1727],{"className":1726,"style":663},[322],[65,1728,1729,1732],{"style":666},[65,1730],{"className":1731,"style":330},[329],[65,1733,1735],{"className":1734},[334,335,336,337],[65,1736,256],{"className":1737},[279,337],[65,1739,591],{"className":1740},[679],[65,1742],{"className":1743,"style":683},[285],[65,1745],{"className":1746,"style":687},[285],[65,1748,1750,1753,1812],{"className":1749},[279],[65,1751],{"className":1752},[883,884],[65,1754,1756],{"className":1755},[795],[65,1757,1759,1804],{"className":1758},[314,891],[65,1760,1762,1801],{"className":1761},[318],[65,1763,1765,1779,1787],{"className":1764,"style":898},[322],[65,1766,1767,1770],{"style":901},[65,1768],{"className":1769,"style":905},[329],[65,1771,1773,1776],{"className":1772},[279],[65,1774,801],{"className":1775},[279,280],[65,1777,1553],{"className":1778,"style":1568},[279,280],[65,1780,1781,1784],{"style":917},[65,1782],{"className":1783,"style":905},[329],[65,1785],{"className":1786,"style":925},[924],[65,1788,1789,1792],{"style":928},[65,1790],{"className":1791,"style":905},[329],[65,1793,1795,1798],{"className":1794},[279],[65,1796,801],{"className":1797},[279,280],[65,1799,1587],{"className":1800},[279,280],[65,1802,945],{"className":1803},[944],[65,1805,1807],{"className":1806},[318],[65,1808,1810],{"className":1809,"style":952},[322],[65,1811],{},[65,1813],{"className":1814},[958,884],[65,1816],{"className":1817,"style":286},[285],[65,1819,93],{"className":1820},[290],[65,1822],{"className":1823,"style":286},[285],[65,1825,1827,1830,1833,1836,1839,1842,1845],{"className":1826},[270],[65,1828],{"className":1829,"style":1657},[274],[65,1831,256],{"className":1832},[279],[65,1834,103],{"className":1835},[883],[65,1837,1553],{"className":1838,"style":1568},[279,280],[65,1840],{"className":1841,"style":1137},[285],[65,1843,1604],{"className":1844},[1141],[65,1846],{"className":1847,"style":1137},[285],[65,1849,1851,1854,1857],{"className":1850},[270],[65,1852],{"className":1853,"style":1657},[274],[65,1855,863],{"className":1856},[279],[65,1858,1595],{"className":1859},[958],[13,1861,1862],{},"先计算损失和梯度，再沿梯度的反方向更新：",[56,1864,1866],{"className":58,"code":1865,"language":60,"meta":61,"style":61},"import torch\n\nw = torch.tensor(0.0, requires_grad=True)\nlearn_rate = 0.1\n\nloss = (w - 3) ** 2\nloss.backward()\n\nwith torch.no_grad():\n    w -= learn_rate * w.grad\n\nnew_loss = (w - 3) ** 2\nprint(f\"Old loss: {loss.item()}, New loss: {new_loss.item()}\")\n",[26,1867,1868,1874,1878,1904,1914,1918,1943,1952,1956,1969,1985,1989,2009],{"__ignoreMap":61},[65,1869,1870,1872],{"class":67,"line":68},[65,1871,72],{"class":71},[65,1873,76],{"class":75},[65,1875,1876],{"class":67,"line":79},[65,1877,83],{"emptyLinePlaceholder":82},[65,1879,1880,1883,1885,1887,1889,1891,1894,1896,1898,1900,1902],{"class":67,"line":86},[65,1881,1882],{"class":75},"w ",[65,1884,93],{"class":92},[65,1886,96],{"class":75},[65,1888,100],{"class":99},[65,1890,103],{"class":75},[65,1892,1893],{"class":106},"0.0",[65,1895,110],{"class":75},[65,1897,114],{"class":113},[65,1899,93],{"class":92},[65,1901,119],{"class":106},[65,1903,122],{"class":75},[65,1905,1906,1909,1911],{"class":67,"line":125},[65,1907,1908],{"class":75},"learn_rate ",[65,1910,93],{"class":92},[65,1912,1913],{"class":106}," 0.1\n",[65,1915,1916],{"class":67,"line":142},[65,1917,83],{"emptyLinePlaceholder":82},[65,1919,1920,1923,1925,1928,1931,1934,1937,1940],{"class":67,"line":147},[65,1921,1922],{"class":75},"loss ",[65,1924,93],{"class":92},[65,1926,1927],{"class":75}," (w ",[65,1929,1930],{"class":92},"-",[65,1932,1933],{"class":106}," 3",[65,1935,1936],{"class":75},") ",[65,1938,1939],{"class":92},"**",[65,1941,1942],{"class":106}," 2\n",[65,1944,1945,1948,1950],{"class":67,"line":169},[65,1946,1947],{"class":75},"loss.",[65,1949,193],{"class":99},[65,1951,196],{"class":75},[65,1953,1954],{"class":67,"line":182},[65,1955,83],{"emptyLinePlaceholder":82},[65,1957,1958,1961,1963,1966],{"class":67,"line":187},[65,1959,1960],{"class":71},"with",[65,1962,96],{"class":75},[65,1964,1965],{"class":99},"no_grad",[65,1967,1968],{"class":75},"():\n",[65,1970,1971,1974,1977,1980,1982],{"class":67,"line":199},[65,1972,1973],{"class":75},"    w ",[65,1975,1976],{"class":92},"-=",[65,1978,1979],{"class":75}," learn_rate ",[65,1981,136],{"class":92},[65,1983,1984],{"class":75}," w.grad\n",[65,1986,1987],{"class":67,"line":204},[65,1988,83],{"emptyLinePlaceholder":82},[65,1990,1992,1995,1997,1999,2001,2003,2005,2007],{"class":67,"line":1991},12,[65,1993,1994],{"class":75},"new_loss ",[65,1996,93],{"class":92},[65,1998,1927],{"class":75},[65,2000,1930],{"class":92},[65,2002,1933],{"class":106},[65,2004,1936],{"class":75},[65,2006,1939],{"class":92},[65,2008,1942],{"class":106},[65,2010,2012,2014,2016,2020,2023,2027,2029,2031,2034,2037,2040,2042,2045,2047,2049,2051,2054],{"class":67,"line":2011},13,[65,2013,151],{"class":150},[65,2015,103],{"class":75},[65,2017,2019],{"class":2018},"sST-V","f",[65,2021,2022],{"class":156},"\"Old loss: ",[65,2024,2026],{"class":2025},"s0kv9","{",[65,2028,1947],{"class":75},[65,2030,163],{"class":99},[65,2032,2033],{"class":75},"()",[65,2035,2036],{"class":2025},"}",[65,2038,2039],{"class":156},", New loss: ",[65,2041,2026],{"class":2025},[65,2043,2044],{"class":75},"new_loss.",[65,2046,163],{"class":99},[65,2048,2033],{"class":75},[65,2050,2036],{"class":2025},[65,2052,2053],{"class":156},"\"",[65,2055,122],{"class":75},[13,2057,2058],{},"我运行后的输出是：",[56,2060,2065],{"className":2061,"code":2063,"language":2064,"meta":61},[2062],"language-text","Old loss: 9.0, New loss: 5.760000228881836\n","text",[26,2066,2063],{"__ignoreMap":61},[13,2068,2069,2070,2105],{},"这一步可以手算核对。初始梯度是 ",[65,2071,2073,2090],{"className":2072},[225],[65,2074,2076],{"className":2075},[229],[231,2077,2078],{"xmlns":233},[235,2079,2080,2087],{},[238,2081,2082,2084],{},[245,2083,1604],{},[254,2085,2086],{},"6",[258,2088,2089],{"encoding":260},"-6",[65,2091,2093],{"className":2092,"ariaHidden":266},[265],[65,2094,2096,2099,2102],{"className":2095},[270],[65,2097],{"className":2098,"style":1444},[274],[65,2100,1604],{"className":2101},[279],[65,2103,2086],{"className":2104},[279],"，学习率为 0.1，因此：",[65,2107,2109],{"className":2108},[565],[65,2110,2112,2165],{"className":2111},[225],[65,2113,2115],{"className":2114},[229],[231,2116,2117],{"xmlns":233,"display":574},[235,2118,2119,2162],{},[238,2120,2121,2137,2139,2142,2144,2147,2149,2151,2153,2155,2157,2159],{},[2122,2123,2124,2126],"msub",{},[241,2125,1553],{},[238,2127,2128,2132,2135],{},[241,2129,2131],{"mathvariant":2130},"normal","n",[241,2133,2134],{"mathvariant":2130},"e",[241,2136,1553],{"mathvariant":2130},[245,2138,93],{},[254,2140,2141],{},"0",[245,2143,1604],{},[254,2145,2146],{},"0.1",[245,2148,1420],{},[245,2150,103],{"stretchy":1590},[245,2152,1604],{},[254,2154,2086],{},[245,2156,1595],{"stretchy":1590},[245,2158,93],{},[254,2160,2161],{},"0.6",[258,2163,2164],{"encoding":260},"w_{\\mathrm{new}}=0-0.1\\times(-6)=0.6",[65,2166,2168,2236,2254,2272,2299],{"className":2167,"ariaHidden":266},[265],[65,2169,2171,2175,2227,2230,2233],{"className":2170},[270],[65,2172],{"className":2173,"style":2174},[274],"height:0.5806em;vertical-align:-0.15em;",[65,2176,2178,2181],{"className":2177},[279],[65,2179,1553],{"className":2180,"style":1568},[279,280],[65,2182,2184],{"className":2183},[310],[65,2185,2187,2218],{"className":2186},[314,891],[65,2188,2190,2215],{"className":2189},[318],[65,2191,2194],{"className":2192,"style":2193},[322],"height:0.1514em;",[65,2195,2197,2200],{"style":2196},"top:-2.55em;margin-left:-0.0269em;margin-right:0.05em;",[65,2198],{"className":2199,"style":330},[329],[65,2201,2203],{"className":2202},[334,335,336,337],[65,2204,2206],{"className":2205},[279,337],[65,2207,2209],{"className":2208},[279,337],[65,2210,2214],{"className":2211,"style":2213},[279,2212,337],"mathrm","margin-right:0.0139em;","new",[65,2216,945],{"className":2217},[944],[65,2219,2221],{"className":2220},[318],[65,2222,2225],{"className":2223,"style":2224},[322],"height:0.15em;",[65,2226],{},[65,2228],{"className":2229,"style":286},[285],[65,2231,93],{"className":2232},[290],[65,2234],{"className":2235,"style":286},[285],[65,2237,2239,2242,2245,2248,2251],{"className":2238},[270],[65,2240],{"className":2241,"style":1444},[274],[65,2243,2141],{"className":2244},[279],[65,2246],{"className":2247,"style":1137},[285],[65,2249,1604],{"className":2250},[1141],[65,2252],{"className":2253,"style":1137},[285],[65,2255,2257,2260,2263,2266,2269],{"className":2256},[270],[65,2258],{"className":2259,"style":1444},[274],[65,2261,2146],{"className":2262},[279],[65,2264],{"className":2265,"style":1137},[285],[65,2267,1420],{"className":2268},[1141],[65,2270],{"className":2271,"style":1137},[285],[65,2273,2275,2278,2281,2284,2287,2290,2293,2296],{"className":2274},[270],[65,2276],{"className":2277,"style":1657},[274],[65,2279,103],{"className":2280},[883],[65,2282,1604],{"className":2283},[279],[65,2285,2086],{"className":2286},[279],[65,2288,1595],{"className":2289},[958],[65,2291],{"className":2292,"style":286},[285],[65,2294,93],{"className":2295},[290],[65,2297],{"className":2298,"style":286},[285],[65,2300,2302,2305],{"className":2301},[270],[65,2303],{"className":2304,"style":370},[274],[65,2306,2161],{"className":2307},[279],[13,2309,2310,2311,2426],{},"更新后的损失是 ",[65,2312,2314,2345],{"className":2313},[225],[65,2315,2317],{"className":2316},[229],[231,2318,2319],{"xmlns":233},[235,2320,2321,2342],{},[238,2322,2323,2325,2327,2329,2331,2337,2339],{},[245,2324,103],{"stretchy":1590},[254,2326,2161],{},[245,2328,1604],{},[254,2330,863],{},[248,2332,2333,2335],{},[245,2334,1595],{"stretchy":1590},[254,2336,256],{},[245,2338,93],{},[254,2340,2341],{},"5.76",[258,2343,2344],{"encoding":260},"(0.6-3)^2=5.76",[65,2346,2348,2369,2417],{"className":2347,"ariaHidden":266},[265],[65,2349,2351,2354,2357,2360,2363,2366],{"className":2350},[270],[65,2352],{"className":2353,"style":1657},[274],[65,2355,103],{"className":2356},[883],[65,2358,2161],{"className":2359},[279],[65,2361],{"className":2362,"style":1137},[285],[65,2364,1604],{"className":2365},[1141],[65,2367],{"className":2368,"style":1137},[285],[65,2370,2372,2376,2379,2408,2411,2414],{"className":2371},[270],[65,2373],{"className":2374,"style":2375},[274],"height:1.0641em;vertical-align:-0.25em;",[65,2377,863],{"className":2378},[279],[65,2380,2382,2385],{"className":2381},[958],[65,2383,1595],{"className":2384},[958],[65,2386,2388],{"className":2387},[310],[65,2389,2391],{"className":2390},[314],[65,2392,2394],{"className":2393},[318],[65,2395,2397],{"className":2396,"style":300},[322],[65,2398,2399,2402],{"style":325},[65,2400],{"className":2401,"style":330},[329],[65,2403,2405],{"className":2404},[334,335,336,337],[65,2406,256],{"className":2407},[279,337],[65,2409],{"className":2410,"style":286},[285],[65,2412,93],{"className":2413},[290],[65,2415],{"className":2416,"style":286},[285],[65,2418,2420,2423],{"className":2419},[270],[65,2421],{"className":2422,"style":370},[274],[65,2424,2341],{"className":2425},[279],"，与输出一致。打印出来的长小数来自浮点数表示。",[2428,2429,2431],"h3",{"id":2430},"两种学习率比较相同的-10-次更新","两种学习率，比较相同的 10 次更新",[13,2433,2434,2435,2438],{},"我把单次更新写成循环，每轮都重新计算损失和梯度。求导前用 ",[26,2436,2437],{},"w.grad = None"," 清除上一轮的梯度，再分别用 0.1 和 0.01 运行 10 次。",[2440,2441,2442,2518],"table",{},[2443,2444,2445],"thead",{},[2446,2447,2448,2452,2484,2515],"tr",{},[2449,2450,2451],"th",{},"学习率",[2449,2453,2455,2456],{"align":2454},"right","第 1 次更新后的 ",[65,2457,2459,2472],{"className":2458},[225],[65,2460,2462],{"className":2461},[229],[231,2463,2464],{"xmlns":233},[235,2465,2466,2470],{},[238,2467,2468],{},[241,2469,1553],{},[258,2471,1553],{"encoding":260},[65,2473,2475],{"className":2474,"ariaHidden":266},[265],[65,2476,2478,2481],{"className":2477},[270],[65,2479],{"className":2480,"style":408},[274],[65,2482,1553],{"className":2483,"style":1568},[279,280],[2449,2485,2486,2487],{"align":2454},"第 10 次更新后的 ",[65,2488,2490,2503],{"className":2489},[225],[65,2491,2493],{"className":2492},[229],[231,2494,2495],{"xmlns":233},[235,2496,2497,2501],{},[238,2498,2499],{},[241,2500,1553],{},[258,2502,1553],{"encoding":260},[65,2504,2506],{"className":2505,"ariaHidden":266},[265],[65,2507,2509,2512],{"className":2508},[270],[65,2510],{"className":2511,"style":408},[274],[65,2513,1553],{"className":2514,"style":1568},[279,280],[2449,2516,2517],{"align":2454},"第 10 次更新后的损失",[2519,2520,2521,2535],"tbody",{},[2446,2522,2523,2526,2529,2532],{},[2524,2525,2146],"td",{},[2524,2527,2528],{"align":2454},"0.6000",[2524,2530,2531],{"align":2454},"2.6779",[2524,2533,2534],{"align":2454},"0.1038",[2446,2536,2537,2540,2543,2546],{},[2524,2538,2539],{},"0.01",[2524,2541,2542],{"align":2454},"0.0600",[2524,2544,2545],{"align":2454},"0.5488",[2524,2547,2548],{"align":2454},"6.0085",[13,2550,2551],{},"在这个函数、相同初值和更新次数下，0.1 的损失下降更快。这次结果说明了两个学习率在这个实验中的差别；学习率过大仍可能造成震荡或发散。",[13,2553,2554,2555,2557,2558,2591,2592,2643,2644,2679,2680,2715],{},"循环中清除梯度很关键，因为 ",[26,2556,437],{}," 会累加梯度。第一轮梯度是 ",[65,2559,2561,2576],{"className":2560},[225],[65,2562,2564],{"className":2563},[229],[231,2565,2566],{"xmlns":233},[235,2567,2568,2574],{},[238,2569,2570,2572],{},[245,2571,1604],{},[254,2573,2086],{},[258,2575,2089],{"encoding":260},[65,2577,2579],{"className":2578,"ariaHidden":266},[265],[65,2580,2582,2585,2588],{"className":2581},[270],[65,2583],{"className":2584,"style":1444},[274],[65,2586,1604],{"className":2587},[279],[65,2589,2086],{"className":2590},[279],"，更新到 ",[65,2593,2595,2613],{"className":2594},[225],[65,2596,2598],{"className":2597},[229],[231,2599,2600],{"xmlns":233},[235,2601,2602,2610],{},[238,2603,2604,2606,2608],{},[241,2605,1553],{},[245,2607,93],{},[254,2609,2161],{},[258,2611,2612],{"encoding":260},"w=0.6",[65,2614,2616,2634],{"className":2615,"ariaHidden":266},[265],[65,2617,2619,2622,2625,2628,2631],{"className":2618},[270],[65,2620],{"className":2621,"style":408},[274],[65,2623,1553],{"className":2624,"style":1568},[279,280],[65,2626],{"className":2627,"style":286},[285],[65,2629,93],{"className":2630},[290],[65,2632],{"className":2633,"style":286},[285],[65,2635,2637,2640],{"className":2636},[270],[65,2638],{"className":2639,"style":370},[274],[65,2641,2161],{"className":2642},[279]," 后，第二轮梯度应是 ",[65,2645,2647,2664],{"className":2646},[225],[65,2648,2650],{"className":2649},[229],[231,2651,2652],{"xmlns":233},[235,2653,2654,2661],{},[238,2655,2656,2658],{},[245,2657,1604],{},[254,2659,2660],{},"4.8",[258,2662,2663],{"encoding":260},"-4.8",[65,2665,2667],{"className":2666,"ariaHidden":266},[265],[65,2668,2670,2673,2676],{"className":2669},[270],[65,2671],{"className":2672,"style":1444},[274],[65,2674,1604],{"className":2675},[279],[65,2677,2660],{"className":2678},[279],"。如果不清除，保存的梯度会变成 ",[65,2681,2683,2700],{"className":2682},[225],[65,2684,2686],{"className":2685},[229],[231,2687,2688],{"xmlns":233},[235,2689,2690,2697],{},[238,2691,2692,2694],{},[245,2693,1604],{},[254,2695,2696],{},"10.8",[258,2698,2699],{"encoding":260},"-10.8",[65,2701,2703],{"className":2702,"ariaHidden":266},[265],[65,2704,2706,2709,2712],{"className":2705},[270],[65,2707],{"className":2708,"style":1444},[274],[65,2710,1604],{"className":2711},[279],[65,2713,2696],{"className":2714},[279],"，就不再只是当前位置这一轮的梯度。",[13,2717,2718,2719,2722,2723,2726],{},"压缩包里的 ",[26,2720,2721],{},"01_autograd.py"," 是连续更新版本，保留了最后一次实验使用的 ",[26,2724,2725],{},"learn_rate = 0.01","。把它改为 0.1，就可以复现另一组结果。",[49,2728,2730],{"id":2729},"换成-sgd-后每一步负责什么","换成 SGD 后，每一步负责什么",[13,2732,2733],{},"接下来用 PyTorch 的优化器替代手动更新：",[56,2735,2737],{"className":58,"code":2736,"language":60,"meta":61,"style":61},"optimizer = torch.optim.SGD([w], lr=0.1)\n",[26,2738,2739],{"__ignoreMap":61},[65,2740,2741,2744,2746,2749,2752,2755,2758,2760,2762],{"class":67,"line":68},[65,2742,2743],{"class":75},"optimizer ",[65,2745,93],{"class":92},[65,2747,2748],{"class":75}," torch.optim.",[65,2750,2751],{"class":99},"SGD",[65,2753,2754],{"class":75},"([w], ",[65,2756,2757],{"class":113},"lr",[65,2759,93],{"class":92},[65,2761,2146],{"class":106},[65,2763,122],{"class":75},[13,2765,2766,2769],{},[26,2767,2768],{},"[w]"," 告诉优化器需要更新哪些参数。这次使用的普通 SGD 没有配置动量和权重衰减，更新规则是：",[65,2771,2773],{"className":2772},[565],[65,2774,2776,2840],{"className":2775},[225],[65,2777,2779],{"className":2778},[229],[231,2780,2781],{"xmlns":233,"display":574},[235,2782,2783,2837],{},[238,2784,2785,2800,2802,2808,2810,2813,2827,2829,2835],{},[2122,2786,2787,2789],{},[241,2788,1553],{},[238,2790,2791,2794,2797],{},[241,2792,2793],{},"t",[245,2795,2796],{},"+",[254,2798,2799],{},"1",[245,2801,93],{},[2122,2803,2804,2806],{},[241,2805,1553],{},[241,2807,2793],{},[245,2809,1604],{},[241,2811,2812],{},"η",[795,2814,2815,2821],{},[238,2816,2817,2819],{},[241,2818,801],{},[241,2820,1587],{},[238,2822,2823,2825],{},[241,2824,801],{},[241,2826,1553],{},[245,2828,103],{"stretchy":1590},[2122,2830,2831,2833],{},[241,2832,1553],{},[241,2834,2793],{},[245,2836,1595],{"stretchy":1590},[258,2838,2839],{"encoding":260},"w_{t+1}=w_t-\\eta\\frac{dL}{dw}(w_t)",[65,2841,2843,2910,2967],{"className":2842,"ariaHidden":266},[265],[65,2844,2846,2850,2901,2904,2907],{"className":2845},[270],[65,2847],{"className":2848,"style":2849},[274],"height:0.6389em;vertical-align:-0.2083em;",[65,2851,2853,2856],{"className":2852},[279],[65,2854,1553],{"className":2855,"style":1568},[279,280],[65,2857,2859],{"className":2858},[310],[65,2860,2862,2892],{"className":2861},[314,891],[65,2863,2865,2889],{"className":2864},[318],[65,2866,2869],{"className":2867,"style":2868},[322],"height:0.3011em;",[65,2870,2871,2874],{"style":2196},[65,2872],{"className":2873,"style":330},[329],[65,2875,2877],{"className":2876},[334,335,336,337],[65,2878,2880,2883,2886],{"className":2879},[279,337],[65,2881,2793],{"className":2882},[279,280,337],[65,2884,2796],{"className":2885},[1141,337],[65,2887,2799],{"className":2888},[279,337],[65,2890,945],{"className":2891},[944],[65,2893,2895],{"className":2894},[318],[65,2896,2899],{"className":2897,"style":2898},[322],"height:0.2083em;",[65,2900],{},[65,2902],{"className":2903,"style":286},[285],[65,2905,93],{"className":2906},[290],[65,2908],{"className":2909,"style":286},[285],[65,2911,2913,2917,2958,2961,2964],{"className":2912},[270],[65,2914],{"className":2915,"style":2916},[274],"height:0.7333em;vertical-align:-0.15em;",[65,2918,2920,2923],{"className":2919},[279],[65,2921,1553],{"className":2922,"style":1568},[279,280],[65,2924,2926],{"className":2925},[310],[65,2927,2929,2950],{"className":2928},[314,891],[65,2930,2932,2947],{"className":2931},[318],[65,2933,2936],{"className":2934,"style":2935},[322],"height:0.2806em;",[65,2937,2938,2941],{"style":2196},[65,2939],{"className":2940,"style":330},[329],[65,2942,2944],{"className":2943},[334,335,336,337],[65,2945,2793],{"className":2946},[279,280,337],[65,2948,945],{"className":2949},[944],[65,2951,2953],{"className":2952},[318],[65,2954,2956],{"className":2955,"style":2224},[322],[65,2957],{},[65,2959],{"className":2960,"style":1137},[285],[65,2962,1604],{"className":2963},[1141],[65,2965],{"className":2966,"style":1137},[285],[65,2968,2970,2973,2976,3044,3047,3087],{"className":2969},[270],[65,2971],{"className":2972,"style":876},[274],[65,2974,2812],{"className":2975,"style":281},[279,280],[65,2977,2979,2982,3041],{"className":2978},[279],[65,2980],{"className":2981},[883,884],[65,2983,2985],{"className":2984},[795],[65,2986,2988,3033],{"className":2987},[314,891],[65,2989,2991,3030],{"className":2990},[318],[65,2992,2994,3008,3016],{"className":2993,"style":898},[322],[65,2995,2996,2999],{"style":901},[65,2997],{"className":2998,"style":905},[329],[65,3000,3002,3005],{"className":3001},[279],[65,3003,801],{"className":3004},[279,280],[65,3006,1553],{"className":3007,"style":1568},[279,280],[65,3009,3010,3013],{"style":917},[65,3011],{"className":3012,"style":905},[329],[65,3014],{"className":3015,"style":925},[924],[65,3017,3018,3021],{"style":928},[65,3019],{"className":3020,"style":905},[329],[65,3022,3024,3027],{"className":3023},[279],[65,3025,801],{"className":3026},[279,280],[65,3028,1587],{"className":3029},[279,280],[65,3031,945],{"className":3032},[944],[65,3034,3036],{"className":3035},[318],[65,3037,3039],{"className":3038,"style":952},[322],[65,3040],{},[65,3042],{"className":3043},[958,884],[65,3045,103],{"className":3046},[883],[65,3048,3050,3053],{"className":3049},[279],[65,3051,1553],{"className":3052,"style":1568},[279,280],[65,3054,3056],{"className":3055},[310],[65,3057,3059,3079],{"className":3058},[314,891],[65,3060,3062,3076],{"className":3061},[318],[65,3063,3065],{"className":3064,"style":2935},[322],[65,3066,3067,3070],{"style":2196},[65,3068],{"className":3069,"style":330},[329],[65,3071,3073],{"className":3072},[334,335,336,337],[65,3074,2793],{"className":3075},[279,280,337],[65,3077,945],{"className":3078},[944],[65,3080,3082],{"className":3081},[318],[65,3083,3085],{"className":3084,"style":2224},[322],[65,3086],{},[65,3088,1595],{"className":3089},[958],[13,3091,3092],{},"对应的训练循环是：",[56,3094,3096],{"className":58,"code":3095,"language":60,"meta":61,"style":61},"for step in range(10):\n    optimizer.zero_grad()\n    loss = (w - 3) ** 2\n    loss.backward()\n    optimizer.step()\n",[26,3097,3098,3120,3130,3149,3158],{"__ignoreMap":61},[65,3099,3100,3103,3106,3109,3112,3114,3117],{"class":67,"line":68},[65,3101,3102],{"class":71},"for",[65,3104,3105],{"class":75}," step ",[65,3107,3108],{"class":71},"in",[65,3110,3111],{"class":150}," range",[65,3113,103],{"class":75},[65,3115,3116],{"class":106},"10",[65,3118,3119],{"class":75},"):\n",[65,3121,3122,3125,3128],{"class":67,"line":79},[65,3123,3124],{"class":75},"    optimizer.",[65,3126,3127],{"class":99},"zero_grad",[65,3129,196],{"class":75},[65,3131,3132,3135,3137,3139,3141,3143,3145,3147],{"class":67,"line":86},[65,3133,3134],{"class":75},"    loss ",[65,3136,93],{"class":92},[65,3138,1927],{"class":75},[65,3140,1930],{"class":92},[65,3142,1933],{"class":106},[65,3144,1936],{"class":75},[65,3146,1939],{"class":92},[65,3148,1942],{"class":106},[65,3150,3151,3154,3156],{"class":67,"line":125},[65,3152,3153],{"class":75},"    loss.",[65,3155,193],{"class":99},[65,3157,196],{"class":75},[65,3159,3160,3162,3165],{"class":67,"line":142},[65,3161,3124],{"class":75},[65,3163,3164],{"class":99},"step",[65,3166,196],{"class":75},[2440,3168,3169,3179],{},[2443,3170,3171],{},[2446,3172,3173,3176],{},[2449,3174,3175],{},"调用",[2449,3177,3178],{},"在这个循环里的作用",[2519,3180,3181,3191,3204],{},[2446,3182,3183,3188],{},[2524,3184,3185],{},[26,3186,3187],{},"optimizer.zero_grad()",[2524,3189,3190],{},"清除优化器管理的参数上一次保存的梯度",[2446,3192,3193,3198],{},[2524,3194,3195],{},[26,3196,3197],{},"loss.backward()",[2524,3199,3200,3201],{},"计算梯度，写入参数的 ",[26,3202,3203],{},".grad",[2446,3205,3206,3211],{},[2524,3207,3208],{},[26,3209,3210],{},"optimizer.step()",[2524,3212,3213],{},"使用梯度和优化器配置更新参数",[13,3215,3216,3217,3219,3220,3248],{},"我当时问 ",[26,3218,3210],{}," 是怎样动态调整的。这个例子里的学习率始终是 0.1，每轮变化的是梯度。",[65,3221,3223,3236],{"className":3222},[225],[65,3224,3226],{"className":3225},[229],[231,3227,3228],{"xmlns":233},[235,3229,3230,3234],{},[238,3231,3232],{},[241,3233,1553],{},[258,3235,1553],{"encoding":260},[65,3237,3239],{"className":3238,"ariaHidden":266},[265],[65,3240,3242,3245],{"className":3241},[270],[65,3243],{"className":3244,"style":408},[274],[65,3246,1553],{"className":3247,"style":1568},[279,280]," 越接近 3，梯度的绝对值越小，更新距离也就越小。",[13,3250,3251],{},"手动更新可以写成：",[56,3253,3255],{"className":58,"code":3254,"language":60,"meta":61,"style":61},"with torch.no_grad():\n    w -= learn_rate * w.grad\n",[26,3256,3257,3267],{"__ignoreMap":61},[65,3258,3259,3261,3263,3265],{"class":67,"line":68},[65,3260,1960],{"class":71},[65,3262,96],{"class":75},[65,3264,1965],{"class":99},[65,3266,1968],{"class":75},[65,3268,3269,3271,3273,3275,3277],{"class":67,"line":79},[65,3270,1973],{"class":75},[65,3272,1976],{"class":92},[65,3274,1979],{"class":75},[65,3276,136],{"class":92},[65,3278,1984],{"class":75},[13,3280,3281,3282,3285,3286,3289],{},"这里是 ",[26,3283,3284],{},"w.grad","，不能写成 ",[26,3287,3288],{},"w.grad()","。它是张量属性，不是函数。",[13,3291,3292,3293,3296,3297,3300,3301,3304,3305,3307],{},"我还把 ",[26,3294,3295],{},"lr=0.1"," 误写成了 ",[26,3298,3299],{},"lr-0.1","，运行时报 ",[26,3302,3303],{},"NameError","。前者是在给优化器传参数，后者会尝试用一个名为 ",[26,3306,2757],{}," 的变量做减法。",[2428,3309,3311,1524,3314,3317],{"id":3310},"zero_grad-和-no_grad-容易混淆",[26,3312,3313],{},"zero_grad()",[26,3315,3316],{},"no_grad()"," 容易混淆",[13,3319,3320,3321,3324],{},"在线性回归的代码里，每隔 10 轮有一个 ",[26,3322,3323],{},"with torch.no_grad():","，我一度以为梯度也是每隔 10 轮才清除。",[13,3326,3327,3328,3330,3331,3333,3334,3337],{},"实际上，",[26,3329,3187],{}," 放在 ",[26,3332,3102],{}," 循环内、",[26,3335,3336],{},"if"," 条件外，每轮都执行。每隔 10 轮执行的是更新后的损失检查和打印。",[13,3339,3340,3343],{},[26,3341,3342],{},"torch.no_grad()"," 暂时停止记录缩进代码块内普通运算的求导关系，计算本身仍会进行，结果也可以保存。它不会清除已有梯度。离开这个代码块后，会恢复进入前的梯度记录模式。",[13,3345,3346],{},"因此，下面的检查不影响前面的训练更新，也不需要为打印结果建立新的求导关系：",[56,3348,3350],{"className":58,"code":3349,"language":60,"meta":61,"style":61},"if (step + 1) % 10 == 0:\n    with torch.no_grad():\n        new_loss = ((w * x - target) ** 2).mean()\n",[26,3351,3352,3381,3392],{"__ignoreMap":61},[65,3353,3354,3356,3359,3361,3364,3366,3369,3372,3375,3378],{"class":67,"line":68},[65,3355,3336],{"class":71},[65,3357,3358],{"class":75}," (step ",[65,3360,2796],{"class":92},[65,3362,3363],{"class":106}," 1",[65,3365,1936],{"class":75},[65,3367,3368],{"class":92},"%",[65,3370,3371],{"class":106}," 10",[65,3373,3374],{"class":92}," ==",[65,3376,3377],{"class":106}," 0",[65,3379,3380],{"class":75},":\n",[65,3382,3383,3386,3388,3390],{"class":67,"line":79},[65,3384,3385],{"class":71},"    with",[65,3387,96],{"class":75},[65,3389,1965],{"class":99},[65,3391,1968],{"class":75},[65,3393,3394,3397,3399,3402,3404,3406,3408,3411,3413,3416,3419,3422],{"class":67,"line":86},[65,3395,3396],{"class":75},"        new_loss ",[65,3398,93],{"class":92},[65,3400,3401],{"class":75}," ((w ",[65,3403,136],{"class":92},[65,3405,133],{"class":75},[65,3407,1930],{"class":92},[65,3409,3410],{"class":75}," target) ",[65,3412,1939],{"class":92},[65,3414,3415],{"class":106}," 2",[65,3417,3418],{"class":75},").",[65,3420,3421],{"class":99},"mean",[65,3423,196],{"class":75},[49,3425,3426],{"id":3426},"用四个样本训练线性回归",[13,3428,3429],{},"标量实验之后，我开始用一组输入和目标训练模型：",[56,3431,3434],{"className":3432,"code":3433,"language":2064,"meta":61},[2062],"x      = [1, 2, 3, 4]\ntarget = [2, 4, 6, 8]\n",[26,3435,3433],{"__ignoreMap":61},[13,3437,3438,3439,3467,3468,445,3662,3665,3666,1524,3668,3671,3672,445],{},"模型只有一个需要学习的参数 ",[65,3440,3442,3455],{"className":3441},[225],[65,3443,3445],{"className":3444},[229],[231,3446,3447],{"xmlns":233},[235,3448,3449,3453],{},[238,3450,3451],{},[241,3452,1553],{},[258,3454,1553],{"encoding":260},[65,3456,3458],{"className":3457,"ariaHidden":266},[265],[65,3459,3461,3464],{"className":3460},[270],[65,3462],{"className":3463,"style":408},[274],[65,3465,1553],{"className":3466,"style":1568},[279,280],"，预测是 ",[65,3469,3471,3506],{"className":3470},[225],[65,3472,3474],{"className":3473},[229],[231,3475,3476],{"xmlns":233},[235,3477,3478,3503],{},[238,3479,3480,3493,3495,3497],{},[2122,3481,3482,3490],{},[3483,3484,3485,3487],"mover",{"accent":266},[241,3486,243],{},[245,3488,3489],{},"^",[241,3491,3492],{},"i",[245,3494,93],{},[241,3496,1553],{},[2122,3498,3499,3501],{},[241,3500,252],{},[241,3502,3492],{},[258,3504,3505],{"encoding":260},"\\hat y_i=wx_i",[65,3507,3509,3612],{"className":3508,"ariaHidden":266},[265],[65,3510,3512,3516,3603,3606,3609],{"className":3511},[270],[65,3513],{"className":3514,"style":3515},[274],"height:0.8889em;vertical-align:-0.1944em;",[65,3517,3519,3567],{"className":3518},[279],[65,3520,3523],{"className":3521},[279,3522],"accent",[65,3524,3526,3558],{"className":3525},[314,891],[65,3527,3529,3555],{"className":3528},[318],[65,3530,3533,3542],{"className":3531,"style":3532},[322],"height:0.6944em;",[65,3534,3536,3539],{"style":3535},"top:-3em;",[65,3537],{"className":3538,"style":905},[329],[65,3540,243],{"className":3541,"style":281},[279,280],[65,3543,3544,3547],{"style":3535},[65,3545],{"className":3546,"style":905},[329],[65,3548,3552],{"className":3549,"style":3551},[3550],"accent-body","left:-0.1944em;",[65,3553,3489],{"className":3554},[279],[65,3556,945],{"className":3557},[944],[65,3559,3561],{"className":3560},[318],[65,3562,3565],{"className":3563,"style":3564},[322],"height:0.1944em;",[65,3566],{},[65,3568,3570],{"className":3569},[310],[65,3571,3573,3595],{"className":3572},[314,891],[65,3574,3576,3592],{"className":3575},[318],[65,3577,3580],{"className":3578,"style":3579},[322],"height:0.3117em;",[65,3581,3583,3586],{"style":3582},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[65,3584],{"className":3585,"style":330},[329],[65,3587,3589],{"className":3588},[334,335,336,337],[65,3590,3492],{"className":3591},[279,280,337],[65,3593,945],{"className":3594},[944],[65,3596,3598],{"className":3597},[318],[65,3599,3601],{"className":3600,"style":2224},[322],[65,3602],{},[65,3604],{"className":3605,"style":286},[285],[65,3607,93],{"className":3608},[290],[65,3610],{"className":3611,"style":286},[285],[65,3613,3615,3618,3621],{"className":3614},[270],[65,3616],{"className":3617,"style":2174},[274],[65,3619,1553],{"className":3620,"style":1568},[279,280],[65,3622,3624,3627],{"className":3623},[279],[65,3625,252],{"className":3626},[279,280],[65,3628,3630],{"className":3629},[310],[65,3631,3633,3654],{"className":3632},[314,891],[65,3634,3636,3651],{"className":3635},[318],[65,3637,3639],{"className":3638,"style":3579},[322],[65,3640,3642,3645],{"style":3641},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[65,3643],{"className":3644,"style":330},[329],[65,3646,3648],{"className":3647},[334,335,336,337],[65,3649,3492],{"className":3650},[279,280,337],[65,3652,945],{"className":3653},[944],[65,3655,3657],{"className":3656},[318],[65,3658,3660],{"className":3659,"style":2224},[322],[65,3661],{},[26,3663,3664],{},"w * x"," 会把同一个参数乘到四个输入上，得到四个预测值。",[26,3667,252],{},[26,3669,3670],{},"target"," 是给定的数据，这次需要求导和更新的是 ",[26,3673,1553],{},[13,3675,3676],{},"损失采用均方误差：",[65,3678,3680],{"className":3679},[565],[65,3681,3683,3751],{"className":3682},[225],[65,3684,3686],{"className":3685},[229],[231,3687,3688],{"xmlns":233,"display":574},[235,3689,3690,3748],{},[238,3691,3692,3694,3696,3698,3700,3702,3708,3724,3726,3728,3734,3736,3742],{},[241,3693,1587],{},[245,3695,103],{"stretchy":1590},[241,3697,1553],{},[245,3699,1595],{"stretchy":1590},[245,3701,93],{},[795,3703,3704,3706],{},[254,3705,2799],{},[254,3707,613],{},[3709,3710,3711,3714,3722],"munderover",{},[245,3712,3713],{},"∑",[238,3715,3716,3718,3720],{},[241,3717,3492],{},[245,3719,93],{},[254,3721,2799],{},[254,3723,613],{},[245,3725,103],{"stretchy":1590},[241,3727,1553],{},[2122,3729,3730,3732],{},[241,3731,252],{},[241,3733,3492],{},[245,3735,1604],{},[2122,3737,3738,3740],{},[241,3739,243],{},[241,3741,3492],{},[248,3743,3744,3746],{},[245,3745,1595],{"stretchy":1590},[254,3747,256],{},[258,3749,3750],{"encoding":260},"L(w)=\\frac{1}{4}\\sum_{i=1}^{4}(wx_i-y_i)^2",[65,3752,3754,3781,3986],{"className":3753,"ariaHidden":266},[265],[65,3755,3757,3760,3763,3766,3769,3772,3775,3778],{"className":3756},[270],[65,3758],{"className":3759,"style":1657},[274],[65,3761,1587],{"className":3762},[279,280],[65,3764,103],{"className":3765},[883],[65,3767,1553],{"className":3768,"style":1568},[279,280],[65,3770,1595],{"className":3771},[958],[65,3773],{"className":3774,"style":286},[285],[65,3776,93],{"className":3777},[290],[65,3779],{"className":3780,"style":286},[285],[65,3782,3784,3788,3851,3854,3931,3934,3937,3977,3980,3983],{"className":3783},[270],[65,3785],{"className":3786,"style":3787},[274],"height:3.0788em;vertical-align:-1.2777em;",[65,3789,3791,3794,3848],{"className":3790},[279],[65,3792],{"className":3793},[883,884],[65,3795,3797],{"className":3796},[795],[65,3798,3800,3840],{"className":3799},[314,891],[65,3801,3803,3837],{"className":3802},[318],[65,3804,3807,3818,3826],{"className":3805,"style":3806},[322],"height:1.3214em;",[65,3808,3809,3812],{"style":901},[65,3810],{"className":3811,"style":905},[329],[65,3813,3815],{"className":3814},[279],[65,3816,613],{"className":3817},[279],[65,3819,3820,3823],{"style":917},[65,3821],{"className":3822,"style":905},[329],[65,3824],{"className":3825,"style":925},[924],[65,3827,3828,3831],{"style":928},[65,3829],{"className":3830,"style":905},[329],[65,3832,3834],{"className":3833},[279],[65,3835,2799],{"className":3836},[279],[65,3838,945],{"className":3839},[944],[65,3841,3843],{"className":3842},[318],[65,3844,3846],{"className":3845,"style":952},[322],[65,3847],{},[65,3849],{"className":3850},[958,884],[65,3852],{"className":3853,"style":687},[285],[65,3855,3859],{"className":3856},[3857,3858],"mop","op-limits",[65,3860,3862,3922],{"className":3861},[314,891],[65,3863,3865,3919],{"className":3864},[318],[65,3866,3869,3891,3904],{"className":3867,"style":3868},[322],"height:1.8011em;",[65,3870,3872,3876],{"style":3871},"top:-1.8723em;margin-left:0em;",[65,3873],{"className":3874,"style":3875},[329],"height:3.05em;",[65,3877,3879],{"className":3878},[334,335,336,337],[65,3880,3882,3885,3888],{"className":3881},[279,337],[65,3883,3492],{"className":3884},[279,280,337],[65,3886,93],{"className":3887},[290,337],[65,3889,2799],{"className":3890},[279,337],[65,3892,3894,3897],{"style":3893},"top:-3.05em;",[65,3895],{"className":3896,"style":3875},[329],[65,3898,3899],{},[65,3900,3713],{"className":3901},[3857,3902,3903],"op-symbol","large-op",[65,3905,3907,3910],{"style":3906},"top:-4.3em;margin-left:0em;",[65,3908],{"className":3909,"style":3875},[329],[65,3911,3913],{"className":3912},[334,335,336,337],[65,3914,3916],{"className":3915},[279,337],[65,3917,613],{"className":3918},[279,337],[65,3920,945],{"className":3921},[944],[65,3923,3925],{"className":3924},[318],[65,3926,3929],{"className":3927,"style":3928},[322],"height:1.2777em;",[65,3930],{},[65,3932,103],{"className":3933},[883],[65,3935,1553],{"className":3936,"style":1568},[279,280],[65,3938,3940,3943],{"className":3939},[279],[65,3941,252],{"className":3942},[279,280],[65,3944,3946],{"className":3945},[310],[65,3947,3949,3969],{"className":3948},[314,891],[65,3950,3952,3966],{"className":3951},[318],[65,3953,3955],{"className":3954,"style":3579},[322],[65,3956,3957,3960],{"style":3641},[65,3958],{"className":3959,"style":330},[329],[65,3961,3963],{"className":3962},[334,335,336,337],[65,3964,3492],{"className":3965},[279,280,337],[65,3967,945],{"className":3968},[944],[65,3970,3972],{"className":3971},[318],[65,3973,3975],{"className":3974,"style":2224},[322],[65,3976],{},[65,3978],{"className":3979,"style":1137},[285],[65,3981,1604],{"className":3982},[1141],[65,3984],{"className":3985,"style":1137},[285],[65,3987,3989,3993,4033],{"className":3988},[270],[65,3990],{"className":3991,"style":3992},[274],"height:1.1141em;vertical-align:-0.25em;",[65,3994,3996,3999],{"className":3995},[279],[65,3997,243],{"className":3998,"style":281},[279,280],[65,4000,4002],{"className":4001},[310],[65,4003,4005,4025],{"className":4004},[314,891],[65,4006,4008,4022],{"className":4007},[318],[65,4009,4011],{"className":4010,"style":3579},[322],[65,4012,4013,4016],{"style":3582},[65,4014],{"className":4015,"style":330},[329],[65,4017,4019],{"className":4018},[334,335,336,337],[65,4020,3492],{"className":4021},[279,280,337],[65,4023,945],{"className":4024},[944],[65,4026,4028],{"className":4027},[318],[65,4029,4031],{"className":4030,"style":2224},[322],[65,4032],{},[65,4034,4036,4039],{"className":4035},[958],[65,4037,1595],{"className":4038},[958],[65,4040,4042],{"className":4041},[310],[65,4043,4045],{"className":4044},[314],[65,4046,4048],{"className":4047},[318],[65,4049,4051],{"className":4050,"style":663},[322],[65,4052,4053,4056],{"style":666},[65,4054],{"className":4055,"style":330},[329],[65,4057,4059],{"className":4058},[334,335,336,337],[65,4060,256],{"className":4061},[279,337],[13,4063,4064,4065,4068],{},"对应代码是 ",[26,4066,4067],{},"((prediction - target) ** 2).mean()","：先计算每个样本的误差，再平方并求平均。",[13,4070,4071,4072,4075],{},"最终运行通过的代码如下，与附件中的 ",[26,4073,4074],{},"03_linear.py"," 使用相同的计算式和训练配置：",[56,4077,4079],{"className":58,"code":4078,"language":60,"meta":61,"style":61},"import torch\n\nw = torch.tensor(0.0, requires_grad=True)\nx = torch.tensor([1.0, 2.0, 3.0, 4.0])\ntarget = torch.tensor([2.0, 4.0, 6.0, 8.0])\noptimizer = torch.optim.SGD([w], lr=0.01)\n\nfor step in range(100):\n    optimizer.zero_grad()\n    prediction = w * x\n    loss = ((prediction - target) ** 2).mean()\n    loss.backward()\n    optimizer.step()\n\n    if (step + 1) % 10 == 0:\n        with torch.no_grad():\n            new_loss = ((w * x - target) ** 2).mean()\n\n        print(\n            f\"第{step + 1:3d}次：\"\n            f\"w={w.item():.4f},\"\n            f\"loss={new_loss.item():.4f}\"\n        )\n\nwith torch.no_grad():\n    test_x = torch.tensor([5.0, 6.0])\n    print(\"新输入：\", test_x)\n    print(\"预测值：\", w * test_x)\n",[26,4080,4081,4087,4091,4115,4148,4179,4199,4203,4220,4228,4242,4265,4273,4281,4286,4310,4322,4350,4355,4364,4390,4415,4438,4444,4449,4460,4483,4497],{"__ignoreMap":61},[65,4082,4083,4085],{"class":67,"line":68},[65,4084,72],{"class":71},[65,4086,76],{"class":75},[65,4088,4089],{"class":67,"line":79},[65,4090,83],{"emptyLinePlaceholder":82},[65,4092,4093,4095,4097,4099,4101,4103,4105,4107,4109,4111,4113],{"class":67,"line":86},[65,4094,1882],{"class":75},[65,4096,93],{"class":92},[65,4098,96],{"class":75},[65,4100,100],{"class":99},[65,4102,103],{"class":75},[65,4104,1893],{"class":106},[65,4106,110],{"class":75},[65,4108,114],{"class":113},[65,4110,93],{"class":92},[65,4112,119],{"class":106},[65,4114,122],{"class":75},[65,4116,4117,4119,4121,4123,4125,4128,4131,4133,4135,4137,4140,4142,4145],{"class":67,"line":125},[65,4118,89],{"class":75},[65,4120,93],{"class":92},[65,4122,96],{"class":75},[65,4124,100],{"class":99},[65,4126,4127],{"class":75},"([",[65,4129,4130],{"class":106},"1.0",[65,4132,110],{"class":75},[65,4134,107],{"class":106},[65,4136,110],{"class":75},[65,4138,4139],{"class":106},"3.0",[65,4141,110],{"class":75},[65,4143,4144],{"class":106},"4.0",[65,4146,4147],{"class":75},"])\n",[65,4149,4150,4153,4155,4157,4159,4161,4163,4165,4167,4169,4172,4174,4177],{"class":67,"line":142},[65,4151,4152],{"class":75},"target ",[65,4154,93],{"class":92},[65,4156,96],{"class":75},[65,4158,100],{"class":99},[65,4160,4127],{"class":75},[65,4162,107],{"class":106},[65,4164,110],{"class":75},[65,4166,4144],{"class":106},[65,4168,110],{"class":75},[65,4170,4171],{"class":106},"6.0",[65,4173,110],{"class":75},[65,4175,4176],{"class":106},"8.0",[65,4178,4147],{"class":75},[65,4180,4181,4183,4185,4187,4189,4191,4193,4195,4197],{"class":67,"line":147},[65,4182,2743],{"class":75},[65,4184,93],{"class":92},[65,4186,2748],{"class":75},[65,4188,2751],{"class":99},[65,4190,2754],{"class":75},[65,4192,2757],{"class":113},[65,4194,93],{"class":92},[65,4196,2539],{"class":106},[65,4198,122],{"class":75},[65,4200,4201],{"class":67,"line":169},[65,4202,83],{"emptyLinePlaceholder":82},[65,4204,4205,4207,4209,4211,4213,4215,4218],{"class":67,"line":182},[65,4206,3102],{"class":71},[65,4208,3105],{"class":75},[65,4210,3108],{"class":71},[65,4212,3111],{"class":150},[65,4214,103],{"class":75},[65,4216,4217],{"class":106},"100",[65,4219,3119],{"class":75},[65,4221,4222,4224,4226],{"class":67,"line":187},[65,4223,3124],{"class":75},[65,4225,3127],{"class":99},[65,4227,196],{"class":75},[65,4229,4230,4233,4235,4238,4240],{"class":67,"line":199},[65,4231,4232],{"class":75},"    prediction ",[65,4234,93],{"class":92},[65,4236,4237],{"class":75}," w ",[65,4239,136],{"class":92},[65,4241,139],{"class":75},[65,4243,4244,4246,4248,4251,4253,4255,4257,4259,4261,4263],{"class":67,"line":204},[65,4245,3134],{"class":75},[65,4247,93],{"class":92},[65,4249,4250],{"class":75}," ((prediction ",[65,4252,1930],{"class":92},[65,4254,3410],{"class":75},[65,4256,1939],{"class":92},[65,4258,3415],{"class":106},[65,4260,3418],{"class":75},[65,4262,3421],{"class":99},[65,4264,196],{"class":75},[65,4266,4267,4269,4271],{"class":67,"line":1991},[65,4268,3153],{"class":75},[65,4270,193],{"class":99},[65,4272,196],{"class":75},[65,4274,4275,4277,4279],{"class":67,"line":2011},[65,4276,3124],{"class":75},[65,4278,3164],{"class":99},[65,4280,196],{"class":75},[65,4282,4284],{"class":67,"line":4283},14,[65,4285,83],{"emptyLinePlaceholder":82},[65,4287,4289,4292,4294,4296,4298,4300,4302,4304,4306,4308],{"class":67,"line":4288},15,[65,4290,4291],{"class":71},"    if",[65,4293,3358],{"class":75},[65,4295,2796],{"class":92},[65,4297,3363],{"class":106},[65,4299,1936],{"class":75},[65,4301,3368],{"class":92},[65,4303,3371],{"class":106},[65,4305,3374],{"class":92},[65,4307,3377],{"class":106},[65,4309,3380],{"class":75},[65,4311,4313,4316,4318,4320],{"class":67,"line":4312},16,[65,4314,4315],{"class":71},"        with",[65,4317,96],{"class":75},[65,4319,1965],{"class":99},[65,4321,1968],{"class":75},[65,4323,4325,4328,4330,4332,4334,4336,4338,4340,4342,4344,4346,4348],{"class":67,"line":4324},17,[65,4326,4327],{"class":75},"            new_loss ",[65,4329,93],{"class":92},[65,4331,3401],{"class":75},[65,4333,136],{"class":92},[65,4335,133],{"class":75},[65,4337,1930],{"class":92},[65,4339,3410],{"class":75},[65,4341,1939],{"class":92},[65,4343,3415],{"class":106},[65,4345,3418],{"class":75},[65,4347,3421],{"class":99},[65,4349,196],{"class":75},[65,4351,4353],{"class":67,"line":4352},18,[65,4354,83],{"emptyLinePlaceholder":82},[65,4356,4358,4361],{"class":67,"line":4357},19,[65,4359,4360],{"class":150},"        print",[65,4362,4363],{"class":75},"(\n",[65,4365,4367,4370,4373,4375,4378,4380,4382,4385,4387],{"class":67,"line":4366},20,[65,4368,4369],{"class":2018},"            f",[65,4371,4372],{"class":156},"\"第",[65,4374,2026],{"class":2025},[65,4376,4377],{"class":75},"step ",[65,4379,2796],{"class":92},[65,4381,3363],{"class":106},[65,4383,4384],{"class":2018},":3d",[65,4386,2036],{"class":2025},[65,4388,4389],{"class":156},"次：\"\n",[65,4391,4393,4395,4398,4400,4403,4405,4407,4410,4412],{"class":67,"line":4392},21,[65,4394,4369],{"class":2018},[65,4396,4397],{"class":156},"\"w=",[65,4399,2026],{"class":2025},[65,4401,4402],{"class":75},"w.",[65,4404,163],{"class":99},[65,4406,2033],{"class":75},[65,4408,4409],{"class":2018},":.4f",[65,4411,2036],{"class":2025},[65,4413,4414],{"class":156},",\"\n",[65,4416,4418,4420,4423,4425,4427,4429,4431,4433,4435],{"class":67,"line":4417},22,[65,4419,4369],{"class":2018},[65,4421,4422],{"class":156},"\"loss=",[65,4424,2026],{"class":2025},[65,4426,2044],{"class":75},[65,4428,163],{"class":99},[65,4430,2033],{"class":75},[65,4432,4409],{"class":2018},[65,4434,2036],{"class":2025},[65,4436,4437],{"class":156},"\"\n",[65,4439,4441],{"class":67,"line":4440},23,[65,4442,4443],{"class":75},"        )\n",[65,4445,4447],{"class":67,"line":4446},24,[65,4448,83],{"emptyLinePlaceholder":82},[65,4450,4452,4454,4456,4458],{"class":67,"line":4451},25,[65,4453,1960],{"class":71},[65,4455,96],{"class":75},[65,4457,1965],{"class":99},[65,4459,1968],{"class":75},[65,4461,4463,4466,4468,4470,4472,4474,4477,4479,4481],{"class":67,"line":4462},26,[65,4464,4465],{"class":75},"    test_x ",[65,4467,93],{"class":92},[65,4469,96],{"class":75},[65,4471,100],{"class":99},[65,4473,4127],{"class":75},[65,4475,4476],{"class":106},"5.0",[65,4478,110],{"class":75},[65,4480,4171],{"class":106},[65,4482,4147],{"class":75},[65,4484,4486,4489,4491,4494],{"class":67,"line":4485},27,[65,4487,4488],{"class":150},"    print",[65,4490,103],{"class":75},[65,4492,4493],{"class":156},"\"新输入：\"",[65,4495,4496],{"class":75},", test_x)\n",[65,4498,4500,4502,4504,4507,4510,4512],{"class":67,"line":4499},28,[65,4501,4488],{"class":150},[65,4503,103],{"class":75},[65,4505,4506],{"class":156},"\"预测值：\"",[65,4508,4509],{"class":75},", w ",[65,4511,136],{"class":92},[65,4513,4514],{"class":75}," test_x)\n",[13,4516,4517],{},"我实际运行后得到：",[56,4519,4522],{"className":4520,"code":4521,"language":2064,"meta":61},[2062],"第 10次：w=1.6063,loss=1.1628\n第 20次：w=1.9225,loss=0.0451\n第 30次：w=1.9847,loss=0.0017\n第 40次：w=1.9970,loss=0.0001\n第 50次：w=1.9994,loss=0.0000\n第 60次：w=1.9999,loss=0.0000\n第 70次：w=2.0000,loss=0.0000\n第 80次：w=2.0000,loss=0.0000\n第 90次：w=2.0000,loss=0.0000\n第100次：w=2.0000,loss=0.0000\n新输入： tensor([5., 6.])\n预测值： tensor([10.0000, 12.0000])\n",[26,4523,4521],{"__ignoreMap":61},[13,4525,4526,4554,4555,4558,4559,445],{},[65,4527,4529,4542],{"className":4528},[225],[65,4530,4532],{"className":4531},[229],[231,4533,4534],{"xmlns":233},[235,4535,4536,4540],{},[238,4537,4538],{},[241,4539,1553],{},[258,4541,1553],{"encoding":260},[65,4543,4545],{"className":4544,"ariaHidden":266},[265],[65,4546,4548,4551],{"className":4547},[270],[65,4549],{"className":4550,"style":408},[274],[65,4552,1553],{"className":4553,"style":1568},[279,280]," 逐渐接近 2，在训练时没出现过的输入 ",[26,4556,4557],{},"[5, 6]"," 上，预测接近 ",[26,4560,4561],{},"[10, 12]",[13,4563,4564,4565,4567,4568,4570,4571,4574],{},"输出中的 ",[26,4566,4384],{}," 表示整数至少占 3 个字符，不够时补空格；",[26,4569,4409],{}," 表示保留小数点后 4 位。显示为 ",[26,4572,4573],{},"loss=0.0000"," 是格式化后的结果，不能据此认定误差已经严格为零。",[49,4576,4578,4579,4582],{"id":4577},"写成-w-4-后为什么停在-125","写成 ",[26,4580,4581],{},"w * 4"," 后，为什么停在 1.25",[13,4584,4585],{},"这个实验中，我把训练时的预测误写成了：",[56,4587,4589],{"className":58,"code":4588,"language":60,"meta":61,"style":61},"prediction = w * 4\n",[26,4590,4591],{"__ignoreMap":61},[65,4592,4593,4596,4598,4600,4602],{"class":67,"line":68},[65,4594,4595],{"class":75},"prediction ",[65,4597,93],{"class":92},[65,4599,4237],{"class":75},[65,4601,136],{"class":92},[65,4603,4604],{"class":106}," 4\n",[13,4606,4607,4608,4642,4643,445],{},"这样得到的是一个标量，和四个目标计算误差时会发生广播。模型给所有样本提供同一个预测值 ",[65,4609,4611,4627],{"className":4610},[225],[65,4612,4614],{"className":4613},[229],[231,4615,4616],{"xmlns":233},[235,4617,4618,4624],{},[238,4619,4620,4622],{},[254,4621,613],{},[241,4623,1553],{},[258,4625,4626],{"encoding":260},"4w",[65,4628,4630],{"className":4629,"ariaHidden":266},[265],[65,4631,4633,4636,4639],{"className":4632},[270],[65,4634],{"className":4635,"style":370},[274],[65,4637,613],{"className":4638},[279],[65,4640,1553],{"className":4641,"style":1568},[279,280],"，完全没有使用输入 ",[26,4644,252],{},[13,4646,4647,4648,4651],{},"四个目标是 ",[26,4649,4650],{},"[2, 4, 6, 8]","。平方误差最小的统一预测值是它们的平均值 5，因此：",[65,4653,4655],{"className":4654},[565],[65,4656,4658,4717],{"className":4657},[225],[65,4659,4661],{"className":4660},[229],[231,4662,4663],{"xmlns":233,"display":574},[235,4664,4665,4714],{},[238,4666,4667,4669,4671,4673,4694,4696,4699,4702,4705,4707,4709,4711],{},[254,4668,613],{},[241,4670,1553],{},[245,4672,93],{},[795,4674,4675,4692],{},[238,4676,4677,4679,4681,4683,4685,4687,4689],{},[254,4678,256],{},[245,4680,2796],{},[254,4682,613],{},[245,4684,2796],{},[254,4686,2086],{},[245,4688,2796],{},[254,4690,4691],{},"8",[254,4693,613],{},[245,4695,93],{},[254,4697,4698],{},"5",[285,4700],{"width":4701},"1em",[245,4703,4704],{},"⇒",[285,4706],{"width":4701},[241,4708,1553],{},[245,4710,93],{},[254,4712,4713],{},"1.25",[258,4715,4716],{"encoding":260},"4w=\\frac{2+4+6+8}{4}=5\n\\quad\\Rightarrow\\quad w=1.25",[65,4718,4720,4741,4855,4880,4898],{"className":4719,"ariaHidden":266},[265],[65,4721,4723,4726,4729,4732,4735,4738],{"className":4722},[270],[65,4724],{"className":4725,"style":370},[274],[65,4727,613],{"className":4728},[279],[65,4730,1553],{"className":4731,"style":1568},[279,280],[65,4733],{"className":4734,"style":286},[285],[65,4736,93],{"className":4737},[290],[65,4739],{"className":4740,"style":286},[285],[65,4742,4744,4748,4846,4849,4852],{"className":4743},[270],[65,4745],{"className":4746,"style":4747},[274],"height:2.0074em;vertical-align:-0.686em;",[65,4749,4751,4754,4843],{"className":4750},[279],[65,4752],{"className":4753},[883,884],[65,4755,4757],{"className":4756},[795],[65,4758,4760,4835],{"className":4759},[314,891],[65,4761,4763,4832],{"className":4762},[318],[65,4764,4766,4777,4785],{"className":4765,"style":3806},[322],[65,4767,4768,4771],{"style":901},[65,4769],{"className":4770,"style":905},[329],[65,4772,4774],{"className":4773},[279],[65,4775,613],{"className":4776},[279],[65,4778,4779,4782],{"style":917},[65,4780],{"className":4781,"style":905},[329],[65,4783],{"className":4784,"style":925},[924],[65,4786,4787,4790],{"style":928},[65,4788],{"className":4789,"style":905},[329],[65,4791,4793,4796,4799,4802,4805,4808,4811,4814,4817,4820,4823,4826,4829],{"className":4792},[279],[65,4794,256],{"className":4795},[279],[65,4797],{"className":4798,"style":1137},[285],[65,4800,2796],{"className":4801},[1141],[65,4803],{"className":4804,"style":1137},[285],[65,4806,613],{"className":4807},[279],[65,4809],{"className":4810,"style":1137},[285],[65,4812,2796],{"className":4813},[1141],[65,4815],{"className":4816,"style":1137},[285],[65,4818,2086],{"className":4819},[279],[65,4821],{"className":4822,"style":1137},[285],[65,4824,2796],{"className":4825},[1141],[65,4827],{"className":4828,"style":1137},[285],[65,4830,4691],{"className":4831},[279],[65,4833,945],{"className":4834},[944],[65,4836,4838],{"className":4837},[318],[65,4839,4841],{"className":4840,"style":952},[322],[65,4842],{},[65,4844],{"className":4845},[958,884],[65,4847],{"className":4848,"style":286},[285],[65,4850,93],{"className":4851},[290],[65,4853],{"className":4854,"style":286},[285],[65,4856,4858,4861,4864,4868,4871,4874,4877],{"className":4857},[270],[65,4859],{"className":4860,"style":370},[274],[65,4862,4698],{"className":4863},[279],[65,4865],{"className":4866,"style":4867},[285],"margin-right:1em;",[65,4869],{"className":4870,"style":286},[285],[65,4872,4704],{"className":4873},[290],[65,4875],{"className":4876,"style":4867},[285],[65,4878],{"className":4879,"style":286},[285],[65,4881,4883,4886,4889,4892,4895],{"className":4882},[270],[65,4884],{"className":4885,"style":408},[274],[65,4887,1553],{"className":4888,"style":1568},[279,280],[65,4890],{"className":4891,"style":286},[285],[65,4893,93],{"className":4894},[290],[65,4896],{"className":4897,"style":286},[285],[65,4899,4901,4904],{"className":4900},[270],[65,4902],{"className":4903,"style":370},[274],[65,4905,4713],{"className":4906},[279],[13,4908,4909],{},"所以参数稳定在 1.25 有具体原因：代码训练出来的是统一预测值为 5 的模型。",[13,4911,4912,4913,4915,4916,4918,4919,4921],{},"我后来只把检查损失的公式改成了 ",[26,4914,3664],{},"，训练部分仍是 ",[26,4917,4581],{},"。这时打印的损失变了，但参数还是停在 1.25，因为参数更新依然依据传给 ",[26,4920,437],{}," 的那个训练损失。",[13,4923,4924],{},"下面是这几次运行的最终输出对比，数值按当时的打印精度记录：",[2440,4926,4927,4976],{},[2443,4928,4929],{},[2446,4930,4931,4934,4937,4967,4970],{},[2449,4932,4933],{},"训练时的预测",[2449,4935,4936],{},"检查损失时的预测",[2449,4938,4939],{"align":2454},[65,4940,4942,4955],{"className":4941},[225],[65,4943,4945],{"className":4944},[229],[231,4946,4947],{"xmlns":233},[235,4948,4949,4953],{},[238,4950,4951],{},[241,4952,1553],{},[258,4954,1553],{"encoding":260},[65,4956,4958],{"className":4957,"ariaHidden":266},[265],[65,4959,4961,4964],{"className":4960},[270],[65,4962],{"className":4963,"style":408},[274],[65,4965,1553],{"className":4966,"style":1568},[279,280],[2449,4968,4969],{"align":2454},"打印的损失",[2449,4971,4972,4973,4975],{},"对 ",[26,4974,4557],{}," 的预测",[2519,4977,4978,4999,5018],{},[2446,4979,4980,4984,4988,4991,4994],{},[2524,4981,4982],{},[26,4983,4581],{},[2524,4985,4986],{},[26,4987,4581],{},[2524,4989,4990],{"align":2454},"1.2500",[2524,4992,4993],{"align":2454},"5.0000",[2524,4995,4996],{},[26,4997,4998],{},"[6.2500, 7.5000]",[2446,5000,5001,5005,5009,5011,5014],{},[2524,5002,5003],{},[26,5004,4581],{},[2524,5006,5007],{},[26,5008,3664],{},[2524,5010,4990],{"align":2454},[2524,5012,5013],{"align":2454},"4.2188",[2524,5015,5016],{},[26,5017,4998],{},[2446,5019,5020,5024,5028,5031,5034],{},[2524,5021,5022],{},[26,5023,3664],{},[2524,5025,5026],{},[26,5027,3664],{},[2524,5029,5030],{"align":2454},"2.0000",[2524,5032,5033],{"align":2454},"0.0000",[2524,5035,5036],{},[26,5037,5038],{},"[10.0000, 12.0000]",[13,5040,5041,5042,5045],{},"把训练中的那一行也改成 ",[26,5043,5044],{},"prediction = w * x"," 后，才得到前面列出的正确输出。这次调试让我记住：训练、检查损失和最终预测必须使用同一套模型计算式。",[49,5047,5048],{"id":5048},"实验代码和复现方式",[13,5050,5051,5054,5055,5058],{},[31,5052,36],{"href":33,"rel":5053},[35],"持续保存源代码；也可以",[31,5056,5057],{"href":40},"下载实验代码压缩包","，解压后包含：",[2440,5060,5061,5071],{},[2443,5062,5063],{},[2446,5064,5065,5068],{},[2449,5066,5067],{},"文件",[2449,5069,5070],{},"内容",[2519,5072,5073,5082,5092,5101,5111],{},[2446,5074,5075,5079],{},[2524,5076,5077],{},[26,5078,2721],{},[2524,5080,5081],{},"手动梯度下降，连续更新 10 次，当前学习率为 0.01",[2446,5083,5084,5089],{},[2524,5085,5086],{},[26,5087,5088],{},"02_SGD.py",[2524,5090,5091],{},"使用 SGD 更新标量参数，学习率为 0.1",[2446,5093,5094,5098],{},[2524,5095,5096],{},[26,5097,4074],{},[2524,5099,5100],{},"修正后的线性回归，训练 100 次并预测新输入",[2446,5102,5103,5108],{},[2524,5104,5105],{},[26,5106,5107],{},"requirements.txt",[2524,5109,5110],{},"实验环境的依赖版本",[2446,5112,5113,5118],{},[2524,5114,5115],{},[26,5116,5117],{},"README.md",[2524,5119,5120],{},"环境建立、运行命令和结果核对说明",[13,5122,5123],{},"进入解压后的目录，在 Python 3.11 环境中运行：",[56,5125,5129],{"className":5126,"code":5127,"language":5128,"meta":61,"style":61},"language-bash shiki shiki-themes github-dark-dimmed github-light houston","python3.11 -m venv .venv\n.venv\u002Fbin\u002Fpython -m pip install -r requirements.txt\n.venv\u002Fbin\u002Fpython 01_autograd.py\n.venv\u002Fbin\u002Fpython 02_SGD.py\n.venv\u002Fbin\u002Fpython 03_linear.py\n","bash",[26,5130,5131,5146,5165,5172,5179],{"__ignoreMap":61},[65,5132,5133,5137,5140,5143],{"class":67,"line":68},[65,5134,5136],{"class":5135},"sLsJe","python3.11",[65,5138,5139],{"class":106}," -m",[65,5141,5142],{"class":156}," venv",[65,5144,5145],{"class":156}," .venv\n",[65,5147,5148,5151,5153,5156,5159,5162],{"class":67,"line":79},[65,5149,5150],{"class":5135},".venv\u002Fbin\u002Fpython",[65,5152,5139],{"class":106},[65,5154,5155],{"class":156}," pip",[65,5157,5158],{"class":156}," install",[65,5160,5161],{"class":106}," -r",[65,5163,5164],{"class":156}," requirements.txt\n",[65,5166,5167,5169],{"class":67,"line":86},[65,5168,5150],{"class":5135},[65,5170,5171],{"class":156}," 01_autograd.py\n",[65,5173,5174,5176],{"class":67,"line":125},[65,5175,5150],{"class":5135},[65,5177,5178],{"class":156}," 02_SGD.py\n",[65,5180,5181,5183],{"class":67,"line":142},[65,5182,5150],{"class":5135},[65,5184,5185],{"class":156}," 03_linear.py\n",[13,5187,5188,5189,494,5248,5307],{},"附件保留了这次练习结束时的三个脚本。初始的 ",[65,5190,5192,5210],{"className":5191},[225],[65,5193,5195],{"className":5194},[229],[231,5196,5197],{"xmlns":233},[235,5198,5199,5207],{},[238,5200,5201],{},[248,5202,5203,5205],{},[241,5204,252],{},[254,5206,256],{},[258,5208,5209],{"encoding":260},"x^2",[65,5211,5213],{"className":5212,"ariaHidden":266},[265],[65,5214,5216,5219],{"className":5215},[270],[65,5217],{"className":5218,"style":300},[274],[65,5220,5222,5225],{"className":5221},[279],[65,5223,252],{"className":5224},[279,280],[65,5226,5228],{"className":5227},[310],[65,5229,5231],{"className":5230},[314],[65,5232,5234],{"className":5233},[318],[65,5235,5237],{"className":5236,"style":300},[322],[65,5238,5239,5242],{"style":325},[65,5240],{"className":5241,"style":330},[329],[65,5243,5245],{"className":5244},[334,335,336,337],[65,5246,256],{"className":5247},[279,337],[65,5249,5251,5269],{"className":5250},[225],[65,5252,5254],{"className":5253},[229],[231,5255,5256],{"xmlns":233},[235,5257,5258,5266],{},[238,5259,5260],{},[248,5261,5262,5264],{},[241,5263,252],{},[254,5265,613],{},[258,5267,5268],{"encoding":260},"x^4",[65,5270,5272],{"className":5271,"ariaHidden":266},[265],[65,5273,5275,5278],{"className":5274},[270],[65,5276],{"className":5277,"style":300},[274],[65,5279,5281,5284],{"className":5280},[279],[65,5282,252],{"className":5283},[279,280],[65,5285,5287],{"className":5286},[310],[65,5288,5290],{"className":5289},[314],[65,5291,5293],{"className":5292},[318],[65,5294,5296],{"className":5295,"style":300},[322],[65,5297,5298,5301],{"style":325},[65,5299],{"className":5300,"style":330},[329],[65,5302,5304],{"className":5303},[334,335,336,337],[65,5305,613],{"className":5306},[279,337]," 求导示例和单次参数更新代码，则保留在这篇记录的正文中。",[5309,5310,5311],"style",{},"html pre.shiki code .slEfO, html code.shiki .slEfO{--shiki-default:#F47067;--shiki-default-font-style:inherit;--shiki-light:#D73A49;--shiki-light-font-style:inherit;--shiki-cyber:#54B9FF;--shiki-cyber-font-style:italic}html pre.shiki code .sWURj, html code.shiki .sWURj{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#EEF0F9}html pre.shiki code .sf2cM, html code.shiki .sf2cM{--shiki-default:#F47067;--shiki-light:#D73A49;--shiki-cyber:#EEF0F9}html pre.shiki code .sq3c-, html code.shiki .sq3c-{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#00DAEF}html pre.shiki code .sgENW, html code.shiki .sgENW{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#FFD493}html pre.shiki code .sA9aF, html code.shiki .sA9aF{--shiki-default:#F69D50;--shiki-default-font-style:inherit;--shiki-light:#E36209;--shiki-light-font-style:inherit;--shiki-cyber:#4BF3C8;--shiki-cyber-font-style:italic}html pre.shiki code .svkvy, html code.shiki .svkvy{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#EEF0F9}html pre.shiki code .sWqY2, html code.shiki .sWqY2{--shiki-default:#96D0FF;--shiki-light:#032F62;--shiki-cyber:#FFD493}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html.cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html pre.shiki code .sST-V, html code.shiki .sST-V{--shiki-default:#F47067;--shiki-light:#D73A49;--shiki-cyber:#54B9FF}html pre.shiki code .s0kv9, html code.shiki .s0kv9{--shiki-default:#F47067;--shiki-light:#005CC5;--shiki-cyber:#FFD493}html pre.shiki code .sLsJe, html code.shiki .sLsJe{--shiki-default:#F69D50;--shiki-light:#6F42C1;--shiki-cyber:#00DAEF}",{"title":61,"searchDepth":79,"depth":86,"links":5313},[5314,5315,5318,5322,5323,5325],{"id":51,"depth":79,"text":51},{"id":1535,"depth":79,"text":1535,"children":5316},[5317],{"id":2430,"depth":86,"text":2431},{"id":2729,"depth":79,"text":2730,"children":5319},[5320],{"id":3310,"depth":86,"text":5321},"zero_grad() 和 no_grad() 容易混淆",{"id":3426,"depth":79,"text":3426},{"id":4577,"depth":79,"text":5324},"写成 w * 4 后，为什么停在 1.25",{"id":5048,"depth":79,"text":5048},[5327],"技术","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression\u002Fcmu-pytorch-autograd-cover-20261015.webp","梯度曲面上的小球沿路径下降，前侧面板展示散点与拟合直线的技术插画","2026-09-03","从自动求导和手动梯度下降开始，比较学习率，再用 SGD 训练线性回归。记录 w * 4 写错后停在 1.25 的原因和修正结果。",false,"md",{"codeUrl":33,"attachmentUrl":40},"\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression",null,{"title":5,"description":5331},"CMU 15-779","posts\u002Fcmu-15-779-pytorch-autograd-linear-regression",[5338,5341,5342,5343],"PyTorch","自动求导","线性回归","5pdX5DpTXWBeeoDAeMvAvjimDFBsN9Phn_mQtOBbL0Y",[5346,5363,5379,5393,5407,5421,5435,5450,5463,5475,5490,5502,5515,5527,5540,5553,5568,5582,5593,5605,5619,5632,5646,5658,5672,5684,5699,5713,5726,5737,5741,5758,5773,5786,5798,5810,5823,5836],{"slug":5347,"path":5348,"title":5349,"date":5350,"tags":5351,"description":5356,"cover":5357,"coverAlt":5358,"excerpt":5359,"draft":5332,"hidden":5332,"published":82,"readingTime":182,"series":5360,"seriesOrder":182,"codeUrl":5361,"attachmentUrl":5362},"cs336-cross-entropy-adamw-training-loop","\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop","Stanford CS336 学习记录 08：数值稳定交叉熵、AdamW、余弦调度与完整训练闭环","2026-10-27",[5352,5341,5353,5354,5355],"CS336","AdamW","CrossEntropy","训练闭环","用 LogSumExp 消去交叉熵中的指数溢出，手写解耦权重衰减的 AdamW 优化器、全局 L2 梯度裁剪、Warmup 余弦退火调度、memmap 数据采样、Checkpoint 序列化与 Top-p 解码，完成 CS336 Assignment 1 全部 48 项测试。","\u002Fimages\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop\u002Fcs336-cross-entropy-adamw-training-loop-cover.webp","滑动窗口采样、余弦退火曲线、双动量优化轨迹与检查点归档构成的训练闭环 AI 辅助概念插画","作为 Stanford CS336 Assignment 1 系列的收官篇，我们已经拥有了分词器（bpe.py、tokenizer.py）和完整的语言模型架构（transformer.py）。要让模型在语料上真正训练并生成文本，还需","Stanford CS336","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fb5c93ad0cc73178b2b2a62e518a0900da47a0fb7","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-08-adamw-training-loop-20261027.zip",{"slug":5364,"path":5365,"title":5366,"date":5367,"tags":5368,"description":5375,"cover":5376,"coverAlt":5377,"excerpt":5378,"draft":5332,"hidden":5332,"published":82,"readingTime":182},"editcache-video-dit-inference-acceleration","\u002Fposts\u002Feditcache-video-dit-inference-acceleration","EditCache：视频 Diffusion Transformer (DiT) 局部编辑的跨步特征复用与区域重算加速","2026-10-25",[5369,5370,5371,5372,5373,5374],"扩散模型","视频生成","AI Infra","推理加速","GPU","论文解读","拆解视频扩散模型（Wan2.1-VACE）局部编辑中的三重算力冗余，通过可重入 Latent、跨步 Block 隐状态缓存复用、带 Halo 边界的局部区域重算与自适应推理起点，在单卡 H100 上实现 2.46× 模型计算加速与 1.49× 严格同步端到端加速。","\u002Fimages\u002Fposts\u002Feditcache-video-dit-inference-acceleration\u002Feditcache-dit-cover-20261025.webp","视频扩散 Transformer 时空三维体素网格中冻结背景缓存与暖橙色局部编辑重算通道的等距技术插画","在基于 Diffusion Transformer（DiT）的视频编辑与局部重绘（Video Inpainting \u002F Local Editing）任务中，用户的典型工作流并不是一次性生成最终成片，而是针对同一段源视频反复调整掩码区",{"slug":5380,"path":5381,"title":5382,"date":5367,"tags":5383,"description":5388,"cover":5389,"coverAlt":5390,"excerpt":5391,"draft":5332,"hidden":5332,"published":82,"readingTime":182,"series":5360,"seriesOrder":169,"codeUrl":5361,"attachmentUrl":5392},"cs336-bpe-tokenizer-training-encoding","\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding","Stanford CS336 学习记录 07：Byte-Level BPE 分词器训练、倒排索引加速与流式编码",[5352,5384,5385,5386,5387],"Tokenizer","BPE","Tiktoken","算法优化","实现 GPT-2 正则预分词、特殊 token 硬边界切分、基于 pair-to-word 倒排索引的增量 BPE 合并算法（\u003C 0.25s 通过速度基准），以及与 tiktoken 完全对齐的 Tokenizer。","\u002Fimages\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding\u002Fcs336-bpe-tokenizer-training-encoding-cover.webp","UTF-8 字节块经过正则切分闸口后沿合并树聚合成子词单元的 AI 辅助概念插画","前面六篇完成了 Transformer 语言模型本身的全部神经网络组件。要让模型真正处理自然语言文本，还需要打通入口处的 ByteLevel BPE（BytePair Encoding）分词器。这一篇记录 cs336basics\u002Fb","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-07-bpe-tokenizer-20261025.zip",{"slug":5394,"path":5395,"title":5396,"date":5397,"tags":5398,"description":5403,"cover":5404,"coverAlt":5405,"excerpt":5406,"draft":5332,"hidden":5332,"published":82,"readingTime":182},"in-situ-autoguidance-editable-scientific-illustration","\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration","从原位自校正引导（In-situ Autoguidance）到可编辑科研插图生成（AutoFigure-Edit）","2026-10-24",[5369,5399,5400,5401,5402,5374],"多模态","Agent","ICML","ACL","串联两项生成式模型与多模态智能体研究：如何通过推理期局部随机微扰在单模型内构造免训练扩散自引导向量（ICML 2025 Workshop 第一作者，arXiv:2510.17136），以及如何通过五阶段视觉-符号混合流水线生成原生可编辑 SVG 科研插图（ACL 2026 Demo，FigureBench 83.0% 胜率）。","\u002Fimages\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration\u002Fautoguidance-autofigure-cover-20261024.webp","左侧扩散流形自校正轨迹矢量场与右侧五阶段可编辑 SVG 科研插图分层网格相衔接的等距技术插画","生成式视觉模型正在经历从“生成高保真像素（Pixel Synthesis）”向“可控结构化生成（Controllable & Editable Generation）”的演进。在这一进程中，我作为第一作者或核心贡献者参与了两项分别处",{"slug":5408,"path":5409,"title":5410,"date":5411,"tags":5412,"description":5416,"cover":5417,"coverAlt":5418,"excerpt":5419,"draft":5332,"hidden":5332,"published":82,"readingTime":182,"series":5360,"seriesOrder":147,"codeUrl":5361,"attachmentUrl":5420},"cs336-pre-norm-transformer-block-lm","\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm","Stanford CS336 学习记录 06：Pre-Norm Transformer Block 与完整语言模型装配","2026-10-23",[5352,5341,5413,5414,5415],"Transformer","Pre-Norm","FLOPs","将 RMSNorm、带 RoPE 的因果多头自注意力和 SwiGLU 组装成 Pre-Norm Transformer Block，搭建完整 TransformerLM，并核算 GPT-2 XL 的参数量与前向 FLOPs。","\u002Fimages\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm\u002Fcs336-pre-norm-transformer-block-lm-cover.webp","干净残差流主干穿过多个 Pre-Norm 子层并汇聚到词表投影层的 AI 辅助概念插画","前五篇把 Linear、Embedding、RMSNorm、SwiGLU、RotaryPositionalEmbedding 和因果 MultiHeadSelfAttention 全部单独打通了。这一篇进入 Assignment 1","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-06-transformer-block-lm-20261023.zip",{"slug":5422,"path":5423,"title":5424,"date":5411,"tags":5425,"description":5431,"cover":5432,"coverAlt":5433,"excerpt":5434,"draft":5332,"hidden":5332,"published":82,"readingTime":4283},"xgrammar-concurrent-cache-generation-fix","\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix","XGrammar 结构化解码缓存剖析：Unlimited 模式并发 Clear 字节记账竞态与代际修复（PR #1011）",[5371,5426,5427,5428,5429,5430],"C++","XGrammar","结构化解码","并发编程","源码分析","拆解 XGrammar GrammarCompiler 的双层缓存架构与 ThreadSafeLRUCache 锁分离实现，定位 Unlimited 模式下 Clear() 字节残留与在途编译（In-flight Computation）跨代污染根因，给出代际号 + 读写锁原子修复与零 sleep 确定性并发 GoogleTest 验证。","\u002Fimages\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix\u002Fxgrammar-cache-fix-cover-20261022.webp","双层磨砂玻璃流道经暖橙色代际闸门过滤后汇入晶格缓存矩阵的等距技术插画","在大语言模型服务（如 SGLang、vLLM）的结构化输出场景中，JSON Schema 或 EBNF 语法需要先编译为词表级掩码缓存（CompiledGrammar），才能在每个 Decode 步以微秒级开销生成合法 token ",{"slug":5436,"path":5437,"title":5438,"date":5439,"tags":5440,"description":5446,"cover":5447,"coverAlt":5448,"excerpt":5449,"draft":5332,"hidden":5332,"published":82,"readingTime":4283},"multi-constraint-agent-routing-dag-eval","\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval","多约束生产级 Agent 系统实践：两阶段工具路由、DAG 局部重规划与轨迹级评测闭环","2026-10-22",[5400,5441,5442,5443,5444,5445],"LLM","工具调用","上下文工程","DPO","强化学习","围绕跨境酒旅等强耦合多约束业务场景，拆解向量 Top-K 召回 + LLM 精排 + 跨语言实体链接与 Schema 门控（96%+ 调用合法率）、基于状态机的 DAG 局部子图重规划、Observation 冗余裁剪（节省 40% Token），以及从线上失败恢复轨迹构造 SFT\u002FDPO 偏好对的评测与训练闭环。","\u002Fimages\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval\u002Fagent-dag-routing-cover-20261020.webp","候选模块经漏斗滤网进入双分支有向无环图轨道并在受阻节点触发暖橙色局部旁路重连的等距技术插画","在玩具级 Demo 里，几个工具加上一段 ReAct 循环就能跑通大多数单步问答。但当 Agent 进入跨境酒旅、复杂交易或研发沙箱等真实业务生产环境时，单靠线性 ReAct 循环很快会撞上四堵墙： 1. 工具空间与参数耦合爆炸：候",{"slug":5451,"path":5452,"title":5453,"date":5454,"tags":5455,"description":5459,"cover":5460,"coverAlt":5461,"excerpt":5462,"draft":5332,"hidden":5332,"published":82,"readingTime":4352},"llm-mrd-distillation-auditable-review-agents","\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents","多视角推理蒸馏（LLM-MRD）与可审计生成式评审系统（DeepReviewer 2.0）深度拆解","2026-10-21",[5456,5399,5457,5400,5458,5374],"知识蒸馏","Qwen2.5-VL","Post-training","对比大模型落地的两条互补路径：如何用 Qwen2.5-VL 离线多视角分析结合 MLP 残差校准与温度缩放 KL 蒸馏，将推理能力压入 358M 学生模型实现 17.2 ms 独立推理（LLM-MRD，DASFAA 2026 Oral）；以及如何通过主张-证据-风险台账、PASA 可比性门控与导出预算校验构建可审计的同行评审 Agent（DeepReviewer 2.0）。","\u002Fimages\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents\u002Fmrd-review-distill-cover-20261016.webp","三块半透明几何棱镜发出平行光束穿过中央校准环汇聚至右侧高密度晶体模块的等距技术插画","把大语言模型或视觉语言模型（VLM）的深度推理能力用到实际系统中，通常会面临一个两难选择： 如果场景受限于毫秒级在线延迟（如高吞吐多模态内容审核与虚假信息检测）：直接在线调用 7B～72B 的多模态大模型做长链条思维链（CoT）生成",{"slug":5464,"path":5465,"title":5466,"date":5454,"tags":5467,"description":5470,"cover":5471,"coverAlt":5472,"excerpt":5473,"draft":5332,"hidden":5332,"published":82,"readingTime":169,"series":5360,"seriesOrder":142,"codeUrl":5361,"attachmentUrl":5474},"cs336-rope-rotary-position-embedding","\u002Fposts\u002Fcs336-rope-rotary-position-embedding","Stanford CS336 学习记录 05：RoPE 旋转位置编码与带位置的多头注意力",[5352,5341,5413,5468,5469],"RoPE","Attention","从二维平面正交旋转推导 RoPE 的奇偶特征配对公式，理解为什么内积只依赖相对位置差 (i - j)，并用 register_buffer 预计算 cos\u002Fsin 缓存接入因果多头自注意力。","\u002Fimages\u002Fposts\u002Fcs336-rope-rotary-position-embedding\u002Fcs336-rope-rotary-position-embedding-cover.webp","二维特征平面中的旋转向量对与预计算余弦正弦网格的 AI 辅助概念插画","上一篇完成了不含位置编码的因果多头自注意力。如果只看那一套矩阵乘法，把输入序列前两个 token 交换顺序（并在因果 mask 允许的范围内看同一个后缀 query 对它们的点积），点积本身并不知道哪个 key 离当前 query ","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-05-rope-mha-20261021.zip",{"slug":5476,"path":5477,"title":5478,"date":5479,"tags":5480,"description":5486,"cover":5487,"coverAlt":5488,"excerpt":5489,"draft":5332,"hidden":5332,"published":82,"readingTime":182},"llm-gateway-semantic-cache-dynamic-batching","\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching","高并发 LLM 推理网关架构实践：两级精确\u002F语义缓存、50ms 有界动态批处理与尾部延迟治理","2026-10-19",[5371,5481,5482,5483,5484,5485],"LLM 推理","分布式系统","Redis","FastAPI","后端","深度拆解生产级大模型推理网关 LLM-Gateway 的核心系统设计：基于 SHA-256 与 Redis 向量余弦相似度的两级精确\u002F语义缓存、50ms 有界等待动态批处理调度器（Dynamic Batching）、令牌桶多租户限流与 Prometheus 全链路可观测性，将命中请求 P99 延迟从 5,307 ms 压降至 17 ms。","\u002Fimages\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching\u002Fllm-gateway-cover-20261019.webp","高并发请求流依次穿过精确哈希滤网、向量球面语义缓存层与 50ms 动态批处理时间窗拼装器的等距系统架构插画","在将大语言模型接入高并发业务系统时，如果让上游微服务直接裸连底层推理引擎（vLLM \u002F SGLang 或外部模型 API），系统很快会暴露出三个典型的生产痛点： 1. 高频重复与近义查询消耗巨量算力：大量用户请求在语义上高度等价（例",{"slug":5491,"path":5492,"title":5493,"date":5479,"tags":5494,"description":5496,"cover":5497,"coverAlt":5498,"excerpt":5499,"draft":5332,"hidden":5332,"published":82,"readingTime":4283,"series":5360,"seriesOrder":125,"codeUrl":5500,"attachmentUrl":5501},"cs336-multihead-attention-shapes","\u002Fposts\u002Fcs336-multihead-attention-shapes","Stanford CS336 学习记录 04：多头注意力怎样拆头、计算和合头",[5352,5341,5413,5469,5495],"张量形状","沿着 (2, 5, 64) 的输入追踪 QKV 投影、reshape、transpose、共享因果 mask 和合头，理解每个头仍处理完整序列，以及最后为什么还需要输出 Linear。","\u002Fimages\u002Fposts\u002Fcs336-multihead-attention-shapes\u002Fcs336-multihead-attention-shapes-cover.webp","宽向量束分成多条并行特征轨道后重新合并的 AI 辅助概念插画","上一篇完成了缩放点积注意力。那个函数接收已经准备好的 Q、K、V，计算匹配分数，应用 mask，再用权重汇总 V。这一篇继续把它放进一个带参数的多头自注意力模块。 这次最需要看清的是“拆头”。代码里只是几行 reshape 和 tr","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002F31363223498fafd1528c550a51efbe0aed231736","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-04-multihead-attention-20261019.zip",{"slug":5503,"path":5504,"title":5505,"date":5506,"tags":5507,"description":5511,"cover":5512,"coverAlt":5513,"excerpt":5514,"draft":5332,"hidden":5332,"published":82,"readingTime":187},"qwen25-sft-dpo-alignment-tax-ablation","\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation","大模型后训练偏好对齐实录：Qwen2.5 SFT 多维消融、动态参考 DPO 与对齐税（Alignment Tax）量化分析","2026-10-18",[5458,5444,5508,5509,5445,5510],"SFT","LoRA","Qwen2.5","基于 Qwen2.5 完整复现 Instruct SFT → Bradley-Terry DPO 两阶段后训练管线：通过 3 随机种子消融 LoRA Rank、学习率与数据规模（GSM8K +12.73pp），实现免双倍权重的动态 Adapter 切换 DPO 训练，并定量揭示偏好对齐对确定性数学推理的对齐税效应。","\u002Fimages\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation\u002Fqwen25-sft-dpo-cover-20261018.webp","左侧监督微调轨迹流经中央 Bradley-Terry 偏好分流天平分离出正负样本通道并连接右侧对齐税监控仪表盘的等距技术插画","在大语言模型后训练（PostTraining）管线中，监督微调（SFT）负责教会模型任务格式与领域推理模式，而直接偏好优化（DPO, Direct Preference Optimization）则利用成对偏好数据 $(x, yw,",{"slug":5516,"path":5517,"title":5518,"date":5506,"tags":5519,"description":5521,"cover":5522,"coverAlt":5523,"excerpt":5524,"draft":5332,"hidden":5332,"published":82,"readingTime":204,"series":5360,"seriesOrder":86,"codeUrl":5525,"attachmentUrl":5526},"cs336-softmax-attention-mask","\u002Fposts\u002Fcs336-softmax-attention-mask","Stanford CS336 学习记录 03：Softmax 的数值稳定与 Attention Mask",[5352,5341,5413,5520,5469],"Softmax","从减去最大值为何不改变 Softmax 开始，理清函数与有参数层的区别，再沿着 Q、K、V 的形状理解缩放点积注意力，以及 mask 为什么必须填负无穷。","\u002Fimages\u002Fposts\u002Fcs336-softmax-attention-mask\u002Fcs336-softmax-attention-mask-cover.webp","注意力分数、权重与因果遮罩的 AI 辅助概念插画","上一篇写到 SwiGLU，接下来进入 Attention。真正开始相乘之前，先做了 Softmax：它把一组分数变成总和为 1 的权重，后面用这些权重汇总不同 token 提供的信息。 这段学习里，有两个很具体的问题值得记下来。一个","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fa39eb684710817f7f389626423fcb26ca0f6d629","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-03-softmax-sdpa-20261018.zip",{"slug":5528,"path":5529,"title":5530,"date":5531,"tags":5532,"description":5535,"cover":5536,"coverAlt":5537,"excerpt":5538,"draft":5332,"hidden":5332,"published":82,"readingTime":187,"series":5360,"seriesOrder":79,"codeUrl":5525,"attachmentUrl":5539},"cs336-rmsnorm-swiglu","\u002Fposts\u002Fcs336-rmsnorm-swiglu","Stanford CS336 学习记录 02：RMSNorm 的广播与 SwiGLU 的门控","2026-10-16",[5352,5341,5413,5533,5534],"RMSNorm","SwiGLU","用一个 token 向量理解均方根归一化，拆开 dim=-1 与 keepdim=True 的作用，再沿着两条分支读懂 SiLU、逐元素乘法和 SwiGLU。","\u002Fimages\u002Fposts\u002Fcs336-rmsnorm-swiglu\u002Fcs336-rmsnorm-swiglu-cover.webp","特征尺度整理和双分支门控的 AI 辅助概念插画","上一篇把 Linear 和 Embedding 的参数形状理顺了。继续往下，最先遇到的是 RMSNorm：公式不算长，但把它落到一批 token 上时，dim=1、keepdim=True 和广播必须一起理解。 然后是 SiLU 和","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-02-rmsnorm-swiglu-20261016.zip",{"slug":5541,"path":5542,"title":5543,"date":5544,"tags":5545,"description":5548,"cover":5549,"coverAlt":5550,"excerpt":5551,"draft":5332,"hidden":5332,"published":82,"readingTime":199,"series":5360,"seriesOrder":68,"codeUrl":5525,"attachmentUrl":5552},"cs336-linear-embedding-shapes","\u002Fposts\u002Fcs336-linear-embedding-shapes","Stanford CS336 学习记录 01：从 Linear、Embedding 搞清张量形状","2026-10-14",[5352,5341,5413,5546,5547],"Linear","Embedding","从 PyTorch 类、参数与 forward 的关系开始，理解 Linear 为什么保存两行三列的权重、计算时为什么要转置，再区分 Embedding 查表与线性变换。","\u002Fimages\u002Fposts\u002Fcs336-linear-embedding-shapes\u002Fcs336-linear-embedding-shapes-cover.webp","token 方块、向量表与矩阵变换的 AI 辅助概念插画","开始沿着 Stanford CS336 的 Assignment 1 一步步实现。BPE 之前已经接触过，这轮先跳过分词器，从 Transformer 的基础层开始。这里记录的是当时写代码时真正卡住的问题，以及对应测试能确认到的范围","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-01-linear-embedding-20261014.zip",{"slug":5554,"path":5555,"title":5556,"date":5557,"tags":5558,"description":5562,"cover":5563,"coverAlt":5564,"excerpt":5565,"draft":5332,"hidden":5332,"published":82,"readingTime":1991,"series":5338,"seriesOrder":4283,"codeUrl":5566,"attachmentUrl":5567},"cmu-15-779-kv-cache-granularity-study","\u002Fposts\u002Fcmu-15-779-kv-cache-granularity-study","CMU 15-779 学习记录 14：KV 页大小、共享边界与 CPU 实现成本","2026-10-12",[5338,5559,5560,5561],"KV cache","PagedAttention","系统实验","用 210 组 CPU 配置比较连续与分页 KV cache，观察 31\u002F32-token 共享前缀的页边界，分别记录活跃槽位、池容量、复制计数和完整调用耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling\u002Fcmu-serving-cover-20261019.webp","浅色手绘分页存储抽屉与共享前缀卡片，旁边连接不同长度的请求队列","个人课程项目中的 CPU 实验记录，延续记录 11 的分页 KV 实现。已完成 210 组配置与 1470 次计时，输出均通过连续存储参考对照；本篇不代表正式作业提交、整门课程或项目全部交付完成。 同样是共享 31 个 token，","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F200faf569eccf94fb125e65110bb5d14094d454a","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fkv-cache-granularity-20261019.zip",{"slug":5569,"path":5570,"title":5571,"date":5572,"tags":5573,"description":5576,"cover":5577,"coverAlt":5578,"excerpt":5579,"draft":5332,"hidden":5332,"published":82,"readingTime":142,"series":5338,"seriesOrder":2011,"codeUrl":5580,"attachmentUrl":5581},"cmu-15-779-moe-routing-capacity","\u002Fposts\u002Fcmu-15-779-moe-routing-capacity","CMU 15-779 学习记录 13：MoE 路由、稀疏分发与容量溢出","2026-10-09",[5338,5574,5575],"MoE","Routing","用 CPU top-2 MoE 检查稀疏分发的输出与梯度，统计 expert-token 行数，并保留容量丢弃会改变输出的反例。","\u002Fimages\u002Fposts\u002Fcmu-15-779-moe-routing-capacity\u002Fcmu-moe-cover-20261019.webp","浅色手绘分拣器将小珠分配到四个专家工作屋，队列呈现负载不均","第 18 讲的个人配套实验。36 号脚本串行运行 CPU 专家，检查 topk 路由后的分发与合并；没有训练 router，也没有执行 GPU grouped GEMM 或跨设备 alltoall。 上一节记录低秩适配，这次转向每个","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fc4851e0cedadf13af7c421867e83c8ff57244fa0","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-parallel-serving-lora-moe-20261019.zip",{"slug":5583,"path":5584,"title":5585,"date":5586,"tags":5587,"description":5589,"cover":5590,"coverAlt":5591,"excerpt":5592,"draft":5332,"hidden":5332,"published":82,"readingTime":142,"series":5338,"seriesOrder":1991,"codeUrl":5580,"attachmentUrl":5581},"cmu-15-779-lora-uniform-quantization","\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization","CMU 15-779 学习记录 12：LoRA 首步梯度、权重合并与均匀 4-bit 打包","2026-10-06",[5338,5509,5588],"Quantization","用一次 CPU SGD 更新检查 LoRA 初始化、冻结基座与合并等价，再分别记录均匀 4-bit 存储字节数和量化误差。","\u002Fimages\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization\u002Fcmu-lora-cover-20261019.webp","浅色手绘的大计算板、两块细长适配板和成对打包的小方块","第 17 讲的个人配套实验，35 号脚本实际执行一次 CPU LoRA 更新，并实现均匀 4bit 打包。实验检查机制与数值误差，不评估微调任务效果，也没有实现完整 QLoRA。 这次把两个问题分开记录：低秩分支是否能合并进原权重，",{"slug":5594,"path":5595,"title":5596,"date":5597,"tags":5598,"description":5602,"cover":5563,"coverAlt":5603,"excerpt":5604,"draft":5332,"hidden":5332,"published":82,"readingTime":169,"series":5338,"seriesOrder":204,"codeUrl":5580,"attachmentUrl":5581},"cmu-15-779-kv-serving-speculative-sampling","\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling","CMU 15-779 学习记录 11：分页 KV、连续批处理与推测采样","2026-10-03",[5338,5599,5600,5601],"LLM Serving","KV Cache","Speculative Decoding","在 CPU 上验证 KV 页共享和写时复制，用单位成本模型检查连续批处理，再用条件概率小模型核对推测采样的接受与补采样。","浅色手绘的共享卡片抽屉与不断补入小珠子的请求托盘","第 15、16 讲的个人配套实验。33 号实现 CPU 分页 KV 存储，37 号模拟请求准入，34 号运行小概率模型的推测采样。它们分别验证存储、调度和分布问题，没有测量真实 LLM 服务吞吐或 GPU 延迟。 记录 08已经对照",{"slug":5606,"path":5607,"title":5608,"date":5609,"tags":5610,"description":5615,"cover":5616,"coverAlt":5617,"excerpt":5618,"draft":5332,"hidden":5332,"published":82,"readingTime":4312},"qwen3-mid-post-train-mlx","\u002Fposts\u002Fqwen3-mid-post-train-mlx","在 M2 Pro 上完成 Qwen3 两阶段训练：Mid Train、SFT 与可复核评估","2026-09-30 11:19:10",[5441,5611,5612,5509,5613,5508,5614],"Qwen3","MLX","持续预训练","Apple Silicon","用 Qwen3 0.6B Base 和 MLX 完成领域持续训练与监督微调，记录数据划分、聊天模板、损失掩码、数值稳定性修复和独立测试结果。","\u002Fimages\u002Fposts\u002Fqwen3-mid-post-train-mlx\u002Fqwen3-mlx-cover-20261018.webp","Apple Silicon 统一内存芯片底座支撑两阶段持续预训练与带精确 Loss 掩码滤网的监督微调桥接插画","在一台 M2 Pro、16 GB 内存的 Mac 上，能否把一个 Base 语言模型接续训练成输出 JSON 的运维事件分级助手？这次项目完成了领域持续训练、监督微调、检查点选择、重新加载和独立测试，并保留了权重、日志和数据哈希。 ",{"slug":5620,"path":5621,"title":5622,"date":5623,"tags":5624,"description":5628,"cover":5629,"coverAlt":5630,"excerpt":5631,"draft":5332,"hidden":5332,"published":82,"readingTime":147,"series":5338,"seriesOrder":199,"codeUrl":5580,"attachmentUrl":5581},"cmu-15-779-cpu-parallelism-pipeline","\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline","CMU 15-779 学习记录 10：加权梯度、状态分片与同步流水线","2026-09-30",[5338,5625,5626,5627],"Parallel Training","Gloo","Pipeline","用双 CPU 进程 Gloo 检查不等数据分片的梯度，推导持久状态字节数，再用依赖模型比较 GPipe 与同步 1F1B 的激活存活。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline\u002Fcmu-parallel-cover-20261019.webp","浅色手绘的四阶段桌面流水线与按数量汇总的托盘","第 12 至 14 讲的个人配套实验，脚本编号 31、32 不对应官方作业。Gloo 实际运行两个 CPU 进程；张量切分、状态字节数和流水线分别按下文的数学或成本模型解释。真实 GPU 实测按学习安排跳过。 上一节检查单个计算图的",{"slug":5633,"path":5634,"title":5635,"date":5636,"tags":5637,"description":5640,"cover":5641,"coverAlt":5642,"excerpt":5643,"draft":5332,"hidden":5332,"published":82,"readingTime":169,"series":5338,"seriesOrder":187,"codeUrl":5644,"attachmentUrl":5645},"cmu-15-779-cpu-compiler-optimizations","\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations","CMU 15-779 学习记录 09：CPU 编译优化、合法重写与候选搜索","2026-09-27",[5338,5638,5639,5341],"ML Compiler","TorchInductor","用 CPU 实验验证 BatchNorm 折叠、RMSNorm 重排与非线性反例，测量分块候选，并运行 TorchInductor，分别记录首调用与预热后的时间。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations\u002Fcmu-cpu-compiler-cover-20261019.webp","淡紫计算积木经过薄荷绿折叠模具变成浅蓝积木，前方摆放四种候选托盘的浅色手绘插画","编译章节的个人配套实验，编号 30 不对应官方作业。图重写、分块候选和 TorchInductor 均在 CPU 上运行；真实 GPU 实验按学习安排跳过。 上一节记录了分块 Attention 和 KV cache。这次转到编译优","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F9261bea6aedf31cdccbbab2541ca693fc97b3743","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-compiler-optimizations-20261019.zip",{"slug":5647,"path":5648,"title":5649,"date":5650,"tags":5651,"description":5653,"cover":5654,"coverAlt":5655,"excerpt":5656,"draft":5332,"hidden":5332,"published":82,"readingTime":1991,"series":5338,"seriesOrder":182,"codeUrl":5566,"attachmentUrl":5657},"cmu-15-779-attention-online-softmax","\u002Fposts\u002Fcmu-15-779-attention-online-softmax","CMU 15-779 学习记录 08：分块 Attention、在线 Softmax 与 KV cache","2026-09-24",[5338,5469,5520,5652,5341],"FlashAttention","在 CPU 上对照完整 Attention、在线 Softmax 分块实现与 PyTorch SDPA，检查因果掩码、浮点误差和分数块大小，理解 FlashAttention 的一部分算法基础。","\u002Fimages\u002Fposts\u002Fcmu-15-779-attention-online-softmax\u002Fcmu-online-attention-cover-20261016.webp","浅紫色卡片分批进入绿色计算托盘，旁边三个状态容器保存累积量的浅色手绘插画","第 5 讲 Transformers & Attention 的个人配套实验。28\u002F29 号是 CPU 单头前向与 KV cache；后续补入 38 号 CUDA 多头合成张量的显式 SDPA MATH\u002FFLASH 对照。没有手写 ","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcuda-triton-sdpa-20261019.zip",{"slug":5659,"path":5660,"title":5661,"date":5662,"tags":5663,"description":5668,"cover":5669,"coverAlt":5670,"excerpt":5671,"draft":5332,"hidden":5332,"published":82,"readingTime":199,"series":5338,"seriesOrder":169,"codeUrl":5566,"attachmentUrl":5657},"cmu-15-779-matmul-tiling-tree-reduction","\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction","CMU 15-779 学习记录 07：矩阵分块、数据复用与树形归约","2026-09-21",[5338,5664,5665,5666,5667],"CUDA","矩阵乘法","数据复用","归约","用 CPU 模型统计矩阵分块的输入读取次数，再比较串行求和和树形归约的依赖深度，记录补零、同步与浮点加法顺序的边界。","\u002Fimages\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction\u002Fcmu-matmul-reduction-cover-20261016.webp","矩阵积木在共享托盘中复用，旁边的小珠沿树形管道逐层合并的浅色手绘插画","GPU Architecture & CUDA Programming 的个人配套实验。首次记录用 CPU 模型检查数据复用与依赖深度；后续补入单卡 GPU 原生 CUDA 矩阵验证，以及 Triton 两阶段归约和矩阵乘法实测。 ",{"slug":5673,"path":5674,"title":5675,"date":5676,"tags":5677,"description":5680,"cover":5681,"coverAlt":5682,"excerpt":5683,"draft":5332,"hidden":5332,"published":82,"readingTime":199,"series":5338,"seriesOrder":147,"codeUrl":5566,"attachmentUrl":5657},"cmu-15-779-cuda-thread-indexing","\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing","CMU 15-779 学习记录 06：CUDA 线程索引与边界检查","2026-09-18",[5338,5664,5373,5678,5679],"线程索引","边界检查","用 CPU 演示 CUDA 的 block、thread 与全局下标：10 个元素如何分给 3 个线程块，为什么要跳过越界线程，以及这些验证能说明什么。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing\u002Fcmu-cuda-indexing-cover-20261015.webp","分组的浅紫色计算方块与末端边界挡板组成线程分配教具的浅色插画","GPU Architecture & CUDA Programming 的个人配套练习。首次记录为 CPU 索引演示；后续在单卡 GPU 环境补入原生 CUDA 编译、运行与 memcheck，以及 Triton 向量加法实测。 上",{"slug":5685,"path":5686,"title":5687,"date":5688,"tags":5689,"description":5693,"cover":5694,"coverAlt":5695,"excerpt":5696,"draft":5332,"hidden":5332,"published":82,"readingTime":199,"series":5338,"seriesOrder":142,"codeUrl":5697,"attachmentUrl":5698},"cmu-15-779-profiler-operator-batching","\u002Fposts\u002Fcmu-15-779-profiler-operator-batching","CMU 15-779 学习记录 05：Profiler、算子调用与批量推理","2026-09-15",[5338,5341,5690,5691,5692],"Profiler","批量推理","性能分析","对照 ML Systems 基础，用 Profiler 拆开两层网络的执行过程，再固定 1000 个样本扫描 batch size，观察算子调用次数、尾批和 CPU 耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-profiler-operator-batching\u002Fcmu-pytorch-profiling-cover-20261015.webp","放大镜观察线性层与 ReLU 模块，旁边面板呈现执行片段的浅色技术教具插画","本篇对照课程官网的 Introduction to ML Systems 整理，是个人配套 CPU 实验，不是课程官方作业。脚本编号是练习顺序，不对应课程讲次。 前几篇一直在补 PyTorch 基础。到了这一步，我在学习聊天里问：现","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fcecf5110d8a95161bb5763eadfca94cd7a0cc50f","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-profiler-batching-20261015.zip",{"slug":5700,"path":5701,"title":5702,"date":5703,"tags":5704,"description":5707,"cover":5708,"coverAlt":5709,"excerpt":5710,"draft":5332,"hidden":5332,"published":82,"readingTime":187,"series":5338,"seriesOrder":125,"codeUrl":5711,"attachmentUrl":5712},"cmu-15-779-pytorch-linear-relu-absolute-value","\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value","CMU 15-779 学习记录 04：多输出线性层、ReLU 与绝对值网络","2026-09-12",[5338,5341,5665,5705,5706],"ReLU","神经网络","把 nn.Linear 对应到矩阵乘法，弄清多个输出怎样计算，再逐层拆开一个绝对值网络，比较手动设置参数与训练得到的结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value\u002Fcmu-pytorch-absolute-cover-20261015.webp","双分支经过折角模块后汇入紫色 V 形曲线面板的浅色技术教具插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的批量推理与矩阵乘法，我继续把 model(x) 拆开看：一层里有多个输出时，每个数是怎么算出来的？两","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F4cc63904a3a583da4a2122c246a14b43aac38528","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-linear-relu-abs-20261015.zip",{"slug":5714,"path":5715,"title":5716,"date":5717,"tags":5718,"description":5721,"cover":5722,"coverAlt":5723,"excerpt":5724,"draft":5332,"hidden":5332,"published":82,"readingTime":187,"series":5338,"seriesOrder":86,"codeUrl":33,"attachmentUrl":5725},"cmu-15-779-pytorch-dataloader-checkpoints-batching","\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching","CMU 15-779 学习记录 03：DataLoader、模型保存与批量推理","2026-09-09",[5338,5341,5719,5691,5720],"DataLoader","模型保存","用 DataLoader 分批训练平方函数，保存并加载模型参数，再比较 1000 个样本与 1 个样本的推理耗时，最后把批量计算对应到矩阵乘法。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching\u002Fcmu-pytorch-batching-cover-20261015.webp","样本卡片按批次进入计算模块，旁边配有参数存储盒和输出矩阵面板的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的平方函数实验，这一节继续练习分批训练、保存参数和独立预测。最后做了一次计时：同一个小网络，逐个处理 ","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-batching-20261015.zip",{"slug":5727,"path":5728,"title":5729,"date":5730,"tags":5731,"description":5732,"cover":5733,"coverAlt":5734,"excerpt":5735,"draft":5332,"hidden":5332,"published":82,"readingTime":187,"series":5338,"seriesOrder":79,"codeUrl":33,"attachmentUrl":5736},"cmu-15-779-pytorch-linear-module-relu","\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu","CMU 15-779 学习记录 02：nn.Linear、张量形状与 ReLU","2026-09-06",[5338,5341,5706,5705],"从带偏置的线性回归开始，弄清 nn.Linear 和 nn.Module 的分工，再用平方函数观察线性模型的限制、ReLU 的作用和测试点的选择。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu\u002Fcmu-pytorch-relu-cover-20261015.webp","输入经过两层神经网络和 ReLU 折角模块，输出曲线拟合结果的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 上一篇里，我用一个参数拟合了 $y=2x$。这一篇继续顺着往下写，先加上偏置，再把手写参数换成 nn.Linea","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-models-20261015.zip",{"slug":5738,"path":5335,"title":5,"date":5330,"tags":5739,"description":5331,"cover":5328,"coverAlt":5329,"excerpt":5740,"draft":5332,"hidden":5332,"published":82,"readingTime":199,"series":5338,"seriesOrder":68,"codeUrl":33,"attachmentUrl":40},"cmu-15-779-pytorch-autograd-linear-regression",[5338,5341,5342,5343],"本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 开始学习 CMU 15779 时，我会 Python，但还没用过 PyTorch。这次先在 Mac 上用 CPU",{"slug":5742,"path":5743,"title":5744,"date":5745,"tags":5746,"description":5752,"cover":5753,"coverAlt":5754,"excerpt":5755,"draft":5332,"hidden":5332,"published":82,"readingTime":2011,"codeUrl":5756,"attachmentUrl":5757},"embodied-pickplace-lab-robustness","\u002Fposts\u002Fembodied-pickplace-lab-robustness","Pickplace Lab 项目复盘：抓取成功率之外的时延与失败分析","2026-08-29",[5747,5748,5749,5341,5750,5751],"具身智能","模仿学习","MuJoCo","机器人","鲁棒性","从 MuJoCo 抓取搬运闭环出发，记录 22,100 个评测回合。动作平滑在自建任务的 50 ms 延迟下有效，换到 Panda Lift 却没有支持同样的收益。","\u002Fimages\u002Fposts\u002Fembodied-pickplace-lab-robustness\u002Fpickplace-grasp-cover-20261016.webp","MuJoCo 实际渲染的笛卡尔夹爪抓取方块场景，桌面上标出放置目标","Pickplace Lab 从一个简单任务开始：让夹爪抓起方块，搬到指定位置，放下，再退开。先把仿真、示范采集、训练和闭环评测接起来，随后再问：干净条件下成功的策略，换一种执行条件，还能不能完成任务？ 最值得讨论的结果出现在指令延迟","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Ftree\u002Fv0.1.0","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Freleases\u002Fdownload\u002Fv0.1.0\u002Fpickplace-lab-research-v1.zip",{"slug":5759,"path":5760,"title":5761,"date":5762,"tags":5763,"description":5768,"cover":5769,"coverAlt":5770,"excerpt":5771,"draft":5332,"hidden":5332,"published":82,"readingTime":5772},"qwen25-vl-sglang-verl-single-gpu-lab","\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab","从 Mac 到单卡 GPU：Qwen2.5-VL 部署、GRPO 与计数 LoRA 实践","2026-08-19",[5457,5399,5764,5765,5766,5767,5612],"SGLang","verl","GRPO","推理优化","在 M2 Pro 和云端单卡上部署 Qwen2.5-VL-3B，实测连续批处理、前缀缓存与 SGLang 参数调优，再用 verl 完成多模态 GRPO 更新和同条件前后评测，记录有效优化、兼容性问题与未观察到提升的结果。","\u002Fimages\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab\u002Fqvl-lab-cover-20261010.webp","笔记本与单卡计算模块通过图像和 token 光轨连接的实验台插画","这次实验从一个很具体的问题开始：M2 Pro、16 GB 统一内存，能不能在本机跑 Qwen2.5VL？跑起来之后，我又想把图文问答、推理性能测量和强化学习训练接成一条完整流程。 本机的 MLX 4bit 服务和云端的 SGLang",29,{"slug":5774,"path":5775,"title":5776,"date":5777,"tags":5778,"description":5782,"cover":5783,"coverAlt":5784,"excerpt":5785,"draft":5332,"hidden":5332,"published":82,"readingTime":2011},"cycleresearcher-post-training","\u002Fposts\u002Fcycleresearcher-post-training","从 DeepReviewer 2.0 到 CycleResearcher：科研 Agent 如何真正做 Post-training","2026-08-07 17:33:05",[5441,5458,5400,5779,5780,5781],"SimPO","Reward Model","ICLR","区分推理时 Agent 编排与模型后训练，并拆解 CycleResearcher 的领域 SFT、生成式 Reward Model、Iterative SimPO 和评测风险。","\u002Fimages\u002Fposts\u002Fcycleresearcher-post-training\u002Fcycleresearcher-cover-20261018.webp","策略生成模块产出多份候选文稿经多维评审棱镜排序后通过偏好梯度反馈弧闭环优化的等距技术插画","看到一个系统会检索、规划、反思和调用工具，不能据此判断它做过 Posttraining。Agent 描述的是模型在运行时如何完成任务；Posttraining 描述的是预训练之后，是否继续用监督数据、偏好数据或奖励信号更新模型参数。",{"slug":5787,"path":5788,"title":5789,"date":5790,"tags":5791,"description":5793,"cover":5794,"coverAlt":5795,"excerpt":5796,"draft":5332,"hidden":5332,"published":82,"readingTime":4283,"series":5797,"seriesOrder":68},"cuda-fused-rmsnorm-kernel","\u002Fposts\u002Fcuda-fused-rmsnorm-kernel","从 PyTorch 到 CUDA Kernel：融合 RMSNorm 在 RTX 4090D 上加速 4.27 倍","2026-07-22 18:16:50",[5664,5371,5373,5341,5481,5792],"Kernel 优化","从零实现 PyTorch CUDA 扩展，拆解 RMSNorm 的访存瓶颈、warp 归约、向量化加载、算子融合与冷热缓存 benchmark。","\u002Fimages\u002Fposts\u002Fcuda-fused-rmsnorm-kernel\u002Fcuda-rmsnorm-cover-20261018.webp","融合残差加法与向量化 float4 访存经 Warp Shuffle 蝴蝶树归约输出归一化张量的等距硬件架构插画","为了准备 AI Infra 岗位，我做了一个尽量小而完整的 CUDA 项目：不追求堆很多算子，而是选一个真实的 LLM 推理路径，把实现、正确性、benchmark 和性能解释全部做扎实。 项目实现了两个 PyTorch CUDA ","AI Infra 实战",{"slug":5799,"path":5800,"title":5801,"date":5802,"tags":5803,"description":5805,"cover":5806,"coverAlt":5807,"excerpt":5808,"draft":5332,"hidden":5332,"published":82,"readingTime":182,"series":5809,"seriesOrder":68},"agent-memory","\u002Fposts\u002Fagent-memory","Agent 上下文工程与分层记忆架构：从 Observation 剪枝、滚动摘要到向量召回闭环","2026-04-24",[5441,5400,5804,5371],"RAG","结合生产级多约束 Agent 与超 100K Token 科研智能体实践，系统拆解长程会话中的上下文膨胀与注意力稀释问题：涵盖 Observation 结构化剪枝（降低约 40% Token 开销）、滚动工作摘要、向量检索长期记忆及 Prefix Cache 友好的上下文组装。","\u002Fimages\u002Fposts\u002Fagent-memory\u002Fagent-memory-cover-20261018.webp","底层高噪数据经剪枝滤网进入中期滑动窗口与金色状态台账并由顶层向量星座按需召回的三层记忆架构插画","在构建多步工具调用（ToolUse）与长周期自主规划（Autonomous Planning）的 Agent 系统时，上下文管理（Context Engineering）往往比单纯改写 System Prompt 更决定线上稳定性。","Agent 上下文工程",{"slug":5811,"path":5812,"title":5813,"date":5814,"tags":5815,"description":5818,"cover":5819,"coverAlt":5820,"excerpt":5821,"draft":5332,"hidden":5332,"published":82,"readingTime":142,"series":5822,"seriesOrder":79},"backend-stack-deep-dive","\u002Fposts\u002Fbackend-stack-deep-dive","高并发 AI 平台后端基础设施实践：FastAPI 异步网关、SQLAlchemy 2.0 Async、PostgreSQL 与 Docker 沙箱编排","2026-04-07",[5485,5482,5484,5816,5817,5371],"PostgreSQL","Docker","结合 DeepScientist 生产级 AI 科研平台实践，拆解 FastAPI ASGI 异步事件循环、SQLAlchemy 2.0 + AsyncPG 连接池调优、PostgreSQL + MinIO 存储分层与 Docker SDK 动态沙箱容器编排的职责边界与核心设计。","\u002Fimages\u002Fposts\u002Fbackend-stack-deep-dive\u002Fbackend-stack-cover-20261018.webp","ASGI 异步事件循环涡轮将非阻塞协程流分发至 PostgreSQL 连接池、MinIO 存储矩阵与动态 Docker 沙箱的等距架构插画","结合 DeepScientist 生产级 AI 科研平台（服务 7,000+ 用户）的实际工程落地，本文系统拆解支撑高并发流式推理与容器沙箱调度的后端基础设施栈——FastAPI（ASGI 异步网关 + SSE 流式推送）、SQLA","DeepScientist 工程复盘",{"slug":5824,"path":5825,"title":5826,"date":5827,"tags":5828,"description":5831,"cover":5832,"coverAlt":5833,"excerpt":5834,"draft":5332,"hidden":5332,"published":82,"readingTime":204,"series":5835,"seriesOrder":68},"minicode-source-analysis","\u002Fposts\u002Fminicode-source-analysis","MiniCode 架构与工具评测深度拆解：开源终端 Agentic Coding 引擎（1k+ Stars）的 ReAct、MCP 与 30 项基准验证","2026-04-06",[5400,5829,5830,5430,5371],"TypeScript","MCP","深度剖析开源终端 AI 编程智能体 MiniCode（1k+ GitHub Stars）的核心架构：从零重型依赖的 ReAct 规划循环、声明式工具注册表与权限拦截层、MCP 协议帧自动协商，到 30 项可执行 Tool-Use 评测基准（86.7% 完成率、100% 错误恢复率）的失败分布归因与闭环优化。","\u002Fimages\u002Fposts\u002Fminicode-source-analysis\u002Fminicode-cover-20261018.webp","终端编程智能体控制台被 ReAct 状态环、Unified Diff 补丁预览门与 MCP 协议插槽环绕的等距技术插画","MiniCode 是我作为核心开发者参与构建的开源终端 Agentic Coding 引擎（目前已在 GitHub 获得 1,000+ Stars）。项目的核心设计目标是：在不引入庞大运行时框架的前提下，用约 5,000 行高内聚的","AI 项目源码导读",{"slug":5837,"path":5838,"title":5839,"date":5827,"tags":5840,"description":5841,"cover":5842,"coverAlt":5843,"excerpt":5844,"draft":5332,"hidden":5332,"published":82,"readingTime":199,"series":5822,"seriesOrder":68},"deepscientist-tech-stack","\u002Fposts\u002Fdeepscientist-tech-stack","DeepScientist 架构与运行时全解析：超 100K 上下文科研智能体、MCP 沙箱与高并发流式系统",[5400,5371,5830,5817,5484,5482],"深度拆解服务 7,000+ 用户的科研自主智能体平台 DeepScientist：从一题一仓库（Quest）持久化状态机、MCP 工具栈（memory \u002F artifact \u002F bash_exec）、Docker SDK 动态容器编排，到 SSE + WebSocket 双流通信与 PostgreSQL \u002F Redis \u002F Prometheus 生产级基础设施。","\u002Fimages\u002Fposts\u002Fdeepscientist-tech-stack\u002Fdeepscientist-cover-20261018.webp","科研智能体编排核心通过 memory、artifact 与隔离容器沙箱三座协议桥连接双流工作台的等距架构插画","DeepScientist（deepscientist.cc）是我们在西湖大学自然语言处理实验室研发的自主科学发现与科研管理平台，目前累计服务 7,000+ 科研用户。与单轮对话或纯文本文献问答工具不同，科研智能体需要在数小时乃至数",1795088943522]