[{"data":1,"prerenderedAt":3516},["ShallowReactive",2],{"post-\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching":3,"archive-public-posts":3017},{"id":4,"title":5,"body":6,"categories":2997,"cover":2999,"coverAlt":3000,"date":3001,"description":3002,"draft":3003,"extension":3004,"haloPublished":1172,"haloSync":1172,"hidden":3003,"legacy":3003,"meta":3005,"navigation":1172,"path":3006,"published":3003,"readingTime":3007,"seo":3008,"series":3007,"seriesOrder":3007,"stem":3009,"tags":3010,"__hash__":3016},"posts\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching.md","高并发 LLM 推理网关架构实践：两级精确\u002F语义缓存、50ms 有界动态批处理与尾部延迟治理",{"type":7,"value":8,"toc":2981},"minimark",[9,13,36,69,72,77,80,90,92,96,101,104,537,903,922,926,1126,1936,2077,2079,2083,2100,2160,2252,2269,2323,2326,2473,2558,2561,2827,2895,2897,2901,2977],[10,11,12],"p",{},"在将大语言模型接入高并发业务系统时，如果让上游微服务直接裸连底层推理引擎（vLLM \u002F SGLang 或外部模型 API），系统很快会暴露出三个典型的生产痛点：",[14,15,16,24,30],"ol",{},[17,18,19,23],"li",{},[20,21,22],"strong",{},"高频重复与近义查询消耗巨量算力","：大量用户请求在语义上高度等价（例如“如何重置登录密码？”与“忘记密码怎么找回？”），每次都穿透到底层跑完整的 Prefill + Decode，不仅浪费昂贵的 GPU 算力，还拉高了整体排队水位；",[17,25,26,29],{},[20,27,28],{},"碎片化小请求导致 GPU 批次利用率低下","：突发流量往往以单条请求（Batch Size = 1）零散到达，缺乏网关层的微批聚合（Micro-batching），底层算子长期处于 Memory-Bound 状态；",[17,31,32,35],{},[20,33,34],{},"长尾请求缺乏多租户隔离与熔断观测","：少数超长 Prompt 请求占满并发槽位，导致普通短请求的 P99 尾部延迟飙升至数十秒。",[10,37,38,39,48,49,53,54,57,58,57,61,64,65,68],{},"为此，我设计并实现了开源推理服务网关 ",[20,40,41],{},[42,43,47],"a",{"href":44,"rel":45},"https:\u002F\u002Fgithub.com\u002FLiangQianXing\u002Fllm-gateway",[46],"nofollow","LLM-Gateway","（配套压测产物保存在 ",[50,51,52],"code",{},"experiments\u002Fgateway-bench\u002Fresults.json","）。本文系统拆解其四大核心组件——",[20,55,56],{},"两级精确\u002F语义缓存（Two-Tier Exact & Semantic Cache）","、",[20,59,60],{},"50 ms 有界等待动态批处理调度器（Bounded-Wait Dynamic Batcher）",[20,62,63],{},"令牌桶限流（Token-Bucket Rate Limiter）","与 ",[20,66,67],{},"Prometheus 指标体系","，并给出多组并发压测下的 P50 \u002F P95 \u002F P99 实测数据。",[70,71],"hr",{},[73,74,76],"h2",{"id":75},"一整体架构四层流水线网关拓扑","一、整体架构：四层流水线网关拓扑",[10,78,79],{},"LLM-Gateway 位于业务调用方与底层推理集群之间，所有请求按以下异步非阻塞流水线依次流转：",[81,82,88],"pre",{"className":83,"code":85,"language":86,"meta":87},[84],"language-text","客户端并发请求 (HTTP \u002F SSE)\n   │\n   ▼\n[1. 令牌桶限流与配额网关] ──► 超出租户 QPS \u002F Token 水位时快速返回 HTTP 429\n   │\n   ▼\n[2. L1 精确哈希缓存 (Redis String)] ──► SHA-256(model + normalized_messages + temp) 命中 (~2 ms)\n   │ Miss\n   ▼\n[3. L2 向量语义缓存 (Embedding + Cosine ≥ τ)] ──► 语义等价查询命中 (~15 ms)\n   │ Miss\n   ▼\n[4. 50ms 有界动态批处理调度器] ──► 聚合时间窗内同模型请求为 Micro-batch ──► 底层推理引擎\n   │\n   └──► 异步回填 L1 & L2 缓存 + 暴露 Prometheus 直方图指标 (\u002Fmetrics)\n","text","",[50,89,85],{"__ignoreMap":87},[70,91],{},[73,93,95],{"id":94},"二核心模块设计与源码实现","二、核心模块设计与源码实现",[97,98,100],"h3",{"id":99},"_1-l1-精确哈希-l2-向量余弦相似度两级缓存","1. L1 精确哈希 + L2 向量余弦相似度两级缓存",[10,102,103],{},"为什么不能只做精确字符串匹配，或者只做向量检索？",[105,106,107,189],"ul",{},[17,108,109,112,113,184,185,188],{},[20,110,111],{},"L1 精确缓存","基于规范化后的请求签名计算确定性 SHA-256 哈希，时间复杂度为 ",[114,115,118,153],"span",{"className":116},[117],"katex",[114,119,122],{"className":120},[121],"katex-mathml",[123,124,126],"math",{"xmlns":125},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[127,128,129,148],"semantics",{},[130,131,132,136,141,145],"mrow",{},[133,134,135],"mi",{},"O",[137,138,140],"mo",{"stretchy":139},"false","(",[142,143,144],"mn",{},"1",[137,146,147],{"stretchy":139},")",[149,150,152],"annotation",{"encoding":151},"application\u002Fx-tex","O(1)",[114,154,158],{"className":155,"ariaHidden":157},[156],"katex-html","true",[114,159,162,167,173,177,180],{"className":160},[161],"base",[114,163],{"className":164,"style":166},[165],"strut","height:1em;vertical-align:-0.25em;",[114,168,135],{"className":169,"style":172},[170,171],"mord","mathnormal","margin-right:0.0278em;",[114,174,140],{"className":175},[176],"mopen",[114,178,144],{"className":179},[170],[114,181,147],{"className":182},[183],"mclose","，不需要运行任何 Embedding 模型，可将完全相同请求的响应压缩到 ",[20,186,187],{},"1～3 ms","；",[17,190,191,194,195,351,352,503,504,536],{},[20,192,193],{},"L2 语义缓存","针对 L1 未命中的自然语言查询，将其编码为单位长度稠密向量 ",[114,196,198,233],{"className":197},[117],[114,199,201],{"className":200},[121],[123,202,203],{"xmlns":125},[127,204,205,230],{},[130,206,207,217,220],{},[208,209,210,214],"msub",{},[133,211,213],{"mathvariant":212},"bold","e",[133,215,216],{},"q",[137,218,219],{},"∈",[221,222,223,227],"msup",{},[133,224,226],{"mathvariant":225},"double-struck","R",[133,228,229],{},"d",[149,231,232],{"encoding":151},"\\mathbf{e}_q \\in \\mathbb{R}^d",[114,234,236,313],{"className":235,"ariaHidden":157},[156],[114,237,239,243,301,306,310],{"className":238},[161],[114,240],{"className":241,"style":242},[165],"height:0.8252em;vertical-align:-0.2861em;",[114,244,246,250],{"className":245},[170],[114,247,213],{"className":248},[170,249],"mathbf",[114,251,254],{"className":252},[253],"msupsub",[114,255,259,292],{"className":256},[257,258],"vlist-t","vlist-t2",[114,260,263,287],{"className":261},[262],"vlist-r",[114,264,268],{"className":265,"style":267},[266],"vlist","height:0.1514em;",[114,269,271,276],{"style":270},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[114,272],{"className":273,"style":275},[274],"pstrut","height:2.7em;",[114,277,283],{"className":278},[279,280,281,282],"sizing","reset-size6","size3","mtight",[114,284,216],{"className":285,"style":286},[170,171,282],"margin-right:0.0359em;",[114,288,291],{"className":289},[290],"vlist-s","​",[114,293,295],{"className":294},[262],[114,296,299],{"className":297,"style":298},[266],"height:0.2861em;",[114,300],{},[114,302],{"className":303,"style":305},[304],"mspace","margin-right:0.2778em;",[114,307,219],{"className":308},[309],"mrel",[114,311],{"className":312,"style":305},[304],[114,314,316,320],{"className":315},[161],[114,317],{"className":318,"style":319},[165],"height:0.8491em;",[114,321,323,327],{"className":322},[170],[114,324,226],{"className":325},[170,326],"mathbb",[114,328,330],{"className":329},[253],[114,331,333],{"className":332},[257],[114,334,336],{"className":335},[262],[114,337,339],{"className":338,"style":319},[266],[114,340,342,345],{"style":341},"top:-3.063em;margin-right:0.05em;",[114,343],{"className":344,"style":275},[274],[114,346,348],{"className":347},[279,280,281,282],[114,349,229],{"className":350},[170,171,282],"（",[114,353,355,389],{"className":354},[117],[114,356,358],{"className":357},[121],[123,359,360],{"xmlns":125},[127,361,362,386],{},[130,363,364,368,374,381,384],{},[133,365,367],{"mathvariant":366},"normal","∥",[208,369,370,372],{},[133,371,213],{"mathvariant":212},[133,373,216],{},[208,375,376,378],{},[133,377,367],{"mathvariant":366},[142,379,380],{},"2",[137,382,383],{},"=",[142,385,144],{},[149,387,388],{"encoding":151},"\\|\\mathbf{e}_q\\|_2 = 1",[114,390,392,493],{"className":391,"ariaHidden":157},[156],[114,393,395,399,402,442,484,487,490],{"className":394},[161],[114,396],{"className":397,"style":398},[165],"height:1.0361em;vertical-align:-0.2861em;",[114,400,367],{"className":401},[170],[114,403,405,408],{"className":404},[170],[114,406,213],{"className":407},[170,249],[114,409,411],{"className":410},[253],[114,412,414,434],{"className":413},[257,258],[114,415,417,431],{"className":416},[262],[114,418,420],{"className":419,"style":267},[266],[114,421,422,425],{"style":270},[114,423],{"className":424,"style":275},[274],[114,426,428],{"className":427},[279,280,281,282],[114,429,216],{"className":430,"style":286},[170,171,282],[114,432,291],{"className":433},[290],[114,435,437],{"className":436},[262],[114,438,440],{"className":439,"style":298},[266],[114,441],{},[114,443,445,448],{"className":444},[170],[114,446,367],{"className":447},[170],[114,449,451],{"className":450},[253],[114,452,454,475],{"className":453},[257,258],[114,455,457,472],{"className":456},[262],[114,458,461],{"className":459,"style":460},[266],"height:0.3011em;",[114,462,463,466],{"style":270},[114,464],{"className":465,"style":275},[274],[114,467,469],{"className":468},[279,280,281,282],[114,470,380],{"className":471},[170,282],[114,473,291],{"className":474},[290],[114,476,478],{"className":477},[262],[114,479,482],{"className":480,"style":481},[266],"height:0.15em;",[114,483],{},[114,485],{"className":486,"style":305},[304],[114,488,383],{"className":489},[309],[114,491],{"className":492,"style":305},[304],[114,494,496,500],{"className":495},[161],[114,497],{"className":498,"style":499},[165],"height:0.6444em;",[114,501,144],{"className":502},[170],"），在 Redis 语义索引中检索满足余弦相似度阈值 ",[114,505,507,522],{"className":506},[117],[114,508,510],{"className":509},[121],[123,511,512],{"xmlns":125},[127,513,514,519],{},[130,515,516],{},[133,517,518],{},"τ",[149,520,521],{"encoding":151},"\\tau",[114,523,525],{"className":524,"ariaHidden":157},[156],[114,526,528,532],{"className":527},[161],[114,529],{"className":530,"style":531},[165],"height:0.4306em;",[114,533,518],{"className":534,"style":535},[170,171],"margin-right:0.1132em;"," 的历史问答对：",[114,538,541],{"className":539},[540],"katex-display",[114,542,544,621],{"className":543},[117],[114,545,547],{"className":546},[121],[123,548,550],{"xmlns":125,"display":549},"block",[127,551,552,618],{},[130,553,554,558,560,566,569,576,578,580,590,596,599,601,604,606,609,611,613,616],{},[555,556,557],"mtext",{},"sim",[137,559,140],{"stretchy":139},[208,561,562,564],{},[133,563,213],{"mathvariant":212},[133,565,216],{},[137,567,568],{"separator":157},",",[208,570,571,573],{},[133,572,213],{"mathvariant":212},[133,574,575],{},"i",[137,577,147],{"stretchy":139},[137,579,383],{},[581,582,583,585,587],"msubsup",{},[133,584,213],{"mathvariant":212},[133,586,216],{},[133,588,589],{"mathvariant":366},"⊤",[208,591,592,594],{},[133,593,213],{"mathvariant":212},[133,595,575],{},[137,597,598],{},"≥",[133,600,518],{},[304,602],{"width":603},"1em",[137,605,140],{"stretchy":139},[555,607,608],{},"默认 ",[133,610,518],{},[137,612,383],{},[142,614,615],{},"0.92",[137,617,147],{"stretchy":139},[149,619,620],{"encoding":151},"\\text{sim}(\\mathbf{e}_q, \\mathbf{e}_i) = \\mathbf{e}_q^\\top \\mathbf{e}_i \\ge \\tau \\quad (\\text{默认 }\\tau = 0.92)",[114,622,624,740,851,891],{"className":623,"ariaHidden":157},[156],[114,625,627,630,636,639,679,683,687,728,731,734,737],{"className":626},[161],[114,628],{"className":629,"style":398},[165],[114,631,633],{"className":632},[170,86],[114,634,557],{"className":635},[170],[114,637,140],{"className":638},[176],[114,640,642,645],{"className":641},[170],[114,643,213],{"className":644},[170,249],[114,646,648],{"className":647},[253],[114,649,651,671],{"className":650},[257,258],[114,652,654,668],{"className":653},[262],[114,655,657],{"className":656,"style":267},[266],[114,658,659,662],{"style":270},[114,660],{"className":661,"style":275},[274],[114,663,665],{"className":664},[279,280,281,282],[114,666,216],{"className":667,"style":286},[170,171,282],[114,669,291],{"className":670},[290],[114,672,674],{"className":673},[262],[114,675,677],{"className":676,"style":298},[266],[114,678],{},[114,680,568],{"className":681},[682],"mpunct",[114,684],{"className":685,"style":686},[304],"margin-right:0.1667em;",[114,688,690,693],{"className":689},[170],[114,691,213],{"className":692},[170,249],[114,694,696],{"className":695},[253],[114,697,699,720],{"className":698},[257,258],[114,700,702,717],{"className":701},[262],[114,703,706],{"className":704,"style":705},[266],"height:0.3117em;",[114,707,708,711],{"style":270},[114,709],{"className":710,"style":275},[274],[114,712,714],{"className":713},[279,280,281,282],[114,715,575],{"className":716},[170,171,282],[114,718,291],{"className":719},[290],[114,721,723],{"className":722},[262],[114,724,726],{"className":725,"style":481},[266],[114,727],{},[114,729,147],{"className":730},[183],[114,732],{"className":733,"style":305},[304],[114,735,383],{"className":736},[309],[114,738],{"className":739,"style":305},[304],[114,741,743,747,802,842,845,848],{"className":742},[161],[114,744],{"className":745,"style":746},[165],"height:1.2822em;vertical-align:-0.3831em;",[114,748,750,753],{"className":749},[170],[114,751,213],{"className":752},[170,249],[114,754,756],{"className":755},[253],[114,757,759,793],{"className":758},[257,258],[114,760,762,790],{"className":761},[262],[114,763,766,778],{"className":764,"style":765},[266],"height:0.8991em;",[114,767,769,772],{"style":768},"top:-2.453em;margin-left:0em;margin-right:0.05em;",[114,770],{"className":771,"style":275},[274],[114,773,775],{"className":774},[279,280,281,282],[114,776,216],{"className":777,"style":286},[170,171,282],[114,779,781,784],{"style":780},"top:-3.113em;margin-right:0.05em;",[114,782],{"className":783,"style":275},[274],[114,785,787],{"className":786},[279,280,281,282],[114,788,589],{"className":789},[170,282],[114,791,291],{"className":792},[290],[114,794,796],{"className":795},[262],[114,797,800],{"className":798,"style":799},[266],"height:0.3831em;",[114,801],{},[114,803,805,808],{"className":804},[170],[114,806,213],{"className":807},[170,249],[114,809,811],{"className":810},[253],[114,812,814,834],{"className":813},[257,258],[114,815,817,831],{"className":816},[262],[114,818,820],{"className":819,"style":705},[266],[114,821,822,825],{"style":270},[114,823],{"className":824,"style":275},[274],[114,826,828],{"className":827},[279,280,281,282],[114,829,575],{"className":830},[170,171,282],[114,832,291],{"className":833},[290],[114,835,837],{"className":836},[262],[114,838,840],{"className":839,"style":481},[266],[114,841],{},[114,843],{"className":844,"style":305},[304],[114,846,598],{"className":847},[309],[114,849],{"className":850,"style":305},[304],[114,852,854,857,860,864,867,879,882,885,888],{"className":853},[161],[114,855],{"className":856,"style":166},[165],[114,858,518],{"className":859,"style":535},[170,171],[114,861],{"className":862,"style":863},[304],"margin-right:1em;",[114,865,140],{"className":866},[176],[114,868,870,875],{"className":869},[170,86],[114,871,874],{"className":872},[170,873],"cjk_fallback","默认",[114,876,878],{"className":877},[170]," ",[114,880,518],{"className":881,"style":535},[170,171],[114,883],{"className":884,"style":305},[304],[114,886,383],{"className":887},[309],[114,889],{"className":890,"style":305},[304],[114,892,894,897,900],{"className":893},[161],[114,895],{"className":896,"style":166},[165],[114,898,615],{"className":899},[170],[114,901,147],{"className":902},[183],[10,904,905,906,909,910,913,914,917,918,921],{},"需要注意的是，",[20,907,908],{},"并非所有请求都允许进入语义缓存","：当请求包含实时工具调用结果（",[50,911,912],{},"role == \"tool\"","）、高随机性采样温度（",[50,915,916],{},"temperature > 0.3","）或显式指定 ",[50,919,920],{},"cache_control: no-cache"," 时，网关会自动旁路 L2 语义层，防止跨上下文串话。",[97,923,925],{"id":924},"_2-50-ms-有界等待动态批处理调度器bounded-wait-dynamic-batcher","2. 50 ms 有界等待动态批处理调度器（Bounded-Wait Dynamic Batcher）",[10,927,928,929,932,933,936,937,1033,1034,1125],{},"对于穿透两级缓存的真实推理请求，网关通过 ",[50,930,931],{},"asyncio.Queue"," 与 ",[50,934,935],{},"asyncio.Future"," 实现了一个无锁异步动态批处理器。其核心停止条件由",[20,938,939,940],{},"最大批次容量 ",[114,941,943,966],{"className":942},[117],[114,944,946],{"className":945},[121],[123,947,948],{"xmlns":125},[127,949,950,963],{},[130,951,952],{},[208,953,954,957,960],{},[133,955,956],{},"B",[133,958,959],{},"max",[137,961,962],{},"⁡",[149,964,965],{"encoding":151},"B_{\\max}",[114,967,969],{"className":968,"ariaHidden":157},[156],[114,970,972,976],{"className":971},[161],[114,973],{"className":974,"style":975},[165],"height:0.8333em;vertical-align:-0.15em;",[114,977,979,983],{"className":978},[170],[114,980,956],{"className":981,"style":982},[170,171],"margin-right:0.0502em;",[114,984,986],{"className":985},[253],[114,987,989,1025],{"className":988},[257,258],[114,990,992,1022],{"className":991},[262],[114,993,995],{"className":994,"style":267},[266],[114,996,998,1001],{"style":997},"top:-2.55em;margin-left:-0.0502em;margin-right:0.05em;",[114,999],{"className":1000,"style":275},[274],[114,1002,1004],{"className":1003},[279,280,281,282],[114,1005,1007],{"className":1006},[170,282],[114,1008,1011,1015,1018],{"className":1009},[1010,282],"mop",[114,1012,1014],{"className":1013},[282],"m",[114,1016,42],{"className":1017},[282],[114,1019,1021],{"className":1020},[282],"x",[114,1023,291],{"className":1024},[290],[114,1026,1028],{"className":1027},[262],[114,1029,1031],{"className":1030,"style":481},[266],[114,1032],{}," 与",[20,1035,1036,1037,1124],{},"最大等待时间窗 ",[114,1038,1040,1061],{"className":1039},[117],[114,1041,1043],{"className":1042},[121],[123,1044,1045],{"xmlns":125},[127,1046,1047,1058],{},[130,1048,1049],{},[208,1050,1051,1054,1056],{},[133,1052,1053],{},"T",[133,1055,959],{},[137,1057,962],{},[149,1059,1060],{"encoding":151},"T_{\\max}",[114,1062,1064],{"className":1063,"ariaHidden":157},[156],[114,1065,1067,1070],{"className":1066},[161],[114,1068],{"className":1069,"style":975},[165],[114,1071,1073,1077],{"className":1072},[170],[114,1074,1053],{"className":1075,"style":1076},[170,171],"margin-right:0.1389em;",[114,1078,1080],{"className":1079},[253],[114,1081,1083,1116],{"className":1082},[257,258],[114,1084,1086,1113],{"className":1085},[262],[114,1087,1089],{"className":1088,"style":267},[266],[114,1090,1092,1095],{"style":1091},"top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;",[114,1093],{"className":1094,"style":275},[274],[114,1096,1098],{"className":1097},[279,280,281,282],[114,1099,1101],{"className":1100},[170,282],[114,1102,1104,1107,1110],{"className":1103},[1010,282],[114,1105,1014],{"className":1106},[282],[114,1108,42],{"className":1109},[282],[114,1111,1021],{"className":1112},[282],[114,1114,291],{"className":1115},[290],[114,1117,1119],{"className":1118},[262],[114,1120,1122],{"className":1121,"style":481},[266],[114,1123],{},"（默认 50 ms）","共同约束：",[81,1127,1131],{"className":1128,"code":1129,"language":1130,"meta":87,"style":87},"language-python shiki shiki-themes github-dark-dimmed github-light houston","import asyncio\nimport time\nfrom dataclasses import dataclass\n\n@dataclass\nclass PendingRequest:\n    payload: dict\n    future: asyncio.Future\n    enqueue_ts: float\n\nclass DynamicBatchScheduler:\n    def __init__(self, backend_client, max_batch_size: int = 16, max_wait_ms: float = 50.0):\n        self.backend = backend_client\n        self.max_batch_size = max_batch_size\n        self.max_wait_sec = max_wait_ms \u002F 1000.0\n        self.queue: asyncio.Queue[PendingRequest] = asyncio.Queue()\n\n    async def submit(self, payload: dict) -> dict:\n        loop = asyncio.get_running_loop()\n        fut = loop.create_future()\n        await self.queue.put(PendingRequest(payload=payload, future=fut, enqueue_ts=time.perf_counter()))\n        return await fut\n\n    async def worker_loop(self) -> None:\n        while True:\n            # 1. 阻塞等待批次的首个请求到达\n            first_req = await self.queue.get()\n            batch = [first_req]\n            deadline = time.perf_counter() + self.max_wait_sec\n\n            # 2. 在 50ms 有界时间窗内尽可能凑满 max_batch_size\n            while len(batch) \u003C self.max_batch_size:\n                remaining = deadline - time.perf_counter()\n                if remaining \u003C= 0:\n                    break\n                try:\n                    nxt = await asyncio.wait_for(self.queue.get(), timeout=remaining)\n                    batch.append(nxt)\n                except asyncio.TimeoutError:\n                    break\n\n            # 3. 批量下发到底层推理后端并按序唤醒各请求的 Future\n            asyncio.create_task(self._dispatch_batch(batch))\n\n    async def _dispatch_batch(self, batch: list[PendingRequest]) -> None:\n        try:\n            responses = await self.backend.generate_batch([item.payload for item in batch])\n            for item, resp in zip(batch, responses):\n                if not item.future.done():\n                    item.future.set_result(resp)\n        except Exception as exc:\n            for item in batch:\n                if not item.future.done():\n                    item.future.set_exception(exc)\n","python",[50,1132,1133,1145,1153,1167,1174,1181,1195,1205,1211,1220,1225,1235,1294,1309,1322,1341,1361,1366,1399,1414,1430,1482,1494,1499,1520,1531,1538,1557,1568,1592,1597,1603,1623,1643,1660,1666,1674,1708,1720,1729,1734,1739,1745,1767,1772,1798,1806,1839,1856,1873,1885,1900,1912,1925],{"__ignoreMap":87},[114,1134,1137,1141],{"class":1135,"line":1136},"line",1,[114,1138,1140],{"class":1139},"slEfO","import",[114,1142,1144],{"class":1143},"sWURj"," asyncio\n",[114,1146,1148,1150],{"class":1135,"line":1147},2,[114,1149,1140],{"class":1139},[114,1151,1152],{"class":1143}," time\n",[114,1154,1156,1159,1162,1164],{"class":1135,"line":1155},3,[114,1157,1158],{"class":1139},"from",[114,1160,1161],{"class":1143}," dataclasses ",[114,1163,1140],{"class":1139},[114,1165,1166],{"class":1143}," dataclass\n",[114,1168,1170],{"class":1135,"line":1169},4,[114,1171,1173],{"emptyLinePlaceholder":1172},true,"\n",[114,1175,1177],{"class":1135,"line":1176},5,[114,1178,1180],{"class":1179},"sS_0i","@dataclass\n",[114,1182,1184,1188,1192],{"class":1135,"line":1183},6,[114,1185,1187],{"class":1186},"sST-V","class",[114,1189,1191],{"class":1190},"sBBmv"," PendingRequest",[114,1193,1194],{"class":1143},":\n",[114,1196,1198,1201],{"class":1135,"line":1197},7,[114,1199,1200],{"class":1143},"    payload: ",[114,1202,1204],{"class":1203},"svkvy","dict\n",[114,1206,1208],{"class":1135,"line":1207},8,[114,1209,1210],{"class":1143},"    future: asyncio.Future\n",[114,1212,1214,1217],{"class":1135,"line":1213},9,[114,1215,1216],{"class":1143},"    enqueue_ts: ",[114,1218,1219],{"class":1203},"float\n",[114,1221,1223],{"class":1135,"line":1222},10,[114,1224,1173],{"emptyLinePlaceholder":1172},[114,1226,1228,1230,1233],{"class":1135,"line":1227},11,[114,1229,1187],{"class":1186},[114,1231,1232],{"class":1190}," DynamicBatchScheduler",[114,1234,1194],{"class":1143},[114,1236,1238,1241,1244,1246,1250,1253,1257,1259,1262,1265,1268,1272,1276,1278,1281,1283,1286,1288,1291],{"class":1135,"line":1237},12,[114,1239,1240],{"class":1186},"    def",[114,1242,1243],{"class":1203}," __init__",[114,1245,140],{"class":1143},[114,1247,1249],{"class":1248},"sbU-9","self",[114,1251,1252],{"class":1143},", ",[114,1254,1256],{"class":1255},"s9NhK","backend_client",[114,1258,1252],{"class":1143},[114,1260,1261],{"class":1255},"max_batch_size",[114,1263,1264],{"class":1143},": ",[114,1266,1267],{"class":1203},"int",[114,1269,1271],{"class":1270},"sf2cM"," =",[114,1273,1275],{"class":1274},"sgENW"," 16",[114,1277,1252],{"class":1143},[114,1279,1280],{"class":1255},"max_wait_ms",[114,1282,1264],{"class":1143},[114,1284,1285],{"class":1203},"float",[114,1287,1271],{"class":1270},[114,1289,1290],{"class":1274}," 50.0",[114,1292,1293],{"class":1143},"):\n",[114,1295,1297,1301,1304,1306],{"class":1135,"line":1296},13,[114,1298,1300],{"class":1299},"sRGqv","        self",[114,1302,1303],{"class":1143},".backend ",[114,1305,383],{"class":1270},[114,1307,1308],{"class":1143}," backend_client\n",[114,1310,1312,1314,1317,1319],{"class":1135,"line":1311},14,[114,1313,1300],{"class":1299},[114,1315,1316],{"class":1143},".max_batch_size ",[114,1318,383],{"class":1270},[114,1320,1321],{"class":1143}," max_batch_size\n",[114,1323,1325,1327,1330,1332,1335,1338],{"class":1135,"line":1324},15,[114,1326,1300],{"class":1299},[114,1328,1329],{"class":1143},".max_wait_sec ",[114,1331,383],{"class":1270},[114,1333,1334],{"class":1143}," max_wait_ms ",[114,1336,1337],{"class":1270},"\u002F",[114,1339,1340],{"class":1274}," 1000.0\n",[114,1342,1344,1346,1349,1351,1354,1358],{"class":1135,"line":1343},16,[114,1345,1300],{"class":1299},[114,1347,1348],{"class":1143},".queue: asyncio.Queue[PendingRequest] ",[114,1350,383],{"class":1270},[114,1352,1353],{"class":1143}," asyncio.",[114,1355,1357],{"class":1356},"sq3c-","Queue",[114,1359,1360],{"class":1143},"()\n",[114,1362,1364],{"class":1135,"line":1363},17,[114,1365,1173],{"emptyLinePlaceholder":1172},[114,1367,1369,1372,1375,1378,1380,1382,1384,1387,1389,1392,1395,1397],{"class":1135,"line":1368},18,[114,1370,1371],{"class":1186},"    async",[114,1373,1374],{"class":1186}," def",[114,1376,1377],{"class":1179}," submit",[114,1379,140],{"class":1143},[114,1381,1249],{"class":1248},[114,1383,1252],{"class":1143},[114,1385,1386],{"class":1255},"payload",[114,1388,1264],{"class":1143},[114,1390,1391],{"class":1203},"dict",[114,1393,1394],{"class":1143},") -> ",[114,1396,1391],{"class":1203},[114,1398,1194],{"class":1143},[114,1400,1402,1405,1407,1409,1412],{"class":1135,"line":1401},19,[114,1403,1404],{"class":1143},"        loop ",[114,1406,383],{"class":1270},[114,1408,1353],{"class":1143},[114,1410,1411],{"class":1356},"get_running_loop",[114,1413,1360],{"class":1143},[114,1415,1417,1420,1422,1425,1428],{"class":1135,"line":1416},20,[114,1418,1419],{"class":1143},"        fut ",[114,1421,383],{"class":1270},[114,1423,1424],{"class":1143}," loop.",[114,1426,1427],{"class":1356},"create_future",[114,1429,1360],{"class":1143},[114,1431,1433,1436,1439,1442,1445,1447,1450,1452,1455,1457,1460,1463,1465,1468,1471,1473,1476,1479],{"class":1135,"line":1432},21,[114,1434,1435],{"class":1139},"        await",[114,1437,1438],{"class":1299}," self",[114,1440,1441],{"class":1143},".queue.",[114,1443,1444],{"class":1356},"put",[114,1446,140],{"class":1143},[114,1448,1449],{"class":1356},"PendingRequest",[114,1451,140],{"class":1143},[114,1453,1386],{"class":1454},"sA9aF",[114,1456,383],{"class":1270},[114,1458,1459],{"class":1143},"payload, ",[114,1461,1462],{"class":1454},"future",[114,1464,383],{"class":1270},[114,1466,1467],{"class":1143},"fut, ",[114,1469,1470],{"class":1454},"enqueue_ts",[114,1472,383],{"class":1270},[114,1474,1475],{"class":1143},"time.",[114,1477,1478],{"class":1356},"perf_counter",[114,1480,1481],{"class":1143},"()))\n",[114,1483,1485,1488,1491],{"class":1135,"line":1484},22,[114,1486,1487],{"class":1139},"        return",[114,1489,1490],{"class":1139}," await",[114,1492,1493],{"class":1143}," fut\n",[114,1495,1497],{"class":1135,"line":1496},23,[114,1498,1173],{"emptyLinePlaceholder":1172},[114,1500,1502,1504,1506,1509,1511,1513,1515,1518],{"class":1135,"line":1501},24,[114,1503,1371],{"class":1186},[114,1505,1374],{"class":1186},[114,1507,1508],{"class":1179}," worker_loop",[114,1510,140],{"class":1143},[114,1512,1249],{"class":1248},[114,1514,1394],{"class":1143},[114,1516,1517],{"class":1274},"None",[114,1519,1194],{"class":1143},[114,1521,1523,1526,1529],{"class":1135,"line":1522},25,[114,1524,1525],{"class":1139},"        while",[114,1527,1528],{"class":1274}," True",[114,1530,1194],{"class":1143},[114,1532,1534],{"class":1135,"line":1533},26,[114,1535,1537],{"class":1536},"sCxhR","            # 1. 阻塞等待批次的首个请求到达\n",[114,1539,1541,1544,1546,1548,1550,1552,1555],{"class":1135,"line":1540},27,[114,1542,1543],{"class":1143},"            first_req ",[114,1545,383],{"class":1270},[114,1547,1490],{"class":1139},[114,1549,1438],{"class":1299},[114,1551,1441],{"class":1143},[114,1553,1554],{"class":1356},"get",[114,1556,1360],{"class":1143},[114,1558,1560,1563,1565],{"class":1135,"line":1559},28,[114,1561,1562],{"class":1143},"            batch ",[114,1564,383],{"class":1270},[114,1566,1567],{"class":1143}," [first_req]\n",[114,1569,1571,1574,1576,1579,1581,1584,1587,1589],{"class":1135,"line":1570},29,[114,1572,1573],{"class":1143},"            deadline ",[114,1575,383],{"class":1270},[114,1577,1578],{"class":1143}," time.",[114,1580,1478],{"class":1356},[114,1582,1583],{"class":1143},"() ",[114,1585,1586],{"class":1270},"+",[114,1588,1438],{"class":1299},[114,1590,1591],{"class":1143},".max_wait_sec\n",[114,1593,1595],{"class":1135,"line":1594},30,[114,1596,1173],{"emptyLinePlaceholder":1172},[114,1598,1600],{"class":1135,"line":1599},31,[114,1601,1602],{"class":1536},"            # 2. 在 50ms 有界时间窗内尽可能凑满 max_batch_size\n",[114,1604,1606,1609,1612,1615,1618,1620],{"class":1135,"line":1605},32,[114,1607,1608],{"class":1139},"            while",[114,1610,1611],{"class":1203}," len",[114,1613,1614],{"class":1143},"(batch) ",[114,1616,1617],{"class":1270},"\u003C",[114,1619,1438],{"class":1299},[114,1621,1622],{"class":1143},".max_batch_size:\n",[114,1624,1626,1629,1631,1634,1637,1639,1641],{"class":1135,"line":1625},33,[114,1627,1628],{"class":1143},"                remaining ",[114,1630,383],{"class":1270},[114,1632,1633],{"class":1143}," deadline ",[114,1635,1636],{"class":1270},"-",[114,1638,1578],{"class":1143},[114,1640,1478],{"class":1356},[114,1642,1360],{"class":1143},[114,1644,1646,1649,1652,1655,1658],{"class":1135,"line":1645},34,[114,1647,1648],{"class":1139},"                if",[114,1650,1651],{"class":1143}," remaining ",[114,1653,1654],{"class":1270},"\u003C=",[114,1656,1657],{"class":1274}," 0",[114,1659,1194],{"class":1143},[114,1661,1663],{"class":1135,"line":1662},35,[114,1664,1665],{"class":1139},"                    break\n",[114,1667,1669,1672],{"class":1135,"line":1668},36,[114,1670,1671],{"class":1139},"                try",[114,1673,1194],{"class":1143},[114,1675,1677,1680,1682,1684,1686,1689,1691,1693,1695,1697,1700,1703,1705],{"class":1135,"line":1676},37,[114,1678,1679],{"class":1143},"                    nxt ",[114,1681,383],{"class":1270},[114,1683,1490],{"class":1139},[114,1685,1353],{"class":1143},[114,1687,1688],{"class":1356},"wait_for",[114,1690,140],{"class":1143},[114,1692,1249],{"class":1299},[114,1694,1441],{"class":1143},[114,1696,1554],{"class":1356},[114,1698,1699],{"class":1143},"(), ",[114,1701,1702],{"class":1454},"timeout",[114,1704,383],{"class":1270},[114,1706,1707],{"class":1143},"remaining)\n",[114,1709,1711,1714,1717],{"class":1135,"line":1710},38,[114,1712,1713],{"class":1143},"                    batch.",[114,1715,1716],{"class":1356},"append",[114,1718,1719],{"class":1143},"(nxt)\n",[114,1721,1723,1726],{"class":1135,"line":1722},39,[114,1724,1725],{"class":1139},"                except",[114,1727,1728],{"class":1143}," asyncio.TimeoutError:\n",[114,1730,1732],{"class":1135,"line":1731},40,[114,1733,1665],{"class":1139},[114,1735,1737],{"class":1135,"line":1736},41,[114,1738,1173],{"emptyLinePlaceholder":1172},[114,1740,1742],{"class":1135,"line":1741},42,[114,1743,1744],{"class":1536},"            # 3. 批量下发到底层推理后端并按序唤醒各请求的 Future\n",[114,1746,1748,1751,1754,1756,1758,1761,1764],{"class":1135,"line":1747},43,[114,1749,1750],{"class":1143},"            asyncio.",[114,1752,1753],{"class":1356},"create_task",[114,1755,140],{"class":1143},[114,1757,1249],{"class":1299},[114,1759,1760],{"class":1143},".",[114,1762,1763],{"class":1356},"_dispatch_batch",[114,1765,1766],{"class":1143},"(batch))\n",[114,1768,1770],{"class":1135,"line":1769},44,[114,1771,1173],{"emptyLinePlaceholder":1172},[114,1773,1775,1777,1779,1782,1784,1786,1788,1791,1794,1796],{"class":1135,"line":1774},45,[114,1776,1371],{"class":1186},[114,1778,1374],{"class":1186},[114,1780,1781],{"class":1179}," _dispatch_batch",[114,1783,140],{"class":1143},[114,1785,1249],{"class":1248},[114,1787,1252],{"class":1143},[114,1789,1790],{"class":1255},"batch",[114,1792,1793],{"class":1143},": list[PendingRequest]) -> ",[114,1795,1517],{"class":1274},[114,1797,1194],{"class":1143},[114,1799,1801,1804],{"class":1135,"line":1800},46,[114,1802,1803],{"class":1139},"        try",[114,1805,1194],{"class":1143},[114,1807,1809,1812,1814,1816,1818,1821,1824,1827,1830,1833,1836],{"class":1135,"line":1808},47,[114,1810,1811],{"class":1143},"            responses ",[114,1813,383],{"class":1270},[114,1815,1490],{"class":1139},[114,1817,1438],{"class":1299},[114,1819,1820],{"class":1143},".backend.",[114,1822,1823],{"class":1356},"generate_batch",[114,1825,1826],{"class":1143},"([item.payload ",[114,1828,1829],{"class":1139},"for",[114,1831,1832],{"class":1143}," item ",[114,1834,1835],{"class":1139},"in",[114,1837,1838],{"class":1143}," batch])\n",[114,1840,1842,1845,1848,1850,1853],{"class":1135,"line":1841},48,[114,1843,1844],{"class":1139},"            for",[114,1846,1847],{"class":1143}," item, resp ",[114,1849,1835],{"class":1139},[114,1851,1852],{"class":1203}," zip",[114,1854,1855],{"class":1143},"(batch, responses):\n",[114,1857,1859,1861,1864,1867,1870],{"class":1135,"line":1858},49,[114,1860,1648],{"class":1139},[114,1862,1863],{"class":1186}," not",[114,1865,1866],{"class":1143}," item.future.",[114,1868,1869],{"class":1356},"done",[114,1871,1872],{"class":1143},"():\n",[114,1874,1876,1879,1882],{"class":1135,"line":1875},50,[114,1877,1878],{"class":1143},"                    item.future.",[114,1880,1881],{"class":1356},"set_result",[114,1883,1884],{"class":1143},"(resp)\n",[114,1886,1888,1891,1894,1897],{"class":1135,"line":1887},51,[114,1889,1890],{"class":1139},"        except",[114,1892,1893],{"class":1203}," Exception",[114,1895,1896],{"class":1139}," as",[114,1898,1899],{"class":1143}," exc:\n",[114,1901,1903,1905,1907,1909],{"class":1135,"line":1902},52,[114,1904,1844],{"class":1139},[114,1906,1832],{"class":1143},[114,1908,1835],{"class":1139},[114,1910,1911],{"class":1143}," batch:\n",[114,1913,1915,1917,1919,1921,1923],{"class":1135,"line":1914},53,[114,1916,1648],{"class":1139},[114,1918,1863],{"class":1186},[114,1920,1866],{"class":1143},[114,1922,1869],{"class":1356},[114,1924,1872],{"class":1143},[114,1926,1928,1930,1933],{"class":1135,"line":1927},54,[114,1929,1878],{"class":1143},[114,1931,1932],{"class":1356},"set_exception",[114,1934,1935],{"class":1143},"(exc)\n",[10,1937,1938,1939,2076],{},"这种“首包触发计时 + 容量\u002F超时双阈值截断”的设计保证了：",[20,1940,1941,1942,1991,1992,2075],{},"在低峰期（单请求稀疏到达），请求最多额外等待 50 ms（相比数秒的 LLM 推理耗时仅占 ",[114,1943,1945,1964],{"className":1944},[117],[114,1946,1948],{"className":1947},[121],[123,1949,1950],{"xmlns":125},[127,1951,1952,1961],{},[130,1953,1954,1956,1958],{},[137,1955,1617],{},[142,1957,380],{},[133,1959,1960],{"mathvariant":366},"%",[149,1962,1963],{"encoding":151},"\u003C2\\%",[114,1965,1967,1980],{"className":1966,"ariaHidden":157},[156],[114,1968,1970,1974,1977],{"className":1969},[161],[114,1971],{"className":1972,"style":1973},[165],"height:0.5782em;vertical-align:-0.0391em;",[114,1975,1617],{"className":1976},[309],[114,1978],{"className":1979,"style":305},[304],[114,1981,1983,1987],{"className":1982},[161],[114,1984],{"className":1985,"style":1986},[165],"height:0.8056em;vertical-align:-0.0556em;",[114,1988,1990],{"className":1989},[170],"2%","）；而在高峰期（并发涌入），队列可在几毫秒内凑满 ",[114,1993,1995,2014],{"className":1994},[117],[114,1996,1998],{"className":1997},[121],[123,1999,2000],{"xmlns":125},[127,2001,2002,2012],{},[130,2003,2004],{},[208,2005,2006,2008,2010],{},[133,2007,956],{},[133,2009,959],{},[137,2011,962],{},[149,2013,965],{"encoding":151},[114,2015,2017],{"className":2016,"ariaHidden":157},[156],[114,2018,2020,2023],{"className":2019},[161],[114,2021],{"className":2022,"style":975},[165],[114,2024,2026,2029],{"className":2025},[170],[114,2027,956],{"className":2028,"style":982},[170,171],[114,2030,2032],{"className":2031},[253],[114,2033,2035,2067],{"className":2034},[257,258],[114,2036,2038,2064],{"className":2037},[262],[114,2039,2041],{"className":2040,"style":267},[266],[114,2042,2043,2046],{"style":997},[114,2044],{"className":2045,"style":275},[274],[114,2047,2049],{"className":2048},[279,280,281,282],[114,2050,2052],{"className":2051},[170,282],[114,2053,2055,2058,2061],{"className":2054},[1010,282],[114,2056,1014],{"className":2057},[282],[114,2059,42],{"className":2060},[282],[114,2062,1021],{"className":2063},[282],[114,2065,291],{"className":2066},[290],[114,2068,2070],{"className":2069},[262],[114,2071,2073],{"className":2072,"style":481},[266],[114,2074],{}," 立即发车","。",[70,2078],{},[73,2080,2082],{"id":2081},"三受控压测与尾部延迟p50-p95-p99对比","三、受控压测与尾部延迟（P50 \u002F P95 \u002F P99）对比",[10,2084,2085,2086,2088,2089,57,2092,2095,2096,2099],{},"我们在真实 LLM 推理负载下对网关进行了三组受控压测（完整原始数据保存在 ",[50,2087,52],{},"），分别考察",[20,2090,2091],{},"精确缓存热命中",[20,2093,2094],{},"混合流量下的语义缓存","以及",[20,2097,2098],{},"高并发突发流量","：",[97,2101,2103,2104,2159],{"id":2102},"_1-场景-al1-精确缓存冷启动-vs-热命中50-请求并发度-c5c5c5","1. 场景 A：L1 精确缓存冷启动 vs. 热命中（50 请求，并发度 ",[114,2105,2107,2127],{"className":2106},[117],[114,2108,2110],{"className":2109},[121],[123,2111,2112],{"xmlns":125},[127,2113,2114,2124],{},[130,2115,2116,2119,2121],{},[133,2117,2118],{},"C",[137,2120,383],{},[142,2122,2123],{},"5",[149,2125,2126],{"encoding":151},"C=5",[114,2128,2130,2150],{"className":2129,"ariaHidden":157},[156],[114,2131,2133,2137,2141,2144,2147],{"className":2132},[161],[114,2134],{"className":2135,"style":2136},[165],"height:0.6833em;",[114,2138,2118],{"className":2139,"style":2140},[170,171],"margin-right:0.0715em;",[114,2142],{"className":2143,"style":305},[304],[114,2145,383],{"className":2146},[309],[114,2148],{"className":2149,"style":305},[304],[114,2151,2153,2156],{"className":2152},[161],[114,2154],{"className":2155,"style":499},[165],[114,2157,2123],{"className":2158},[170],"）",[2161,2162,2163,2192],"table",{},[2164,2165,2166],"thead",{},[2167,2168,2169,2173,2177,2180,2183,2186,2189],"tr",{},[2170,2171,2172],"th",{},"压测阶段",[2170,2174,2176],{"align":2175},"right","缓存命中率",[2170,2178,2179],{"align":2175},"吞吐量 (RPS)",[2170,2181,2182],{"align":2175},"平均延迟 (ms)",[2170,2184,2185],{"align":2175},"P50 延迟 (ms)",[2170,2187,2188],{"align":2175},"P95 延迟 (ms)",[2170,2190,2191],{"align":2175},"P99 延迟 (ms)",[2193,2194,2195,2223],"tbody",{},[2167,2196,2197,2203,2206,2209,2212,2215,2218],{},[2198,2199,2200],"td",{},[20,2201,2202],{},"冷缓存穿透（Cold Pass）",[2198,2204,2205],{"align":2175},"0.0%",[2198,2207,2208],{"align":2175},"1.38 req\u002Fs",[2198,2210,2211],{"align":2175},"3,522.6 ms",[2198,2213,2214],{"align":2175},"3,372.6 ms",[2198,2216,2217],{"align":2175},"4,768.2 ms",[2198,2219,2220],{"align":2175},[20,2221,2222],{},"5,307.0 ms",[2167,2224,2225,2230,2233,2238,2241,2244,2247],{},[2198,2226,2227],{},[20,2228,2229],{},"热缓存命中（Warm Exact Hit）",[2198,2231,2232],{"align":2175},"100.0%",[2198,2234,2235],{"align":2175},[20,2236,2237],{},"316.58 req\u002Fs",[2198,2239,2240],{"align":2175},"14.5 ms",[2198,2242,2243],{"align":2175},"15.6 ms",[2198,2245,2246],{"align":2175},"16.9 ms",[2198,2248,2249],{"align":2175},[20,2250,2251],{},"17.0 ms",[10,2253,2254,2255,2258,2259,2262,2263,2076],{},"在热点问题重复到达时，L1 精确缓存将 P99 尾部延迟从 ",[20,2256,2257],{},"5,307.0 ms 直接压降至 17.0 ms（降低 99.7%）","，系统吞吐量从 ",[50,2260,2261],{},"1.38 RPS"," 跃升至 ",[20,2264,2265,2268],{},[50,2266,2267],{},"316.58 RPS","（提升 229×）",[97,2270,2272,2273,2159],{"id":2271},"_2-场景-b混合流量下的两级精确-语义缓存60-请求并发度-c5c5c5","2. 场景 B：混合流量下的两级精确 + 语义缓存（60 请求，并发度 ",[114,2274,2276,2293],{"className":2275},[117],[114,2277,2279],{"className":2278},[121],[123,2280,2281],{"xmlns":125},[127,2282,2283,2291],{},[130,2284,2285,2287,2289],{},[133,2286,2118],{},[137,2288,383],{},[142,2290,2123],{},[149,2292,2126],{"encoding":151},[114,2294,2296,2314],{"className":2295,"ariaHidden":157},[156],[114,2297,2299,2302,2305,2308,2311],{"className":2298},[161],[114,2300],{"className":2301,"style":2136},[165],[114,2303,2118],{"className":2304,"style":2140},[170,171],[114,2306],{"className":2307,"style":305},[304],[114,2309,383],{"className":2310},[309],[114,2312],{"className":2313,"style":305},[304],[114,2315,2317,2320],{"className":2316},[161],[114,2318],{"className":2319,"style":499},[165],[114,2321,2123],{"className":2322},[170],[10,2324,2325],{},"在包含 20 条首次唯一查询、20 条精确重复查询与 20 条句式改写近义查询的混合流量中：",[2161,2327,2328,2344],{},[2164,2329,2330],{},[2167,2331,2332,2335,2338,2341],{},[2170,2333,2334],{},"指标项",[2170,2336,2337],{"align":2175},"无缓存基线（Direct LLM）",[2170,2339,2340],{"align":2175},"开启两级缓存（L1 Exact + L2 Semantic）",[2170,2342,2343],{},"核心收益对比",[2193,2345,2346,2367,2385,2400],{},[2167,2347,2348,2353,2356,2361],{},[2198,2349,2350],{},[20,2351,2352],{},"总耗时（Wall Time）",[2198,2354,2355],{"align":2175},"84.52 s",[2198,2357,2358],{"align":2175},[20,2359,2360],{},"30.52 s",[2198,2362,2363,2364],{},"端到端总耗时缩短 ",[20,2365,2366],{},"63.9%",[2167,2368,2369,2374,2376,2382],{},[2198,2370,2371],{},[20,2372,2373],{},"整体缓存拦截率",[2198,2375,2205],{"align":2175},[2198,2377,2378,2381],{"align":2175},[20,2379,2380],{},"65.0%","（39 \u002F 60 命中）",[2198,2383,2384],{},"直接节省约三分之二的底层 GPU 推理调用",[2167,2386,2387,2392,2394,2397],{},[2198,2388,2389],{},[20,2390,2391],{},"未命中请求 P50 延迟",[2198,2393,2214],{"align":2175},[2198,2395,2396],{"align":2175},"3,332.9 ms",[2198,2398,2399],{},"缓存未命中路径几乎零额外损耗",[2167,2401,2402,2407,2410,2470],{},[2198,2403,2404],{},[20,2405,2406],{},"语义缓存命中延迟（L2 Hit）",[2198,2408,2409],{"align":2175},"—",[2198,2411,2412],{"align":2175},[20,2413,2414,2415,2159],{},"15.1 ms（P99 ",[114,2416,2418,2439],{"className":2417},[117],[114,2419,2421],{"className":2420},[121],[123,2422,2423],{"xmlns":125},[127,2424,2425,2436],{},[130,2426,2427,2430,2433],{},[137,2428,2429],{},"≤",[142,2431,2432],{},"37.0",[555,2434,2435],{}," ms",[149,2437,2438],{"encoding":151},"\\le 37.0\\text{ ms}",[114,2440,2442,2455],{"className":2441,"ariaHidden":157},[156],[114,2443,2445,2449,2452],{"className":2444},[161],[114,2446],{"className":2447,"style":2448},[165],"height:0.7719em;vertical-align:-0.136em;",[114,2450,2429],{"className":2451},[309],[114,2453],{"className":2454,"style":305},[304],[114,2456,2458,2461,2464],{"className":2457},[161],[114,2459],{"className":2460,"style":499},[165],[114,2462,2432],{"className":2463},[170],[114,2465,2467],{"className":2466},[170,86],[114,2468,2435],{"className":2469},[170],[2198,2471,2472],{},"在本地完成向量编码与余弦匹配，免除秒级生成等待",[97,2474,2476,2477,2159],{"id":2475},"_3-场景-c高并发压力测试100-请求并发度-c102050c10-20-50c102050","3. 场景 C：高并发压力测试（100 请求，并发度 ",[114,2478,2480,2509],{"className":2479},[117],[114,2481,2483],{"className":2482},[121],[123,2484,2485],{"xmlns":125},[127,2486,2487,2506],{},[130,2488,2489,2491,2493,2496,2498,2501,2503],{},[133,2490,2118],{},[137,2492,383],{},[142,2494,2495],{},"10",[137,2497,568],{"separator":157},[142,2499,2500],{},"20",[137,2502,568],{"separator":157},[142,2504,2505],{},"50",[149,2507,2508],{"encoding":151},"C=10, 20, 50",[114,2510,2512,2530],{"className":2511,"ariaHidden":157},[156],[114,2513,2515,2518,2521,2524,2527],{"className":2514},[161],[114,2516],{"className":2517,"style":2136},[165],[114,2519,2118],{"className":2520,"style":2140},[170,171],[114,2522],{"className":2523,"style":305},[304],[114,2525,383],{"className":2526},[309],[114,2528],{"className":2529,"style":305},[304],[114,2531,2533,2537,2540,2543,2546,2549,2552,2555],{"className":2532},[161],[114,2534],{"className":2535,"style":2536},[165],"height:0.8389em;vertical-align:-0.1944em;",[114,2538,2495],{"className":2539},[170],[114,2541,568],{"className":2542},[682],[114,2544],{"className":2545,"style":686},[304],[114,2547,2500],{"className":2548},[170],[114,2550,568],{"className":2551},[682],[114,2553],{"className":2554,"style":686},[304],[114,2556,2505],{"className":2557},[170],[10,2559,2560],{},"在 65% 稳态缓存命中率的混合生产流量模拟下，逐步拉高客户端并发度：",[2161,2562,2563,2613],{},[2164,2564,2565],{},[2167,2566,2567,2598,2601,2604,2607,2610],{},[2170,2568,2569,2570,147],{"align":2175},"客户端并发度 (",[114,2571,2573,2586],{"className":2572},[117],[114,2574,2576],{"className":2575},[121],[123,2577,2578],{"xmlns":125},[127,2579,2580,2584],{},[130,2581,2582],{},[133,2583,2118],{},[149,2585,2118],{"encoding":151},[114,2587,2589],{"className":2588,"ariaHidden":157},[156],[114,2590,2592,2595],{"className":2591},[161],[114,2593],{"className":2594,"style":2136},[165],[114,2596,2118],{"className":2597,"style":2140},[170,171],[2170,2599,2600],{"align":2175},"成功率",[2170,2602,2603],{"align":2175},"网关总吞吐 (RPS)",[2170,2605,2606],{"align":2175},"整体 P50 延迟",[2170,2608,2609],{"align":2175},"缓存命中路径 P99 延迟",[2170,2611,2612],{"align":2175},"穿透推理路径 P99 延迟",[2193,2614,2615,2684,2752],{},[2167,2616,2617,2670,2672,2675,2678,2681],{},[2198,2618,2619],{"align":2175},[114,2620,2622,2640],{"className":2621},[117],[114,2623,2625],{"className":2624},[121],[123,2626,2627],{"xmlns":125},[127,2628,2629,2637],{},[130,2630,2631,2633,2635],{},[133,2632,2118],{},[137,2634,383],{},[142,2636,2495],{},[149,2638,2639],{"encoding":151},"C = 10",[114,2641,2643,2661],{"className":2642,"ariaHidden":157},[156],[114,2644,2646,2649,2652,2655,2658],{"className":2645},[161],[114,2647],{"className":2648,"style":2136},[165],[114,2650,2118],{"className":2651,"style":2140},[170,171],[114,2653],{"className":2654,"style":305},[304],[114,2656,383],{"className":2657},[309],[114,2659],{"className":2660,"style":305},[304],[114,2662,2664,2667],{"className":2663},[161],[114,2665],{"className":2666,"style":499},[165],[114,2668,2495],{"className":2669},[170],[2198,2671,2232],{"align":2175},[2198,2673,2674],{"align":2175},"4.86 req\u002Fs",[2198,2676,2677],{"align":2175},"1.9 ms",[2198,2679,2680],{"align":2175},"18.5 ms",[2198,2682,2683],{"align":2175},"7,280.7 ms",[2167,2685,2686,2739,2741,2743,2746,2749],{},[2198,2687,2688],{"align":2175},[114,2689,2691,2709],{"className":2690},[117],[114,2692,2694],{"className":2693},[121],[123,2695,2696],{"xmlns":125},[127,2697,2698,2706],{},[130,2699,2700,2702,2704],{},[133,2701,2118],{},[137,2703,383],{},[142,2705,2500],{},[149,2707,2708],{"encoding":151},"C = 20",[114,2710,2712,2730],{"className":2711,"ariaHidden":157},[156],[114,2713,2715,2718,2721,2724,2727],{"className":2714},[161],[114,2716],{"className":2717,"style":2136},[165],[114,2719,2118],{"className":2720,"style":2140},[170,171],[114,2722],{"className":2723,"style":305},[304],[114,2725,383],{"className":2726},[309],[114,2728],{"className":2729,"style":305},[304],[114,2731,2733,2736],{"className":2732},[161],[114,2734],{"className":2735,"style":499},[165],[114,2737,2500],{"className":2738},[170],[2198,2740,2232],{"align":2175},[2198,2742,2674],{"align":2175},[2198,2744,2745],{"align":2175},"1.6 ms",[2198,2747,2748],{"align":2175},"24.3 ms",[2198,2750,2751],{"align":2175},"14,193.3 ms",[2167,2753,2754,2807,2809,2814,2819,2824],{},[2198,2755,2756],{"align":2175},[114,2757,2759,2777],{"className":2758},[117],[114,2760,2762],{"className":2761},[121],[123,2763,2764],{"xmlns":125},[127,2765,2766,2774],{},[130,2767,2768,2770,2772],{},[133,2769,2118],{},[137,2771,383],{},[142,2773,2505],{},[149,2775,2776],{"encoding":151},"C = 50",[114,2778,2780,2798],{"className":2779,"ariaHidden":157},[156],[114,2781,2783,2786,2789,2792,2795],{"className":2782},[161],[114,2784],{"className":2785,"style":2136},[165],[114,2787,2118],{"className":2788,"style":2140},[170,171],[114,2790],{"className":2791,"style":305},[304],[114,2793,383],{"className":2794},[309],[114,2796],{"className":2797,"style":305},[304],[114,2799,2801,2804],{"className":2800},[161],[114,2802],{"className":2803,"style":499},[165],[114,2805,2505],{"className":2806},[170],[2198,2808,2232],{"align":2175},[2198,2810,2811],{"align":2175},[20,2812,2813],{},"5.55 req\u002Fs",[2198,2815,2816],{"align":2175},[20,2817,2818],{},"2.2 ms",[2198,2820,2821],{"align":2175},[20,2822,2823],{},"37.0 ms",[2198,2825,2826],{"align":2175},"21,963.1 ms",[10,2828,2829,2830,2881,2882,2884,2885,2894],{},"可以看到，即使在 ",[114,2831,2833,2851],{"className":2832},[117],[114,2834,2836],{"className":2835},[121],[123,2837,2838],{"xmlns":125},[127,2839,2840,2848],{},[130,2841,2842,2844,2846],{},[133,2843,2118],{},[137,2845,383],{},[142,2847,2505],{},[149,2849,2850],{"encoding":151},"C=50",[114,2852,2854,2872],{"className":2853,"ariaHidden":157},[156],[114,2855,2857,2860,2863,2866,2869],{"className":2856},[161],[114,2858],{"className":2859,"style":2136},[165],[114,2861,2118],{"className":2862,"style":2140},[170,171],[114,2864],{"className":2865,"style":305},[304],[114,2867,383],{"className":2868},[309],[114,2870],{"className":2871,"style":305},[304],[114,2873,2875,2878],{"className":2874},[161],[114,2876],{"className":2877,"style":499},[165],[114,2879,2505],{"className":2880},[170]," 的突发洪峰下（底层 LLM 因并发排队导致穿透路径 P99 达到 ",[50,2883,2826],{},"），网关层的异步事件循环依然未被阻塞，",[20,2886,2887,2888,2890,2891,2893],{},"全部命中缓存的请求 P50 保持在 ",[50,2889,2818],{},"、P99 稳定在 ",[50,2892,2823],{}," 以内","，实现了快慢路径的彻底解耦。",[70,2896],{},[73,2898,2900],{"id":2899},"四工程设计总结","四、工程设计总结",[14,2902,2903,2909,2967],{},[17,2904,2905,2908],{},[20,2906,2907],{},"快慢路径解耦是治理 LLM 尾部延迟的第一要务","：通过前置的 L1 精确哈希与 L2 向量语义缓存拦截 60%+ 的重复与同义长尾流量，能防止底层 GPU 队列被低价值重复计算打满；",[17,2910,2911,2914,2915,2966],{},[20,2912,2913],{},"语义缓存必须受控启用","：仅对确定性、无状态问答开启高阈值（",[114,2916,2918,2936],{"className":2917},[117],[114,2919,2921],{"className":2920},[121],[123,2922,2923],{"xmlns":125},[127,2924,2925,2933],{},[130,2926,2927,2929,2931],{},[133,2928,518],{},[137,2930,598],{},[142,2932,615],{},[149,2934,2935],{"encoding":151},"\\tau \\ge 0.92",[114,2937,2939,2957],{"className":2938,"ariaHidden":157},[156],[114,2940,2942,2945,2948,2951,2954],{"className":2941},[161],[114,2943],{"className":2944,"style":2448},[165],[114,2946,518],{"className":2947,"style":535},[170,171],[114,2949],{"className":2950,"style":305},[304],[114,2952,598],{"className":2953},[309],[114,2955],{"className":2956,"style":305},[304],[114,2958,2960,2963],{"className":2959},[161],[114,2961],{"className":2962,"style":499},[165],[114,2964,615],{"className":2965},[170],"）余弦匹配，对带实时 Tool Observation 或高采样温度的请求严格旁路；",[17,2968,2969,2972,2973,2976],{},[20,2970,2971],{},"有界时间窗（50 ms）动态批处理兼顾了延迟与吞吐","：用微小的 50 ms 聚合窗口将零散并发请求打包下发，配合 Prometheus ",[50,2974,2975],{},"\u002Fmetrics"," 暴露的缓存命中率与队列水位直方图，构成了生产级推理网关的标准底座。",[2978,2979,2980],"style",{},"html pre.shiki code .slEfO, html code.shiki .slEfO{--shiki-default:#F47067;--shiki-default-font-style:inherit;--shiki-light:#D73A49;--shiki-light-font-style:inherit;--shiki-cyber:#54B9FF;--shiki-cyber-font-style:italic}html pre.shiki code .sWURj, html code.shiki .sWURj{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#EEF0F9}html pre.shiki code .sS_0i, html code.shiki .sS_0i{--shiki-default:#DCBDFB;--shiki-light:#6F42C1;--shiki-cyber:#00DAEF}html pre.shiki code .sST-V, html code.shiki .sST-V{--shiki-default:#F47067;--shiki-light:#D73A49;--shiki-cyber:#54B9FF}html pre.shiki code .sBBmv, html code.shiki .sBBmv{--shiki-default:#F69D50;--shiki-light:#6F42C1;--shiki-cyber:#ACAFFF}html pre.shiki code .svkvy, html code.shiki .svkvy{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#EEF0F9}html pre.shiki code .sbU-9, html code.shiki .sbU-9{--shiki-default:#ADBAC7;--shiki-default-font-style:inherit;--shiki-light:#24292E;--shiki-light-font-style:inherit;--shiki-cyber:#ACAFFF;--shiki-cyber-font-style:italic}html pre.shiki code .s9NhK, html code.shiki .s9NhK{--shiki-default:#ADBAC7;--shiki-default-font-style:inherit;--shiki-light:#24292E;--shiki-light-font-style:inherit;--shiki-cyber:#FFD493;--shiki-cyber-font-style:italic}html pre.shiki code .sf2cM, html code.shiki .sf2cM{--shiki-default:#F47067;--shiki-light:#D73A49;--shiki-cyber:#EEF0F9}html pre.shiki code .sgENW, html code.shiki .sgENW{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#FFD493}html pre.shiki code .sRGqv, html code.shiki .sRGqv{--shiki-default:#6CB6FF;--shiki-light:#005CC5;--shiki-cyber:#ACAFFF}html pre.shiki code .sq3c-, html code.shiki .sq3c-{--shiki-default:#ADBAC7;--shiki-light:#24292E;--shiki-cyber:#00DAEF}html pre.shiki code .sA9aF, html code.shiki .sA9aF{--shiki-default:#F69D50;--shiki-default-font-style:inherit;--shiki-light:#E36209;--shiki-light-font-style:inherit;--shiki-cyber:#4BF3C8;--shiki-cyber-font-style:italic}html pre.shiki code .sCxhR, html code.shiki .sCxhR{--shiki-default:#768390;--shiki-default-font-style:inherit;--shiki-light:#6A737D;--shiki-light-font-style:inherit;--shiki-cyber:#EEF0F98F;--shiki-cyber-font-style:italic}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}html.cyber .shiki span {color: var(--shiki-cyber);background: var(--shiki-cyber-bg);font-style: var(--shiki-cyber-font-style);font-weight: var(--shiki-cyber-font-weight);text-decoration: var(--shiki-cyber-text-decoration);}",{"title":87,"searchDepth":1147,"depth":1155,"links":2982},[2983,2984,2988,2996],{"id":75,"depth":1147,"text":76},{"id":94,"depth":1147,"text":95,"children":2985},[2986,2987],{"id":99,"depth":1155,"text":100},{"id":924,"depth":1155,"text":925},{"id":2081,"depth":1147,"text":2082,"children":2989},[2990,2992,2994],{"id":2102,"depth":1155,"text":2991},"1. 场景 A：L1 精确缓存冷启动 vs. 热命中（50 请求，并发度 C=5C=5C=5）",{"id":2271,"depth":1155,"text":2993},"2. 场景 B：混合流量下的两级精确 + 语义缓存（60 请求，并发度 C=5C=5C=5）",{"id":2475,"depth":1155,"text":2995},"3. 场景 C：高并发压力测试（100 请求，并发度 C=10,20,50C=10, 20, 50C=10,20,50）",{"id":2899,"depth":1147,"text":2900},[2998],"AI Infra","\u002Fimages\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching\u002Fllm-gateway-cover-20261019.webp","高并发请求流依次穿过精确哈希滤网、向量球面语义缓存层与 50ms 动态批处理时间窗拼装器的等距系统架构插画","2026-10-19","深度拆解生产级大模型推理网关 LLM-Gateway 的核心系统设计：基于 SHA-256 与 Redis 向量余弦相似度的两级精确\u002F语义缓存、50ms 有界等待动态批处理调度器（Dynamic Batching）、令牌桶多租户限流与 Prometheus 全链路可观测性，将命中请求 P99 延迟从 5,307 ms 压降至 17 ms。",false,"md",{"mathjax":1172},"\u002Fposts\u002Fllm-gateway-semantic-cache-dynamic-batching",null,{"title":5,"description":3002},"posts\u002Fllm-gateway-semantic-cache-dynamic-batching",[2998,3011,3012,3013,3014,3015],"LLM 推理","分布式系统","Redis","FastAPI","后端","Rtm-qCu2G5FdRyaHfaeP5NkM4m6Xd-AE2llW0a3l-p0",[3018,3036,3051,3065,3079,3093,3107,3122,3135,3147,3151,3163,3176,3188,3201,3214,3230,3244,3255,3267,3281,3294,3308,3320,3334,3346,3361,3375,3389,3400,3413,3430,3444,3457,3469,3481,3494,3507],{"slug":3019,"path":3020,"title":3021,"date":3022,"tags":3023,"description":3029,"cover":3030,"coverAlt":3031,"excerpt":3032,"draft":3003,"hidden":3003,"published":1172,"readingTime":1207,"series":3033,"seriesOrder":1207,"codeUrl":3034,"attachmentUrl":3035},"cs336-cross-entropy-adamw-training-loop","\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop","Stanford CS336 学习记录 08：数值稳定交叉熵、AdamW、余弦调度与完整训练闭环","2026-10-27",[3024,3025,3026,3027,3028],"CS336","PyTorch","AdamW","CrossEntropy","训练闭环","用 LogSumExp 消去交叉熵中的指数溢出，手写解耦权重衰减的 AdamW 优化器、全局 L2 梯度裁剪、Warmup 余弦退火调度、memmap 数据采样、Checkpoint 序列化与 Top-p 解码，完成 CS336 Assignment 1 全部 48 项测试。","\u002Fimages\u002Fposts\u002Fcs336-cross-entropy-adamw-training-loop\u002Fcs336-cross-entropy-adamw-training-loop-cover.webp","滑动窗口采样、余弦退火曲线、双动量优化轨迹与检查点归档构成的训练闭环 AI 辅助概念插画","作为 Stanford CS336 Assignment 1 系列的收官篇，我们已经拥有了分词器（bpe.py、tokenizer.py）和完整的语言模型架构（transformer.py）。要让模型在语料上真正训练并生成文本，还需","Stanford CS336","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fb5c93ad0cc73178b2b2a62e518a0900da47a0fb7","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-08-adamw-training-loop-20261027.zip",{"slug":3037,"path":3038,"title":3039,"date":3040,"tags":3041,"description":3047,"cover":3048,"coverAlt":3049,"excerpt":3050,"draft":3003,"hidden":3003,"published":1172,"readingTime":1207},"editcache-video-dit-inference-acceleration","\u002Fposts\u002Feditcache-video-dit-inference-acceleration","EditCache：视频 Diffusion Transformer (DiT) 局部编辑的跨步特征复用与区域重算加速","2026-10-25",[3042,3043,2998,3044,3045,3046],"扩散模型","视频生成","推理加速","GPU","论文解读","拆解视频扩散模型（Wan2.1-VACE）局部编辑中的三重算力冗余，通过可重入 Latent、跨步 Block 隐状态缓存复用、带 Halo 边界的局部区域重算与自适应推理起点，在单卡 H100 上实现 2.46× 模型计算加速与 1.49× 严格同步端到端加速。","\u002Fimages\u002Fposts\u002Feditcache-video-dit-inference-acceleration\u002Feditcache-dit-cover-20261025.webp","视频扩散 Transformer 时空三维体素网格中冻结背景缓存与暖橙色局部编辑重算通道的等距技术插画","在基于 Diffusion Transformer（DiT）的视频编辑与局部重绘（Video Inpainting \u002F Local Editing）任务中，用户的典型工作流并不是一次性生成最终成片，而是针对同一段源视频反复调整掩码区",{"slug":3052,"path":3053,"title":3054,"date":3040,"tags":3055,"description":3060,"cover":3061,"coverAlt":3062,"excerpt":3063,"draft":3003,"hidden":3003,"published":1172,"readingTime":1207,"series":3033,"seriesOrder":1197,"codeUrl":3034,"attachmentUrl":3064},"cs336-bpe-tokenizer-training-encoding","\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding","Stanford CS336 学习记录 07：Byte-Level BPE 分词器训练、倒排索引加速与流式编码",[3024,3056,3057,3058,3059],"Tokenizer","BPE","Tiktoken","算法优化","实现 GPT-2 正则预分词、特殊 token 硬边界切分、基于 pair-to-word 倒排索引的增量 BPE 合并算法（\u003C 0.25s 通过速度基准），以及与 tiktoken 完全对齐的 Tokenizer。","\u002Fimages\u002Fposts\u002Fcs336-bpe-tokenizer-training-encoding\u002Fcs336-bpe-tokenizer-training-encoding-cover.webp","UTF-8 字节块经过正则切分闸口后沿合并树聚合成子词单元的 AI 辅助概念插画","前面六篇完成了 Transformer 语言模型本身的全部神经网络组件。要让模型真正处理自然语言文本，还需要打通入口处的 ByteLevel BPE（BytePair Encoding）分词器。这一篇记录 cs336basics\u002Fb","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-07-bpe-tokenizer-20261025.zip",{"slug":3066,"path":3067,"title":3068,"date":3069,"tags":3070,"description":3075,"cover":3076,"coverAlt":3077,"excerpt":3078,"draft":3003,"hidden":3003,"published":1172,"readingTime":1207},"in-situ-autoguidance-editable-scientific-illustration","\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration","从原位自校正引导（In-situ Autoguidance）到可编辑科研插图生成（AutoFigure-Edit）","2026-10-24",[3042,3071,3072,3073,3074,3046],"多模态","Agent","ICML","ACL","串联两项生成式模型与多模态智能体研究：如何通过推理期局部随机微扰在单模型内构造免训练扩散自引导向量（ICML 2025 Workshop 第一作者，arXiv:2510.17136），以及如何通过五阶段视觉-符号混合流水线生成原生可编辑 SVG 科研插图（ACL 2026 Demo，FigureBench 83.0% 胜率）。","\u002Fimages\u002Fposts\u002Fin-situ-autoguidance-editable-scientific-illustration\u002Fautoguidance-autofigure-cover-20261024.webp","左侧扩散流形自校正轨迹矢量场与右侧五阶段可编辑 SVG 科研插图分层网格相衔接的等距技术插画","生成式视觉模型正在经历从“生成高保真像素（Pixel Synthesis）”向“可控结构化生成（Controllable & Editable Generation）”的演进。在这一进程中，我作为第一作者或核心贡献者参与了两项分别处",{"slug":3080,"path":3081,"title":3082,"date":3083,"tags":3084,"description":3088,"cover":3089,"coverAlt":3090,"excerpt":3091,"draft":3003,"hidden":3003,"published":1172,"readingTime":1207,"series":3033,"seriesOrder":1183,"codeUrl":3034,"attachmentUrl":3092},"cs336-pre-norm-transformer-block-lm","\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm","Stanford CS336 学习记录 06：Pre-Norm Transformer Block 与完整语言模型装配","2026-10-23",[3024,3025,3085,3086,3087],"Transformer","Pre-Norm","FLOPs","将 RMSNorm、带 RoPE 的因果多头自注意力和 SwiGLU 组装成 Pre-Norm Transformer Block，搭建完整 TransformerLM，并核算 GPT-2 XL 的参数量与前向 FLOPs。","\u002Fimages\u002Fposts\u002Fcs336-pre-norm-transformer-block-lm\u002Fcs336-pre-norm-transformer-block-lm-cover.webp","干净残差流主干穿过多个 Pre-Norm 子层并汇聚到词表投影层的 AI 辅助概念插画","前五篇把 Linear、Embedding、RMSNorm、SwiGLU、RotaryPositionalEmbedding 和因果 MultiHeadSelfAttention 全部单独打通了。这一篇进入 Assignment 1","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-06-transformer-block-lm-20261023.zip",{"slug":3094,"path":3095,"title":3096,"date":3083,"tags":3097,"description":3103,"cover":3104,"coverAlt":3105,"excerpt":3106,"draft":3003,"hidden":3003,"published":1172,"readingTime":1311},"xgrammar-concurrent-cache-generation-fix","\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix","XGrammar 结构化解码缓存剖析：Unlimited 模式并发 Clear 字节记账竞态与代际修复（PR #1011）",[2998,3098,3099,3100,3101,3102],"C++","XGrammar","结构化解码","并发编程","源码分析","拆解 XGrammar GrammarCompiler 的双层缓存架构与 ThreadSafeLRUCache 锁分离实现，定位 Unlimited 模式下 Clear() 字节残留与在途编译（In-flight Computation）跨代污染根因，给出代际号 + 读写锁原子修复与零 sleep 确定性并发 GoogleTest 验证。","\u002Fimages\u002Fposts\u002Fxgrammar-concurrent-cache-generation-fix\u002Fxgrammar-cache-fix-cover-20261022.webp","双层磨砂玻璃流道经暖橙色代际闸门过滤后汇入晶格缓存矩阵的等距技术插画","在大语言模型服务（如 SGLang、vLLM）的结构化输出场景中，JSON Schema 或 EBNF 语法需要先编译为词表级掩码缓存（CompiledGrammar），才能在每个 Decode 步以微秒级开销生成合法 token ",{"slug":3108,"path":3109,"title":3110,"date":3111,"tags":3112,"description":3118,"cover":3119,"coverAlt":3120,"excerpt":3121,"draft":3003,"hidden":3003,"published":1172,"readingTime":1311},"multi-constraint-agent-routing-dag-eval","\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval","多约束生产级 Agent 系统实践：两阶段工具路由、DAG 局部重规划与轨迹级评测闭环","2026-10-22",[3072,3113,3114,3115,3116,3117],"LLM","工具调用","上下文工程","DPO","强化学习","围绕跨境酒旅等强耦合多约束业务场景，拆解向量 Top-K 召回 + LLM 精排 + 跨语言实体链接与 Schema 门控（96%+ 调用合法率）、基于状态机的 DAG 局部子图重规划、Observation 冗余裁剪（节省 40% Token），以及从线上失败恢复轨迹构造 SFT\u002FDPO 偏好对的评测与训练闭环。","\u002Fimages\u002Fposts\u002Fmulti-constraint-agent-routing-dag-eval\u002Fagent-dag-routing-cover-20261020.webp","候选模块经漏斗滤网进入双分支有向无环图轨道并在受阻节点触发暖橙色局部旁路重连的等距技术插画","在玩具级 Demo 里，几个工具加上一段 ReAct 循环就能跑通大多数单步问答。但当 Agent 进入跨境酒旅、复杂交易或研发沙箱等真实业务生产环境时，单靠线性 ReAct 循环很快会撞上四堵墙： 1. 工具空间与参数耦合爆炸：候",{"slug":3123,"path":3124,"title":3125,"date":3126,"tags":3127,"description":3131,"cover":3132,"coverAlt":3133,"excerpt":3134,"draft":3003,"hidden":3003,"published":1172,"readingTime":1368},"llm-mrd-distillation-auditable-review-agents","\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents","多视角推理蒸馏（LLM-MRD）与可审计生成式评审系统（DeepReviewer 2.0）深度拆解","2026-10-21",[3128,3071,3129,3072,3130,3046],"知识蒸馏","Qwen2.5-VL","Post-training","对比大模型落地的两条互补路径：如何用 Qwen2.5-VL 离线多视角分析结合 MLP 残差校准与温度缩放 KL 蒸馏，将推理能力压入 358M 学生模型实现 17.2 ms 独立推理（LLM-MRD，DASFAA 2026 Oral）；以及如何通过主张-证据-风险台账、PASA 可比性门控与导出预算校验构建可审计的同行评审 Agent（DeepReviewer 2.0）。","\u002Fimages\u002Fposts\u002Fllm-mrd-distillation-auditable-review-agents\u002Fmrd-review-distill-cover-20261016.webp","三块半透明几何棱镜发出平行光束穿过中央校准环汇聚至右侧高密度晶体模块的等距技术插画","把大语言模型或视觉语言模型（VLM）的深度推理能力用到实际系统中，通常会面临一个两难选择： 如果场景受限于毫秒级在线延迟（如高吞吐多模态内容审核与虚假信息检测）：直接在线调用 7B～72B 的多模态大模型做长链条思维链（CoT）生成",{"slug":3136,"path":3137,"title":3138,"date":3126,"tags":3139,"description":3142,"cover":3143,"coverAlt":3144,"excerpt":3145,"draft":3003,"hidden":3003,"published":1172,"readingTime":1197,"series":3033,"seriesOrder":1176,"codeUrl":3034,"attachmentUrl":3146},"cs336-rope-rotary-position-embedding","\u002Fposts\u002Fcs336-rope-rotary-position-embedding","Stanford CS336 学习记录 05：RoPE 旋转位置编码与带位置的多头注意力",[3024,3025,3085,3140,3141],"RoPE","Attention","从二维平面正交旋转推导 RoPE 的奇偶特征配对公式，理解为什么内积只依赖相对位置差 (i - j)，并用 register_buffer 预计算 cos\u002Fsin 缓存接入因果多头自注意力。","\u002Fimages\u002Fposts\u002Fcs336-rope-rotary-position-embedding\u002Fcs336-rope-rotary-position-embedding-cover.webp","二维特征平面中的旋转向量对与预计算余弦正弦网格的 AI 辅助概念插画","上一篇完成了不含位置编码的因果多头自注意力。如果只看那一套矩阵乘法，把输入序列前两个 token 交换顺序（并在因果 mask 允许的范围内看同一个后缀 query 对它们的点积），点积本身并不知道哪个 key 离当前 query ","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-05-rope-mha-20261021.zip",{"slug":3148,"path":3006,"title":5,"date":3001,"tags":3149,"description":3002,"cover":2999,"coverAlt":3000,"excerpt":3150,"draft":3003,"hidden":3003,"published":1172,"readingTime":1207},"llm-gateway-semantic-cache-dynamic-batching",[2998,3011,3012,3013,3014,3015],"在将大语言模型接入高并发业务系统时，如果让上游微服务直接裸连底层推理引擎（vLLM \u002F SGLang 或外部模型 API），系统很快会暴露出三个典型的生产痛点： 1. 高频重复与近义查询消耗巨量算力：大量用户请求在语义上高度等价（例",{"slug":3152,"path":3153,"title":3154,"date":3001,"tags":3155,"description":3157,"cover":3158,"coverAlt":3159,"excerpt":3160,"draft":3003,"hidden":3003,"published":1172,"readingTime":1311,"series":3033,"seriesOrder":1169,"codeUrl":3161,"attachmentUrl":3162},"cs336-multihead-attention-shapes","\u002Fposts\u002Fcs336-multihead-attention-shapes","Stanford CS336 学习记录 04：多头注意力怎样拆头、计算和合头",[3024,3025,3085,3141,3156],"张量形状","沿着 (2, 5, 64) 的输入追踪 QKV 投影、reshape、transpose、共享因果 mask 和合头，理解每个头仍处理完整序列，以及最后为什么还需要输出 Linear。","\u002Fimages\u002Fposts\u002Fcs336-multihead-attention-shapes\u002Fcs336-multihead-attention-shapes-cover.webp","宽向量束分成多条并行特征轨道后重新合并的 AI 辅助概念插画","上一篇完成了缩放点积注意力。那个函数接收已经准备好的 Q、K、V，计算匹配分数，应用 mask，再用权重汇总 V。这一篇继续把它放进一个带参数的多头自注意力模块。 这次最需要看清的是“拆头”。代码里只是几行 reshape 和 tr","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002F31363223498fafd1528c550a51efbe0aed231736","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-04-multihead-attention-20261019.zip",{"slug":3164,"path":3165,"title":3166,"date":3167,"tags":3168,"description":3172,"cover":3173,"coverAlt":3174,"excerpt":3175,"draft":3003,"hidden":3003,"published":1172,"readingTime":1213},"qwen25-sft-dpo-alignment-tax-ablation","\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation","大模型后训练偏好对齐实录：Qwen2.5 SFT 多维消融、动态参考 DPO 与对齐税（Alignment Tax）量化分析","2026-10-18",[3130,3116,3169,3170,3117,3171],"SFT","LoRA","Qwen2.5","基于 Qwen2.5 完整复现 Instruct SFT → Bradley-Terry DPO 两阶段后训练管线：通过 3 随机种子消融 LoRA Rank、学习率与数据规模（GSM8K +12.73pp），实现免双倍权重的动态 Adapter 切换 DPO 训练，并定量揭示偏好对齐对确定性数学推理的对齐税效应。","\u002Fimages\u002Fposts\u002Fqwen25-sft-dpo-alignment-tax-ablation\u002Fqwen25-sft-dpo-cover-20261018.webp","左侧监督微调轨迹流经中央 Bradley-Terry 偏好分流天平分离出正负样本通道并连接右侧对齐税监控仪表盘的等距技术插画","在大语言模型后训练（PostTraining）管线中，监督微调（SFT）负责教会模型任务格式与领域推理模式，而直接偏好优化（DPO, Direct Preference Optimization）则利用成对偏好数据 $(x, yw,",{"slug":3177,"path":3178,"title":3179,"date":3167,"tags":3180,"description":3182,"cover":3183,"coverAlt":3184,"excerpt":3185,"draft":3003,"hidden":3003,"published":1172,"readingTime":1227,"series":3033,"seriesOrder":1155,"codeUrl":3186,"attachmentUrl":3187},"cs336-softmax-attention-mask","\u002Fposts\u002Fcs336-softmax-attention-mask","Stanford CS336 学习记录 03：Softmax 的数值稳定与 Attention Mask",[3024,3025,3085,3181,3141],"Softmax","从减去最大值为何不改变 Softmax 开始，理清函数与有参数层的区别，再沿着 Q、K、V 的形状理解缩放点积注意力，以及 mask 为什么必须填负无穷。","\u002Fimages\u002Fposts\u002Fcs336-softmax-attention-mask\u002Fcs336-softmax-attention-mask-cover.webp","注意力分数、权重与因果遮罩的 AI 辅助概念插画","上一篇写到 SwiGLU，接下来进入 Attention。真正开始相乘之前，先做了 Softmax：它把一组分数变成总和为 1 的权重，后面用这些权重汇总不同 token 提供的信息。 这段学习里，有两个很具体的问题值得记下来。一个","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcs336-labs\u002Ftree\u002Fa39eb684710817f7f389626423fcb26ca0f6d629","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-03-softmax-sdpa-20261018.zip",{"slug":3189,"path":3190,"title":3191,"date":3192,"tags":3193,"description":3196,"cover":3197,"coverAlt":3198,"excerpt":3199,"draft":3003,"hidden":3003,"published":1172,"readingTime":1213,"series":3033,"seriesOrder":1147,"codeUrl":3186,"attachmentUrl":3200},"cs336-rmsnorm-swiglu","\u002Fposts\u002Fcs336-rmsnorm-swiglu","Stanford CS336 学习记录 02：RMSNorm 的广播与 SwiGLU 的门控","2026-10-16",[3024,3025,3085,3194,3195],"RMSNorm","SwiGLU","用一个 token 向量理解均方根归一化，拆开 dim=-1 与 keepdim=True 的作用，再沿着两条分支读懂 SiLU、逐元素乘法和 SwiGLU。","\u002Fimages\u002Fposts\u002Fcs336-rmsnorm-swiglu\u002Fcs336-rmsnorm-swiglu-cover.webp","特征尺度整理和双分支门控的 AI 辅助概念插画","上一篇把 Linear 和 Embedding 的参数形状理顺了。继续往下，最先遇到的是 RMSNorm：公式不算长，但把它落到一批 token 上时，dim=1、keepdim=True 和广播必须一起理解。 然后是 SiLU 和","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-02-rmsnorm-swiglu-20261016.zip",{"slug":3202,"path":3203,"title":3204,"date":3205,"tags":3206,"description":3209,"cover":3210,"coverAlt":3211,"excerpt":3212,"draft":3003,"hidden":3003,"published":1172,"readingTime":1222,"series":3033,"seriesOrder":1136,"codeUrl":3186,"attachmentUrl":3213},"cs336-linear-embedding-shapes","\u002Fposts\u002Fcs336-linear-embedding-shapes","Stanford CS336 学习记录 01：从 Linear、Embedding 搞清张量形状","2026-10-14",[3024,3025,3085,3207,3208],"Linear","Embedding","从 PyTorch 类、参数与 forward 的关系开始，理解 Linear 为什么保存两行三列的权重、计算时为什么要转置，再区分 Embedding 查表与线性变换。","\u002Fimages\u002Fposts\u002Fcs336-linear-embedding-shapes\u002Fcs336-linear-embedding-shapes-cover.webp","token 方块、向量表与矩阵变换的 AI 辅助概念插画","开始沿着 Stanford CS336 的 Assignment 1 一步步实现。BPE 之前已经接触过，这轮先跳过分词器，从 Transformer 的基础层开始。这里记录的是当时写代码时真正卡住的问题，以及对应测试能确认到的范围","\u002Fmaterials\u002Fcs336\u002Fexperiments\u002Fcs336-01-linear-embedding-20261014.zip",{"slug":3215,"path":3216,"title":3217,"date":3218,"tags":3219,"description":3224,"cover":3225,"coverAlt":3226,"excerpt":3227,"draft":3003,"hidden":3003,"published":1172,"readingTime":1237,"series":3220,"seriesOrder":1311,"codeUrl":3228,"attachmentUrl":3229},"cmu-15-779-kv-cache-granularity-study","\u002Fposts\u002Fcmu-15-779-kv-cache-granularity-study","CMU 15-779 学习记录 14：KV 页大小、共享边界与 CPU 实现成本","2026-10-12",[3220,3221,3222,3223],"CMU 15-779","KV cache","PagedAttention","系统实验","用 210 组 CPU 配置比较连续与分页 KV cache，观察 31\u002F32-token 共享前缀的页边界，分别记录活跃槽位、池容量、复制计数和完整调用耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling\u002Fcmu-serving-cover-20261019.webp","浅色手绘分页存储抽屉与共享前缀卡片，旁边连接不同长度的请求队列","个人课程项目中的 CPU 实验记录，延续记录 11 的分页 KV 实现。已完成 210 组配置与 1470 次计时，输出均通过连续存储参考对照；本篇不代表正式作业提交、整门课程或项目全部交付完成。 同样是共享 31 个 token，","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F200faf569eccf94fb125e65110bb5d14094d454a","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fkv-cache-granularity-20261019.zip",{"slug":3231,"path":3232,"title":3233,"date":3234,"tags":3235,"description":3238,"cover":3239,"coverAlt":3240,"excerpt":3241,"draft":3003,"hidden":3003,"published":1172,"readingTime":1176,"series":3220,"seriesOrder":1296,"codeUrl":3242,"attachmentUrl":3243},"cmu-15-779-moe-routing-capacity","\u002Fposts\u002Fcmu-15-779-moe-routing-capacity","CMU 15-779 学习记录 13：MoE 路由、稀疏分发与容量溢出","2026-10-09",[3220,3236,3237],"MoE","Routing","用 CPU top-2 MoE 检查稀疏分发的输出与梯度，统计 expert-token 行数，并保留容量丢弃会改变输出的反例。","\u002Fimages\u002Fposts\u002Fcmu-15-779-moe-routing-capacity\u002Fcmu-moe-cover-20261019.webp","浅色手绘分拣器将小珠分配到四个专家工作屋，队列呈现负载不均","第 18 讲的个人配套实验。36 号脚本串行运行 CPU 专家，检查 topk 路由后的分发与合并；没有训练 router，也没有执行 GPU grouped GEMM 或跨设备 alltoall。 上一节记录低秩适配，这次转向每个","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fc4851e0cedadf13af7c421867e83c8ff57244fa0","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-parallel-serving-lora-moe-20261019.zip",{"slug":3245,"path":3246,"title":3247,"date":3248,"tags":3249,"description":3251,"cover":3252,"coverAlt":3253,"excerpt":3254,"draft":3003,"hidden":3003,"published":1172,"readingTime":1176,"series":3220,"seriesOrder":1237,"codeUrl":3242,"attachmentUrl":3243},"cmu-15-779-lora-uniform-quantization","\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization","CMU 15-779 学习记录 12：LoRA 首步梯度、权重合并与均匀 4-bit 打包","2026-10-06",[3220,3170,3250],"Quantization","用一次 CPU SGD 更新检查 LoRA 初始化、冻结基座与合并等价，再分别记录均匀 4-bit 存储字节数和量化误差。","\u002Fimages\u002Fposts\u002Fcmu-15-779-lora-uniform-quantization\u002Fcmu-lora-cover-20261019.webp","浅色手绘的大计算板、两块细长适配板和成对打包的小方块","第 17 讲的个人配套实验，35 号脚本实际执行一次 CPU LoRA 更新，并实现均匀 4bit 打包。实验检查机制与数值误差，不评估微调任务效果，也没有实现完整 QLoRA。 这次把两个问题分开记录：低秩分支是否能合并进原权重，",{"slug":3256,"path":3257,"title":3258,"date":3259,"tags":3260,"description":3264,"cover":3225,"coverAlt":3265,"excerpt":3266,"draft":3003,"hidden":3003,"published":1172,"readingTime":1197,"series":3220,"seriesOrder":1227,"codeUrl":3242,"attachmentUrl":3243},"cmu-15-779-kv-serving-speculative-sampling","\u002Fposts\u002Fcmu-15-779-kv-serving-speculative-sampling","CMU 15-779 学习记录 11：分页 KV、连续批处理与推测采样","2026-10-03",[3220,3261,3262,3263],"LLM Serving","KV Cache","Speculative Decoding","在 CPU 上验证 KV 页共享和写时复制，用单位成本模型检查连续批处理，再用条件概率小模型核对推测采样的接受与补采样。","浅色手绘的共享卡片抽屉与不断补入小珠子的请求托盘","第 15、16 讲的个人配套实验。33 号实现 CPU 分页 KV 存储，37 号模拟请求准入，34 号运行小概率模型的推测采样。它们分别验证存储、调度和分布问题，没有测量真实 LLM 服务吞吐或 GPU 延迟。 记录 08已经对照",{"slug":3268,"path":3269,"title":3270,"date":3271,"tags":3272,"description":3277,"cover":3278,"coverAlt":3279,"excerpt":3280,"draft":3003,"hidden":3003,"published":1172,"readingTime":1343},"qwen3-mid-post-train-mlx","\u002Fposts\u002Fqwen3-mid-post-train-mlx","在 M2 Pro 上完成 Qwen3 两阶段训练：Mid Train、SFT 与可复核评估","2026-09-30 11:19:10",[3113,3273,3274,3170,3275,3169,3276],"Qwen3","MLX","持续预训练","Apple Silicon","用 Qwen3 0.6B Base 和 MLX 完成领域持续训练与监督微调，记录数据划分、聊天模板、损失掩码、数值稳定性修复和独立测试结果。","\u002Fimages\u002Fposts\u002Fqwen3-mid-post-train-mlx\u002Fqwen3-mlx-cover-20261018.webp","Apple Silicon 统一内存芯片底座支撑两阶段持续预训练与带精确 Loss 掩码滤网的监督微调桥接插画","在一台 M2 Pro、16 GB 内存的 Mac 上，能否把一个 Base 语言模型接续训练成输出 JSON 的运维事件分级助手？这次项目完成了领域持续训练、监督微调、检查点选择、重新加载和独立测试，并保留了权重、日志和数据哈希。 ",{"slug":3282,"path":3283,"title":3284,"date":3285,"tags":3286,"description":3290,"cover":3291,"coverAlt":3292,"excerpt":3293,"draft":3003,"hidden":3003,"published":1172,"readingTime":1183,"series":3220,"seriesOrder":1222,"codeUrl":3242,"attachmentUrl":3243},"cmu-15-779-cpu-parallelism-pipeline","\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline","CMU 15-779 学习记录 10：加权梯度、状态分片与同步流水线","2026-09-30",[3220,3287,3288,3289],"Parallel Training","Gloo","Pipeline","用双 CPU 进程 Gloo 检查不等数据分片的梯度，推导持久状态字节数，再用依赖模型比较 GPipe 与同步 1F1B 的激活存活。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-parallelism-pipeline\u002Fcmu-parallel-cover-20261019.webp","浅色手绘的四阶段桌面流水线与按数量汇总的托盘","第 12 至 14 讲的个人配套实验，脚本编号 31、32 不对应官方作业。Gloo 实际运行两个 CPU 进程；张量切分、状态字节数和流水线分别按下文的数学或成本模型解释。真实 GPU 实测按学习安排跳过。 上一节检查单个计算图的",{"slug":3295,"path":3296,"title":3297,"date":3298,"tags":3299,"description":3302,"cover":3303,"coverAlt":3304,"excerpt":3305,"draft":3003,"hidden":3003,"published":1172,"readingTime":1197,"series":3220,"seriesOrder":1213,"codeUrl":3306,"attachmentUrl":3307},"cmu-15-779-cpu-compiler-optimizations","\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations","CMU 15-779 学习记录 09：CPU 编译优化、合法重写与候选搜索","2026-09-27",[3220,3300,3301,3025],"ML Compiler","TorchInductor","用 CPU 实验验证 BatchNorm 折叠、RMSNorm 重排与非线性反例，测量分块候选，并运行 TorchInductor，分别记录首调用与预热后的时间。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cpu-compiler-optimizations\u002Fcmu-cpu-compiler-cover-20261019.webp","淡紫计算积木经过薄荷绿折叠模具变成浅蓝积木，前方摆放四种候选托盘的浅色手绘插画","编译章节的个人配套实验，编号 30 不对应官方作业。图重写、分块候选和 TorchInductor 均在 CPU 上运行；真实 GPU 实验按学习安排跳过。 上一节记录了分块 Attention 和 KV cache。这次转到编译优","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F9261bea6aedf31cdccbbab2541ca693fc97b3743","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcpu-compiler-optimizations-20261019.zip",{"slug":3309,"path":3310,"title":3311,"date":3312,"tags":3313,"description":3315,"cover":3316,"coverAlt":3317,"excerpt":3318,"draft":3003,"hidden":3003,"published":1172,"readingTime":1237,"series":3220,"seriesOrder":1207,"codeUrl":3228,"attachmentUrl":3319},"cmu-15-779-attention-online-softmax","\u002Fposts\u002Fcmu-15-779-attention-online-softmax","CMU 15-779 学习记录 08：分块 Attention、在线 Softmax 与 KV cache","2026-09-24",[3220,3141,3181,3314,3025],"FlashAttention","在 CPU 上对照完整 Attention、在线 Softmax 分块实现与 PyTorch SDPA，检查因果掩码、浮点误差和分数块大小，理解 FlashAttention 的一部分算法基础。","\u002Fimages\u002Fposts\u002Fcmu-15-779-attention-online-softmax\u002Fcmu-online-attention-cover-20261016.webp","浅紫色卡片分批进入绿色计算托盘，旁边三个状态容器保存累积量的浅色手绘插画","第 5 讲 Transformers & Attention 的个人配套实验。28\u002F29 号是 CPU 单头前向与 KV cache；后续补入 38 号 CUDA 多头合成张量的显式 SDPA MATH\u002FFLASH 对照。没有手写 ","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fcuda-triton-sdpa-20261019.zip",{"slug":3321,"path":3322,"title":3323,"date":3324,"tags":3325,"description":3330,"cover":3331,"coverAlt":3332,"excerpt":3333,"draft":3003,"hidden":3003,"published":1172,"readingTime":1222,"series":3220,"seriesOrder":1197,"codeUrl":3228,"attachmentUrl":3319},"cmu-15-779-matmul-tiling-tree-reduction","\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction","CMU 15-779 学习记录 07：矩阵分块、数据复用与树形归约","2026-09-21",[3220,3326,3327,3328,3329],"CUDA","矩阵乘法","数据复用","归约","用 CPU 模型统计矩阵分块的输入读取次数，再比较串行求和和树形归约的依赖深度，记录补零、同步与浮点加法顺序的边界。","\u002Fimages\u002Fposts\u002Fcmu-15-779-matmul-tiling-tree-reduction\u002Fcmu-matmul-reduction-cover-20261016.webp","矩阵积木在共享托盘中复用，旁边的小珠沿树形管道逐层合并的浅色手绘插画","GPU Architecture & CUDA Programming 的个人配套实验。首次记录用 CPU 模型检查数据复用与依赖深度；后续补入单卡 GPU 原生 CUDA 矩阵验证，以及 Triton 两阶段归约和矩阵乘法实测。 ",{"slug":3335,"path":3336,"title":3337,"date":3338,"tags":3339,"description":3342,"cover":3343,"coverAlt":3344,"excerpt":3345,"draft":3003,"hidden":3003,"published":1172,"readingTime":1222,"series":3220,"seriesOrder":1183,"codeUrl":3228,"attachmentUrl":3319},"cmu-15-779-cuda-thread-indexing","\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing","CMU 15-779 学习记录 06：CUDA 线程索引与边界检查","2026-09-18",[3220,3326,3045,3340,3341],"线程索引","边界检查","用 CPU 演示 CUDA 的 block、thread 与全局下标：10 个元素如何分给 3 个线程块，为什么要跳过越界线程，以及这些验证能说明什么。","\u002Fimages\u002Fposts\u002Fcmu-15-779-cuda-thread-indexing\u002Fcmu-cuda-indexing-cover-20261015.webp","分组的浅紫色计算方块与末端边界挡板组成线程分配教具的浅色插画","GPU Architecture & CUDA Programming 的个人配套练习。首次记录为 CPU 索引演示；后续在单卡 GPU 环境补入原生 CUDA 编译、运行与 memcheck，以及 Triton 向量加法实测。 上",{"slug":3347,"path":3348,"title":3349,"date":3350,"tags":3351,"description":3355,"cover":3356,"coverAlt":3357,"excerpt":3358,"draft":3003,"hidden":3003,"published":1172,"readingTime":1222,"series":3220,"seriesOrder":1176,"codeUrl":3359,"attachmentUrl":3360},"cmu-15-779-profiler-operator-batching","\u002Fposts\u002Fcmu-15-779-profiler-operator-batching","CMU 15-779 学习记录 05：Profiler、算子调用与批量推理","2026-09-15",[3220,3025,3352,3353,3354],"Profiler","批量推理","性能分析","对照 ML Systems 基础，用 Profiler 拆开两层网络的执行过程，再固定 1000 个样本扫描 batch size，观察算子调用次数、尾批和 CPU 耗时。","\u002Fimages\u002Fposts\u002Fcmu-15-779-profiler-operator-batching\u002Fcmu-pytorch-profiling-cover-20261015.webp","放大镜观察线性层与 ReLU 模块，旁边面板呈现执行片段的浅色技术教具插画","本篇对照课程官网的 Introduction to ML Systems 整理，是个人配套 CPU 实验，不是课程官方作业。脚本编号是练习顺序，不对应课程讲次。 前几篇一直在补 PyTorch 基础。到了这一步，我在学习聊天里问：现","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002Fcecf5110d8a95161bb5763eadfca94cd7a0cc50f","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-profiler-batching-20261015.zip",{"slug":3362,"path":3363,"title":3364,"date":3365,"tags":3366,"description":3369,"cover":3370,"coverAlt":3371,"excerpt":3372,"draft":3003,"hidden":3003,"published":1172,"readingTime":1213,"series":3220,"seriesOrder":1169,"codeUrl":3373,"attachmentUrl":3374},"cmu-15-779-pytorch-linear-relu-absolute-value","\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value","CMU 15-779 学习记录 04：多输出线性层、ReLU 与绝对值网络","2026-09-12",[3220,3025,3327,3367,3368],"ReLU","神经网络","把 nn.Linear 对应到矩阵乘法，弄清多个输出怎样计算，再逐层拆开一个绝对值网络，比较手动设置参数与训练得到的结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-relu-absolute-value\u002Fcmu-pytorch-absolute-cover-20261015.webp","双分支经过折角模块后汇入紫色 V 形曲线面板的浅色技术教具插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的批量推理与矩阵乘法，我继续把 model(x) 拆开看：一层里有多个输出时，每个数是怎么算出来的？两","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs\u002Ftree\u002F4cc63904a3a583da4a2122c246a14b43aac38528","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-linear-relu-abs-20261015.zip",{"slug":3376,"path":3377,"title":3378,"date":3379,"tags":3380,"description":3383,"cover":3384,"coverAlt":3385,"excerpt":3386,"draft":3003,"hidden":3003,"published":1172,"readingTime":1213,"series":3220,"seriesOrder":1155,"codeUrl":3387,"attachmentUrl":3388},"cmu-15-779-pytorch-dataloader-checkpoints-batching","\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching","CMU 15-779 学习记录 03：DataLoader、模型保存与批量推理","2026-09-09",[3220,3025,3381,3353,3382],"DataLoader","模型保存","用 DataLoader 分批训练平方函数，保存并加载模型参数，再比较 1000 个样本与 1 个样本的推理耗时，最后把批量计算对应到矩阵乘法。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-dataloader-checkpoints-batching\u002Fcmu-pytorch-batching-cover-20261015.webp","样本卡片按批次进入计算模块，旁边配有参数存储盒和输出矩阵面板的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 接着前一篇的平方函数实验，这一节继续练习分批训练、保存参数和独立预测。最后做了一次计时：同一个小网络，逐个处理 ","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fcmu-15-779-labs","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-batching-20261015.zip",{"slug":3390,"path":3391,"title":3392,"date":3393,"tags":3394,"description":3395,"cover":3396,"coverAlt":3397,"excerpt":3398,"draft":3003,"hidden":3003,"published":1172,"readingTime":1213,"series":3220,"seriesOrder":1147,"codeUrl":3387,"attachmentUrl":3399},"cmu-15-779-pytorch-linear-module-relu","\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu","CMU 15-779 学习记录 02：nn.Linear、张量形状与 ReLU","2026-09-06",[3220,3025,3368,3367],"从带偏置的线性回归开始，弄清 nn.Linear 和 nn.Module 的分工，再用平方函数观察线性模型的限制、ReLU 的作用和测试点的选择。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-linear-module-relu\u002Fcmu-pytorch-relu-cover-20261015.webp","输入经过两层神经网络和 ReLU 折角模块，输出曲线拟合结果的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 上一篇里，我用一个参数拟合了 $y=2x$。这一篇继续顺着往下写，先加上偏置，再把手写参数换成 nn.Linea","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-models-20261015.zip",{"slug":3401,"path":3402,"title":3403,"date":3404,"tags":3405,"description":3408,"cover":3409,"coverAlt":3410,"excerpt":3411,"draft":3003,"hidden":3003,"published":1172,"readingTime":1222,"series":3220,"seriesOrder":1136,"codeUrl":3387,"attachmentUrl":3412},"cmu-15-779-pytorch-autograd-linear-regression","\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression","CMU 15-779 学习记录 01：PyTorch 自动求导与线性回归","2026-09-03",[3220,3025,3406,3407],"自动求导","线性回归","从自动求导和手动梯度下降开始，比较学习率，再用 SGD 训练线性回归。记录 w * 4 写错后停在 1.25 的原因和修正结果。","\u002Fimages\u002Fposts\u002Fcmu-15-779-pytorch-autograd-linear-regression\u002Fcmu-pytorch-autograd-cover-20261015.webp","梯度曲面上的小球沿路径下降，前侧面板展示散点与拟合直线的技术插画","本篇整理的是学习 CMU 15779 时补充的 PyTorch 预备练习。脚本编号不对应课程讲次，也不是课程官方作业。 开始学习 CMU 15779 时，我会 Python，但还没用过 PyTorch。这次先在 Mac 上用 CPU","\u002Fmaterials\u002Fcmu-15-779\u002Fexperiments\u002Fpytorch-basics-20261013.zip",{"slug":3414,"path":3415,"title":3416,"date":3417,"tags":3418,"description":3424,"cover":3425,"coverAlt":3426,"excerpt":3427,"draft":3003,"hidden":3003,"published":1172,"readingTime":1296,"codeUrl":3428,"attachmentUrl":3429},"embodied-pickplace-lab-robustness","\u002Fposts\u002Fembodied-pickplace-lab-robustness","Pickplace Lab 项目复盘：抓取成功率之外的时延与失败分析","2026-08-29",[3419,3420,3421,3025,3422,3423],"具身智能","模仿学习","MuJoCo","机器人","鲁棒性","从 MuJoCo 抓取搬运闭环出发，记录 22,100 个评测回合。动作平滑在自建任务的 50 ms 延迟下有效，换到 Panda Lift 却没有支持同样的收益。","\u002Fimages\u002Fposts\u002Fembodied-pickplace-lab-robustness\u002Fpickplace-grasp-cover-20261016.webp","MuJoCo 实际渲染的笛卡尔夹爪抓取方块场景，桌面上标出放置目标","Pickplace Lab 从一个简单任务开始：让夹爪抓起方块，搬到指定位置，放下，再退开。先把仿真、示范采集、训练和闭环评测接起来，随后再问：干净条件下成功的策略，换一种执行条件，还能不能完成任务？ 最值得讨论的结果出现在指令延迟","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Ftree\u002Fv0.1.0","https:\u002F\u002Fgithub.com\u002Fliangqianxing\u002Fembodied-pickplace-lab\u002Freleases\u002Fdownload\u002Fv0.1.0\u002Fpickplace-lab-research-v1.zip",{"slug":3431,"path":3432,"title":3433,"date":3434,"tags":3435,"description":3440,"cover":3441,"coverAlt":3442,"excerpt":3443,"draft":3003,"hidden":3003,"published":1172,"readingTime":1570},"qwen25-vl-sglang-verl-single-gpu-lab","\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab","从 Mac 到单卡 GPU：Qwen2.5-VL 部署、GRPO 与计数 LoRA 实践","2026-08-19",[3129,3071,3436,3437,3438,3439,3274],"SGLang","verl","GRPO","推理优化","在 M2 Pro 和云端单卡上部署 Qwen2.5-VL-3B，实测连续批处理、前缀缓存与 SGLang 参数调优，再用 verl 完成多模态 GRPO 更新和同条件前后评测，记录有效优化、兼容性问题与未观察到提升的结果。","\u002Fimages\u002Fposts\u002Fqwen25-vl-sglang-verl-single-gpu-lab\u002Fqvl-lab-cover-20261010.webp","笔记本与单卡计算模块通过图像和 token 光轨连接的实验台插画","这次实验从一个很具体的问题开始：M2 Pro、16 GB 统一内存，能不能在本机跑 Qwen2.5VL？跑起来之后，我又想把图文问答、推理性能测量和强化学习训练接成一条完整流程。 本机的 MLX 4bit 服务和云端的 SGLang",{"slug":3445,"path":3446,"title":3447,"date":3448,"tags":3449,"description":3453,"cover":3454,"coverAlt":3455,"excerpt":3456,"draft":3003,"hidden":3003,"published":1172,"readingTime":1296},"cycleresearcher-post-training","\u002Fposts\u002Fcycleresearcher-post-training","从 DeepReviewer 2.0 到 CycleResearcher：科研 Agent 如何真正做 Post-training","2026-08-07 17:33:05",[3113,3130,3072,3450,3451,3452],"SimPO","Reward Model","ICLR","区分推理时 Agent 编排与模型后训练，并拆解 CycleResearcher 的领域 SFT、生成式 Reward Model、Iterative SimPO 和评测风险。","\u002Fimages\u002Fposts\u002Fcycleresearcher-post-training\u002Fcycleresearcher-cover-20261018.webp","策略生成模块产出多份候选文稿经多维评审棱镜排序后通过偏好梯度反馈弧闭环优化的等距技术插画","看到一个系统会检索、规划、反思和调用工具，不能据此判断它做过 Posttraining。Agent 描述的是模型在运行时如何完成任务；Posttraining 描述的是预训练之后，是否继续用监督数据、偏好数据或奖励信号更新模型参数。",{"slug":3458,"path":3459,"title":3460,"date":3461,"tags":3462,"description":3464,"cover":3465,"coverAlt":3466,"excerpt":3467,"draft":3003,"hidden":3003,"published":1172,"readingTime":1311,"series":3468,"seriesOrder":1136},"cuda-fused-rmsnorm-kernel","\u002Fposts\u002Fcuda-fused-rmsnorm-kernel","从 PyTorch 到 CUDA Kernel：融合 RMSNorm 在 RTX 4090D 上加速 4.27 倍","2026-07-22 18:16:50",[3326,2998,3045,3025,3011,3463],"Kernel 优化","从零实现 PyTorch CUDA 扩展，拆解 RMSNorm 的访存瓶颈、warp 归约、向量化加载、算子融合与冷热缓存 benchmark。","\u002Fimages\u002Fposts\u002Fcuda-fused-rmsnorm-kernel\u002Fcuda-rmsnorm-cover-20261018.webp","融合残差加法与向量化 float4 访存经 Warp Shuffle 蝴蝶树归约输出归一化张量的等距硬件架构插画","为了准备 AI Infra 岗位，我做了一个尽量小而完整的 CUDA 项目：不追求堆很多算子，而是选一个真实的 LLM 推理路径，把实现、正确性、benchmark 和性能解释全部做扎实。 项目实现了两个 PyTorch CUDA ","AI Infra 实战",{"slug":3470,"path":3471,"title":3472,"date":3473,"tags":3474,"description":3476,"cover":3477,"coverAlt":3478,"excerpt":3479,"draft":3003,"hidden":3003,"published":1172,"readingTime":1207,"series":3480,"seriesOrder":1136},"agent-memory","\u002Fposts\u002Fagent-memory","Agent 上下文工程与分层记忆架构：从 Observation 剪枝、滚动摘要到向量召回闭环","2026-04-24",[3113,3072,3475,2998],"RAG","结合生产级多约束 Agent 与超 100K Token 科研智能体实践，系统拆解长程会话中的上下文膨胀与注意力稀释问题：涵盖 Observation 结构化剪枝（降低约 40% Token 开销）、滚动工作摘要、向量检索长期记忆及 Prefix Cache 友好的上下文组装。","\u002Fimages\u002Fposts\u002Fagent-memory\u002Fagent-memory-cover-20261018.webp","底层高噪数据经剪枝滤网进入中期滑动窗口与金色状态台账并由顶层向量星座按需召回的三层记忆架构插画","在构建多步工具调用（ToolUse）与长周期自主规划（Autonomous Planning）的 Agent 系统时，上下文管理（Context Engineering）往往比单纯改写 System Prompt 更决定线上稳定性。","Agent 上下文工程",{"slug":3482,"path":3483,"title":3484,"date":3485,"tags":3486,"description":3489,"cover":3490,"coverAlt":3491,"excerpt":3492,"draft":3003,"hidden":3003,"published":1172,"readingTime":1176,"series":3493,"seriesOrder":1147},"backend-stack-deep-dive","\u002Fposts\u002Fbackend-stack-deep-dive","高并发 AI 平台后端基础设施实践：FastAPI 异步网关、SQLAlchemy 2.0 Async、PostgreSQL 与 Docker 沙箱编排","2026-04-07",[3015,3012,3014,3487,3488,2998],"PostgreSQL","Docker","结合 DeepScientist 生产级 AI 科研平台实践，拆解 FastAPI ASGI 异步事件循环、SQLAlchemy 2.0 + AsyncPG 连接池调优、PostgreSQL + MinIO 存储分层与 Docker SDK 动态沙箱容器编排的职责边界与核心设计。","\u002Fimages\u002Fposts\u002Fbackend-stack-deep-dive\u002Fbackend-stack-cover-20261018.webp","ASGI 异步事件循环涡轮将非阻塞协程流分发至 PostgreSQL 连接池、MinIO 存储矩阵与动态 Docker 沙箱的等距架构插画","结合 DeepScientist 生产级 AI 科研平台（服务 7,000+ 用户）的实际工程落地，本文系统拆解支撑高并发流式推理与容器沙箱调度的后端基础设施栈——FastAPI（ASGI 异步网关 + SSE 流式推送）、SQLA","DeepScientist 工程复盘",{"slug":3495,"path":3496,"title":3497,"date":3498,"tags":3499,"description":3502,"cover":3503,"coverAlt":3504,"excerpt":3505,"draft":3003,"hidden":3003,"published":1172,"readingTime":1227,"series":3506,"seriesOrder":1136},"minicode-source-analysis","\u002Fposts\u002Fminicode-source-analysis","MiniCode 架构与工具评测深度拆解：开源终端 Agentic Coding 引擎（1k+ Stars）的 ReAct、MCP 与 30 项基准验证","2026-04-06",[3072,3500,3501,3102,2998],"TypeScript","MCP","深度剖析开源终端 AI 编程智能体 MiniCode（1k+ GitHub Stars）的核心架构：从零重型依赖的 ReAct 规划循环、声明式工具注册表与权限拦截层、MCP 协议帧自动协商，到 30 项可执行 Tool-Use 评测基准（86.7% 完成率、100% 错误恢复率）的失败分布归因与闭环优化。","\u002Fimages\u002Fposts\u002Fminicode-source-analysis\u002Fminicode-cover-20261018.webp","终端编程智能体控制台被 ReAct 状态环、Unified Diff 补丁预览门与 MCP 协议插槽环绕的等距技术插画","MiniCode 是我作为核心开发者参与构建的开源终端 Agentic Coding 引擎（目前已在 GitHub 获得 1,000+ Stars）。项目的核心设计目标是：在不引入庞大运行时框架的前提下，用约 5,000 行高内聚的","AI 项目源码导读",{"slug":3508,"path":3509,"title":3510,"date":3498,"tags":3511,"description":3512,"cover":3513,"coverAlt":3514,"excerpt":3515,"draft":3003,"hidden":3003,"published":1172,"readingTime":1222,"series":3493,"seriesOrder":1136},"deepscientist-tech-stack","\u002Fposts\u002Fdeepscientist-tech-stack","DeepScientist 架构与运行时全解析：超 100K 上下文科研智能体、MCP 沙箱与高并发流式系统",[3072,2998,3501,3488,3014,3012],"深度拆解服务 7,000+ 用户的科研自主智能体平台 DeepScientist：从一题一仓库（Quest）持久化状态机、MCP 工具栈（memory \u002F artifact \u002F bash_exec）、Docker SDK 动态容器编排，到 SSE + WebSocket 双流通信与 PostgreSQL \u002F Redis \u002F Prometheus 生产级基础设施。","\u002Fimages\u002Fposts\u002Fdeepscientist-tech-stack\u002Fdeepscientist-cover-20261018.webp","科研智能体编排核心通过 memory、artifact 与隔离容器沙箱三座协议桥连接双流工作台的等距架构插画","DeepScientist（deepscientist.cc）是我们在西湖大学自然语言处理实验室研发的自主科学发现与科研管理平台，目前累计服务 7,000+ 科研用户。与单轮对话或纯文本文献问答工具不同，科研智能体需要在数小时乃至数",1795088944150]