[{"data":1,"prerenderedAt":1467},["ShallowReactive",2],{"post-\u002Fposts\u002Fgpu-inference-acceleration-week2":3,"all-posts-nav":1184},{"id":4,"title":5,"body":6,"categories":1165,"date":1166,"description":1167,"draft":1168,"extension":1169,"hidden":1168,"meta":1170,"navigation":1171,"path":1172,"published":1168,"seo":1173,"stem":1174,"tags":1175,"__hash__":1183},"posts\u002Fposts\u002Fgpu-inference-acceleration-week2.md","Week 2：GPU 与推理加速——从 Kernel、算子融合到 LLM Serving",{"type":7,"value":8,"toc":1128},"minimark",[9,18,25,32,37,40,59,62,79,83,86,89,100,103,108,111,114,128,135,139,142,153,157,160,163,167,170,237,243,247,250,265,268,274,277,280,284,287,290,310,320,323,329,332,336,339,370,373,379,382,390,393,397,403,406,412,415,429,432,443,446,450,453,459,462,465,479,482,521,524,528,531,535,538,541,555,559,562,564,578,581,587,590,594,598,601,604,610,613,616,633,637,640,643,649,652,655,663,666,680,683,689,692,696,699,702,719,722,726,729,735,738,741,745,748,751,757,760,774,777,781,784,787,807,810,830,833,837,840,843,849,852,866,869,889,892,896,899,903,906,912,915,919,922,928,931,935,938,944,947,951,954,958,961,978,981,1075,1079,1082,1114,1118,1121,1124],[10,11,12,13,17],"p",{},"Week 1 我们从 Autograd 理解了深度学习框架的训练本质：Tensor、计算图、反向传播和内存优化。Week 2 要切到更贴近论文和系统落地的部分：",[14,15,16],"strong",{},"GPU 与推理加速","。",[10,19,20,21,24],{},"如果说训练框架的核心问题是“如何自动求梯度”，那么推理系统的核心问题就是：",[14,22,23],{},"如何把有限的 GPU 算力、显存带宽和请求调度能力，用在最多的 token 上","。尤其是 LLM 推理，慢往往不是因为“计算公式复杂”，而是因为 memory bound、KV cache、batch scheduling、streaming generation 共同决定了端到端延迟。",[10,26,27],{},[28,29],"img",{"alt":30,"src":31},"LLM inference pipeline","\u002Fimages\u002Fposts\u002Fgpu-inference\u002Fllm-inference.svg",[33,34,36],"h2",{"id":35},"_1-week-2-学什么","1. Week 2 学什么",[10,38,39],{},"这一周只看 CMU 10-414 中和系统性能最相关的模块：",[41,42,43,47,50,53,56],"ul",{},[44,45,46],"li",{},"GPU architecture basics：理解 GPU 为什么适合大规模并行；",[44,48,49],{},"Kernel launch overhead：理解小算子为什么慢；",[44,51,52],{},"Operator fusion：理解为什么要融合算子；",[44,54,55],{},"Batching \u002F throughput vs latency：理解服务系统的核心取舍；",[44,57,58],{},"Distributed training overview：只看概念，理解数据并行、张量并行、流水并行的基本动机。",[10,60,61],{},"学完后你应该能解释：",[41,63,64,67,70,73,76],{},[44,65,66],{},"为什么 LLM 推理经常是 memory bound；",[44,68,69],{},"KV cache 为什么既加速 attention，又吃掉大量显存；",[44,71,72],{},"batch scheduling 为什么决定 serving 系统吞吐；",[44,74,75],{},"streaming generation 为什么提升体感速度但不减少总计算；",[44,77,78],{},"vLLM、TensorRT、diffusion acceleration 分别在优化什么。",[33,80,82],{"id":81},"_2-gpu-架构基础gpu-快在哪里","2. GPU 架构基础：GPU 快在哪里",[10,84,85],{},"CPU 擅长复杂控制流、低延迟任务和通用逻辑；GPU 擅长把同一个操作并行应用到大量数据上。深度学习里的矩阵乘法、卷积、attention 都天然符合这种模式。",[10,87,88],{},"一个非常简化的 GPU 层级可以理解为：",[90,91,97],"pre",{"className":92,"code":94,"language":95,"meta":96},[93],"language-text","GPU\n  -> 多个 SM（Streaming Multiprocessor）\n      -> 多个 warp\n          -> 多个 thread\n  -> HBM 显存\n  -> L2 cache \u002F shared memory \u002F register\n","text","",[98,99,94],"code",{"__ignoreMap":96},[10,101,102],{},"几个关键词必须理解。",[104,105,107],"h3",{"id":106},"_21-thread","2.1 Thread",[10,109,110],{},"Thread 是最细粒度的执行单元。一个 kernel 会启动大量 thread，每个 thread 处理一小块数据。",[10,112,113],{},"例如向量加法：",[90,115,119],{"className":116,"code":117,"language":118,"meta":96,"style":96},"language-cuda shiki shiki-themes github-dark-dimmed github-light","c[i] = a[i] + b[i]\n","cuda",[98,120,121],{"__ignoreMap":96},[122,123,126],"span",{"class":124,"line":125},"line",1,[122,127,117],{},[10,129,130,131,134],{},"每个 thread 可以负责一个 ",[98,132,133],{},"i","。如果向量有一百万个元素，就可以启动大量线程并行处理。",[104,136,138],{"id":137},"_22-warp","2.2 Warp",[10,140,141],{},"NVIDIA GPU 中，一个 warp 通常是 32 个线程。Warp 内线程执行同一条指令，但处理不同数据，这叫 SIMT：Single Instruction, Multiple Threads。",[10,143,144,145,148,149,152],{},"如果 warp 内线程走不同分支，例如一半执行 ",[98,146,147],{},"if","，一半执行 ",[98,150,151],{},"else","，就会产生 warp divergence，实际执行效率下降。",[104,154,156],{"id":155},"_23-sm","2.3 SM",[10,158,159],{},"SM 可以理解为 GPU 的计算核心。每个 SM 管理多个 warp，并在 warp 之间切换，用大量并发隐藏内存访问延迟。",[10,161,162],{},"GPU 高吞吐的关键不是单个线程很快，而是同时跑很多线程。",[104,164,166],{"id":165},"_24-register-shared-memory-hbm","2.4 Register \u002F Shared Memory \u002F HBM",[10,168,169],{},"不同内存层级速度差别很大：",[171,172,173,189],"table",{},[174,175,176],"thead",{},[177,178,179,183,186],"tr",{},[180,181,182],"th",{},"层级",[180,184,185],{},"特点",[180,187,188],{},"用法",[190,191,192,204,215,226],"tbody",{},[177,193,194,198,201],{},[195,196,197],"td",{},"Register",[195,199,200],{},"最快，线程私有",[195,202,203],{},"保存局部变量",[177,205,206,209,212],{},[195,207,208],{},"Shared Memory",[195,210,211],{},"很快，block 内共享",[195,213,214],{},"tile 复用、局部缓存",[177,216,217,220,223],{},[195,218,219],{},"L2 Cache",[195,221,222],{},"GPU 全局缓存",[195,224,225],{},"缓解重复访问",[177,227,228,231,234],{},[195,229,230],{},"HBM",[195,232,233],{},"容量大但慢于片上存储",[195,235,236],{},"存参数、激活、KV cache",[10,238,239,240,17],{},"推理优化的一个核心目标就是：",[14,241,242],{},"尽量把数据复用发生在 register\u002Fshared memory\u002Fcache，而不是频繁往返 HBM",[33,244,246],{"id":245},"_3-roofline为什么要区分-compute-bound-和-memory-bound","3. Roofline：为什么要区分 compute bound 和 memory bound",[10,248,249],{},"一个算子慢，可能有两种原因：",[251,252,253,259],"ol",{},[44,254,255,258],{},[14,256,257],{},"Compute bound","：计算单元忙不过来，瓶颈是 FLOPs；",[44,260,261,264],{},[14,262,263],{},"Memory bound","：计算单元在等数据，瓶颈是显存带宽。",[10,266,267],{},"判断关键是 arithmetic intensity：",[90,269,272],{"className":270,"code":271,"language":95,"meta":96},[93],"arithmetic intensity = FLOPs \u002F bytes moved\n",[98,273,271],{"__ignoreMap":96},[10,275,276],{},"如果每读 1 byte 数据能做很多计算，例如大矩阵乘法，通常更接近 compute bound。如果每读很多数据只做很少计算，例如 LayerNorm、Elementwise Add、Decode 阶段的小 batch GEMV，就容易 memory bound。",[10,278,279],{},"LLM 推理尤其在 decode 阶段经常 memory bound。因为每生成一个 token，都要读一遍大量模型权重，但 batch 可能很小，每个权重参与的计算复用不够。",[33,281,283],{"id":282},"_4-kernel-是什么","4. Kernel 是什么",[10,285,286],{},"Kernel 是运行在 GPU 上的函数。Python 代码本身不在 GPU 上执行，它只是通过 CUDA runtime \u002F driver 让 GPU 启动 kernel。",[10,288,289],{},"例如 PyTorch：",[90,291,295],{"className":292,"code":293,"language":294,"meta":96,"style":96},"language-python shiki shiki-themes github-dark-dimmed github-light","y = torch.relu(x)\n","python",[98,296,297],{"__ignoreMap":96},[122,298,299,303,307],{"class":124,"line":125},[122,300,302],{"class":301},"ssh_m","y ",[122,304,306],{"class":305},"s6PUj","=",[122,308,309],{"class":301}," torch.relu(x)\n",[10,311,312,313,316,317,17],{},"背后通常会触发一个 GPU kernel：对 ",[98,314,315],{},"x"," 的每个元素并行执行 ",[98,318,319],{},"max(x, 0)",[10,321,322],{},"一次 kernel launch 包含固定开销：",[90,324,327],{"className":325,"code":326,"language":95,"meta":96},[93],"Python 调用\n  -> C++ dispatcher\n  -> CUDA runtime \u002F driver\n  -> GPU 排队执行 kernel\n  -> kernel 真正运行\n",[98,328,326],{"__ignoreMap":96},[10,330,331],{},"如果 kernel 本身工作量很大，例如大矩阵乘，启动开销可以忽略。如果 kernel 很小，例如几个 elementwise 操作，启动开销和显存读写可能比计算本身更贵。",[33,333,335],{"id":334},"_5-kernel-launch-overhead为什么小算子很慢","5. Kernel Launch Overhead：为什么小算子很慢",[10,337,338],{},"假设你写：",[90,340,342],{"className":292,"code":341,"language":294,"meta":96,"style":96},"y = gelu(x + bias)\nz = dropout(y)\n",[98,343,344,359],{"__ignoreMap":96},[122,345,346,348,350,353,356],{"class":124,"line":125},[122,347,302],{"class":301},[122,349,306],{"class":305},[122,351,352],{"class":301}," gelu(x ",[122,354,355],{"class":305},"+",[122,357,358],{"class":301}," bias)\n",[122,360,362,365,367],{"class":124,"line":361},2,[122,363,364],{"class":301},"z ",[122,366,306],{"class":305},[122,368,369],{"class":301}," dropout(y)\n",[10,371,372],{},"如果没有融合，可能触发多个 kernel：",[90,374,377],{"className":375,"code":376,"language":95,"meta":96},[93],"add kernel\n  -> 写中间结果到 HBM\ngelu kernel\n  -> 从 HBM 读中间结果，再写回 HBM\ndropout kernel\n  -> 再读再写\n",[98,378,376],{"__ignoreMap":96},[10,380,381],{},"问题有两个：",[251,383,384,387],{},[44,385,386],{},"每个 kernel launch 都有固定调度成本；",[44,388,389],{},"中间结果反复读写 HBM，浪费带宽。",[10,391,392],{},"这就是为什么深度学习编译器和推理引擎都非常重视 operator fusion。",[33,394,396],{"id":395},"_6-operator-fusion算子融合","6. Operator Fusion：算子融合",[10,398,399],{},[28,400],{"alt":401,"src":402},"Operator fusion","\u002Fimages\u002Fposts\u002Fgpu-inference\u002Ffusion.svg",[10,404,405],{},"算子融合就是把多个算子合成一个 kernel。例如：",[90,407,410],{"className":408,"code":409,"language":95,"meta":96},[93],"未融合：MatMul -> BiasAdd -> GELU -> Dropout\n融合后：FusedMatMulBiasGELUDropout\n",[98,411,409],{"__ignoreMap":96},[10,413,414],{},"融合的收益：",[41,416,417,420,423,426],{},[44,418,419],{},"减少 kernel launch 次数；",[44,421,422],{},"减少中间激活写回 HBM；",[44,424,425],{},"增加寄存器和 shared memory 内的数据复用；",[44,427,428],{},"给编译器更多优化空间。",[10,430,431],{},"但融合也有代价：",[41,433,434,437,440],{},[44,435,436],{},"kernel 更复杂，开发和调试难度更高；",[44,438,439],{},"动态 shape、复杂控制流会降低融合机会；",[44,441,442],{},"过度融合可能导致 register pressure 过大，反而降低 occupancy。",[10,444,445],{},"TensorRT、XLA、TVM、TorchInductor、Triton 都在不同层面做 fusion。你看到推理框架性能大幅提升，很多时候不是数学变了，而是执行计划变了。",[33,447,449],{"id":448},"_7-batching吞吐与延迟的永恒取舍","7. Batching：吞吐与延迟的永恒取舍",[10,451,452],{},"推理服务面对的不是一个固定输入，而是持续到来的请求流。Batching 的作用是把多个请求合在一起，提高 GPU 利用率。",[90,454,457],{"className":455,"code":456,"language":95,"meta":96},[93],"request A: prompt length 20\nrequest B: prompt length 300\nrequest C: prompt length 80\n",[98,458,456],{"__ignoreMap":96},[10,460,461],{},"如果一个一个跑，GPU 可能吃不满。如果合成 batch，矩阵乘规模变大，权重复用更好，吞吐上升。",[10,463,464],{},"但 batch 太大也会增加延迟：",[41,466,467,470,473,476],{},[44,468,469],{},"请求要在队列里等待凑 batch；",[44,471,472],{},"长 prompt 可能拖慢短 prompt；",[44,474,475],{},"decode 阶段每个请求生成长度不同，会产生调度碎片；",[44,477,478],{},"显存中的 KV cache 随 batch 和序列长度增长。",[10,480,481],{},"所以 serving 系统要在两个指标间取舍：",[171,483,484,497],{},[174,485,486],{},[177,487,488,491,494],{},[180,489,490],{},"指标",[180,492,493],{},"含义",[180,495,496],{},"偏好",[190,498,499,510],{},[177,500,501,504,507],{},[195,502,503],{},"Throughput",[195,505,506],{},"单位时间处理多少 token\u002Frequest",[195,508,509],{},"大 batch、有队列等待",[177,511,512,515,518],{},[195,513,514],{},"Latency",[195,516,517],{},"单个请求多久返回",[195,519,520],{},"小 batch、少等待",[10,522,523],{},"在线聊天更看重低延迟，离线批处理更看重高吞吐。",[33,525,527],{"id":526},"_8-llm-推理流程prefill-与-decode","8. LLM 推理流程：Prefill 与 Decode",[10,529,530],{},"LLM 生成可以分成两个阶段。",[104,532,534],{"id":533},"_81-prefill","8.1 Prefill",[10,536,537],{},"Prefill 处理输入 prompt，一次性计算 prompt 中所有 token 的 hidden states，并建立 KV cache。",[10,539,540],{},"特点：",[41,542,543,546,549,552],{},[44,544,545],{},"输入长度可能很长；",[44,547,548],{},"attention 可以并行处理整个 prompt；",[44,550,551],{},"更像大矩阵乘，GPU 利用率通常较高；",[44,553,554],{},"决定 time to first token 的重要部分。",[104,556,558],{"id":557},"_82-decode","8.2 Decode",[10,560,561],{},"Decode 每次生成一个新 token，并把这个 token 的 K\u002FV 追加到 KV cache。",[10,563,540],{},[41,565,566,569,572,575],{},[44,567,568],{},"每轮只新增一个 token；",[44,570,571],{},"必须自回归，不能把未来 token 并行算出来；",[44,573,574],{},"batch 小时很容易 memory bound；",[44,576,577],{},"端到端生成时间主要由 decode token 数决定。",[10,579,580],{},"完整流程：",[90,582,585],{"className":583,"code":584,"language":95,"meta":96},[93],"prompt tokens\n  -> prefill\n  -> first token\n  -> decode step 1\n  -> decode step 2\n  -> ...\n  -> EOS \u002F max_tokens\n",[98,586,584],{"__ignoreMap":96},[10,588,589],{},"这解释了为什么输入很长会影响首 token 延迟，而输出很长会影响总生成时间。",[33,591,593],{"id":592},"_9-为什么-llm-推理慢","9. 为什么 LLM 推理慢",[104,595,597],{"id":596},"_91-memory-bound","9.1 Memory Bound",[10,599,600],{},"LLM 参数巨大。每生成一个 token，Transformer 的每一层都要访问大量权重。如果 batch 很小，这些权重读出来后只服务少量 token，复用不足。",[10,602,603],{},"以 decode 为例，很多操作更接近矩阵向量乘或小 batch 矩阵乘：",[90,605,608],{"className":606,"code":607,"language":95,"meta":96},[93],"hidden: [batch, hidden_dim]\nweight: [hidden_dim, 4 * hidden_dim]\n",[98,609,607],{"__ignoreMap":96},[10,611,612],{},"当 batch 小时，读 weight 的成本很高，计算单元可能等数据。这就是 memory bound。",[10,614,615],{},"优化方向包括：",[41,617,618,621,624,627,630],{},[44,619,620],{},"增大 continuous batching，提高权重复用；",[44,622,623],{},"量化权重，减少 bytes moved；",[44,625,626],{},"使用更高效 kernel，减少额外内存访问；",[44,628,629],{},"KV cache 分页管理，减少显存碎片；",[44,631,632],{},"speculative decoding，减少大模型调用步数。",[104,634,636],{"id":635},"_92-kv-cache","9.2 KV Cache",[10,638,639],{},"Transformer attention 每个 token 都需要看之前的 token。如果每一步都重新计算历史 token 的 K\u002FV，成本会爆炸。",[10,641,642],{},"KV cache 保存每层历史 token 的 Key 和 Value：",[90,644,647],{"className":645,"code":646,"language":95,"meta":96},[93],"layer 1: K_cache, V_cache\nlayer 2: K_cache, V_cache\n...\nlayer L: K_cache, V_cache\n",[98,648,646],{"__ignoreMap":96},[10,650,651],{},"生成第 t 个 token 时，只需要计算新 token 的 K\u002FV，然后和历史 K\u002FV 做 attention。",[10,653,654],{},"KV cache 的好处：",[41,656,657,660],{},[44,658,659],{},"避免重复计算历史 token；",[44,661,662],{},"让自回归 decode 可用。",[10,664,665],{},"KV cache 的问题：",[41,667,668,671,674,677],{},[44,669,670],{},"显存占用随 batch、层数、hidden size、序列长度线性增长；",[44,672,673],{},"请求长度不同会造成碎片；",[44,675,676],{},"cache 读写本身也会产生带宽压力；",[44,678,679],{},"长上下文推理时 KV cache 可能比权重更难管理。",[10,681,682],{},"一个粗略估算：",[90,684,687],{"className":685,"code":686,"language":95,"meta":96},[93],"KV cache bytes ≈ batch_size * seq_len * num_layers * 2(K,V) * hidden_size * bytes_per_element\n",[98,688,686],{"__ignoreMap":96},[10,690,691],{},"如果用了多头注意力，还要考虑 head 数、head dim、GQA\u002FMQA 等结构。GQA 和 MQA 的重要收益之一就是减少 KV cache 规模。",[104,693,695],{"id":694},"_93-batch-scheduling","9.3 Batch Scheduling",[10,697,698],{},"LLM 请求是动态的：有人 prompt 长，有人 prompt 短；有人生成 20 token，有人生成 2000 token。传统静态 batch 很容易低效。",[10,700,701],{},"难点包括：",[41,703,704,707,710,713,716],{},[44,705,706],{},"新请求什么时候插入正在 decode 的 batch；",[44,708,709],{},"已完成请求如何从 batch 中移除；",[44,711,712],{},"不同长度序列如何管理 KV cache；",[44,714,715],{},"如何避免短请求被长请求拖死；",[44,717,718],{},"如何在吞吐和 P99 延迟之间平衡。",[10,720,721],{},"这就是 vLLM、TGI、TensorRT-LLM 等 serving 系统的核心竞争点之一。",[104,723,725],{"id":724},"_94-streaming-generation","9.4 Streaming Generation",[10,727,728],{},"Streaming generation 是边生成边返回：",[90,730,733],{"className":731,"code":732,"language":95,"meta":96},[93],"用户看到：你 -> 好 -> ， -> 我 -> 来 -> 帮 -> 你\n",[98,734,732],{"__ignoreMap":96},[10,736,737],{},"它的优点是降低用户体感延迟，特别是 time to first token 很重要。",[10,739,740],{},"但 streaming 不会减少总计算量。模型还是要一个 token 一个 token 地自回归生成。它优化的是交互体验，不是数学成本。",[33,742,744],{"id":743},"_10-vllm-在优化什么","10. vLLM 在优化什么",[10,746,747],{},"vLLM 最出名的是 PagedAttention。它的动机来自一个非常工程的问题：KV cache 很大，而且不同请求长度不同，如果用连续大块显存保存，很容易碎片化和浪费。",[10,749,750],{},"PagedAttention 借鉴操作系统虚拟内存的思想，把 KV cache 切成 block\u002Fpage：",[90,752,755],{"className":753,"code":754,"language":95,"meta":96},[93],"logical sequence blocks -> physical KV cache blocks\n",[98,756,754],{"__ignoreMap":96},[10,758,759],{},"好处：",[41,761,762,765,768,771],{},[44,763,764],{},"不要求每个请求的 KV cache 在显存中连续；",[44,766,767],{},"可以更灵活地增长、释放、复用 KV block；",[44,769,770],{},"支持更高效的 continuous batching；",[44,772,773],{},"降低显存碎片，提高可服务 batch 数。",[10,775,776],{},"vLLM 的核心不是“让模型少算公式”，而是让 serving 系统能把更多请求稳定塞进 GPU，并减少 KV cache 管理浪费。最终表现为吞吐提升、并发提升、显存利用率提升。",[33,778,780],{"id":779},"_11-tensorrt-tensorrt-llm-在优化什么","11. TensorRT \u002F TensorRT-LLM 在优化什么",[10,782,783],{},"TensorRT 是 NVIDIA 的高性能推理优化引擎。它更像一个面向部署的编译器和 runtime。",[10,785,786],{},"常见优化包括：",[41,788,789,792,795,798,801,804],{},[44,790,791],{},"图优化：删除无用节点、常量折叠、算子融合；",[44,793,794],{},"kernel auto-tuning：为具体 shape 选择最快 kernel；",[44,796,797],{},"精度优化：FP16、BF16、INT8、FP8；",[44,799,800],{},"内存规划：复用 buffer，减少峰值显存；",[44,802,803],{},"插件 kernel：为 attention、layernorm、gemm 等提供特化实现；",[44,805,806],{},"多 GPU 推理：张量并行、流水并行等。",[10,808,809],{},"TensorRT-LLM 则进一步针对 LLM：",[41,811,812,815,818,821,824,827],{},[44,813,814],{},"fused attention；",[44,816,817],{},"paged KV cache；",[44,819,820],{},"inflight batching；",[44,822,823],{},"quantization；",[44,825,826],{},"tensor parallel；",[44,828,829],{},"speculative decoding 支持。",[10,831,832],{},"如果 vLLM 的关键词是 serving scheduler + KV cache 管理，那么 TensorRT 的关键词是 graph optimization + kernel optimization + deployment runtime。",[33,834,836],{"id":835},"_12-diffusion-acceleration-在优化什么","12. Diffusion Acceleration 在优化什么",[10,838,839],{},"Diffusion 模型和 LLM 都是生成模型，但瓶颈不完全一样。",[10,841,842],{},"Diffusion 生成图片通常需要多步 denoising：",[90,844,847],{"className":845,"code":846,"language":95,"meta":96},[93],"noise x_T\n  -> denoise step T\n  -> denoise step T-1\n  -> ...\n  -> image x_0\n",[98,848,846],{"__ignoreMap":96},[10,850,851],{},"慢的原因：",[41,853,854,857,860,863],{},[44,855,856],{},"U-Net \u002F DiT 要重复执行很多步；",[44,858,859],{},"每一步都有大量卷积或 attention；",[44,861,862],{},"高分辨率图片带来巨大的激活和计算量；",[44,864,865],{},"classifier-free guidance 可能让一次 step 跑两次网络。",[10,867,868],{},"加速方向：",[41,870,871,874,877,880,883,886],{},[44,872,873],{},"减少采样步数：DDIM、DPM-Solver、LCM、Turbo 类模型；",[44,875,876],{},"蒸馏：把多步模型蒸馏成少步模型；",[44,878,879],{},"算子优化：fused attention、xFormers、FlashAttention；",[44,881,882],{},"量化：FP16、INT8、FP8；",[44,884,885],{},"编译优化：TensorRT、TorchInductor、ONNX Runtime；",[44,887,888],{},"缓存复用：在视频或交互编辑里复用部分特征。",[10,890,891],{},"所以你看 diffusion acceleration 时，要先判断它是在减少 step 数，还是在加速每一步 kernel，还是在减少内存访问。",[33,893,895],{"id":894},"_13-distributed-training-overview只看概念","13. Distributed Training Overview：只看概念",[10,897,898],{},"虽然这一周重点是推理，但分布式训练的概念也要知道，因为很多推理并行策略来自训练。",[104,900,902],{"id":901},"_131-data-parallel","13.1 Data Parallel",[10,904,905],{},"每张 GPU 放一份完整模型，处理不同 batch，反向后同步梯度。",[90,907,910],{"className":908,"code":909,"language":95,"meta":96},[93],"GPU0: model + batch0\nGPU1: model + batch1\nall-reduce gradients\n",[98,911,909],{"__ignoreMap":96},[10,913,914],{},"优点简单，缺点是每张卡都要放完整模型。",[104,916,918],{"id":917},"_132-tensor-parallel","13.2 Tensor Parallel",[10,920,921],{},"把单层的大矩阵切到多张 GPU 上。例如一个线性层的 weight 按列或按行切分。",[90,923,926],{"className":924,"code":925,"language":95,"meta":96},[93],"W = [W0, W1, W2, W3]\n",[98,927,925],{"__ignoreMap":96},[10,929,930],{},"适合单卡放不下或单层计算太大的模型，但需要 GPU 间通信。",[104,932,934],{"id":933},"_133-pipeline-parallel","13.3 Pipeline Parallel",[10,936,937],{},"把不同层放到不同 GPU 上：",[90,939,942],{"className":940,"code":941,"language":95,"meta":96},[93],"GPU0: layer 0-9\nGPU1: layer 10-19\nGPU2: layer 20-29\n",[98,943,941],{"__ignoreMap":96},[10,945,946],{},"适合层数很深的模型，但会有 pipeline bubble，需要 micro-batch 填流水线。",[104,948,950],{"id":949},"_134-推理中的并行","13.4 推理中的并行",[10,952,953],{},"LLM 推理也会用 tensor parallel 和 pipeline parallel，尤其是模型单卡放不下时。但推理还有 serving 特有问题：batch 动态变化、KV cache 分布、跨卡通信延迟、首 token 延迟等。",[33,955,957],{"id":956},"_14-一个性能分析心法","14. 一个性能分析心法",[10,959,960],{},"遇到任何推理加速论文或系统，先问五个问题：",[251,962,963,966,969,972,975],{},[44,964,965],{},"它减少了 FLOPs，还是减少了 bytes moved？",[44,967,968],{},"它优化 prefill，还是优化 decode？",[44,970,971],{},"它优化单请求 latency，还是多请求 throughput？",[44,973,974],{},"它改变模型数学结果，还是只改变执行方式？",[44,976,977],{},"它的代价是什么：精度、显存、工程复杂度、兼容性，还是调度公平性？",[10,979,980],{},"例如：",[171,982,983,996],{},[174,984,985],{},[177,986,987,990,993],{},[180,988,989],{},"技术",[180,991,992],{},"主要优化",[180,994,995],{},"代价",[190,997,998,1009,1020,1031,1042,1053,1064],{},[177,999,1000,1003,1006],{},[195,1001,1002],{},"Quantization",[195,1004,1005],{},"减少权重\u002F激活 bytes",[195,1007,1008],{},"可能掉精度",[177,1010,1011,1014,1017],{},[195,1012,1013],{},"FlashAttention",[195,1015,1016],{},"减少 attention HBM 读写",[195,1018,1019],{},"kernel 更复杂",[177,1021,1022,1025,1028],{},[195,1023,1024],{},"Operator Fusion",[195,1026,1027],{},"减少 launch 和中间写回",[195,1029,1030],{},"编译\u002F调试复杂",[177,1032,1033,1036,1039],{},[195,1034,1035],{},"Continuous Batching",[195,1037,1038],{},"提高吞吐和权重复用",[195,1040,1041],{},"调度复杂，可能影响延迟",[177,1043,1044,1047,1050],{},[195,1045,1046],{},"PagedAttention",[195,1048,1049],{},"降低 KV cache 碎片",[195,1051,1052],{},"cache 管理更复杂",[177,1054,1055,1058,1061],{},[195,1056,1057],{},"Speculative Decoding",[195,1059,1060],{},"减少大模型 decode 步数",[195,1062,1063],{},"需要小模型\u002F草稿模型",[177,1065,1066,1069,1072],{},[195,1067,1068],{},"Distillation",[195,1070,1071],{},"减少模型或采样步数",[195,1073,1074],{},"训练成本和质量风险",[33,1076,1078],{"id":1077},"_15-week-2-学完应该能讲清楚","15. Week 2 学完应该能讲清楚",[10,1080,1081],{},"这一周不要求你会手写 CUDA kernel，但要能建立系统直觉：",[41,1083,1084,1087,1090,1093,1096,1099,1102,1105,1108,1111],{},[44,1085,1086],{},"GPU 的并行来自大量线程、warp、SM，而不是单线程快；",[44,1088,1089],{},"kernel launch 有固定成本，小算子多会拖慢端到端；",[44,1091,1092],{},"operator fusion 的本质是减少 launch 和 HBM 往返；",[44,1094,1095],{},"compute bound 和 memory bound 要用 arithmetic intensity 区分；",[44,1097,1098],{},"LLM prefill 和 decode 的性能特征完全不同；",[44,1100,1101],{},"KV cache 是 LLM 自回归推理的关键状态，也是显存管理难点；",[44,1103,1104],{},"batch scheduling 是 serving 系统吞吐与延迟的核心；",[44,1106,1107],{},"streaming generation 提升体感速度，但不减少总计算；",[44,1109,1110],{},"vLLM 更偏 serving 和 KV cache 管理，TensorRT 更偏编译和 kernel 优化；",[44,1112,1113],{},"diffusion acceleration 要区分减少采样步数和加速单步网络。",[33,1115,1117],{"id":1116},"_16-最后总结","16. 最后总结",[10,1119,1120],{},"GPU 推理加速不是单一技术，而是一组围绕硬件瓶颈展开的系统工程：算子层面要减少 kernel launch 和显存访问，图层面要做 fusion 和内存规划，服务层面要做 batching 和 KV cache 管理，模型层面要做量化、蒸馏或结构改造。",[10,1122,1123],{},"理解这些之后，再看 vLLM、TensorRT、FlashAttention、diffusion turbo、speculative decoding，你会更容易判断它们到底在优化哪一层、为什么有效、代价是什么。这也是读推理系统论文时最重要的底层框架。",[1125,1126,1127],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html pre.shiki code .ssh_m, html code.shiki .ssh_m{--shiki-default:#ADBAC7;--shiki-light:#24292E}html pre.shiki code .s6PUj, html code.shiki .s6PUj{--shiki-default:#F47067;--shiki-light:#D73A49}",{"title":96,"searchDepth":361,"depth":1129,"links":1130},3,[1131,1132,1138,1139,1140,1141,1142,1143,1147,1153,1154,1155,1156,1162,1163,1164],{"id":35,"depth":361,"text":36},{"id":81,"depth":361,"text":82,"children":1133},[1134,1135,1136,1137],{"id":106,"depth":1129,"text":107},{"id":137,"depth":1129,"text":138},{"id":155,"depth":1129,"text":156},{"id":165,"depth":1129,"text":166},{"id":245,"depth":361,"text":246},{"id":282,"depth":361,"text":283},{"id":334,"depth":361,"text":335},{"id":395,"depth":361,"text":396},{"id":448,"depth":361,"text":449},{"id":526,"depth":361,"text":527,"children":1144},[1145,1146],{"id":533,"depth":1129,"text":534},{"id":557,"depth":1129,"text":558},{"id":592,"depth":361,"text":593,"children":1148},[1149,1150,1151,1152],{"id":596,"depth":1129,"text":597},{"id":635,"depth":1129,"text":636},{"id":694,"depth":1129,"text":695},{"id":724,"depth":1129,"text":725},{"id":743,"depth":361,"text":744},{"id":779,"depth":361,"text":780},{"id":835,"depth":361,"text":836},{"id":894,"depth":361,"text":895,"children":1157},[1158,1159,1160,1161],{"id":901,"depth":1129,"text":902},{"id":917,"depth":1129,"text":918},{"id":933,"depth":1129,"text":934},{"id":949,"depth":1129,"text":950},{"id":956,"depth":361,"text":957},{"id":1077,"depth":361,"text":1078},{"id":1116,"depth":361,"text":1117},[989],"2026-05-05 10:00:00","Week 1 我们从 Autograd 理解了深度学习框架的训练本质：Tensor、计算图、反向传播和内存优化。Week 2 要切到更贴近论文和系统落地的部分：GPU 与推理加速。",false,"md",{},true,"\u002Fposts\u002Fgpu-inference-acceleration-week2",{"title":5,"description":1167},"posts\u002Fgpu-inference-acceleration-week2",[1176,1177,1178,1179,1180,1181,1182],"深度学习","GPU","推理加速","LLM","CMU 10-414","vLLM","TensorRT","4elcgLnK7zQAazTvgiErWUD5EngRfySbSwKElLN-Y1M",[1185,1199,1209,1216,1229,1239,1249,1260,1271,1280,1290,1302,1315,1327,1336,1345,1358,1369,1372,1380,1391,1398,1404,1410,1418,1428,1436,1442,1450,1458],{"slug":1186,"path":1187,"title":1188,"date":1189,"tags":1190,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1198},"multimodal-rag-from-scratch","\u002Fposts\u002Fmultimodal-rag-from-scratch","从零实现多模态 RAG：BM25、Dense 检索、RRF 融合、MMR 重排全部手写","2026-06-30 18:00:00",[1191,1192,1193,1194,1195,1196,1197],"RAG","多模态","AI Infra","BM25","向量检索","混合检索","实习求职",9,{"slug":1200,"path":1201,"title":1202,"date":1203,"tags":1204,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1208},"mini-llm-engine-deep-dive","\u002Fposts\u002Fmini-llm-engine-deep-dive","讲透 mini-llm-engine：从显存碎片到六大推理优化","2026-06-30 14:00:00",[1179,1193,1181,1046,1205,1206,1207,1197],"KV Cache","推理优化","投机解码",11,{"slug":1210,"path":1211,"title":1212,"date":1213,"tags":1214,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1215},"mini-llm-engine-from-scratch","\u002Fposts\u002Fmini-llm-engine-from-scratch","从零实现 LLM 推理引擎：深挖 vLLM 的六大核心优化","2026-06-30 10:00:00",[1179,1193,1181,1046,1206,1207,1197],10,{"slug":1217,"path":1218,"title":1219,"date":1220,"tags":1221,"description":1227,"draft":1168,"hidden":1168,"published":1171,"readingTime":1228},"bytedance-recommendation-architecture-intern-interview","\u002Fposts\u002Fbytedance-recommendation-architecture-intern-interview","字节推荐架构实习生 Data 面试准备：推荐系统、实时特征与高并发八股","2026-06-09",[1222,1223,1224,1225,1226],"面试","推荐系统","后端架构","实时计算","字节跳动","面向字节跳动推荐架构团队实习岗位的八股准备清单，覆盖推荐系统链路、实时数据、特征服务、高并发后端、分布式系统与项目包装。",18,{"slug":1230,"path":1231,"title":1232,"date":1220,"tags":1233,"description":1237,"draft":1168,"hidden":1168,"published":1171,"readingTime":1238},"high-concurrency-rate-limiting-algorithms","\u002Fposts\u002Fhigh-concurrency-rate-limiting-algorithms","高并发限流算法：固定窗口、滑动窗口、漏桶与令牌桶",[1234,1235,1224,1236,1222],"高并发","限流","系统设计","面试和工程都常见的限流算法总结，讲清楚固定窗口、滑动窗口、漏桶、令牌桶、并发数限制以及分布式限流如何落地。",12,{"slug":1240,"path":1241,"title":1242,"date":1220,"tags":1243,"description":1247,"draft":1168,"hidden":1168,"published":1171,"readingTime":1248},"kafka-producer-broker-consumer","\u002Fposts\u002Fkafka-producer-broker-consumer","Kafka 入门：生产者、Broker、消费者和“消费”到底是什么意思",[1244,1245,1246,1222,1223],"Kafka","消息队列","后端","用推荐系统里的用户行为日志为例，讲清楚 Kafka 的作用、Producer、Broker、Consumer、Topic、Partition、Offset 和消费语义。",8,{"slug":1250,"path":1251,"title":1252,"date":1220,"tags":1253,"description":1258,"draft":1168,"hidden":1168,"published":1171,"readingTime":1259},"leetcode-lru-merge-k-reverse-list","\u002Fposts\u002Fleetcode-lru-merge-k-reverse-list","链表与缓存高频题：LRU Cache、合并 K 个有序链表、反转链表",[1254,1255,1256,1257,1222],"算法","链表","LRU","LeetCode","面试高频算法题速记，整理 LRU Cache、合并 K 个有序链表、反转链表的核心思路、复杂度和 C++ 代码。",6,{"slug":1261,"path":1262,"title":1263,"date":1264,"tags":1265,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1270},"meddococr-interpreter-source-analysis","\u002Fposts\u002Fmeddococr-interpreter-source-analysis","MedDocOCR-Interpreter 源码导读：医疗文档 OCR、结构化抽取与报告解读原型","2026-05-22 10:30:00",[1266,1192,1267,1191,1268,1269],"OCR","医疗 AI","Python","源码分析",16,{"slug":1272,"path":1273,"title":1274,"date":1275,"tags":1276,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1215},"cet6-writing-model-essays","\u002Fposts\u002Fcet6-writing-model-essays","六级写作范文背诵包：10 个高频话题","2026-05-20 09:00:00",[1277,1278,1279],"English","CET6","Writing",{"slug":1281,"path":1282,"title":1283,"date":1284,"tags":1285,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1289},"claude-code-context-management","\u002Fposts\u002Fclaude-code-context-management","Claude Code 上下文管理机制：从 Microcompact 到 Auto Compact","2026-05-19 10:00:00",[1286,1287,1179,1193,1288],"Claude Code","Agent","上下文工程",27,{"slug":1291,"path":1292,"title":1293,"date":1294,"tags":1295,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1301},"nanotron-llm-pretraining-framework-analysis","\u002Fposts\u002Fnanotron-llm-pretraining-framework-analysis","Nanotron 项目详解：Hugging Face 的大模型预训练框架怎么做分布式训练","2026-05-10 12:10:00",[1179,1296,1297,1298,1299,1193,1300],"大模型训练","分布式训练","Nanotron","Hugging Face","PyTorch",19,{"slug":1303,"path":1304,"title":1305,"date":1306,"tags":1307,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1314},"gofoundry-go-backend-foundation-framework","\u002Fposts\u002Fgofoundry-go-backend-foundation-framework","GoFoundry 项目详解：基于 Go 的后端基础框架套件设计","2026-05-10 11:20:00",[1308,1309,1310,1311,1312,1245,1313],"Go","后端框架","ORM","分布式缓存","分布式锁","项目架构",25,{"slug":1316,"path":1317,"title":1318,"date":1319,"tags":1320,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1289},"cloudvault-go-cloud-storage-system","\u002Fposts\u002Fcloudvault-go-cloud-storage-system","CloudVault 项目详解：基于 Go 的云端存储与网盘系统架构设计","2026-05-10 10:30:00",[1308,1321,1322,1323,1313,1324,1325,1326],"云存储","网盘系统","分布式系统","Redis","RabbitMQ","Elasticsearch",{"slug":1328,"path":1329,"title":1330,"date":1331,"tags":1332,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1335},"openclaw-source-code-analysis","\u002Fposts\u002Fopenclaw-source-code-analysis","OpenClaw 源码导读：个人 AI 助手的网关、通道、插件与运行时架构","2026-05-08 16:30:00",[1333,1287,1193,1334,1269],"OpenClaw","TypeScript",20,{"slug":1337,"path":1338,"title":1339,"date":1340,"tags":1341,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1215},"flow-matching-generative-models","\u002Fposts\u002Fflow-matching-generative-models","Flow Matching：从噪声到数据的连续流生成模型","2026-05-07 00:00:00",[1342,1343,1344,1176],"生成模型","Diffusion","Flow Matching",{"slug":1346,"path":1347,"title":1348,"date":1349,"tags":1350,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1357},"database-ai-week4","\u002Fposts\u002Fdatabase-ai-week4","Week 4：数据库速成——从 Storage、Index、Query Optimization 到 Vector DB 与 RAG","2026-05-05 12:00:00",[1351,1352,1353,1191,1354,1355,1356],"数据库","CMU 15-445","Vector DB","LLM Memory","Query Optimization","Caching",15,{"slug":1359,"path":1360,"title":1361,"date":1362,"tags":1363,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1270},"distributed-systems-week3","\u002Fposts\u002Fdistributed-systems-week3","Week 3：分布式系统速成——MapReduce、Raft、容错与 Distributed KV Store","2026-05-05 11:00:00",[1323,1364,1365,1366,1367,1368,1287],"MIT 6.824","MapReduce","Raft","KV Store","Ray",{"slug":1370,"path":1172,"title":5,"date":1166,"tags":1371,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1357},"gpu-inference-acceleration-week2",[1176,1177,1178,1179,1180,1181,1182],{"slug":1373,"path":1374,"title":1375,"date":1376,"tags":1377,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1228},"dl-framework-autograd-mini","\u002Fposts\u002Fdl-framework-autograd-mini","Week 1：DL 框架与 Autograd——从计算图、反向传播到 Mini Autograd 实现","2026-05-05 09:00:00",[1176,1378,1300,1180,1379],"Autograd","Mini Framework",{"slug":1381,"path":1382,"title":1383,"date":1384,"tags":1385,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1238},"lock-free-concurrency-notes","\u002Fposts\u002Flock-free-concurrency-notes","无锁并发入门：从 CAS 到 Atomic Ring Buffer","2026-04-25",[1386,1387,1388,1389,1390],"C++","并发","无锁编程","性能优化","量化开发",{"slug":1392,"path":1393,"title":1394,"date":1395,"tags":1396,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1397},"agent-memory","\u002Fposts\u002Fagent-memory","Agent 对话记忆化：从原理到实现","2026-04-24",[1179,1287,1191,1222],5,{"slug":1399,"path":1400,"title":1401,"date":1395,"tags":1402,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1259},"llm-context-compression","\u002Fposts\u002Fllm-context-compression","LLM 上下文五层压缩机制详解",[1179,1287,1403,1222],"上下文压缩",{"slug":1405,"path":1406,"title":1407,"date":1408,"tags":1409,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1198},"cpp-concurrency-basics","\u002Fposts\u002Fcpp-concurrency-basics","C++ 并发编程入门：从数据竞争到线程池","2026-04-15",[1386,1387,1222,1390],{"slug":1411,"path":1412,"title":1413,"date":1414,"tags":1415,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":125},"travel-in-shenzhen","\u002Fposts\u002Ftravel-in-shenzhen","XCPC 深圳游记","2026-04-13",[1416,1386,1417],"XCPC","比赛",{"slug":1419,"path":1420,"title":1421,"date":1422,"tags":1423,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1427},"backend-stack-deep-dive","\u002Fposts\u002Fbackend-stack-deep-dive","后端五件套：FastAPI \u002F Node.js \u002F SQLAlchemy async \u002F PostgreSQL \u002F Docker 面试速通","2026-04-07",[1246,1424,1425,1426,1222],"FastAPI","PostgreSQL","Docker",7,{"slug":1429,"path":1430,"title":1431,"date":1432,"tags":1433,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1259},"deepscientist-tech-stack","\u002Fposts\u002Fdeepscientist-tech-stack","DeepScientist 技术栈全解析：一个 AI 科研平台的架构设计","2026-04-06",[1434,1424,1435,1425,1222],"全栈","Next.js",{"slug":1437,"path":1438,"title":1439,"date":1432,"tags":1440,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1248},"minicode-source-analysis","\u002Fposts\u002Fminicode-source-analysis","MiniCode 源码解析：用 5000 行 TypeScript 实现一个 AI 编程助手",[1334,1441,1179,1269,1222],"CLI",{"slug":1443,"path":1444,"title":1445,"date":1432,"tags":1446,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1397},"nova-theme-implementation","\u002Fposts\u002Fnova-theme-implementation","我是怎么从零实现 Nova 主题的",[1447,1448,1449],"Hexo","前端","开源",{"slug":1451,"path":1452,"title":1453,"date":1454,"tags":1455,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":361},"git-cheatsheet","\u002Fposts\u002Fgit-cheatsheet","Git 常用操作备忘","2026-04-05 14:00:00",[1456,1457],"Git","工具",{"slug":1459,"path":1460,"title":1461,"date":1462,"tags":1463,"description":96,"draft":1168,"hidden":1168,"published":1171,"readingTime":1129},"github-actions-intro","\u002Fposts\u002Fgithub-actions-intro","GitHub Actions 入门：自动化你的工作流","2026-04-04 09:00:00",[1464,1465,1466],"GitHub Actions","CI\u002FCD","自动化",1782796012169]