[{"data":1,"prerenderedAt":1771},["ShallowReactive",2],{"post-\u002Fposts\u002Fnanotron-llm-pretraining-framework-analysis":3,"all-posts-nav":1487},{"id":4,"title":5,"body":6,"categories":1468,"date":1470,"description":1471,"draft":1472,"extension":1473,"hidden":1472,"meta":1474,"navigation":1475,"path":1476,"published":1472,"seo":1477,"stem":1478,"tags":1479,"__hash__":1486},"posts\u002Fposts\u002Fnanotron-llm-pretraining-framework-analysis.md","Nanotron 项目详解：Hugging Face 的大模型预训练框架怎么做分布式训练",{"type":7,"value":8,"toc":1410},"minimark",[9,18,27,34,37,41,44,50,53,58,62,65,84,87,90,93,104,107,139,142,146,149,155,158,179,183,186,227,230,262,269,272,275,281,284,301,305,308,314,317,321,324,327,333,336,339,350,353,364,367,371,374,377,383,386,392,395,406,408,419,422,426,429,432,438,441,444,455,457,468,472,475,481,484,502,505,509,512,518,521,541,544,547,553,557,560,565,568,574,577,580,583,587,590,596,599,601,612,615,619,622,625,631,634,637,640,646,649,652,656,659,662,665,671,674,678,681,687,690,693,707,710,714,717,720,726,729,746,753,757,760,763,769,772,775,778,782,785,788,814,817,821,824,827,833,836,839,843,846,849,872,875,879,882,959,962,966,969,972,978,981,1014,1025,1028,1035,1041,1044,1047,1070,1073,1076,1080,1083,1100,1103,1120,1124,1126,1143,1145,1162,1166,1168,1184,1186,1203,1207,1210,1213,1219,1222,1226,1229,1233,1236,1239,1242,1246,1249,1289,1293,1296,1301,1304,1308,1311,1315,1318,1322,1325,1329,1336,1340,1343,1347,1350,1354,1357,1361,1364,1368,1371,1375,1378,1381,1384,1389,1392,1397,1400,1403,1406],[10,11,12,13,17],"p",{},"这篇文章选一个真正和“大模型训练”强相关的开源项目来讲：Hugging Face 的 ",[14,15,16],"strong",{},"Nanotron","。",[10,19,20,21],{},"项目地址：",[22,23,24],"a",{"href":24,"rel":25},"https:\u002F\u002Fgithub.com\u002Fhuggingface\u002Fnanotron",[26],"nofollow",[10,28,29,30,33],{},"Nanotron 是 Hugging Face 开源的大模型预训练框架，定位是 ",[14,31,32],{},"Pretraining transformer models made easy","。它不是一个简单的训练脚本，而是一个面向大规模 Transformer 预训练的训练基础设施项目，核心能力包括 3D 并行（DP + TP + PP）、MoE Expert Parallelism、Pipeline Parallel schedule、ZeRO-1、FP32 梯度累积、参数 tying\u002Fsharding、大模型 checkpoint、CUDA event 性能计时等。",[10,35,36],{},"如果你想理解“大模型训练项目到底在做什么”，Nanotron 是一个非常适合拆解的项目。",[38,39,40],"h2",{"id":40},"项目一句话介绍",[10,42,43],{},"Nanotron 可以概括为：",[45,46,47],"blockquote",{},[10,48,49],{},"Nanotron 是 Hugging Face 开源的 Transformer 大模型预训练框架，围绕分布式并行、训练配置、数据加载、模型切分、优化器、checkpoint 和性能监控构建了一套可扩展的大模型训练基础设施。",[10,51,52],{},"如果放到面试里，可以这样介绍：",[45,54,55],{},[10,56,57],{},"Nanotron 不是普通的单卡训练脚本，而是一个面向 LLM pretraining 的分布式训练框架。它通过数据并行、张量并行和流水线并行组成 3D parallelism，把模型参数、计算和 batch 拆到多张 GPU 甚至多节点上；通过 ZeRO-1 降低优化器状态内存；通过 pipeline schedule 提升多 stage 训练效率；通过 YAML 配置统一管理模型、数据、优化器、并行策略、checkpoint 和日志，从而让大模型预训练流程工程化、可复现、可扩展。",[38,59,61],{"id":60},"为什么选择-nanotron","为什么选择 Nanotron",[10,63,64],{},"大模型训练项目有很多，例如 Megatron-LM、DeepSpeed、Colossal-AI、LLaMA-Factory、Axolotl 等。这里选择 Nanotron 的原因是：",[66,67,68,72,75,78,81],"ol",{},[69,70,71],"li",{},"它来自 Hugging Face，生态链接很强。",[69,73,74],{},"它专注于 pretraining，而不只是 SFT\u002FLoRA 微调。",[69,76,77],{},"它显式支持 DP、TP、PP 的 3D 并行。",[69,79,80],{},"它的训练入口、配置和文档比较清晰。",[69,82,83],{},"它适合用来学习“大模型训练基础设施”而不是只会调 trainer。",[10,85,86],{},"如果你的目标是理解“从零训练一个 Transformer 大模型需要哪些系统能力”，Nanotron 比很多只做微调的项目更合适。",[38,88,89],{"id":89},"大模型训练为什么复杂",[10,91,92],{},"普通 PyTorch 训练大概是：",[94,95,101],"pre",{"className":96,"code":98,"language":99,"meta":100},[97],"language-text","model -> dataloader -> forward -> loss -> backward -> optimizer.step\n","text","",[102,103,98],"code",{"__ignoreMap":100},[10,105,106],{},"但大模型训练会遇到完全不同的问题：",[66,108,109,112,115,118,121,124,127,130,133,136],{},[69,110,111],{},"模型参数太大，一张 GPU 放不下。",[69,113,114],{},"激活值太大，batch 稍大就 OOM。",[69,116,117],{},"优化器状态通常是参数量的数倍。",[69,119,120],{},"多 GPU 通信开销很高。",[69,122,123],{},"多节点训练容易出现 straggler 和网络瓶颈。",[69,125,126],{},"checkpoint 很大，保存和恢复都很慢。",[69,128,129],{},"数据吞吐必须跟得上 GPU 消耗。",[69,131,132],{},"训练中断后必须能恢复。",[69,134,135],{},"loss spike、梯度溢出、nan 都要监控。",[69,137,138],{},"性能不能只看能不能跑，还要看 MFU、吞吐和尾部延迟。",[10,140,141],{},"Nanotron 的价值就是把这些复杂问题拆成一个个可配置的训练模块。",[38,143,145],{"id":144},"nanotron-的整体训练链路","Nanotron 的整体训练链路",[10,147,148],{},"一个 Nanotron 训练任务大致是：",[94,150,153],{"className":151,"code":152,"language":99,"meta":100},[97],"读取 YAML 配置\n  -> 初始化 torch.distributed\n  -> 构建并行拓扑 DP \u002F TP \u002F PP\n  -> 构建 tokenizer 和 dataloader\n  -> 构建 Transformer 模型\n  -> 按 TP \u002F PP 切分模型\n  -> 构建 optimizer \u002F scheduler\n  -> 加载 checkpoint 或从头训练\n  -> 训练循环 forward \u002F backward \u002F step\n  -> 定期保存 checkpoint\n  -> 记录 loss、吞吐、显存、计时指标\n",[102,154,152],{"__ignoreMap":100},[10,156,157],{},"相比一个普通训练脚本，它多出来的核心是：",[159,160,161,164,167,170,173,176],"ul",{},[69,162,163],{},"分布式进程组管理。",[69,165,166],{},"模型并行切分。",[69,168,169],{},"Pipeline 调度。",[69,171,172],{},"优化器状态分片。",[69,174,175],{},"大规模 checkpoint 管理。",[69,177,178],{},"性能监控和 benchmark。",[38,180,182],{"id":181},"quick-start-怎么跑","Quick Start 怎么跑",[10,184,185],{},"Nanotron README 给出的最小训练命令是：",[94,187,191],{"className":188,"code":189,"language":190,"meta":100,"style":100},"language-bash shiki shiki-themes github-dark-dimmed github-light","CUDA_DEVICE_MAX_CONNECTIONS=1 torchrun --nproc_per_node=8 run_train.py --config-file examples\u002Fconfig_tiny_llama.yaml\n","bash",[102,192,193],{"__ignoreMap":100},[194,195,198,202,206,210,214,218,221,224],"span",{"class":196,"line":197},"line",1,[194,199,201],{"class":200},"ssh_m","CUDA_DEVICE_MAX_CONNECTIONS",[194,203,205],{"class":204},"s6PUj","=",[194,207,209],{"class":208},"sXfbr","1",[194,211,213],{"class":212},"sqRhv"," torchrun",[194,215,217],{"class":216},"swcJU"," --nproc_per_node=8",[194,219,220],{"class":208}," run_train.py",[194,222,223],{"class":216}," --config-file",[194,225,226],{"class":208}," examples\u002Fconfig_tiny_llama.yaml\n",[10,228,229],{},"这个命令里有几个关键点：",[159,231,232,238,244,250,256],{},[69,233,234,237],{},[102,235,236],{},"torchrun","：PyTorch 官方分布式启动器。",[69,239,240,243],{},[102,241,242],{},"--nproc_per_node=8","：单节点启动 8 个进程，通常对应 8 张 GPU。",[69,245,246,249],{},[102,247,248],{},"run_train.py","：Nanotron 的训练入口。",[69,251,252,255],{},[102,253,254],{},"--config-file","：训练配置文件。",[69,257,258,261],{},[102,259,260],{},"CUDA_DEVICE_MAX_CONNECTIONS=1","：对某些分布式通信和 kernel 调度有影响，常用于提升特定并行模式下的稳定性或性能。",[10,263,264,265,268],{},"这说明 Nanotron 的训练方式不是在 Python 里手动 ",[102,266,267],{},"for gpu in gpus","，而是使用标准分布式多进程模型。",[38,270,271],{"id":271},"配置驱动训练",[10,273,274],{},"Nanotron 使用 YAML 配置训练参数。一个大模型训练任务通常包括：",[94,276,279],{"className":277,"code":278,"language":99,"meta":100},[97],"model:\n  architecture\n  hidden_size\n  num_layers\n  num_attention_heads\n  sequence_length\n\ntokens:\n  tokenizer\n  vocab_size\n\ndata:\n  dataset\n  num_loading_workers\n  seed\n\nparallelism:\n  dp\n  tp\n  pp\n\noptimizer:\n  learning_rate\n  weight_decay\n  betas\n  grad_clip\n\nscheduler:\n  warmup_steps\n  total_steps\n\ncheckpoints:\n  checkpoint_interval\n  save_dir\n\nlogging:\n  wandb\n  log_interval\n",[102,280,278],{"__ignoreMap":100},[10,282,283],{},"配置驱动的好处：",[66,285,286,289,292,295,298],{},[69,287,288],{},"实验可复现。",[69,290,291],{},"多组实验只需要改配置。",[69,293,294],{},"训练脚本不用写死参数。",[69,296,297],{},"多人协作时更容易审查训练设置。",[69,299,300],{},"方便在 Slurm 多节点环境里提交任务。",[38,302,304],{"id":303},"_3d-parallelismdp-tp-pp","3D Parallelism：DP + TP + PP",[10,306,307],{},"Nanotron 的核心能力之一是 3D 并行，也就是：",[94,309,312],{"className":310,"code":311,"language":99,"meta":100},[97],"Data Parallelism + Tensor Parallelism + Pipeline Parallelism\n",[102,313,311],{"__ignoreMap":100},[10,315,316],{},"这是大模型训练里最重要的概念之一。",[38,318,320],{"id":319},"dp数据并行","DP：数据并行",[10,322,323],{},"数据并行是最容易理解的并行方式。",[10,325,326],{},"每张 GPU 放一份完整模型，但喂不同 batch：",[94,328,331],{"className":329,"code":330,"language":99,"meta":100},[97],"GPU0: model + batch0\nGPU1: model + batch1\nGPU2: model + batch2\nGPU3: model + batch3\n",[102,332,330],{"__ignoreMap":100},[10,334,335],{},"每张卡独立 forward\u002Fbackward，然后通过 AllReduce 同步梯度。",[10,337,338],{},"优点：",[159,340,341,344,347],{},[69,342,343],{},"实现简单。",[69,345,346],{},"扩展 batch size 直接。",[69,348,349],{},"适合模型能放进单卡的情况。",[10,351,352],{},"缺点：",[159,354,355,358,361],{},[69,356,357],{},"每张卡都要保存完整模型参数。",[69,359,360],{},"模型太大时单卡放不下。",[69,362,363],{},"优化器状态也会重复保存。",[10,365,366],{},"所以大模型不能只靠 DP。",[38,368,370],{"id":369},"tp张量并行","TP：张量并行",[10,372,373],{},"张量并行是把单层内部的矩阵计算拆到多张 GPU 上。",[10,375,376],{},"以 Transformer 里的线性层为例：",[94,378,381],{"className":379,"code":380,"language":99,"meta":100},[97],"Y = XW\n",[102,382,380],{"__ignoreMap":100},[10,384,385],{},"如果 W 太大，可以按列或按行切分：",[94,387,390],{"className":388,"code":389,"language":99,"meta":100},[97],"W = [W0, W1, W2, W3]\nGPU0 计算 XW0\nGPU1 计算 XW1\nGPU2 计算 XW2\nGPU3 计算 XW3\n最后再通信合并\n",[102,391,389],{"__ignoreMap":100},[10,393,394],{},"TP 的优点：",[159,396,397,400,403],{},[69,398,399],{},"单层参数可以拆到多张卡。",[69,401,402],{},"适合超大 hidden size 和 attention\u002FMLP 层。",[69,404,405],{},"降低单卡参数和激活压力。",[10,407,352],{},[159,409,410,413,416],{},[69,411,412],{},"层内通信频繁。",[69,414,415],{},"对 GPU 间互联要求高，例如 NVLink \u002F NVSwitch。",[69,417,418],{},"跨节点 TP 通常性能较差。",[10,420,421],{},"因此 TP 通常放在单节点内做。",[38,423,425],{"id":424},"pp流水线并行","PP：流水线并行",[10,427,428],{},"流水线并行是按层切分模型。",[10,430,431],{},"例如一个 32 层 Transformer，可以分成 4 个 stage：",[94,433,436],{"className":434,"code":435,"language":99,"meta":100},[97],"GPU0: layers 0-7\nGPU1: layers 8-15\nGPU2: layers 16-23\nGPU3: layers 24-31\n",[102,437,435],{"__ignoreMap":100},[10,439,440],{},"数据从 stage 0 流到 stage 3，反向传播再从 stage 3 回到 stage 0。",[10,442,443],{},"PP 的优点：",[159,445,446,449,452],{},[69,447,448],{},"适合层数很多的大模型。",[69,450,451],{},"每张卡只保存一部分层。",[69,453,454],{},"可以跨节点扩展。",[10,456,352],{},[159,458,459,462,465],{},[69,460,461],{},"有 pipeline bubble，也就是部分 GPU 在等待。",[69,463,464],{},"需要 micro-batch 切分提高流水线利用率。",[69,466,467],{},"调度复杂，debug 难度高。",[38,469,471],{"id":470},"_3d-并行怎么组合","3D 并行怎么组合",[10,473,474],{},"假设有 64 张 GPU，可以组合为：",[94,476,479],{"className":477,"code":478,"language":99,"meta":100},[97],"dp = 4\ntp = 4\npp = 4\n总 GPU = dp * tp * pp = 64\n",[102,480,478],{"__ignoreMap":100},[10,482,483],{},"含义是：",[159,485,486,493,496,499],{},[69,487,488,489,492],{},"每个数据并行副本有 ",[102,490,491],{},"tp * pp = 16"," 张 GPU。",[69,494,495],{},"模型在一个副本内部被 TP 和 PP 切开。",[69,497,498],{},"一共有 4 个这样的副本处理不同数据。",[69,500,501],{},"不同 DP 副本之间同步梯度。",[10,503,504],{},"Nanotron 的配置里就需要明确这些并行维度。",[38,506,508],{"id":507},"global-batch-size-怎么算","Global Batch Size 怎么算",[10,510,511],{},"Nanotron 文档里提到，global batch size 通常是：",[94,513,516],{"className":514,"code":515,"language":99,"meta":100},[97],"micro_batch_size * batch_accumulation_per_replica * dp\n",[102,517,515],{"__ignoreMap":100},[10,519,520],{},"这里每个变量的含义是：",[159,522,523,529,535],{},[69,524,525,528],{},[102,526,527],{},"micro_batch_size","：每次 forward 的小 batch。",[69,530,531,534],{},[102,532,533],{},"batch_accumulation_per_replica","：梯度累积步数。",[69,536,537,540],{},[102,538,539],{},"dp","：数据并行副本数量。",[10,542,543],{},"为什么需要梯度累积？因为显存有限，不能一次塞很大的 batch。可以多次 forward\u002Fbackward 累积梯度，再执行一次 optimizer step。",[10,545,546],{},"例如：",[94,548,551],{"className":549,"code":550,"language":99,"meta":100},[97],"micro_batch_size = 2\nbatch_accumulation_per_replica = 8\ndp = 16\nglobal_batch_size = 2 * 8 * 16 = 256\n",[102,552,550],{"__ignoreMap":100},[38,554,556],{"id":555},"pipeline-scheduleafab-和-1f1b","Pipeline Schedule：AFAB 和 1F1B",[10,558,559],{},"Nanotron 支持 Pipeline Parallel 的 schedule，例如 AFAB 和 1F1B。",[561,562,564],"h3",{"id":563},"afab","AFAB",[10,566,567],{},"AFAB 可以理解为：",[94,569,572],{"className":570,"code":571,"language":99,"meta":100},[97],"All Forward, All Backward\n",[102,573,571],{"__ignoreMap":100},[10,575,576],{},"先把所有 micro-batch 的 forward 做完，再做 backward。",[10,578,579],{},"优点：简单。",[10,581,582],{},"缺点：激活保存多，显存压力大，pipeline bubble 可能明显。",[561,584,586],{"id":585},"_1f1b","1F1B",[10,588,589],{},"1F1B 是：",[94,591,594],{"className":592,"code":593,"language":99,"meta":100},[97],"One Forward, One Backward\n",[102,595,593],{"__ignoreMap":100},[10,597,598],{},"流水线填满后，每个 stage 交替做一个 forward 和一个 backward。",[10,600,338],{},[159,602,603,606,609],{},[69,604,605],{},"显存更友好。",[69,607,608],{},"Pipeline 利用率更高。",[69,610,611],{},"大模型训练更常用。",[10,613,614],{},"缺点：实现复杂，调度和通信更难 debug。",[38,616,618],{"id":617},"zero-1优化器状态分片","ZeRO-1：优化器状态分片",[10,620,621],{},"大模型训练里，优化器状态非常占显存。",[10,623,624],{},"以 Adam 为例，每个参数除了权重本身，还要维护：",[94,626,629],{"className":627,"code":628,"language":99,"meta":100},[97],"param\nGradient\nMomentum m\nVariance v\n",[102,630,628],{"__ignoreMap":100},[10,632,633],{},"如果用混合精度，还可能有 FP32 master weight。",[10,635,636],{},"ZeRO 的核心思想是把冗余状态切分到不同 DP rank 上。",[10,638,639],{},"ZeRO-1 主要分片 optimizer states：",[94,641,644],{"className":642,"code":643,"language":99,"meta":100},[97],"DP rank0 保存一部分 optimizer state\nDP rank1 保存一部分 optimizer state\nDP rank2 保存一部分 optimizer state\n...\n",[102,645,643],{"__ignoreMap":100},[10,647,648],{},"这样可以减少每张 GPU 上重复保存的优化器状态。",[10,650,651],{},"Nanotron 当前支持 ZeRO-1，roadmap 中有 ZeRO-3 \u002F FSDP。",[38,653,655],{"id":654},"fp32-梯度累积","FP32 梯度累积",[10,657,658],{},"Nanotron 支持 FP32 gradient accumulation。",[10,660,661],{},"这点对训练稳定性很重要。大模型训练通常会用 BF16\u002FFP16 做 forward\u002Fbackward，但梯度累积如果精度太低，可能带来数值误差。",[10,663,664],{},"FP32 累积的含义是：",[94,666,669],{"className":667,"code":668,"language":99,"meta":100},[97],"每个 micro-batch 得到梯度\n  -> 转成 \u002F 保持 FP32 累积\n  -> 累积若干步\n  -> optimizer step\n",[102,670,668],{"__ignoreMap":100},[10,672,673],{},"这样可以提升训练稳定性，尤其是在大 batch、长序列和深层网络下。",[38,675,677],{"id":676},"parameter-tying-和-sharding","Parameter Tying 和 Sharding",[10,679,680],{},"Transformer 语言模型中常见参数 tying：",[94,682,685],{"className":683,"code":684,"language":99,"meta":100},[97],"input embedding weight == output lm_head weight\n",[102,686,684],{"__ignoreMap":100},[10,688,689],{},"这样可以减少参数量，也可能提升泛化。",[10,691,692],{},"但在 TP\u002FPP\u002FDP 并行下，参数 tying 会复杂很多：",[159,694,695,698,701,704],{},[69,696,697],{},"tied 参数可能在不同 pipeline stage。",[69,699,700],{},"参数可能被 tensor parallel 切分。",[69,702,703],{},"checkpoint 保存和加载要知道它们共享同一份权重。",[69,705,706],{},"梯度同步不能重复或漏掉。",[10,708,709],{},"Nanotron 支持 parameter tying\u002Fsharding，说明它不是只处理单卡模型，而是考虑了大模型并行训练中的真实边界情况。",[38,711,713],{"id":712},"checkpoint大模型训练的生命线","Checkpoint：大模型训练的生命线",[10,715,716],{},"大模型训练动辄跑几天、几周甚至几个月，checkpoint 是生命线。",[10,718,719],{},"Checkpoint 至少要保存：",[94,721,724],{"className":722,"code":723,"language":99,"meta":100},[97],"模型参数\n优化器状态\nscheduler 状态\n训练步数\n随机数状态\n并行切分信息\ntokenizer \u002F config\n",[102,725,723],{"__ignoreMap":100},[10,727,728],{},"在 3D 并行下，checkpoint 更复杂：",[159,730,731,734,737,740,743],{},[69,732,733],{},"每个 rank 只保存自己负责的参数 shard。",[69,735,736],{},"恢复时必须匹配 TP\u002FPP\u002FDP 拓扑。",[69,738,739],{},"如果并行配置变化，还需要 reshard。",[69,741,742],{},"保存太频繁会拖慢训练。",[69,744,745],{},"保存太少则失败恢复成本高。",[10,747,748,749,752],{},"Nanotron 配置中可以设置 ",[102,750,751],{},"checkpoint_interval"," 控制保存频率。",[38,754,756],{"id":755},"数据加载吞吐不能拖后腿","数据加载：吞吐不能拖后腿",[10,758,759],{},"大模型训练时 GPU 很贵，最怕 GPU 等数据。",[10,761,762],{},"一个训练数据链路通常是：",[94,764,767],{"className":765,"code":766,"language":99,"meta":100},[97],"原始文本数据\n  -> 清洗、去重、过滤\n  -> tokenizer 编码\n  -> packed sequence\n  -> dataloader 多进程加载\n  -> batch 送入 GPU\n",[102,768,766],{"__ignoreMap":100},[10,770,771],{},"Nanotron 支持 Hugging Face datasets，也支持自定义 dataloader。",[10,773,774],{},"文档里提到，如果要使用自定义数据加载器，可以把 dataset 配置设为 null，然后自己实现 dataloader。",[10,776,777],{},"这在真实训练中很常见，因为公司内部数据格式、数据清洗逻辑和采样策略往往是定制的。",[38,779,781],{"id":780},"自定义-dataloader-为什么重要","自定义 Dataloader 为什么重要",[10,783,784],{},"大模型训练的数据不是简单的图片分类数据集，而是海量 token 流。",[10,786,787],{},"自定义 dataloader 可能要处理：",[159,789,790,793,796,799,802,805,808,811],{},[69,791,792],{},"多数据源混合采样。",[69,794,795],{},"数据权重配比。",[69,797,798],{},"文档边界。",[69,800,801],{},"sequence packing。",[69,803,804],{},"tokenizer 版本。",[69,806,807],{},"curriculum learning。",[69,809,810],{},"数据去重标记。",[69,812,813],{},"断点恢复时的数据位置。",[10,815,816],{},"如果数据加载不可控，训练结果就不可控。",[38,818,820],{"id":819},"性能监控不能只看-loss","性能监控：不能只看 loss",[10,822,823],{},"Nanotron 支持 CUDA event-based timing，用于更准确地测量 GPU 性能。",[10,825,826],{},"大模型训练要关注的指标包括：",[94,828,831],{"className":829,"code":830,"language":99,"meta":100},[97],"loss\nlearning rate\ngrad norm\ntokens\u002Fsec\nsamples\u002Fsec\nGPU memory\nMFU\nstep time\ncommunication time\npipeline bubble\ncheckpoint time\n",[102,832,830],{"__ignoreMap":100},[10,834,835],{},"其中 MFU 是 Model FLOPS Utilization，表示模型实际使用了理论峰值算力的多少。",[10,837,838],{},"高质量训练系统不只要 loss 降，还要知道训练资源利用率怎么样。",[38,840,842],{"id":841},"mfu-为什么重要","MFU 为什么重要",[10,844,845],{},"如果 100 张 H100 只跑出很低的 MFU，就意味着大量钱花在空等通信、数据加载或 pipeline bubble 上。",[10,847,848],{},"MFU 低可能来自：",[159,850,851,854,857,860,863,866,869],{},[69,852,853],{},"batch 太小。",[69,855,856],{},"TP 通信过重。",[69,858,859],{},"PP bubble 太大。",[69,861,862],{},"dataloader 跟不上。",[69,864,865],{},"checkpoint 阻塞训练。",[69,867,868],{},"kernel 没有融合。",[69,870,871],{},"序列长度和模型大小不匹配硬件。",[10,873,874],{},"Nanotron README 中也强调了 benchmark 和 Ultrascale Playbook，说明它很关注训练效率。",[38,876,878],{"id":877},"和-megatron-lm-deepspeed-的关系","和 Megatron-LM \u002F DeepSpeed 的关系",[10,880,881],{},"Nanotron 的很多思想和 Megatron-LM、DeepSpeed 类似。",[883,884,885,901],"table",{},[886,887,888],"thead",{},[889,890,891,895,898],"tr",{},[892,893,894],"th",{},"项目",[892,896,897],{},"重点",[892,899,900],{},"适合学习什么",[902,903,904,916,927,937,948],"tbody",{},[889,905,906,910,913],{},[907,908,909],"td",{},"Megatron-LM",[907,911,912],{},"NVIDIA 大规模 Transformer 训练",[907,914,915],{},"TP\u002FPP、GPT 预训练、高性能 kernel",[889,917,918,921,924],{},[907,919,920],{},"DeepSpeed",[907,922,923],{},"分布式训练优化库",[907,925,926],{},"ZeRO、offload、训练系统优化",[889,928,929,931,934],{},[907,930,16],{},[907,932,933],{},"Hugging Face 预训练框架",[907,935,936],{},"配置化 LLM pretraining、3D 并行、训练工程化",[889,938,939,942,945],{},[907,940,941],{},"LLaMA-Factory",[907,943,944],{},"微调框架",[907,946,947],{},"SFT、LoRA、DPO、数据格式适配",[889,949,950,953,956],{},[907,951,952],{},"Axolotl",[907,954,955],{},"微调配置框架",[907,957,958],{},"指令微调、多模型配置",[10,960,961],{},"所以 Nanotron 更适合作为“大模型预训练系统”的学习案例，而不是“怎么快速微调一个模型”的工具。",[38,963,965],{"id":964},"训练一个-tiny-llama-的流程","训练一个 tiny Llama 的流程",[10,967,968],{},"Nanotron Quick Start 是训练 tiny Llama。",[10,970,971],{},"完整理解可以拆成：",[94,973,976],{"className":974,"code":975,"language":99,"meta":100},[97],"1. 安装 Python \u002F PyTorch \u002F Nanotron\n2. 安装 datasets、transformers、wandb、flash-attn 等依赖\n3. 登录 Hugging Face 和 W&B\n4. 准备 config_tiny_llama.yaml\n5. torchrun 启动 8 GPU 训练\n6. 训练中保存 checkpoint\n7. 使用 run_generate.py 从 checkpoint 生成文本\n",[102,977,975],{"__ignoreMap":100},[10,979,980],{},"生成命令类似：",[94,982,984],{"className":188,"code":983,"language":190,"meta":100,"style":100},"torchrun --nproc_per_node=1 run_generate.py --ckpt-path checkpoints\u002F{checkpoint_number}\u002F --tp 1 --pp 1\n",[102,985,986],{"__ignoreMap":100},[194,987,988,990,993,996,999,1002,1005,1008,1011],{"class":196,"line":197},[194,989,236],{"class":212},[194,991,992],{"class":216}," --nproc_per_node=1",[194,994,995],{"class":208}," run_generate.py",[194,997,998],{"class":216}," --ckpt-path",[194,1000,1001],{"class":208}," checkpoints\u002F{checkpoint_number}\u002F",[194,1003,1004],{"class":216}," --tp",[194,1006,1007],{"class":216}," 1",[194,1009,1010],{"class":216}," --pp",[194,1012,1013],{"class":216}," 1\n",[10,1015,1016,1017,1020,1021,1024],{},"这里 ",[102,1018,1019],{},"--tp"," 和 ",[102,1022,1023],{},"--pp"," 也可以用于生成阶段的并行设置。",[38,1026,1027],{"id":1027},"多节点训练",[10,1029,1030,1031,1034],{},"单节点训练只需要 ",[102,1032,1033],{},"torchrun --nproc_per_node=8","，多节点训练通常还需要：",[94,1036,1039],{"className":1037,"code":1038,"language":99,"meta":100},[97],"nnodes\nnode_rank\nmaster_addr\nmaster_port\n",[102,1040,1038],{"__ignoreMap":100},[10,1042,1043],{},"以及集群调度系统，例如 Slurm。",[10,1045,1046],{},"多节点训练难点包括：",[159,1048,1049,1052,1055,1058,1061,1064,1067],{},[69,1050,1051],{},"节点间网络延迟。",[69,1053,1054],{},"NCCL 配置。",[69,1056,1057],{},"master 节点发现。",[69,1059,1060],{},"rank 映射。",[69,1062,1063],{},"故障恢复。",[69,1065,1066],{},"日志聚合。",[69,1068,1069],{},"checkpoint 存储共享。",[10,1071,1072],{},"Nanotron 提供 multi-node training 文档，说明它不是只考虑单机多卡。",[38,1074,1075],{"id":1075},"大模型训练中的常见故障",[561,1077,1079],{"id":1078},"oom","OOM",[10,1081,1082],{},"原因可能是：",[159,1084,1085,1088,1091,1094,1097],{},[69,1086,1087],{},"micro batch 太大。",[69,1089,1090],{},"sequence length 太长。",[69,1092,1093],{},"TP\u002FPP 配置不合理。",[69,1095,1096],{},"激活 checkpoint 没开。",[69,1098,1099],{},"optimizer state 太大。",[10,1101,1102],{},"解决方式：",[159,1104,1105,1108,1111,1114,1117],{},[69,1106,1107],{},"降低 micro batch。",[69,1109,1110],{},"增加 TP\u002FPP。",[69,1112,1113],{},"使用梯度累积保持 global batch。",[69,1115,1116],{},"使用 checkpointing。",[69,1118,1119],{},"使用 ZeRO。",[561,1121,1123],{"id":1122},"loss-spike","Loss Spike",[10,1125,1082],{},[159,1127,1128,1131,1134,1137,1140],{},[69,1129,1130],{},"学习率过高。",[69,1132,1133],{},"warmup 不足。",[69,1135,1136],{},"数据异常。",[69,1138,1139],{},"梯度裁剪缺失。",[69,1141,1142],{},"混合精度溢出。",[10,1144,1102],{},[159,1146,1147,1150,1153,1156,1159],{},[69,1148,1149],{},"调低学习率。",[69,1151,1152],{},"增加 warmup。",[69,1154,1155],{},"检查数据。",[69,1157,1158],{},"开启 grad clip。",[69,1160,1161],{},"监控 grad norm。",[561,1163,1165],{"id":1164},"gpu-利用率低","GPU 利用率低",[10,1167,1082],{},[159,1169,1170,1173,1176,1179,1181],{},[69,1171,1172],{},"dataloader 慢。",[69,1174,1175],{},"通信开销大。",[69,1177,1178],{},"pipeline bubble。",[69,1180,853],{},[69,1182,1183],{},"checkpoint 阻塞。",[10,1185,1102],{},[159,1187,1188,1191,1194,1197,1200],{},[69,1189,1190],{},"增加 num workers。",[69,1192,1193],{},"优化并行配置。",[69,1195,1196],{},"增加 micro batch 数。",[69,1198,1199],{},"使用更合适的 schedule。",[69,1201,1202],{},"异步或降低 checkpoint 频率。",[38,1204,1206],{"id":1205},"nanotron-的项目难点","Nanotron 的项目难点",[561,1208,1209],{"id":1209},"并行拓扑管理",[10,1211,1212],{},"DP、TP、PP 同时存在时，每个 rank 属于多个进程组。框架必须知道：",[94,1214,1217],{"className":1215,"code":1216,"language":99,"meta":100},[97],"当前 rank 属于哪个 DP group\n当前 rank 属于哪个 TP group\n当前 rank 属于哪个 PP stage\n哪些 rank 需要通信\n哪些参数需要同步\n",[102,1218,1216],{"__ignoreMap":100},[10,1220,1221],{},"这比普通 DDP 复杂得多。",[561,1223,1225],{"id":1224},"pipeline-调度","Pipeline 调度",[10,1227,1228],{},"Pipeline parallel 要处理 micro-batch 的 forward\u002Fbackward 顺序、激活保存、通信、bubble 和梯度累积。schedule 写错会导致死锁、梯度错误或性能很差。",[561,1230,1232],{"id":1231},"参数切分和-checkpoint","参数切分和 checkpoint",[10,1234,1235],{},"模型参数可能被 TP 切分，也可能按 PP 分布在不同 stage。保存和恢复 checkpoint 时必须知道每个 shard 的位置和含义。",[561,1237,1238],{"id":1238},"数据吞吐和训练稳定性",[10,1240,1241],{},"训练系统不仅要能跑，还要稳定跑很多 step。数据加载、随机种子、checkpoint、日志、异常恢复都必须工程化。",[38,1243,1245],{"id":1244},"如果要读源码建议顺序","如果要读源码，建议顺序",[10,1247,1248],{},"读 Nanotron 不建议一开始就钻进所有并行细节，可以按这个顺序：",[66,1250,1251,1257,1263,1268,1271,1274,1277,1280,1283,1286],{},[69,1252,1253,1256],{},[102,1254,1255],{},"README.md","：理解项目定位和 quick start。",[69,1258,1259,1262],{},[102,1260,1261],{},"examples\u002Fconfig_tiny_llama.yaml","：理解配置项。",[69,1264,1265,1267],{},[102,1266,248],{},"：看训练入口。",[69,1269,1270],{},"配置解析相关代码：看 YAML 怎么变成训练对象。",[69,1272,1273],{},"distributed \u002F parallel context：看 DP\u002FTP\u002FPP group 怎么建。",[69,1275,1276],{},"model 构建：看 Llama \u002F Transformer 怎么定义。",[69,1278,1279],{},"pipeline schedule：看 forward\u002Fbackward 怎么调度。",[69,1281,1282],{},"optimizer：看 ZeRO-1 和梯度累积。",[69,1284,1285],{},"checkpoint：看保存和恢复。",[69,1287,1288],{},"dataloader：看数据如何进入训练。",[38,1290,1292],{"id":1291},"面试-1-分钟讲法","面试 1 分钟讲法",[10,1294,1295],{},"如果面试官让你讲 Nanotron，可以这样说：",[45,1297,1298],{},[10,1299,1300],{},"Nanotron 是 Hugging Face 开源的大模型预训练框架，主要用于 Transformer \u002F LLM pretraining。它的核心是把大模型训练工程化：通过 YAML 配置描述模型、数据、优化器、并行策略和 checkpoint；通过 torchrun 启动多进程分布式训练；通过 DP、TP、PP 组成 3D 并行，把 batch、张量和模型层分别拆到不同 GPU 上；通过 ZeRO-1 分片优化器状态降低显存；通过 AFAB\u002F1F1B pipeline schedule 提升流水线训练效率；同时支持 FP32 梯度累积、参数 tying\u002Fsharding、大模型 checkpoint 和 CUDA event 计时。它适合学习从单机训练到多节点 LLM 预训练所需的训练基础设施。",[38,1302,1303],{"id":1303},"面试高频问答",[561,1305,1307],{"id":1306},"q1nanotron-和普通-trainer-有什么区别","Q1：Nanotron 和普通 Trainer 有什么区别？",[10,1309,1310],{},"普通 Trainer 更偏单机或简单 DDP 训练，Nanotron 面向大模型预训练，重点是 3D 并行、pipeline schedule、optimizer state sharding、checkpoint sharding 和性能监控。",[561,1312,1314],{"id":1313},"q2为什么大模型训练不能只用数据并行","Q2：为什么大模型训练不能只用数据并行？",[10,1316,1317],{},"因为数据并行要求每张 GPU 都保存完整模型和优化器状态。模型太大时单卡放不下，即使放得下，优化器状态和激活也会占用大量显存。",[561,1319,1321],{"id":1320},"q3tp-和-pp-的区别是什么","Q3：TP 和 PP 的区别是什么？",[10,1323,1324],{},"TP 是把单层内部的矩阵计算切到多张 GPU；PP 是把模型的不同层切到不同 GPU。TP 更依赖高速互联，PP 更适合按层扩展超深模型。",[561,1326,1328],{"id":1327},"q4global-batch-size-怎么算","Q4：global batch size 怎么算？",[10,1330,1331,1332,1335],{},"通常是 ",[102,1333,1334],{},"micro_batch_size * batch_accumulation_per_replica * dp","。如果显存不够，就减小 micro batch，用梯度累积保持 global batch。",[561,1337,1339],{"id":1338},"q51f1b-为什么比-afab-更常用","Q5：1F1B 为什么比 AFAB 更常用？",[10,1341,1342],{},"1F1B 在流水线填满后交替执行 forward 和 backward，可以减少激活保存压力，并提升 pipeline 利用率。AFAB 简单但显存压力更大。",[561,1344,1346],{"id":1345},"q6zero-1-节省什么","Q6：ZeRO-1 节省什么？",[10,1348,1349],{},"ZeRO-1 主要分片优化器状态，例如 Adam 的 momentum 和 variance，减少数据并行副本间重复保存 optimizer states 的显存开销。",[561,1351,1353],{"id":1352},"q7为什么-checkpoint-在大模型训练里复杂","Q7：为什么 checkpoint 在大模型训练里复杂？",[10,1355,1356],{},"因为参数可能被 TP\u002FPP 切分，不同 rank 保存不同 shard；恢复时必须匹配并行拓扑，还要恢复 optimizer、scheduler、step 和随机状态。",[561,1358,1360],{"id":1359},"q8为什么要-fp32-梯度累积","Q8：为什么要 FP32 梯度累积？",[10,1362,1363],{},"混合精度训练中，FP32 累积能减少数值误差，提高训练稳定性，尤其是在长序列、大 batch 和深层网络中。",[561,1365,1367],{"id":1366},"q9mfu-是什么","Q9：MFU 是什么？",[10,1369,1370],{},"MFU 是 Model FLOPS Utilization，表示模型训练实际利用了理论峰值算力的比例。它能反映训练系统是否高效利用 GPU。",[561,1372,1374],{"id":1373},"q10nanotron-适合微调吗","Q10：Nanotron 适合微调吗？",[10,1376,1377],{},"Nanotron 更偏预训练基础设施。如果目标是 LoRA\u002FSFT\u002FDPO，LLaMA-Factory 或 Axolotl 这类微调框架更直接；如果想学大规模 pretraining，Nanotron 更合适。",[38,1379,1380],{"id":1380},"简历写法建议",[10,1382,1383],{},"如果你想把 Nanotron 源码学习写进简历，可以这样写：",[45,1385,1386],{},[10,1387,1388],{},"深入学习 Hugging Face Nanotron 大模型预训练框架，梳理其基于 torchrun 的分布式训练入口、YAML 配置系统、DP\u002FTP\u002FPP 3D 并行拓扑、Pipeline Parallel schedule、ZeRO-1 优化器状态分片、FP32 梯度累积、参数 tying\u002Fsharding、checkpoint 保存恢复和 CUDA event 性能计时机制；理解 LLM pretraining 中显存、通信、吞吐、checkpoint 与训练稳定性的系统性权衡。",[10,1390,1391],{},"如果是做项目复现，可以写：",[45,1393,1394],{},[10,1395,1396],{},"基于 Nanotron 复现 tiny Llama 预训练流程，完成训练配置构建、数据加载、8 卡 torchrun 启动、checkpoint 保存与生成测试，分析 DP\u002FTP\u002FPP 并行配置对 global batch size、显存占用和训练吞吐的影响。",[38,1398,1399],{"id":1399},"总结",[10,1401,1402],{},"Nanotron 是一个非常适合学习大模型训练基础设施的项目。它把 LLM 预训练中最关键的系统问题都摆在了台面上：模型太大怎么办、显存不够怎么办、多卡怎么切、多节点怎么跑、pipeline 怎么调度、优化器状态怎么省、checkpoint 怎么存、数据怎么喂、性能怎么量化。",[10,1404,1405],{},"如果只是想“把模型跑起来”，可能用一个 Trainer 就够了；但如果想理解“大模型训练工程到底怎么做”，Nanotron 这种项目更值得认真读。",[1407,1408,1409],"style",{},"html pre.shiki code .ssh_m, html code.shiki .ssh_m{--shiki-default:#ADBAC7;--shiki-light:#24292E}html pre.shiki code .s6PUj, html code.shiki .s6PUj{--shiki-default:#F47067;--shiki-light:#D73A49}html pre.shiki code .sXfbr, html code.shiki .sXfbr{--shiki-default:#96D0FF;--shiki-light:#032F62}html pre.shiki code .sqRhv, html code.shiki .sqRhv{--shiki-default:#F69D50;--shiki-light:#6F42C1}html pre.shiki code .swcJU, html code.shiki .swcJU{--shiki-default:#6CB6FF;--shiki-light:#005CC5}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}",{"title":100,"searchDepth":1411,"depth":1412,"links":1413},2,3,[1414,1415,1416,1417,1418,1419,1420,1421,1422,1423,1424,1425,1426,1430,1431,1432,1433,1434,1435,1436,1437,1438,1439,1440,1441,1446,1452,1453,1454,1466,1467],{"id":40,"depth":1411,"text":40},{"id":60,"depth":1411,"text":61},{"id":89,"depth":1411,"text":89},{"id":144,"depth":1411,"text":145},{"id":181,"depth":1411,"text":182},{"id":271,"depth":1411,"text":271},{"id":303,"depth":1411,"text":304},{"id":319,"depth":1411,"text":320},{"id":369,"depth":1411,"text":370},{"id":424,"depth":1411,"text":425},{"id":470,"depth":1411,"text":471},{"id":507,"depth":1411,"text":508},{"id":555,"depth":1411,"text":556,"children":1427},[1428,1429],{"id":563,"depth":1412,"text":564},{"id":585,"depth":1412,"text":586},{"id":617,"depth":1411,"text":618},{"id":654,"depth":1411,"text":655},{"id":676,"depth":1411,"text":677},{"id":712,"depth":1411,"text":713},{"id":755,"depth":1411,"text":756},{"id":780,"depth":1411,"text":781},{"id":819,"depth":1411,"text":820},{"id":841,"depth":1411,"text":842},{"id":877,"depth":1411,"text":878},{"id":964,"depth":1411,"text":965},{"id":1027,"depth":1411,"text":1027},{"id":1075,"depth":1411,"text":1075,"children":1442},[1443,1444,1445],{"id":1078,"depth":1412,"text":1079},{"id":1122,"depth":1412,"text":1123},{"id":1164,"depth":1412,"text":1165},{"id":1205,"depth":1411,"text":1206,"children":1447},[1448,1449,1450,1451],{"id":1209,"depth":1412,"text":1209},{"id":1224,"depth":1412,"text":1225},{"id":1231,"depth":1412,"text":1232},{"id":1238,"depth":1412,"text":1238},{"id":1244,"depth":1411,"text":1245},{"id":1291,"depth":1411,"text":1292},{"id":1303,"depth":1411,"text":1303,"children":1455},[1456,1457,1458,1459,1460,1461,1462,1463,1464,1465],{"id":1306,"depth":1412,"text":1307},{"id":1313,"depth":1412,"text":1314},{"id":1320,"depth":1412,"text":1321},{"id":1327,"depth":1412,"text":1328},{"id":1338,"depth":1412,"text":1339},{"id":1345,"depth":1412,"text":1346},{"id":1352,"depth":1412,"text":1353},{"id":1359,"depth":1412,"text":1360},{"id":1366,"depth":1412,"text":1367},{"id":1373,"depth":1412,"text":1374},{"id":1380,"depth":1411,"text":1380},{"id":1399,"depth":1411,"text":1399},[1469],"AI","2026-05-10 12:10:00","这篇文章选一个真正和“大模型训练”强相关的开源项目来讲：Hugging Face 的 Nanotron。",false,"md",{},true,"\u002Fposts\u002Fnanotron-llm-pretraining-framework-analysis",{"title":5,"description":1471},"posts\u002Fnanotron-llm-pretraining-framework-analysis",[1480,1481,1482,16,1483,1484,1485],"LLM","大模型训练","分布式训练","Hugging Face","AI Infra","PyTorch","UiZUP8p_ui12jRPpDL7FbbcXX5IxzN-RSB82_AP4txE",[1488,1501,1513,1520,1533,1543,1553,1564,1575,1584,1594,1598,1611,1623,1632,1642,1655,1666,1676,1684,1695,1702,1708,1714,1722,1732,1740,1746,1754,1762],{"slug":1489,"path":1490,"title":1491,"date":1492,"tags":1493,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1500},"multimodal-rag-from-scratch","\u002Fposts\u002Fmultimodal-rag-from-scratch","从零实现多模态 RAG：BM25、Dense 检索、RRF 融合、MMR 重排全部手写","2026-06-30 18:00:00",[1494,1495,1484,1496,1497,1498,1499],"RAG","多模态","BM25","向量检索","混合检索","实习求职",9,{"slug":1502,"path":1503,"title":1504,"date":1505,"tags":1506,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1512},"mini-llm-engine-deep-dive","\u002Fposts\u002Fmini-llm-engine-deep-dive","讲透 mini-llm-engine：从显存碎片到六大推理优化","2026-06-30 14:00:00",[1480,1484,1507,1508,1509,1510,1511,1499],"vLLM","PagedAttention","KV Cache","推理优化","投机解码",11,{"slug":1514,"path":1515,"title":1516,"date":1517,"tags":1518,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1519},"mini-llm-engine-from-scratch","\u002Fposts\u002Fmini-llm-engine-from-scratch","从零实现 LLM 推理引擎：深挖 vLLM 的六大核心优化","2026-06-30 10:00:00",[1480,1484,1507,1508,1510,1511,1499],10,{"slug":1521,"path":1522,"title":1523,"date":1524,"tags":1525,"description":1531,"draft":1472,"hidden":1472,"published":1475,"readingTime":1532},"bytedance-recommendation-architecture-intern-interview","\u002Fposts\u002Fbytedance-recommendation-architecture-intern-interview","字节推荐架构实习生 Data 面试准备：推荐系统、实时特征与高并发八股","2026-06-09",[1526,1527,1528,1529,1530],"面试","推荐系统","后端架构","实时计算","字节跳动","面向字节跳动推荐架构团队实习岗位的八股准备清单，覆盖推荐系统链路、实时数据、特征服务、高并发后端、分布式系统与项目包装。",18,{"slug":1534,"path":1535,"title":1536,"date":1524,"tags":1537,"description":1541,"draft":1472,"hidden":1472,"published":1475,"readingTime":1542},"high-concurrency-rate-limiting-algorithms","\u002Fposts\u002Fhigh-concurrency-rate-limiting-algorithms","高并发限流算法：固定窗口、滑动窗口、漏桶与令牌桶",[1538,1539,1528,1540,1526],"高并发","限流","系统设计","面试和工程都常见的限流算法总结，讲清楚固定窗口、滑动窗口、漏桶、令牌桶、并发数限制以及分布式限流如何落地。",12,{"slug":1544,"path":1545,"title":1546,"date":1524,"tags":1547,"description":1551,"draft":1472,"hidden":1472,"published":1475,"readingTime":1552},"kafka-producer-broker-consumer","\u002Fposts\u002Fkafka-producer-broker-consumer","Kafka 入门：生产者、Broker、消费者和“消费”到底是什么意思",[1548,1549,1550,1526,1527],"Kafka","消息队列","后端","用推荐系统里的用户行为日志为例，讲清楚 Kafka 的作用、Producer、Broker、Consumer、Topic、Partition、Offset 和消费语义。",8,{"slug":1554,"path":1555,"title":1556,"date":1524,"tags":1557,"description":1562,"draft":1472,"hidden":1472,"published":1475,"readingTime":1563},"leetcode-lru-merge-k-reverse-list","\u002Fposts\u002Fleetcode-lru-merge-k-reverse-list","链表与缓存高频题：LRU Cache、合并 K 个有序链表、反转链表",[1558,1559,1560,1561,1526],"算法","链表","LRU","LeetCode","面试高频算法题速记，整理 LRU Cache、合并 K 个有序链表、反转链表的核心思路、复杂度和 C++ 代码。",6,{"slug":1565,"path":1566,"title":1567,"date":1568,"tags":1569,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1574},"meddococr-interpreter-source-analysis","\u002Fposts\u002Fmeddococr-interpreter-source-analysis","MedDocOCR-Interpreter 源码导读：医疗文档 OCR、结构化抽取与报告解读原型","2026-05-22 10:30:00",[1570,1495,1571,1494,1572,1573],"OCR","医疗 AI","Python","源码分析",16,{"slug":1576,"path":1577,"title":1578,"date":1579,"tags":1580,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1519},"cet6-writing-model-essays","\u002Fposts\u002Fcet6-writing-model-essays","六级写作范文背诵包：10 个高频话题","2026-05-20 09:00:00",[1581,1582,1583],"English","CET6","Writing",{"slug":1585,"path":1586,"title":1587,"date":1588,"tags":1589,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1593},"claude-code-context-management","\u002Fposts\u002Fclaude-code-context-management","Claude Code 上下文管理机制：从 Microcompact 到 Auto Compact","2026-05-19 10:00:00",[1590,1591,1480,1484,1592],"Claude Code","Agent","上下文工程",27,{"slug":1595,"path":1476,"title":5,"date":1470,"tags":1596,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1597},"nanotron-llm-pretraining-framework-analysis",[1480,1481,1482,16,1483,1484,1485],19,{"slug":1599,"path":1600,"title":1601,"date":1602,"tags":1603,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1610},"gofoundry-go-backend-foundation-framework","\u002Fposts\u002Fgofoundry-go-backend-foundation-framework","GoFoundry 项目详解：基于 Go 的后端基础框架套件设计","2026-05-10 11:20:00",[1604,1605,1606,1607,1608,1549,1609],"Go","后端框架","ORM","分布式缓存","分布式锁","项目架构",25,{"slug":1612,"path":1613,"title":1614,"date":1615,"tags":1616,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1593},"cloudvault-go-cloud-storage-system","\u002Fposts\u002Fcloudvault-go-cloud-storage-system","CloudVault 项目详解：基于 Go 的云端存储与网盘系统架构设计","2026-05-10 10:30:00",[1604,1617,1618,1619,1609,1620,1621,1622],"云存储","网盘系统","分布式系统","Redis","RabbitMQ","Elasticsearch",{"slug":1624,"path":1625,"title":1626,"date":1627,"tags":1628,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1631},"openclaw-source-code-analysis","\u002Fposts\u002Fopenclaw-source-code-analysis","OpenClaw 源码导读：个人 AI 助手的网关、通道、插件与运行时架构","2026-05-08 16:30:00",[1629,1591,1484,1630,1573],"OpenClaw","TypeScript",20,{"slug":1633,"path":1634,"title":1635,"date":1636,"tags":1637,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1519},"flow-matching-generative-models","\u002Fposts\u002Fflow-matching-generative-models","Flow Matching：从噪声到数据的连续流生成模型","2026-05-07 00:00:00",[1638,1639,1640,1641],"生成模型","Diffusion","Flow Matching","深度学习",{"slug":1643,"path":1644,"title":1645,"date":1646,"tags":1647,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1654},"database-ai-week4","\u002Fposts\u002Fdatabase-ai-week4","Week 4：数据库速成——从 Storage、Index、Query Optimization 到 Vector DB 与 RAG","2026-05-05 12:00:00",[1648,1649,1650,1494,1651,1652,1653],"数据库","CMU 15-445","Vector DB","LLM Memory","Query Optimization","Caching",15,{"slug":1656,"path":1657,"title":1658,"date":1659,"tags":1660,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1574},"distributed-systems-week3","\u002Fposts\u002Fdistributed-systems-week3","Week 3：分布式系统速成——MapReduce、Raft、容错与 Distributed KV Store","2026-05-05 11:00:00",[1619,1661,1662,1663,1664,1665,1591],"MIT 6.824","MapReduce","Raft","KV Store","Ray",{"slug":1667,"path":1668,"title":1669,"date":1670,"tags":1671,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1654},"gpu-inference-acceleration-week2","\u002Fposts\u002Fgpu-inference-acceleration-week2","Week 2：GPU 与推理加速——从 Kernel、算子融合到 LLM Serving","2026-05-05 10:00:00",[1641,1672,1673,1480,1674,1507,1675],"GPU","推理加速","CMU 10-414","TensorRT",{"slug":1677,"path":1678,"title":1679,"date":1680,"tags":1681,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1532},"dl-framework-autograd-mini","\u002Fposts\u002Fdl-framework-autograd-mini","Week 1：DL 框架与 Autograd——从计算图、反向传播到 Mini Autograd 实现","2026-05-05 09:00:00",[1641,1682,1485,1674,1683],"Autograd","Mini Framework",{"slug":1685,"path":1686,"title":1687,"date":1688,"tags":1689,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1542},"lock-free-concurrency-notes","\u002Fposts\u002Flock-free-concurrency-notes","无锁并发入门：从 CAS 到 Atomic Ring Buffer","2026-04-25",[1690,1691,1692,1693,1694],"C++","并发","无锁编程","性能优化","量化开发",{"slug":1696,"path":1697,"title":1698,"date":1699,"tags":1700,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1701},"agent-memory","\u002Fposts\u002Fagent-memory","Agent 对话记忆化：从原理到实现","2026-04-24",[1480,1591,1494,1526],5,{"slug":1703,"path":1704,"title":1705,"date":1699,"tags":1706,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1563},"llm-context-compression","\u002Fposts\u002Fllm-context-compression","LLM 上下文五层压缩机制详解",[1480,1591,1707,1526],"上下文压缩",{"slug":1709,"path":1710,"title":1711,"date":1712,"tags":1713,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1500},"cpp-concurrency-basics","\u002Fposts\u002Fcpp-concurrency-basics","C++ 并发编程入门：从数据竞争到线程池","2026-04-15",[1690,1691,1526,1694],{"slug":1715,"path":1716,"title":1717,"date":1718,"tags":1719,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":197},"travel-in-shenzhen","\u002Fposts\u002Ftravel-in-shenzhen","XCPC 深圳游记","2026-04-13",[1720,1690,1721],"XCPC","比赛",{"slug":1723,"path":1724,"title":1725,"date":1726,"tags":1727,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1731},"backend-stack-deep-dive","\u002Fposts\u002Fbackend-stack-deep-dive","后端五件套：FastAPI \u002F Node.js \u002F SQLAlchemy async \u002F PostgreSQL \u002F Docker 面试速通","2026-04-07",[1550,1728,1729,1730,1526],"FastAPI","PostgreSQL","Docker",7,{"slug":1733,"path":1734,"title":1735,"date":1736,"tags":1737,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1563},"deepscientist-tech-stack","\u002Fposts\u002Fdeepscientist-tech-stack","DeepScientist 技术栈全解析：一个 AI 科研平台的架构设计","2026-04-06",[1738,1728,1739,1729,1526],"全栈","Next.js",{"slug":1741,"path":1742,"title":1743,"date":1736,"tags":1744,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1552},"minicode-source-analysis","\u002Fposts\u002Fminicode-source-analysis","MiniCode 源码解析：用 5000 行 TypeScript 实现一个 AI 编程助手",[1630,1745,1480,1573,1526],"CLI",{"slug":1747,"path":1748,"title":1749,"date":1736,"tags":1750,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1701},"nova-theme-implementation","\u002Fposts\u002Fnova-theme-implementation","我是怎么从零实现 Nova 主题的",[1751,1752,1753],"Hexo","前端","开源",{"slug":1755,"path":1756,"title":1757,"date":1758,"tags":1759,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1411},"git-cheatsheet","\u002Fposts\u002Fgit-cheatsheet","Git 常用操作备忘","2026-04-05 14:00:00",[1760,1761],"Git","工具",{"slug":1763,"path":1764,"title":1765,"date":1766,"tags":1767,"description":100,"draft":1472,"hidden":1472,"published":1475,"readingTime":1412},"github-actions-intro","\u002Fposts\u002Fgithub-actions-intro","GitHub Actions 入门：自动化你的工作流","2026-04-04 09:00:00",[1768,1769,1770],"GitHub Actions","CI\u002FCD","自动化",1782796012505]