[{"data":1,"prerenderedAt":22089},["ShallowReactive",2],{"learning-material-cmu-15-779-transformer-paper-zh":3},{"courseSlug":4,"slug":5,"fileName":6,"title":7,"description":8,"language":9,"sourcePdf":10,"sourceLanguage":11,"body":12,"toc":22055,"bytes":22087,"downloadPath":22088},"cmu-15-779","transformer-paper-zh","transformer-paper-zh.md","Attention Is All You Need：只需注意力机制","Transformer 原论文中文学习译稿。","zh","\u002Fmaterials\u002Fcmu-15-779\u002Foriginals\u002Ftransformer-paper-original.pdf","en",{"type":13,"children":14},"root",[15,24,36,46,56,65,71,76,81,86,92,97,424,429,434,440,445,450,455,460,465,471,1267,1272,1283,1290,1384,1541,1798,1886,1981,1987,1992,1999,2403,2413,2555,3130,3135,3633,4154,4160,4751,4756,5201,5206,5708,5713,7358,7813,7819,7824,7914,7920,7925,8475,8480,8805,8811,8946,9145,9151,9286,9291,9788,10307,10555,10895,10900,10906,11086,12481,12487,13213,13218,13223,13387,13525,13864,14244,14249,14255,14260,14266,14271,14276,14281,14287,14292,14297,14303,14734,15575,15804,15810,15815,15988,16153,16159,16165,16170,16175,16340,16345,16432,16437,16443,18545,18551,18565,18570,18678,18683,18689,18701,21841,21847,21852,21857,21862,21875,21880,21885,21890,21895,21900,21905,21910,21915,21920,21925,21930,21935,21940,21945,21950,21955,21960,21965,21970,21975,21980,21985,21990,21995,22000,22005,22010,22015,22020,22025,22030,22035,22040,22045,22050],{"type":16,"tag":17,"props":18,"children":20},"element","h1",{"id":19},"attention-is-all-you-need",[21],{"type":22,"value":23},"text","Attention Is All You Need",{"type":16,"tag":25,"props":26,"children":27},"p",{},[28,34],{"type":16,"tag":29,"props":30,"children":31},"strong",{},[32],{"type":22,"value":33},"题目：",{"type":22,"value":35}," 只需注意力机制",{"type":16,"tag":25,"props":37,"children":38},{},[39,44],{"type":16,"tag":29,"props":40,"children":41},{},[42],{"type":22,"value":43},"作者：",{"type":22,"value":45}," Ashish Vaswani、Noam Shazeer、Niki Parmar、Jakob Uszkoreit、Llion Jones、Aidan N. Gomez、Łukasz Kaiser、Illia Polosukhin",{"type":16,"tag":25,"props":47,"children":48},{},[49,54],{"type":16,"tag":29,"props":50,"children":51},{},[52],{"type":22,"value":53},"来源：",{"type":22,"value":55}," NeurIPS 2017，第 31 届神经信息处理系统大会，美国长滩。",{"type":16,"tag":57,"props":58,"children":59},"blockquote",{},[60],{"type":16,"tag":25,"props":61,"children":62},{},[63],{"type":22,"value":64},"说明：本文根据用户提供的论文 PDF 翻译。公式和变量的含义、代码仓库地址和引用编号与原文一致。",{"type":16,"tag":66,"props":67,"children":69},"h2",{"id":68},"摘要",[70],{"type":22,"value":68},{"type":16,"tag":25,"props":72,"children":73},{},[74],{"type":22,"value":75},"主流的序列转换模型建立在复杂的循环神经网络或卷积神经网络之上，通常包含编码器和解码器。性能最好的模型还会通过注意力机制连接编码器和解码器。",{"type":16,"tag":25,"props":77,"children":78},{},[79],{"type":22,"value":80},"我们提出一种新的简单网络架构 Transformer，它完全建立在注意力机制之上，彻底舍弃循环和卷积。两个机器翻译任务上的实验表明，Transformer 在质量上优于已有模型，同时更易于并行化，训练所需时间也显著更少。",{"type":16,"tag":25,"props":82,"children":83},{},[84],{"type":22,"value":85},"在 WMT 2014 英德翻译任务上，我们的模型取得 28.4 BLEU，比包括集成模型在内的已有最佳结果高出 2 个以上 BLEU。在 WMT 2014 英法翻译任务上，模型只使用 8 块 GPU 训练 3.5 天，就取得了 41.0 的单模型最新 BLEU 分数，而训练成本只是文献中最佳模型的一小部分。",{"type":16,"tag":66,"props":87,"children":89},{"id":88},"_1-引言",[90],{"type":22,"value":91},"1 引言",{"type":16,"tag":25,"props":93,"children":94},{},[95],{"type":22,"value":96},"循环神经网络，尤其是长短期记忆网络 [12] 和门控循环神经网络 [7]，已经成为序列建模和序列转换问题（例如语言建模与机器翻译）的事实标准 [29, 2, 5]。此后，大量工作继续推动循环语言模型以及编码器-解码器架构的发展 [31, 21, 13]。",{"type":16,"tag":25,"props":98,"children":99},{},[100,102,270,272,315,317,422],{"type":22,"value":101},"循环模型通常沿输入和输出序列的符号位置分解计算。把序列位置与计算时间步对齐后，它们根据前一时刻的隐藏状态 ",{"type":16,"tag":103,"props":104,"children":107},"span",{"className":105},[106],"katex",[108,163],{"type":16,"tag":103,"props":109,"children":112},{"className":110},[111],"katex-mathml",[113],{"type":16,"tag":114,"props":115,"children":117},"math",{"xmlns":116},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[118],{"type":16,"tag":119,"props":120,"children":121},"semantics",{},[122,156],{"type":16,"tag":123,"props":124,"children":125},"mrow",{},[126],{"type":16,"tag":127,"props":128,"children":129},"msub",{},[130,136],{"type":16,"tag":131,"props":132,"children":133},"mi",{},[134],{"type":22,"value":135},"h",{"type":16,"tag":123,"props":137,"children":138},{},[139,144,150],{"type":16,"tag":131,"props":140,"children":141},{},[142],{"type":22,"value":143},"t",{"type":16,"tag":145,"props":146,"children":147},"mo",{},[148],{"type":22,"value":149},"−",{"type":16,"tag":151,"props":152,"children":153},"mn",{},[154],{"type":22,"value":155},"1",{"type":16,"tag":157,"props":158,"children":160},"annotation",{"encoding":159},"application\u002Fx-tex",[161],{"type":22,"value":162},"h_{t-1}",{"type":16,"tag":103,"props":164,"children":168},{"className":165,"ariaHidden":167},[166],"katex-html","true",[169],{"type":16,"tag":103,"props":170,"children":173},{"className":171},[172],"base",[174,180],{"type":16,"tag":103,"props":175,"children":179},{"className":176,"style":178},[177],"strut","height:0.9028em;vertical-align:-0.2083em;",[],{"type":16,"tag":103,"props":181,"children":184},{"className":182},[183],"mord",[185,191],{"type":16,"tag":103,"props":186,"children":189},{"className":187},[183,188],"mathnormal",[190],{"type":22,"value":135},{"type":16,"tag":103,"props":192,"children":195},{"className":193},[194],"msupsub",[196],{"type":16,"tag":103,"props":197,"children":201},{"className":198},[199,200],"vlist-t","vlist-t2",[202,258],{"type":16,"tag":103,"props":203,"children":206},{"className":204},[205],"vlist-r",[207,251],{"type":16,"tag":103,"props":208,"children":212},{"className":209,"style":211},[210],"vlist","height:0.3011em;",[213],{"type":16,"tag":103,"props":214,"children":216},{"style":215},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[217,223],{"type":16,"tag":103,"props":218,"children":222},{"className":219,"style":221},[220],"pstrut","height:2.7em;",[],{"type":16,"tag":103,"props":224,"children":230},{"className":225},[226,227,228,229],"sizing","reset-size6","size3","mtight",[231],{"type":16,"tag":103,"props":232,"children":234},{"className":233},[183,229],[235,240,246],{"type":16,"tag":103,"props":236,"children":238},{"className":237},[183,188,229],[239],{"type":22,"value":143},{"type":16,"tag":103,"props":241,"children":244},{"className":242},[243,229],"mbin",[245],{"type":22,"value":149},{"type":16,"tag":103,"props":247,"children":249},{"className":248},[183,229],[250],{"type":22,"value":155},{"type":16,"tag":103,"props":252,"children":255},{"className":253},[254],"vlist-s",[256],{"type":22,"value":257},"​",{"type":16,"tag":103,"props":259,"children":261},{"className":260},[205],[262],{"type":16,"tag":103,"props":263,"children":266},{"className":264,"style":265},[210],"height:0.2083em;",[267],{"type":16,"tag":103,"props":268,"children":269},{},[],{"type":22,"value":271}," 和位置 ",{"type":16,"tag":103,"props":273,"children":275},{"className":274},[106],[276,297],{"type":16,"tag":103,"props":277,"children":279},{"className":278},[111],[280],{"type":16,"tag":114,"props":281,"children":282},{"xmlns":116},[283],{"type":16,"tag":119,"props":284,"children":285},{},[286,293],{"type":16,"tag":123,"props":287,"children":288},{},[289],{"type":16,"tag":131,"props":290,"children":291},{},[292],{"type":22,"value":143},{"type":16,"tag":157,"props":294,"children":295},{"encoding":159},[296],{"type":22,"value":143},{"type":16,"tag":103,"props":298,"children":300},{"className":299,"ariaHidden":167},[166],[301],{"type":16,"tag":103,"props":302,"children":304},{"className":303},[172],[305,310],{"type":16,"tag":103,"props":306,"children":309},{"className":307,"style":308},[177],"height:0.6151em;",[],{"type":16,"tag":103,"props":311,"children":313},{"className":312},[183,188],[314],{"type":22,"value":143},{"type":22,"value":316}," 的输入，生成一系列隐藏状态 ",{"type":16,"tag":103,"props":318,"children":320},{"className":319},[106],[321,350],{"type":16,"tag":103,"props":322,"children":324},{"className":323},[111],[325],{"type":16,"tag":114,"props":326,"children":327},{"xmlns":116},[328],{"type":16,"tag":119,"props":329,"children":330},{},[331,345],{"type":16,"tag":123,"props":332,"children":333},{},[334],{"type":16,"tag":127,"props":335,"children":336},{},[337,341],{"type":16,"tag":131,"props":338,"children":339},{},[340],{"type":22,"value":135},{"type":16,"tag":131,"props":342,"children":343},{},[344],{"type":22,"value":143},{"type":16,"tag":157,"props":346,"children":347},{"encoding":159},[348],{"type":22,"value":349},"h_t",{"type":16,"tag":103,"props":351,"children":353},{"className":352,"ariaHidden":167},[166],[354],{"type":16,"tag":103,"props":355,"children":357},{"className":356},[172],[358,363],{"type":16,"tag":103,"props":359,"children":362},{"className":360,"style":361},[177],"height:0.8444em;vertical-align:-0.15em;",[],{"type":16,"tag":103,"props":364,"children":366},{"className":365},[183],[367,372],{"type":16,"tag":103,"props":368,"children":370},{"className":369},[183,188],[371],{"type":22,"value":135},{"type":16,"tag":103,"props":373,"children":375},{"className":374},[194],[376],{"type":16,"tag":103,"props":377,"children":379},{"className":378},[199,200],[380,410],{"type":16,"tag":103,"props":381,"children":383},{"className":382},[205],[384,405],{"type":16,"tag":103,"props":385,"children":388},{"className":386,"style":387},[210],"height:0.2806em;",[389],{"type":16,"tag":103,"props":390,"children":391},{"style":215},[392,396],{"type":16,"tag":103,"props":393,"children":395},{"className":394,"style":221},[220],[],{"type":16,"tag":103,"props":397,"children":399},{"className":398},[226,227,228,229],[400],{"type":16,"tag":103,"props":401,"children":403},{"className":402},[183,188,229],[404],{"type":22,"value":143},{"type":16,"tag":103,"props":406,"children":408},{"className":407},[254],[409],{"type":22,"value":257},{"type":16,"tag":103,"props":411,"children":413},{"className":412},[205],[414],{"type":16,"tag":103,"props":415,"children":418},{"className":416,"style":417},[210],"height:0.15em;",[419],{"type":16,"tag":103,"props":420,"children":421},{},[],{"type":22,"value":423},"。这种天然的顺序性阻止了训练样本内部的并行化；当序列变长时，由于内存限制，跨样本进行批处理的能力也会下降，因此这一问题更加关键。近期工作通过因式分解技巧 [18] 和条件计算 [26] 显著提高了计算效率，后者还改善了模型性能，但顺序计算这一根本限制仍然存在。",{"type":16,"tag":25,"props":425,"children":426},{},[427],{"type":22,"value":428},"注意力机制已经成为许多优秀序列建模和序列转换模型的重要组成部分。它可以在不考虑输入或输出位置之间距离的情况下建模依赖关系 [2, 16]。然而，除少数例外 [22] 外，这些注意力机制通常仍与循环网络结合使用。",{"type":16,"tag":25,"props":430,"children":431},{},[432],{"type":22,"value":433},"本文提出 Transformer：一种舍弃循环、完全依赖注意力机制来建立输入与输出之间全局依赖关系的模型架构。Transformer 可以实现大幅并行化，只需在 8 块 P100 GPU 上训练 12 小时，就能在翻译质量上达到新的最好结果。",{"type":16,"tag":66,"props":435,"children":437},{"id":436},"_2-背景",[438],{"type":22,"value":439},"2 背景",{"type":16,"tag":25,"props":441,"children":442},{},[443],{"type":22,"value":444},"减少顺序计算的目标，也是 Extended Neural GPU [20]、ByteNet [15] 和 ConvS2S [8] 的基础。这些模型都把卷积神经网络作为基本构件，为输入和输出的所有位置并行计算隐藏表示。",{"type":16,"tag":25,"props":446,"children":447},{},[448],{"type":22,"value":449},"在这些模型中，关联任意两个输入或输出位置所需的操作数，会随着位置间距离增长：ConvS2S 呈线性增长，ByteNet 呈对数增长。这使得模型更难学习远距离依赖 [11]。在 Transformer 中，这个操作数被降低为常数，但代价是有效分辨率下降，因为注意力会对不同位置进行加权平均。我们通过第 3.2 节介绍的多头注意力来抵消这一影响。",{"type":16,"tag":25,"props":451,"children":452},{},[453],{"type":22,"value":454},"自注意力（self-attention），有时也称为内部注意力（intra-attention），是一种关联同一序列中不同位置的注意力机制，用于计算整个序列的表示。自注意力已成功用于阅读理解、抽象式摘要、文本蕴含和学习与任务无关的句子表示等多种任务 [4, 22, 23, 19]。",{"type":16,"tag":25,"props":456,"children":457},{},[458],{"type":22,"value":459},"端到端记忆网络使用循环注意力机制，而不是与序列位置对齐的循环结构；它们在简单语言问答和语言建模任务上表现良好 [28]。",{"type":16,"tag":25,"props":461,"children":462},{},[463],{"type":22,"value":464},"据我们所知，Transformer 是第一个完全依靠自注意力计算输入和输出表示、而不使用与序列位置对齐的 RNN 或卷积的序列转换模型。下面我们将描述 Transformer，说明采用自注意力的动机，并讨论它相对于 [14, 15] 和 [8] 等模型的优势。",{"type":16,"tag":66,"props":466,"children":468},{"id":467},"_3-模型架构",[469],{"type":22,"value":470},"3 模型架构",{"type":16,"tag":25,"props":472,"children":473},{},[474,476,718,720,988,990,1032,1034,1265],{"type":22,"value":475},"大多数有竞争力的神经序列转换模型都采用编码器-解码器结构 [5, 2, 29]。编码器把输入符号表示序列 ",{"type":16,"tag":103,"props":477,"children":479},{"className":478},[106],[480,547],{"type":16,"tag":103,"props":481,"children":483},{"className":482},[111],[484],{"type":16,"tag":114,"props":485,"children":486},{"xmlns":116},[487],{"type":16,"tag":119,"props":488,"children":489},{},[490,542],{"type":16,"tag":123,"props":491,"children":492},{},[493,499,511,516,521,525,537],{"type":16,"tag":145,"props":494,"children":496},{"stretchy":495},"false",[497],{"type":22,"value":498},"(",{"type":16,"tag":127,"props":500,"children":501},{},[502,507],{"type":16,"tag":131,"props":503,"children":504},{},[505],{"type":22,"value":506},"x",{"type":16,"tag":151,"props":508,"children":509},{},[510],{"type":22,"value":155},{"type":16,"tag":145,"props":512,"children":513},{"separator":167},[514],{"type":22,"value":515},",",{"type":16,"tag":145,"props":517,"children":518},{},[519],{"type":22,"value":520},"…",{"type":16,"tag":145,"props":522,"children":523},{"separator":167},[524],{"type":22,"value":515},{"type":16,"tag":127,"props":526,"children":527},{},[528,532],{"type":16,"tag":131,"props":529,"children":530},{},[531],{"type":22,"value":506},{"type":16,"tag":131,"props":533,"children":534},{},[535],{"type":22,"value":536},"n",{"type":16,"tag":145,"props":538,"children":539},{"stretchy":495},[540],{"type":22,"value":541},")",{"type":16,"tag":157,"props":543,"children":544},{"encoding":159},[545],{"type":22,"value":546},"(x_1, \\ldots, x_n)",{"type":16,"tag":103,"props":548,"children":550},{"className":549,"ariaHidden":167},[166],[551],{"type":16,"tag":103,"props":552,"children":554},{"className":553},[172],[555,560,566,623,629,635,641,645,650,654,712],{"type":16,"tag":103,"props":556,"children":559},{"className":557,"style":558},[177],"height:1em;vertical-align:-0.25em;",[],{"type":16,"tag":103,"props":561,"children":564},{"className":562},[563],"mopen",[565],{"type":22,"value":498},{"type":16,"tag":103,"props":567,"children":569},{"className":568},[183],[570,575],{"type":16,"tag":103,"props":571,"children":573},{"className":572},[183,188],[574],{"type":22,"value":506},{"type":16,"tag":103,"props":576,"children":578},{"className":577},[194],[579],{"type":16,"tag":103,"props":580,"children":582},{"className":581},[199,200],[583,612],{"type":16,"tag":103,"props":584,"children":586},{"className":585},[205],[587,607],{"type":16,"tag":103,"props":588,"children":590},{"className":589,"style":211},[210],[591],{"type":16,"tag":103,"props":592,"children":593},{"style":215},[594,598],{"type":16,"tag":103,"props":595,"children":597},{"className":596,"style":221},[220],[],{"type":16,"tag":103,"props":599,"children":601},{"className":600},[226,227,228,229],[602],{"type":16,"tag":103,"props":603,"children":605},{"className":604},[183,229],[606],{"type":22,"value":155},{"type":16,"tag":103,"props":608,"children":610},{"className":609},[254],[611],{"type":22,"value":257},{"type":16,"tag":103,"props":613,"children":615},{"className":614},[205],[616],{"type":16,"tag":103,"props":617,"children":619},{"className":618,"style":417},[210],[620],{"type":16,"tag":103,"props":621,"children":622},{},[],{"type":16,"tag":103,"props":624,"children":627},{"className":625},[626],"mpunct",[628],{"type":22,"value":515},{"type":16,"tag":103,"props":630,"children":634},{"className":631,"style":633},[632],"mspace","margin-right:0.1667em;",[],{"type":16,"tag":103,"props":636,"children":639},{"className":637},[638],"minner",[640],{"type":22,"value":520},{"type":16,"tag":103,"props":642,"children":644},{"className":643,"style":633},[632],[],{"type":16,"tag":103,"props":646,"children":648},{"className":647},[626],[649],{"type":22,"value":515},{"type":16,"tag":103,"props":651,"children":653},{"className":652,"style":633},[632],[],{"type":16,"tag":103,"props":655,"children":657},{"className":656},[183],[658,663],{"type":16,"tag":103,"props":659,"children":661},{"className":660},[183,188],[662],{"type":22,"value":506},{"type":16,"tag":103,"props":664,"children":666},{"className":665},[194],[667],{"type":16,"tag":103,"props":668,"children":670},{"className":669},[199,200],[671,701],{"type":16,"tag":103,"props":672,"children":674},{"className":673},[205],[675,696],{"type":16,"tag":103,"props":676,"children":679},{"className":677,"style":678},[210],"height:0.1514em;",[680],{"type":16,"tag":103,"props":681,"children":682},{"style":215},[683,687],{"type":16,"tag":103,"props":684,"children":686},{"className":685,"style":221},[220],[],{"type":16,"tag":103,"props":688,"children":690},{"className":689},[226,227,228,229],[691],{"type":16,"tag":103,"props":692,"children":694},{"className":693},[183,188,229],[695],{"type":22,"value":536},{"type":16,"tag":103,"props":697,"children":699},{"className":698},[254],[700],{"type":22,"value":257},{"type":16,"tag":103,"props":702,"children":704},{"className":703},[205],[705],{"type":16,"tag":103,"props":706,"children":708},{"className":707,"style":417},[210],[709],{"type":16,"tag":103,"props":710,"children":711},{},[],{"type":16,"tag":103,"props":713,"children":716},{"className":714},[715],"mclose",[717],{"type":22,"value":541},{"type":22,"value":719}," 映射为连续表示序列 ",{"type":16,"tag":103,"props":721,"children":723},{"className":722},[106],[724,794],{"type":16,"tag":103,"props":725,"children":727},{"className":726},[111],[728],{"type":16,"tag":114,"props":729,"children":730},{"xmlns":116},[731],{"type":16,"tag":119,"props":732,"children":733},{},[734,789],{"type":16,"tag":123,"props":735,"children":736},{},[737,742,747,751,762,766,770,774,785],{"type":16,"tag":131,"props":738,"children":739},{},[740],{"type":22,"value":741},"z",{"type":16,"tag":145,"props":743,"children":744},{},[745],{"type":22,"value":746},"=",{"type":16,"tag":145,"props":748,"children":749},{"stretchy":495},[750],{"type":22,"value":498},{"type":16,"tag":127,"props":752,"children":753},{},[754,758],{"type":16,"tag":131,"props":755,"children":756},{},[757],{"type":22,"value":741},{"type":16,"tag":151,"props":759,"children":760},{},[761],{"type":22,"value":155},{"type":16,"tag":145,"props":763,"children":764},{"separator":167},[765],{"type":22,"value":515},{"type":16,"tag":145,"props":767,"children":768},{},[769],{"type":22,"value":520},{"type":16,"tag":145,"props":771,"children":772},{"separator":167},[773],{"type":22,"value":515},{"type":16,"tag":127,"props":775,"children":776},{},[777,781],{"type":16,"tag":131,"props":778,"children":779},{},[780],{"type":22,"value":741},{"type":16,"tag":131,"props":782,"children":783},{},[784],{"type":22,"value":536},{"type":16,"tag":145,"props":786,"children":787},{"stretchy":495},[788],{"type":22,"value":541},{"type":16,"tag":157,"props":790,"children":791},{"encoding":159},[792],{"type":22,"value":793},"z = (z_1, \\ldots, z_n)",{"type":16,"tag":103,"props":795,"children":797},{"className":796,"ariaHidden":167},[166],[798,828],{"type":16,"tag":103,"props":799,"children":801},{"className":800},[172],[802,807,813,818,824],{"type":16,"tag":103,"props":803,"children":806},{"className":804,"style":805},[177],"height:0.4306em;",[],{"type":16,"tag":103,"props":808,"children":811},{"className":809,"style":810},[183,188],"margin-right:0.044em;",[812],{"type":22,"value":741},{"type":16,"tag":103,"props":814,"children":817},{"className":815,"style":816},[632],"margin-right:0.2778em;",[],{"type":16,"tag":103,"props":819,"children":822},{"className":820},[821],"mrel",[823],{"type":22,"value":746},{"type":16,"tag":103,"props":825,"children":827},{"className":826,"style":816},[632],[],{"type":16,"tag":103,"props":829,"children":831},{"className":830},[172],[832,836,841,899,904,908,913,917,922,926,983],{"type":16,"tag":103,"props":833,"children":835},{"className":834,"style":558},[177],[],{"type":16,"tag":103,"props":837,"children":839},{"className":838},[563],[840],{"type":22,"value":498},{"type":16,"tag":103,"props":842,"children":844},{"className":843},[183],[845,850],{"type":16,"tag":103,"props":846,"children":848},{"className":847,"style":810},[183,188],[849],{"type":22,"value":741},{"type":16,"tag":103,"props":851,"children":853},{"className":852},[194],[854],{"type":16,"tag":103,"props":855,"children":857},{"className":856},[199,200],[858,888],{"type":16,"tag":103,"props":859,"children":861},{"className":860},[205],[862,883],{"type":16,"tag":103,"props":863,"children":865},{"className":864,"style":211},[210],[866],{"type":16,"tag":103,"props":867,"children":869},{"style":868},"top:-2.55em;margin-left:-0.044em;margin-right:0.05em;",[870,874],{"type":16,"tag":103,"props":871,"children":873},{"className":872,"style":221},[220],[],{"type":16,"tag":103,"props":875,"children":877},{"className":876},[226,227,228,229],[878],{"type":16,"tag":103,"props":879,"children":881},{"className":880},[183,229],[882],{"type":22,"value":155},{"type":16,"tag":103,"props":884,"children":886},{"className":885},[254],[887],{"type":22,"value":257},{"type":16,"tag":103,"props":889,"children":891},{"className":890},[205],[892],{"type":16,"tag":103,"props":893,"children":895},{"className":894,"style":417},[210],[896],{"type":16,"tag":103,"props":897,"children":898},{},[],{"type":16,"tag":103,"props":900,"children":902},{"className":901},[626],[903],{"type":22,"value":515},{"type":16,"tag":103,"props":905,"children":907},{"className":906,"style":633},[632],[],{"type":16,"tag":103,"props":909,"children":911},{"className":910},[638],[912],{"type":22,"value":520},{"type":16,"tag":103,"props":914,"children":916},{"className":915,"style":633},[632],[],{"type":16,"tag":103,"props":918,"children":920},{"className":919},[626],[921],{"type":22,"value":515},{"type":16,"tag":103,"props":923,"children":925},{"className":924,"style":633},[632],[],{"type":16,"tag":103,"props":927,"children":929},{"className":928},[183],[930,935],{"type":16,"tag":103,"props":931,"children":933},{"className":932,"style":810},[183,188],[934],{"type":22,"value":741},{"type":16,"tag":103,"props":936,"children":938},{"className":937},[194],[939],{"type":16,"tag":103,"props":940,"children":942},{"className":941},[199,200],[943,972],{"type":16,"tag":103,"props":944,"children":946},{"className":945},[205],[947,967],{"type":16,"tag":103,"props":948,"children":950},{"className":949,"style":678},[210],[951],{"type":16,"tag":103,"props":952,"children":953},{"style":868},[954,958],{"type":16,"tag":103,"props":955,"children":957},{"className":956,"style":221},[220],[],{"type":16,"tag":103,"props":959,"children":961},{"className":960},[226,227,228,229],[962],{"type":16,"tag":103,"props":963,"children":965},{"className":964},[183,188,229],[966],{"type":22,"value":536},{"type":16,"tag":103,"props":968,"children":970},{"className":969},[254],[971],{"type":22,"value":257},{"type":16,"tag":103,"props":973,"children":975},{"className":974},[205],[976],{"type":16,"tag":103,"props":977,"children":979},{"className":978,"style":417},[210],[980],{"type":16,"tag":103,"props":981,"children":982},{},[],{"type":16,"tag":103,"props":984,"children":986},{"className":985},[715],[987],{"type":22,"value":541},{"type":22,"value":989},"。给定 ",{"type":16,"tag":103,"props":991,"children":993},{"className":992},[106],[994,1015],{"type":16,"tag":103,"props":995,"children":997},{"className":996},[111],[998],{"type":16,"tag":114,"props":999,"children":1000},{"xmlns":116},[1001],{"type":16,"tag":119,"props":1002,"children":1003},{},[1004,1011],{"type":16,"tag":123,"props":1005,"children":1006},{},[1007],{"type":16,"tag":131,"props":1008,"children":1009},{},[1010],{"type":22,"value":741},{"type":16,"tag":157,"props":1012,"children":1013},{"encoding":159},[1014],{"type":22,"value":741},{"type":16,"tag":103,"props":1016,"children":1018},{"className":1017,"ariaHidden":167},[166],[1019],{"type":16,"tag":103,"props":1020,"children":1022},{"className":1021},[172],[1023,1027],{"type":16,"tag":103,"props":1024,"children":1026},{"className":1025,"style":805},[177],[],{"type":16,"tag":103,"props":1028,"children":1030},{"className":1029,"style":810},[183,188],[1031],{"type":22,"value":741},{"type":22,"value":1033}," 后，解码器逐个生成输出符号序列 ",{"type":16,"tag":103,"props":1035,"children":1037},{"className":1036},[106],[1038,1100],{"type":16,"tag":103,"props":1039,"children":1041},{"className":1040},[111],[1042],{"type":16,"tag":114,"props":1043,"children":1044},{"xmlns":116},[1045],{"type":16,"tag":119,"props":1046,"children":1047},{},[1048,1095],{"type":16,"tag":123,"props":1049,"children":1050},{},[1051,1055,1067,1071,1075,1079,1091],{"type":16,"tag":145,"props":1052,"children":1053},{"stretchy":495},[1054],{"type":22,"value":498},{"type":16,"tag":127,"props":1056,"children":1057},{},[1058,1063],{"type":16,"tag":131,"props":1059,"children":1060},{},[1061],{"type":22,"value":1062},"y",{"type":16,"tag":151,"props":1064,"children":1065},{},[1066],{"type":22,"value":155},{"type":16,"tag":145,"props":1068,"children":1069},{"separator":167},[1070],{"type":22,"value":515},{"type":16,"tag":145,"props":1072,"children":1073},{},[1074],{"type":22,"value":520},{"type":16,"tag":145,"props":1076,"children":1077},{"separator":167},[1078],{"type":22,"value":515},{"type":16,"tag":127,"props":1080,"children":1081},{},[1082,1086],{"type":16,"tag":131,"props":1083,"children":1084},{},[1085],{"type":22,"value":1062},{"type":16,"tag":131,"props":1087,"children":1088},{},[1089],{"type":22,"value":1090},"m",{"type":16,"tag":145,"props":1092,"children":1093},{"stretchy":495},[1094],{"type":22,"value":541},{"type":16,"tag":157,"props":1096,"children":1097},{"encoding":159},[1098],{"type":22,"value":1099},"(y_1, \\ldots, y_m)",{"type":16,"tag":103,"props":1101,"children":1103},{"className":1102,"ariaHidden":167},[166],[1104],{"type":16,"tag":103,"props":1105,"children":1107},{"className":1106},[172],[1108,1112,1117,1176,1181,1185,1190,1194,1199,1203,1260],{"type":16,"tag":103,"props":1109,"children":1111},{"className":1110,"style":558},[177],[],{"type":16,"tag":103,"props":1113,"children":1115},{"className":1114},[563],[1116],{"type":22,"value":498},{"type":16,"tag":103,"props":1118,"children":1120},{"className":1119},[183],[1121,1127],{"type":16,"tag":103,"props":1122,"children":1125},{"className":1123,"style":1124},[183,188],"margin-right:0.0359em;",[1126],{"type":22,"value":1062},{"type":16,"tag":103,"props":1128,"children":1130},{"className":1129},[194],[1131],{"type":16,"tag":103,"props":1132,"children":1134},{"className":1133},[199,200],[1135,1165],{"type":16,"tag":103,"props":1136,"children":1138},{"className":1137},[205],[1139,1160],{"type":16,"tag":103,"props":1140,"children":1142},{"className":1141,"style":211},[210],[1143],{"type":16,"tag":103,"props":1144,"children":1146},{"style":1145},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[1147,1151],{"type":16,"tag":103,"props":1148,"children":1150},{"className":1149,"style":221},[220],[],{"type":16,"tag":103,"props":1152,"children":1154},{"className":1153},[226,227,228,229],[1155],{"type":16,"tag":103,"props":1156,"children":1158},{"className":1157},[183,229],[1159],{"type":22,"value":155},{"type":16,"tag":103,"props":1161,"children":1163},{"className":1162},[254],[1164],{"type":22,"value":257},{"type":16,"tag":103,"props":1166,"children":1168},{"className":1167},[205],[1169],{"type":16,"tag":103,"props":1170,"children":1172},{"className":1171,"style":417},[210],[1173],{"type":16,"tag":103,"props":1174,"children":1175},{},[],{"type":16,"tag":103,"props":1177,"children":1179},{"className":1178},[626],[1180],{"type":22,"value":515},{"type":16,"tag":103,"props":1182,"children":1184},{"className":1183,"style":633},[632],[],{"type":16,"tag":103,"props":1186,"children":1188},{"className":1187},[638],[1189],{"type":22,"value":520},{"type":16,"tag":103,"props":1191,"children":1193},{"className":1192,"style":633},[632],[],{"type":16,"tag":103,"props":1195,"children":1197},{"className":1196},[626],[1198],{"type":22,"value":515},{"type":16,"tag":103,"props":1200,"children":1202},{"className":1201,"style":633},[632],[],{"type":16,"tag":103,"props":1204,"children":1206},{"className":1205},[183],[1207,1212],{"type":16,"tag":103,"props":1208,"children":1210},{"className":1209,"style":1124},[183,188],[1211],{"type":22,"value":1062},{"type":16,"tag":103,"props":1213,"children":1215},{"className":1214},[194],[1216],{"type":16,"tag":103,"props":1217,"children":1219},{"className":1218},[199,200],[1220,1249],{"type":16,"tag":103,"props":1221,"children":1223},{"className":1222},[205],[1224,1244],{"type":16,"tag":103,"props":1225,"children":1227},{"className":1226,"style":678},[210],[1228],{"type":16,"tag":103,"props":1229,"children":1230},{"style":1145},[1231,1235],{"type":16,"tag":103,"props":1232,"children":1234},{"className":1233,"style":221},[220],[],{"type":16,"tag":103,"props":1236,"children":1238},{"className":1237},[226,227,228,229],[1239],{"type":16,"tag":103,"props":1240,"children":1242},{"className":1241},[183,188,229],[1243],{"type":22,"value":1090},{"type":16,"tag":103,"props":1245,"children":1247},{"className":1246},[254],[1248],{"type":22,"value":257},{"type":16,"tag":103,"props":1250,"children":1252},{"className":1251},[205],[1253],{"type":16,"tag":103,"props":1254,"children":1256},{"className":1255,"style":417},[210],[1257],{"type":16,"tag":103,"props":1258,"children":1259},{},[],{"type":16,"tag":103,"props":1261,"children":1263},{"className":1262},[715],[1264],{"type":22,"value":541},{"type":22,"value":1266},"。",{"type":16,"tag":25,"props":1268,"children":1269},{},[1270],{"type":22,"value":1271},"在每一步，模型都是自回归的 [9]：生成下一个符号时，把此前已经生成的符号作为额外输入。Transformer 保留了整体的编码器-解码器结构，但编码器和解码器都使用堆叠的自注意力层与逐位置全连接层，如图 1 左右两部分所示。",{"type":16,"tag":25,"props":1273,"children":1274},{},[1275],{"type":16,"tag":1276,"props":1277,"children":1282},"img",{"alt":1278,"src":1279,"width":1280,"height":1281},"Transformer 模型架构（原文图 1）","\u002Fimages\u002Fmaterials\u002Fcmu-15-779\u002Ftransformer\u002Ffigure-01.webp",942,1379,[],{"type":16,"tag":1284,"props":1285,"children":1287},"h3",{"id":1286},"_31-编码器和解码器堆栈",[1288],{"type":22,"value":1289},"3.1 编码器和解码器堆栈",{"type":16,"tag":25,"props":1291,"children":1292},{},[1293,1298,1300,1382],{"type":16,"tag":29,"props":1294,"children":1295},{},[1296],{"type":22,"value":1297},"编码器：",{"type":22,"value":1299}," 编码器由 ",{"type":16,"tag":103,"props":1301,"children":1303},{"className":1302},[106],[1304,1336],{"type":16,"tag":103,"props":1305,"children":1307},{"className":1306},[111],[1308],{"type":16,"tag":114,"props":1309,"children":1310},{"xmlns":116},[1311],{"type":16,"tag":119,"props":1312,"children":1313},{},[1314,1331],{"type":16,"tag":123,"props":1315,"children":1316},{},[1317,1322,1326],{"type":16,"tag":131,"props":1318,"children":1319},{},[1320],{"type":22,"value":1321},"N",{"type":16,"tag":145,"props":1323,"children":1324},{},[1325],{"type":22,"value":746},{"type":16,"tag":151,"props":1327,"children":1328},{},[1329],{"type":22,"value":1330},"6",{"type":16,"tag":157,"props":1332,"children":1333},{"encoding":159},[1334],{"type":22,"value":1335},"N = 6",{"type":16,"tag":103,"props":1337,"children":1339},{"className":1338,"ariaHidden":167},[166],[1340,1368],{"type":16,"tag":103,"props":1341,"children":1343},{"className":1342},[172],[1344,1349,1355,1359,1364],{"type":16,"tag":103,"props":1345,"children":1348},{"className":1346,"style":1347},[177],"height:0.6833em;",[],{"type":16,"tag":103,"props":1350,"children":1353},{"className":1351,"style":1352},[183,188],"margin-right:0.109em;",[1354],{"type":22,"value":1321},{"type":16,"tag":103,"props":1356,"children":1358},{"className":1357,"style":816},[632],[],{"type":16,"tag":103,"props":1360,"children":1362},{"className":1361},[821],[1363],{"type":22,"value":746},{"type":16,"tag":103,"props":1365,"children":1367},{"className":1366,"style":816},[632],[],{"type":16,"tag":103,"props":1369,"children":1371},{"className":1370},[172],[1372,1377],{"type":16,"tag":103,"props":1373,"children":1376},{"className":1374,"style":1375},[177],"height:0.6444em;",[],{"type":16,"tag":103,"props":1378,"children":1380},{"className":1379},[183],[1381],{"type":22,"value":1330},{"type":22,"value":1383}," 个相同层堆叠而成。每层包含两个子层：第一个是多头自注意力机制，第二个是简单的逐位置全连接前馈网络。我们在两个子层外都使用残差连接 [10]，然后进行层归一化 [1]。也就是说，每个子层的输出为：",{"type":16,"tag":103,"props":1385,"children":1388},{"className":1386},[1387],"katex-display",[1389],{"type":16,"tag":103,"props":1390,"children":1392},{"className":1391},[106],[1393,1461],{"type":16,"tag":103,"props":1394,"children":1396},{"className":1395},[111],[1397],{"type":16,"tag":114,"props":1398,"children":1400},{"xmlns":116,"display":1399},"block",[1401],{"type":16,"tag":119,"props":1402,"children":1403},{},[1404,1456],{"type":16,"tag":123,"props":1405,"children":1406},{},[1407,1413,1418,1422,1426,1431,1436,1440,1444,1448,1452],{"type":16,"tag":131,"props":1408,"children":1410},{"mathvariant":1409},"normal",[1411],{"type":22,"value":1412},"LayerNorm",{"type":16,"tag":145,"props":1414,"children":1415},{},[1416],{"type":22,"value":1417},"⁡",{"type":16,"tag":145,"props":1419,"children":1420},{"stretchy":495},[1421],{"type":22,"value":498},{"type":16,"tag":131,"props":1423,"children":1424},{},[1425],{"type":22,"value":506},{"type":16,"tag":145,"props":1427,"children":1428},{},[1429],{"type":22,"value":1430},"+",{"type":16,"tag":131,"props":1432,"children":1433},{"mathvariant":1409},[1434],{"type":22,"value":1435},"Sublayer",{"type":16,"tag":145,"props":1437,"children":1438},{},[1439],{"type":22,"value":1417},{"type":16,"tag":145,"props":1441,"children":1442},{"stretchy":495},[1443],{"type":22,"value":498},{"type":16,"tag":131,"props":1445,"children":1446},{},[1447],{"type":22,"value":506},{"type":16,"tag":145,"props":1449,"children":1450},{"stretchy":495},[1451],{"type":22,"value":541},{"type":16,"tag":145,"props":1453,"children":1454},{"stretchy":495},[1455],{"type":22,"value":541},{"type":16,"tag":157,"props":1457,"children":1458},{"encoding":159},[1459],{"type":22,"value":1460},"\\operatorname{LayerNorm}(x + \\operatorname{Sublayer}(x))",{"type":16,"tag":103,"props":1462,"children":1464},{"className":1463,"ariaHidden":167},[166],[1465,1508],{"type":16,"tag":103,"props":1466,"children":1468},{"className":1467},[172],[1469,1473,1484,1489,1494,1499,1504],{"type":16,"tag":103,"props":1470,"children":1472},{"className":1471,"style":558},[177],[],{"type":16,"tag":103,"props":1474,"children":1477},{"className":1475},[1476],"mop",[1478],{"type":16,"tag":103,"props":1479,"children":1482},{"className":1480},[183,1481],"mathrm",[1483],{"type":22,"value":1412},{"type":16,"tag":103,"props":1485,"children":1487},{"className":1486},[563],[1488],{"type":22,"value":498},{"type":16,"tag":103,"props":1490,"children":1492},{"className":1491},[183,188],[1493],{"type":22,"value":506},{"type":16,"tag":103,"props":1495,"children":1498},{"className":1496,"style":1497},[632],"margin-right:0.2222em;",[],{"type":16,"tag":103,"props":1500,"children":1502},{"className":1501},[243],[1503],{"type":22,"value":1430},{"type":16,"tag":103,"props":1505,"children":1507},{"className":1506,"style":1497},[632],[],{"type":16,"tag":103,"props":1509,"children":1511},{"className":1510},[172],[1512,1516,1525,1530,1535],{"type":16,"tag":103,"props":1513,"children":1515},{"className":1514,"style":558},[177],[],{"type":16,"tag":103,"props":1517,"children":1519},{"className":1518},[1476],[1520],{"type":16,"tag":103,"props":1521,"children":1523},{"className":1522},[183,1481],[1524],{"type":22,"value":1435},{"type":16,"tag":103,"props":1526,"children":1528},{"className":1527},[563],[1529],{"type":22,"value":498},{"type":16,"tag":103,"props":1531,"children":1533},{"className":1532},[183,188],[1534],{"type":22,"value":506},{"type":16,"tag":103,"props":1536,"children":1538},{"className":1537},[715],[1539],{"type":22,"value":1540},"))",{"type":16,"tag":25,"props":1542,"children":1543},{},[1544,1546,1624,1626,1796],{"type":22,"value":1545},"其中 ",{"type":16,"tag":103,"props":1547,"children":1549},{"className":1548},[106],[1550,1588],{"type":16,"tag":103,"props":1551,"children":1553},{"className":1552},[111],[1554],{"type":16,"tag":114,"props":1555,"children":1556},{"xmlns":116},[1557],{"type":16,"tag":119,"props":1558,"children":1559},{},[1560,1583],{"type":16,"tag":123,"props":1561,"children":1562},{},[1563,1567,1571,1575,1579],{"type":16,"tag":131,"props":1564,"children":1565},{"mathvariant":1409},[1566],{"type":22,"value":1435},{"type":16,"tag":145,"props":1568,"children":1569},{},[1570],{"type":22,"value":1417},{"type":16,"tag":145,"props":1572,"children":1573},{"stretchy":495},[1574],{"type":22,"value":498},{"type":16,"tag":131,"props":1576,"children":1577},{},[1578],{"type":22,"value":506},{"type":16,"tag":145,"props":1580,"children":1581},{"stretchy":495},[1582],{"type":22,"value":541},{"type":16,"tag":157,"props":1584,"children":1585},{"encoding":159},[1586],{"type":22,"value":1587},"\\operatorname{Sublayer}(x)",{"type":16,"tag":103,"props":1589,"children":1591},{"className":1590,"ariaHidden":167},[166],[1592],{"type":16,"tag":103,"props":1593,"children":1595},{"className":1594},[172],[1596,1600,1609,1614,1619],{"type":16,"tag":103,"props":1597,"children":1599},{"className":1598,"style":558},[177],[],{"type":16,"tag":103,"props":1601,"children":1603},{"className":1602},[1476],[1604],{"type":16,"tag":103,"props":1605,"children":1607},{"className":1606},[183,1481],[1608],{"type":22,"value":1435},{"type":16,"tag":103,"props":1610,"children":1612},{"className":1611},[563],[1613],{"type":22,"value":498},{"type":16,"tag":103,"props":1615,"children":1617},{"className":1616},[183,188],[1618],{"type":22,"value":506},{"type":16,"tag":103,"props":1620,"children":1622},{"className":1621},[715],[1623],{"type":22,"value":541},{"type":22,"value":1625}," 是该子层本身实现的函数。为了能够使用残差连接，模型中的所有子层以及嵌入层都产生维度为 ",{"type":16,"tag":103,"props":1627,"children":1629},{"className":1628},[106],[1630,1691],{"type":16,"tag":103,"props":1631,"children":1633},{"className":1632},[111],[1634],{"type":16,"tag":114,"props":1635,"children":1636},{"xmlns":116},[1637],{"type":16,"tag":119,"props":1638,"children":1639},{},[1640,1686],{"type":16,"tag":123,"props":1641,"children":1642},{},[1643,1677,1681],{"type":16,"tag":127,"props":1644,"children":1645},{},[1646,1651],{"type":16,"tag":131,"props":1647,"children":1648},{},[1649],{"type":22,"value":1650},"d",{"type":16,"tag":123,"props":1652,"children":1653},{},[1654,1658,1663,1667,1672],{"type":16,"tag":131,"props":1655,"children":1656},{"mathvariant":1409},[1657],{"type":22,"value":1090},{"type":16,"tag":131,"props":1659,"children":1660},{"mathvariant":1409},[1661],{"type":22,"value":1662},"o",{"type":16,"tag":131,"props":1664,"children":1665},{"mathvariant":1409},[1666],{"type":22,"value":1650},{"type":16,"tag":131,"props":1668,"children":1669},{"mathvariant":1409},[1670],{"type":22,"value":1671},"e",{"type":16,"tag":131,"props":1673,"children":1674},{"mathvariant":1409},[1675],{"type":22,"value":1676},"l",{"type":16,"tag":145,"props":1678,"children":1679},{},[1680],{"type":22,"value":746},{"type":16,"tag":151,"props":1682,"children":1683},{},[1684],{"type":22,"value":1685},"512",{"type":16,"tag":157,"props":1687,"children":1688},{"encoding":159},[1689],{"type":22,"value":1690},"d_{\\mathrm{model}} = 512",{"type":16,"tag":103,"props":1692,"children":1694},{"className":1693,"ariaHidden":167},[166],[1695,1783],{"type":16,"tag":103,"props":1696,"children":1698},{"className":1697},[172],[1699,1703,1770,1774,1779],{"type":16,"tag":103,"props":1700,"children":1702},{"className":1701,"style":361},[177],[],{"type":16,"tag":103,"props":1704,"children":1706},{"className":1705},[183],[1707,1712],{"type":16,"tag":103,"props":1708,"children":1710},{"className":1709},[183,188],[1711],{"type":22,"value":1650},{"type":16,"tag":103,"props":1713,"children":1715},{"className":1714},[194],[1716],{"type":16,"tag":103,"props":1717,"children":1719},{"className":1718},[199,200],[1720,1759],{"type":16,"tag":103,"props":1721,"children":1723},{"className":1722},[205],[1724,1754],{"type":16,"tag":103,"props":1725,"children":1728},{"className":1726,"style":1727},[210],"height:0.3361em;",[1729],{"type":16,"tag":103,"props":1730,"children":1731},{"style":215},[1732,1736],{"type":16,"tag":103,"props":1733,"children":1735},{"className":1734,"style":221},[220],[],{"type":16,"tag":103,"props":1737,"children":1739},{"className":1738},[226,227,228,229],[1740],{"type":16,"tag":103,"props":1741,"children":1743},{"className":1742},[183,229],[1744],{"type":16,"tag":103,"props":1745,"children":1747},{"className":1746},[183,229],[1748],{"type":16,"tag":103,"props":1749,"children":1751},{"className":1750},[183,1481,229],[1752],{"type":22,"value":1753},"model",{"type":16,"tag":103,"props":1755,"children":1757},{"className":1756},[254],[1758],{"type":22,"value":257},{"type":16,"tag":103,"props":1760,"children":1762},{"className":1761},[205],[1763],{"type":16,"tag":103,"props":1764,"children":1766},{"className":1765,"style":417},[210],[1767],{"type":16,"tag":103,"props":1768,"children":1769},{},[],{"type":16,"tag":103,"props":1771,"children":1773},{"className":1772,"style":816},[632],[],{"type":16,"tag":103,"props":1775,"children":1777},{"className":1776},[821],[1778],{"type":22,"value":746},{"type":16,"tag":103,"props":1780,"children":1782},{"className":1781,"style":816},[632],[],{"type":16,"tag":103,"props":1784,"children":1786},{"className":1785},[172],[1787,1791],{"type":16,"tag":103,"props":1788,"children":1790},{"className":1789,"style":1375},[177],[],{"type":16,"tag":103,"props":1792,"children":1794},{"className":1793},[183],[1795],{"type":22,"value":1685},{"type":22,"value":1797}," 的输出。",{"type":16,"tag":25,"props":1799,"children":1800},{},[1801,1806,1808,1884],{"type":16,"tag":29,"props":1802,"children":1803},{},[1804],{"type":22,"value":1805},"解码器：",{"type":22,"value":1807}," 解码器同样由 ",{"type":16,"tag":103,"props":1809,"children":1811},{"className":1810},[106],[1812,1841],{"type":16,"tag":103,"props":1813,"children":1815},{"className":1814},[111],[1816],{"type":16,"tag":114,"props":1817,"children":1818},{"xmlns":116},[1819],{"type":16,"tag":119,"props":1820,"children":1821},{},[1822,1837],{"type":16,"tag":123,"props":1823,"children":1824},{},[1825,1829,1833],{"type":16,"tag":131,"props":1826,"children":1827},{},[1828],{"type":22,"value":1321},{"type":16,"tag":145,"props":1830,"children":1831},{},[1832],{"type":22,"value":746},{"type":16,"tag":151,"props":1834,"children":1835},{},[1836],{"type":22,"value":1330},{"type":16,"tag":157,"props":1838,"children":1839},{"encoding":159},[1840],{"type":22,"value":1335},{"type":16,"tag":103,"props":1842,"children":1844},{"className":1843,"ariaHidden":167},[166],[1845,1871],{"type":16,"tag":103,"props":1846,"children":1848},{"className":1847},[172],[1849,1853,1858,1862,1867],{"type":16,"tag":103,"props":1850,"children":1852},{"className":1851,"style":1347},[177],[],{"type":16,"tag":103,"props":1854,"children":1856},{"className":1855,"style":1352},[183,188],[1857],{"type":22,"value":1321},{"type":16,"tag":103,"props":1859,"children":1861},{"className":1860,"style":816},[632],[],{"type":16,"tag":103,"props":1863,"children":1865},{"className":1864},[821],[1866],{"type":22,"value":746},{"type":16,"tag":103,"props":1868,"children":1870},{"className":1869,"style":816},[632],[],{"type":16,"tag":103,"props":1872,"children":1874},{"className":1873},[172],[1875,1879],{"type":16,"tag":103,"props":1876,"children":1878},{"className":1877,"style":1375},[177],[],{"type":16,"tag":103,"props":1880,"children":1882},{"className":1881},[183],[1883],{"type":22,"value":1330},{"type":22,"value":1885}," 个相同层堆叠而成。除了编码器每层中的两个子层外，解码器还插入第三个子层，对编码器堆栈的输出执行多头注意力。",{"type":16,"tag":25,"props":1887,"children":1888},{},[1889,1891,1935,1937,1979],{"type":22,"value":1890},"与编码器类似，我们在每个子层外使用残差连接，再进行层归一化。我们还修改了解码器堆栈中的自注意力子层，使某个位置不能关注后续位置。这种掩码与输出嵌入向右偏移一个位置相结合，可以保证位置 ",{"type":16,"tag":103,"props":1892,"children":1894},{"className":1893},[106],[1895,1917],{"type":16,"tag":103,"props":1896,"children":1898},{"className":1897},[111],[1899],{"type":16,"tag":114,"props":1900,"children":1901},{"xmlns":116},[1902],{"type":16,"tag":119,"props":1903,"children":1904},{},[1905,1913],{"type":16,"tag":123,"props":1906,"children":1907},{},[1908],{"type":16,"tag":131,"props":1909,"children":1910},{},[1911],{"type":22,"value":1912},"i",{"type":16,"tag":157,"props":1914,"children":1915},{"encoding":159},[1916],{"type":22,"value":1912},{"type":16,"tag":103,"props":1918,"children":1920},{"className":1919,"ariaHidden":167},[166],[1921],{"type":16,"tag":103,"props":1922,"children":1924},{"className":1923},[172],[1925,1930],{"type":16,"tag":103,"props":1926,"children":1929},{"className":1927,"style":1928},[177],"height:0.6595em;",[],{"type":16,"tag":103,"props":1931,"children":1933},{"className":1932},[183,188],[1934],{"type":22,"value":1912},{"type":22,"value":1936}," 的预测只能依赖位置 ",{"type":16,"tag":103,"props":1938,"children":1940},{"className":1939},[106],[1941,1962],{"type":16,"tag":103,"props":1942,"children":1944},{"className":1943},[111],[1945],{"type":16,"tag":114,"props":1946,"children":1947},{"xmlns":116},[1948],{"type":16,"tag":119,"props":1949,"children":1950},{},[1951,1958],{"type":16,"tag":123,"props":1952,"children":1953},{},[1954],{"type":16,"tag":131,"props":1955,"children":1956},{},[1957],{"type":22,"value":1912},{"type":16,"tag":157,"props":1959,"children":1960},{"encoding":159},[1961],{"type":22,"value":1912},{"type":16,"tag":103,"props":1963,"children":1965},{"className":1964,"ariaHidden":167},[166],[1966],{"type":16,"tag":103,"props":1967,"children":1969},{"className":1968},[172],[1970,1974],{"type":16,"tag":103,"props":1971,"children":1973},{"className":1972,"style":1928},[177],[],{"type":16,"tag":103,"props":1975,"children":1977},{"className":1976},[183,188],[1978],{"type":22,"value":1912},{"type":22,"value":1980}," 之前已经知道的输出。",{"type":16,"tag":1284,"props":1982,"children":1984},{"id":1983},"_32-注意力",[1985],{"type":22,"value":1986},"3.2 注意力",{"type":16,"tag":25,"props":1988,"children":1989},{},[1990],{"type":22,"value":1991},"注意力函数可以描述为：把一个查询（query）和一组键-值对（key-value pairs）映射为一个输出；查询、键、值和输出都是向量。输出是值的加权和，每个值的权重由查询与对应键之间的兼容性函数计算得到。",{"type":16,"tag":1993,"props":1994,"children":1996},"h4",{"id":1995},"_321-缩放点积注意力",[1997],{"type":22,"value":1998},"3.2.1 缩放点积注意力",{"type":16,"tag":25,"props":2000,"children":2001},{},[2002,2004,2108,2110,2213,2215,2401],{"type":22,"value":2003},"我们使用的具体注意力函数称为“缩放点积注意力”（Scaled Dot-Product Attention），如图 2 所示。输入包含维度为 ",{"type":16,"tag":103,"props":2005,"children":2007},{"className":2006},[106],[2008,2038],{"type":16,"tag":103,"props":2009,"children":2011},{"className":2010},[111],[2012],{"type":16,"tag":114,"props":2013,"children":2014},{"xmlns":116},[2015],{"type":16,"tag":119,"props":2016,"children":2017},{},[2018,2033],{"type":16,"tag":123,"props":2019,"children":2020},{},[2021],{"type":16,"tag":127,"props":2022,"children":2023},{},[2024,2028],{"type":16,"tag":131,"props":2025,"children":2026},{},[2027],{"type":22,"value":1650},{"type":16,"tag":131,"props":2029,"children":2030},{},[2031],{"type":22,"value":2032},"k",{"type":16,"tag":157,"props":2034,"children":2035},{"encoding":159},[2036],{"type":22,"value":2037},"d_k",{"type":16,"tag":103,"props":2039,"children":2041},{"className":2040,"ariaHidden":167},[166],[2042],{"type":16,"tag":103,"props":2043,"children":2045},{"className":2044},[172],[2046,2050],{"type":16,"tag":103,"props":2047,"children":2049},{"className":2048,"style":361},[177],[],{"type":16,"tag":103,"props":2051,"children":2053},{"className":2052},[183],[2054,2059],{"type":16,"tag":103,"props":2055,"children":2057},{"className":2056},[183,188],[2058],{"type":22,"value":1650},{"type":16,"tag":103,"props":2060,"children":2062},{"className":2061},[194],[2063],{"type":16,"tag":103,"props":2064,"children":2066},{"className":2065},[199,200],[2067,2097],{"type":16,"tag":103,"props":2068,"children":2070},{"className":2069},[205],[2071,2092],{"type":16,"tag":103,"props":2072,"children":2074},{"className":2073,"style":1727},[210],[2075],{"type":16,"tag":103,"props":2076,"children":2077},{"style":215},[2078,2082],{"type":16,"tag":103,"props":2079,"children":2081},{"className":2080,"style":221},[220],[],{"type":16,"tag":103,"props":2083,"children":2085},{"className":2084},[226,227,228,229],[2086],{"type":16,"tag":103,"props":2087,"children":2090},{"className":2088,"style":2089},[183,188,229],"margin-right:0.0315em;",[2091],{"type":22,"value":2032},{"type":16,"tag":103,"props":2093,"children":2095},{"className":2094},[254],[2096],{"type":22,"value":257},{"type":16,"tag":103,"props":2098,"children":2100},{"className":2099},[205],[2101],{"type":16,"tag":103,"props":2102,"children":2104},{"className":2103,"style":417},[210],[2105],{"type":16,"tag":103,"props":2106,"children":2107},{},[],{"type":22,"value":2109}," 的查询和键，以及维度为 ",{"type":16,"tag":103,"props":2111,"children":2113},{"className":2112},[106],[2114,2144],{"type":16,"tag":103,"props":2115,"children":2117},{"className":2116},[111],[2118],{"type":16,"tag":114,"props":2119,"children":2120},{"xmlns":116},[2121],{"type":16,"tag":119,"props":2122,"children":2123},{},[2124,2139],{"type":16,"tag":123,"props":2125,"children":2126},{},[2127],{"type":16,"tag":127,"props":2128,"children":2129},{},[2130,2134],{"type":16,"tag":131,"props":2131,"children":2132},{},[2133],{"type":22,"value":1650},{"type":16,"tag":131,"props":2135,"children":2136},{},[2137],{"type":22,"value":2138},"v",{"type":16,"tag":157,"props":2140,"children":2141},{"encoding":159},[2142],{"type":22,"value":2143},"d_v",{"type":16,"tag":103,"props":2145,"children":2147},{"className":2146,"ariaHidden":167},[166],[2148],{"type":16,"tag":103,"props":2149,"children":2151},{"className":2150},[172],[2152,2156],{"type":16,"tag":103,"props":2153,"children":2155},{"className":2154,"style":361},[177],[],{"type":16,"tag":103,"props":2157,"children":2159},{"className":2158},[183],[2160,2165],{"type":16,"tag":103,"props":2161,"children":2163},{"className":2162},[183,188],[2164],{"type":22,"value":1650},{"type":16,"tag":103,"props":2166,"children":2168},{"className":2167},[194],[2169],{"type":16,"tag":103,"props":2170,"children":2172},{"className":2171},[199,200],[2173,2202],{"type":16,"tag":103,"props":2174,"children":2176},{"className":2175},[205],[2177,2197],{"type":16,"tag":103,"props":2178,"children":2180},{"className":2179,"style":678},[210],[2181],{"type":16,"tag":103,"props":2182,"children":2183},{"style":215},[2184,2188],{"type":16,"tag":103,"props":2185,"children":2187},{"className":2186,"style":221},[220],[],{"type":16,"tag":103,"props":2189,"children":2191},{"className":2190},[226,227,228,229],[2192],{"type":16,"tag":103,"props":2193,"children":2195},{"className":2194,"style":1124},[183,188,229],[2196],{"type":22,"value":2138},{"type":16,"tag":103,"props":2198,"children":2200},{"className":2199},[254],[2201],{"type":22,"value":257},{"type":16,"tag":103,"props":2203,"children":2205},{"className":2204},[205],[2206],{"type":16,"tag":103,"props":2207,"children":2209},{"className":2208,"style":417},[210],[2210],{"type":16,"tag":103,"props":2211,"children":2212},{},[],{"type":22,"value":2214}," 的值。我们计算查询与所有键的点积，把每个点积除以 ",{"type":16,"tag":103,"props":2216,"children":2218},{"className":2217},[106],[2219,2252],{"type":16,"tag":103,"props":2220,"children":2222},{"className":2221},[111],[2223],{"type":16,"tag":114,"props":2224,"children":2225},{"xmlns":116},[2226],{"type":16,"tag":119,"props":2227,"children":2228},{},[2229,2247],{"type":16,"tag":123,"props":2230,"children":2231},{},[2232],{"type":16,"tag":2233,"props":2234,"children":2235},"msqrt",{},[2236],{"type":16,"tag":127,"props":2237,"children":2238},{},[2239,2243],{"type":16,"tag":131,"props":2240,"children":2241},{},[2242],{"type":22,"value":1650},{"type":16,"tag":131,"props":2244,"children":2245},{},[2246],{"type":22,"value":2032},{"type":16,"tag":157,"props":2248,"children":2249},{"encoding":159},[2250],{"type":22,"value":2251},"\\sqrt{d_k}",{"type":16,"tag":103,"props":2253,"children":2255},{"className":2254,"ariaHidden":167},[166],[2256],{"type":16,"tag":103,"props":2257,"children":2259},{"className":2258},[172],[2260,2265],{"type":16,"tag":103,"props":2261,"children":2264},{"className":2262,"style":2263},[177],"height:1.04em;vertical-align:-0.1828em;",[],{"type":16,"tag":103,"props":2266,"children":2269},{"className":2267},[183,2268],"sqrt",[2270],{"type":16,"tag":103,"props":2271,"children":2273},{"className":2272},[199,200],[2274,2389],{"type":16,"tag":103,"props":2275,"children":2277},{"className":2276},[205],[2278,2384],{"type":16,"tag":103,"props":2279,"children":2282},{"className":2280,"style":2281},[210],"height:0.8572em;",[2283,2356],{"type":16,"tag":103,"props":2284,"children":2288},{"className":2285,"style":2287},[2286],"svg-align","top:-3em;",[2289,2294],{"type":16,"tag":103,"props":2290,"children":2293},{"className":2291,"style":2292},[220],"height:3em;",[],{"type":16,"tag":103,"props":2295,"children":2298},{"className":2296,"style":2297},[183],"padding-left:0.833em;",[2299],{"type":16,"tag":103,"props":2300,"children":2302},{"className":2301},[183],[2303,2308],{"type":16,"tag":103,"props":2304,"children":2306},{"className":2305},[183,188],[2307],{"type":22,"value":1650},{"type":16,"tag":103,"props":2309,"children":2311},{"className":2310},[194],[2312],{"type":16,"tag":103,"props":2313,"children":2315},{"className":2314},[199,200],[2316,2345],{"type":16,"tag":103,"props":2317,"children":2319},{"className":2318},[205],[2320,2340],{"type":16,"tag":103,"props":2321,"children":2323},{"className":2322,"style":1727},[210],[2324],{"type":16,"tag":103,"props":2325,"children":2326},{"style":215},[2327,2331],{"type":16,"tag":103,"props":2328,"children":2330},{"className":2329,"style":221},[220],[],{"type":16,"tag":103,"props":2332,"children":2334},{"className":2333},[226,227,228,229],[2335],{"type":16,"tag":103,"props":2336,"children":2338},{"className":2337,"style":2089},[183,188,229],[2339],{"type":22,"value":2032},{"type":16,"tag":103,"props":2341,"children":2343},{"className":2342},[254],[2344],{"type":22,"value":257},{"type":16,"tag":103,"props":2346,"children":2348},{"className":2347},[205],[2349],{"type":16,"tag":103,"props":2350,"children":2352},{"className":2351,"style":417},[210],[2353],{"type":16,"tag":103,"props":2354,"children":2355},{},[],{"type":16,"tag":103,"props":2357,"children":2359},{"style":2358},"top:-2.8172em;",[2360,2364],{"type":16,"tag":103,"props":2361,"children":2363},{"className":2362,"style":2292},[220],[],{"type":16,"tag":103,"props":2365,"children":2369},{"className":2366,"style":2368},[2367],"hide-tail","min-width:0.853em;height:1.08em;",[2370],{"type":16,"tag":2371,"props":2372,"children":2378},"svg",{"xmlns":2373,"width":2374,"height":2375,"viewBox":2376,"preserveAspectRatio":2377},"http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","400em","1.08em","0 0 400000 1080","xMinYMin slice",[2379],{"type":16,"tag":2380,"props":2381,"children":2383},"path",{"d":2382},"M95,702\nc-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14\nc0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54\nc44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10\ns173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429\nc69,-144,104.5,-217.7,106.5,-221\nl0 -0\nc5.3,-9.3,12,-14,20,-14\nH400000v40H845.2724\ns-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7\nc-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z\nM834 80h400000v40h-400000z",[],{"type":16,"tag":103,"props":2385,"children":2387},{"className":2386},[254],[2388],{"type":22,"value":257},{"type":16,"tag":103,"props":2390,"children":2392},{"className":2391},[205],[2393],{"type":16,"tag":103,"props":2394,"children":2397},{"className":2395,"style":2396},[210],"height:0.1828em;",[2398],{"type":16,"tag":103,"props":2399,"children":2400},{},[],{"type":22,"value":2402},"，再应用 softmax 函数得到各个值的权重。",{"type":16,"tag":25,"props":2404,"children":2405},{},[2406],{"type":16,"tag":1276,"props":2407,"children":2412},{"alt":2408,"src":2409,"width":2410,"height":2411},"缩放点积注意力与多头注意力（原文图 2）","\u002Fimages\u002Fmaterials\u002Fcmu-15-779\u002Ftransformer\u002Ffigure-02.webp",1367,846,[],{"type":16,"tag":25,"props":2414,"children":2415},{},[2416,2418,2462,2464,2508,2510,2553],{"type":22,"value":2417},"实践中，我们会同时对一组查询计算注意力，并把它们打包成矩阵 ",{"type":16,"tag":103,"props":2419,"children":2421},{"className":2420},[106],[2422,2444],{"type":16,"tag":103,"props":2423,"children":2425},{"className":2424},[111],[2426],{"type":16,"tag":114,"props":2427,"children":2428},{"xmlns":116},[2429],{"type":16,"tag":119,"props":2430,"children":2431},{},[2432,2440],{"type":16,"tag":123,"props":2433,"children":2434},{},[2435],{"type":16,"tag":131,"props":2436,"children":2437},{},[2438],{"type":22,"value":2439},"Q",{"type":16,"tag":157,"props":2441,"children":2442},{"encoding":159},[2443],{"type":22,"value":2439},{"type":16,"tag":103,"props":2445,"children":2447},{"className":2446,"ariaHidden":167},[166],[2448],{"type":16,"tag":103,"props":2449,"children":2451},{"className":2450},[172],[2452,2457],{"type":16,"tag":103,"props":2453,"children":2456},{"className":2454,"style":2455},[177],"height:0.8778em;vertical-align:-0.1944em;",[],{"type":16,"tag":103,"props":2458,"children":2460},{"className":2459},[183,188],[2461],{"type":22,"value":2439},{"type":22,"value":2463},"。键和值也分别打包成矩阵 ",{"type":16,"tag":103,"props":2465,"children":2467},{"className":2466},[106],[2468,2490],{"type":16,"tag":103,"props":2469,"children":2471},{"className":2470},[111],[2472],{"type":16,"tag":114,"props":2473,"children":2474},{"xmlns":116},[2475],{"type":16,"tag":119,"props":2476,"children":2477},{},[2478,2486],{"type":16,"tag":123,"props":2479,"children":2480},{},[2481],{"type":16,"tag":131,"props":2482,"children":2483},{},[2484],{"type":22,"value":2485},"K",{"type":16,"tag":157,"props":2487,"children":2488},{"encoding":159},[2489],{"type":22,"value":2485},{"type":16,"tag":103,"props":2491,"children":2493},{"className":2492,"ariaHidden":167},[166],[2494],{"type":16,"tag":103,"props":2495,"children":2497},{"className":2496},[172],[2498,2502],{"type":16,"tag":103,"props":2499,"children":2501},{"className":2500,"style":1347},[177],[],{"type":16,"tag":103,"props":2503,"children":2506},{"className":2504,"style":2505},[183,188],"margin-right:0.0715em;",[2507],{"type":22,"value":2485},{"type":22,"value":2509}," 和 ",{"type":16,"tag":103,"props":2511,"children":2513},{"className":2512},[106],[2514,2536],{"type":16,"tag":103,"props":2515,"children":2517},{"className":2516},[111],[2518],{"type":16,"tag":114,"props":2519,"children":2520},{"xmlns":116},[2521],{"type":16,"tag":119,"props":2522,"children":2523},{},[2524,2532],{"type":16,"tag":123,"props":2525,"children":2526},{},[2527],{"type":16,"tag":131,"props":2528,"children":2529},{},[2530],{"type":22,"value":2531},"V",{"type":16,"tag":157,"props":2533,"children":2534},{"encoding":159},[2535],{"type":22,"value":2531},{"type":16,"tag":103,"props":2537,"children":2539},{"className":2538,"ariaHidden":167},[166],[2540],{"type":16,"tag":103,"props":2541,"children":2543},{"className":2542},[172],[2544,2548],{"type":16,"tag":103,"props":2545,"children":2547},{"className":2546,"style":1347},[177],[],{"type":16,"tag":103,"props":2549,"children":2551},{"className":2550,"style":1497},[183,188],[2552],{"type":22,"value":2531},{"type":22,"value":2554},"。输出矩阵计算如下：",{"type":16,"tag":103,"props":2556,"children":2558},{"className":2557},[1387],[2559],{"type":16,"tag":103,"props":2560,"children":2562},{"className":2561},[106],[2563,2713],{"type":16,"tag":103,"props":2564,"children":2566},{"className":2565},[111],[2567],{"type":16,"tag":114,"props":2568,"children":2569},{"xmlns":116,"display":1399},[2570],{"type":16,"tag":119,"props":2571,"children":2572},{},[2573,2708],{"type":16,"tag":2574,"props":2575,"children":2577},"mtable",{"width":2576},"100%",[2578],{"type":16,"tag":2579,"props":2580,"children":2581},"mtr",{},[2582,2587,2696,2699],{"type":16,"tag":2583,"props":2584,"children":2586},"mtd",{"width":2585},"50%",[],{"type":16,"tag":2583,"props":2588,"children":2589},{},[2590],{"type":16,"tag":123,"props":2591,"children":2592},{},[2593,2598,2602,2606,2610,2614,2618,2622,2626,2630,2634,2639,2643,2692],{"type":16,"tag":131,"props":2594,"children":2595},{"mathvariant":1409},[2596],{"type":22,"value":2597},"Attention",{"type":16,"tag":145,"props":2599,"children":2600},{},[2601],{"type":22,"value":1417},{"type":16,"tag":145,"props":2603,"children":2604},{"stretchy":495},[2605],{"type":22,"value":498},{"type":16,"tag":131,"props":2607,"children":2608},{},[2609],{"type":22,"value":2439},{"type":16,"tag":145,"props":2611,"children":2612},{"separator":167},[2613],{"type":22,"value":515},{"type":16,"tag":131,"props":2615,"children":2616},{},[2617],{"type":22,"value":2485},{"type":16,"tag":145,"props":2619,"children":2620},{"separator":167},[2621],{"type":22,"value":515},{"type":16,"tag":131,"props":2623,"children":2624},{},[2625],{"type":22,"value":2531},{"type":16,"tag":145,"props":2627,"children":2628},{"stretchy":495},[2629],{"type":22,"value":541},{"type":16,"tag":145,"props":2631,"children":2632},{},[2633],{"type":22,"value":746},{"type":16,"tag":131,"props":2635,"children":2636},{"mathvariant":1409},[2637],{"type":22,"value":2638},"softmax",{"type":16,"tag":145,"props":2640,"children":2641},{},[2642],{"type":22,"value":1417},{"type":16,"tag":123,"props":2644,"children":2645},{},[2646,2650,2688],{"type":16,"tag":145,"props":2647,"children":2648},{"fence":167},[2649],{"type":22,"value":498},{"type":16,"tag":2651,"props":2652,"children":2653},"mfrac",{},[2654,2674],{"type":16,"tag":123,"props":2655,"children":2656},{},[2657,2661],{"type":16,"tag":131,"props":2658,"children":2659},{},[2660],{"type":22,"value":2439},{"type":16,"tag":2662,"props":2663,"children":2664},"msup",{},[2665,2669],{"type":16,"tag":131,"props":2666,"children":2667},{},[2668],{"type":22,"value":2485},{"type":16,"tag":131,"props":2670,"children":2671},{},[2672],{"type":22,"value":2673},"T",{"type":16,"tag":2233,"props":2675,"children":2676},{},[2677],{"type":16,"tag":127,"props":2678,"children":2679},{},[2680,2684],{"type":16,"tag":131,"props":2681,"children":2682},{},[2683],{"type":22,"value":1650},{"type":16,"tag":131,"props":2685,"children":2686},{},[2687],{"type":22,"value":2032},{"type":16,"tag":145,"props":2689,"children":2690},{"fence":167},[2691],{"type":22,"value":541},{"type":16,"tag":131,"props":2693,"children":2694},{},[2695],{"type":22,"value":2531},{"type":16,"tag":2583,"props":2697,"children":2698},{"width":2585},[],{"type":16,"tag":2583,"props":2700,"children":2701},{},[2702],{"type":16,"tag":2703,"props":2704,"children":2705},"mtext",{},[2706],{"type":22,"value":2707},"(1)",{"type":16,"tag":157,"props":2709,"children":2710},{"encoding":159},[2711],{"type":22,"value":2712},"\\operatorname{Attention}(Q, K, V) = \\operatorname{softmax}\\left(\\frac{QK^T}{\\sqrt{d_k}}\\right)V\\tag{1}",{"type":16,"tag":103,"props":2714,"children":2716},{"className":2715,"ariaHidden":167},[166],[2717,2790,3098],{"type":16,"tag":103,"props":2718,"children":2720},{"className":2719},[172],[2721,2725,2734,2739,2744,2749,2753,2758,2763,2767,2772,2777,2781,2786],{"type":16,"tag":103,"props":2722,"children":2724},{"className":2723,"style":558},[177],[],{"type":16,"tag":103,"props":2726,"children":2728},{"className":2727},[1476],[2729],{"type":16,"tag":103,"props":2730,"children":2732},{"className":2731},[183,1481],[2733],{"type":22,"value":2597},{"type":16,"tag":103,"props":2735,"children":2737},{"className":2736},[563],[2738],{"type":22,"value":498},{"type":16,"tag":103,"props":2740,"children":2742},{"className":2741},[183,188],[2743],{"type":22,"value":2439},{"type":16,"tag":103,"props":2745,"children":2747},{"className":2746},[626],[2748],{"type":22,"value":515},{"type":16,"tag":103,"props":2750,"children":2752},{"className":2751,"style":633},[632],[],{"type":16,"tag":103,"props":2754,"children":2756},{"className":2755,"style":2505},[183,188],[2757],{"type":22,"value":2485},{"type":16,"tag":103,"props":2759,"children":2761},{"className":2760},[626],[2762],{"type":22,"value":515},{"type":16,"tag":103,"props":2764,"children":2766},{"className":2765,"style":633},[632],[],{"type":16,"tag":103,"props":2768,"children":2770},{"className":2769,"style":1497},[183,188],[2771],{"type":22,"value":2531},{"type":16,"tag":103,"props":2773,"children":2775},{"className":2774},[715],[2776],{"type":22,"value":541},{"type":16,"tag":103,"props":2778,"children":2780},{"className":2779,"style":816},[632],[],{"type":16,"tag":103,"props":2782,"children":2784},{"className":2783},[821],[2785],{"type":22,"value":746},{"type":16,"tag":103,"props":2787,"children":2789},{"className":2788,"style":816},[632],[],{"type":16,"tag":103,"props":2791,"children":2793},{"className":2792},[172],[2794,2799,2808,2812,3089,3093],{"type":16,"tag":103,"props":2795,"children":2798},{"className":2796,"style":2797},[177],"height:2.4684em;vertical-align:-0.95em;",[],{"type":16,"tag":103,"props":2800,"children":2802},{"className":2801},[1476],[2803],{"type":16,"tag":103,"props":2804,"children":2806},{"className":2805},[183,1481],[2807],{"type":22,"value":2638},{"type":16,"tag":103,"props":2809,"children":2811},{"className":2810,"style":633},[632],[],{"type":16,"tag":103,"props":2813,"children":2815},{"className":2814},[638],[2816,2828,3080],{"type":16,"tag":103,"props":2817,"children":2821},{"className":2818,"style":2820},[563,2819],"delimcenter","top:0em;",[2822],{"type":16,"tag":103,"props":2823,"children":2826},{"className":2824},[2825,228],"delimsizing",[2827],{"type":22,"value":498},{"type":16,"tag":103,"props":2829,"children":2831},{"className":2830},[183],[2832,2837,3076],{"type":16,"tag":103,"props":2833,"children":2836},{"className":2834},[563,2835],"nulldelimiter",[],{"type":16,"tag":103,"props":2838,"children":2840},{"className":2839},[2651],[2841],{"type":16,"tag":103,"props":2842,"children":2844},{"className":2843},[199,200],[2845,3064],{"type":16,"tag":103,"props":2846,"children":2848},{"className":2847},[205],[2849,3059],{"type":16,"tag":103,"props":2850,"children":2853},{"className":2851,"style":2852},[210],"height:1.5183em;",[2854,2984,2998],{"type":16,"tag":103,"props":2855,"children":2857},{"style":2856},"top:-2.2528em;",[2858,2862],{"type":16,"tag":103,"props":2859,"children":2861},{"className":2860,"style":2292},[220],[],{"type":16,"tag":103,"props":2863,"children":2865},{"className":2864},[183],[2866],{"type":16,"tag":103,"props":2867,"children":2869},{"className":2868},[183,2268],[2870],{"type":16,"tag":103,"props":2871,"children":2873},{"className":2872},[199,200],[2874,2973],{"type":16,"tag":103,"props":2875,"children":2877},{"className":2876},[205],[2878,2968],{"type":16,"tag":103,"props":2879,"children":2881},{"className":2880,"style":2281},[210],[2882,2951],{"type":16,"tag":103,"props":2883,"children":2885},{"className":2884,"style":2287},[2286],[2886,2890],{"type":16,"tag":103,"props":2887,"children":2889},{"className":2888,"style":2292},[220],[],{"type":16,"tag":103,"props":2891,"children":2893},{"className":2892,"style":2297},[183],[2894],{"type":16,"tag":103,"props":2895,"children":2897},{"className":2896},[183],[2898,2903],{"type":16,"tag":103,"props":2899,"children":2901},{"className":2900},[183,188],[2902],{"type":22,"value":1650},{"type":16,"tag":103,"props":2904,"children":2906},{"className":2905},[194],[2907],{"type":16,"tag":103,"props":2908,"children":2910},{"className":2909},[199,200],[2911,2940],{"type":16,"tag":103,"props":2912,"children":2914},{"className":2913},[205],[2915,2935],{"type":16,"tag":103,"props":2916,"children":2918},{"className":2917,"style":1727},[210],[2919],{"type":16,"tag":103,"props":2920,"children":2921},{"style":215},[2922,2926],{"type":16,"tag":103,"props":2923,"children":2925},{"className":2924,"style":221},[220],[],{"type":16,"tag":103,"props":2927,"children":2929},{"className":2928},[226,227,228,229],[2930],{"type":16,"tag":103,"props":2931,"children":2933},{"className":2932,"style":2089},[183,188,229],[2934],{"type":22,"value":2032},{"type":16,"tag":103,"props":2936,"children":2938},{"className":2937},[254],[2939],{"type":22,"value":257},{"type":16,"tag":103,"props":2941,"children":2943},{"className":2942},[205],[2944],{"type":16,"tag":103,"props":2945,"children":2947},{"className":2946,"style":417},[210],[2948],{"type":16,"tag":103,"props":2949,"children":2950},{},[],{"type":16,"tag":103,"props":2952,"children":2953},{"style":2358},[2954,2958],{"type":16,"tag":103,"props":2955,"children":2957},{"className":2956,"style":2292},[220],[],{"type":16,"tag":103,"props":2959,"children":2961},{"className":2960,"style":2368},[2367],[2962],{"type":16,"tag":2371,"props":2963,"children":2964},{"xmlns":2373,"width":2374,"height":2375,"viewBox":2376,"preserveAspectRatio":2377},[2965],{"type":16,"tag":2380,"props":2966,"children":2967},{"d":2382},[],{"type":16,"tag":103,"props":2969,"children":2971},{"className":2970},[254],[2972],{"type":22,"value":257},{"type":16,"tag":103,"props":2974,"children":2976},{"className":2975},[205],[2977],{"type":16,"tag":103,"props":2978,"children":2980},{"className":2979,"style":2396},[210],[2981],{"type":16,"tag":103,"props":2982,"children":2983},{},[],{"type":16,"tag":103,"props":2985,"children":2987},{"style":2986},"top:-3.23em;",[2988,2992],{"type":16,"tag":103,"props":2989,"children":2991},{"className":2990,"style":2292},[220],[],{"type":16,"tag":103,"props":2993,"children":2997},{"className":2994,"style":2996},[2995],"frac-line","border-bottom-width:0.04em;",[],{"type":16,"tag":103,"props":2999,"children":3001},{"style":3000},"top:-3.677em;",[3002,3006],{"type":16,"tag":103,"props":3003,"children":3005},{"className":3004,"style":2292},[220],[],{"type":16,"tag":103,"props":3007,"children":3009},{"className":3008},[183],[3010,3015],{"type":16,"tag":103,"props":3011,"children":3013},{"className":3012},[183,188],[3014],{"type":22,"value":2439},{"type":16,"tag":103,"props":3016,"children":3018},{"className":3017},[183],[3019,3024],{"type":16,"tag":103,"props":3020,"children":3022},{"className":3021,"style":2505},[183,188],[3023],{"type":22,"value":2485},{"type":16,"tag":103,"props":3025,"children":3027},{"className":3026},[194],[3028],{"type":16,"tag":103,"props":3029,"children":3031},{"className":3030},[199],[3032],{"type":16,"tag":103,"props":3033,"children":3035},{"className":3034},[205],[3036],{"type":16,"tag":103,"props":3037,"children":3040},{"className":3038,"style":3039},[210],"height:0.8413em;",[3041],{"type":16,"tag":103,"props":3042,"children":3044},{"style":3043},"top:-3.063em;margin-right:0.05em;",[3045,3049],{"type":16,"tag":103,"props":3046,"children":3048},{"className":3047,"style":221},[220],[],{"type":16,"tag":103,"props":3050,"children":3052},{"className":3051},[226,227,228,229],[3053],{"type":16,"tag":103,"props":3054,"children":3057},{"className":3055,"style":3056},[183,188,229],"margin-right:0.1389em;",[3058],{"type":22,"value":2673},{"type":16,"tag":103,"props":3060,"children":3062},{"className":3061},[254],[3063],{"type":22,"value":257},{"type":16,"tag":103,"props":3065,"children":3067},{"className":3066},[205],[3068],{"type":16,"tag":103,"props":3069,"children":3072},{"className":3070,"style":3071},[210],"height:0.93em;",[3073],{"type":16,"tag":103,"props":3074,"children":3075},{},[],{"type":16,"tag":103,"props":3077,"children":3079},{"className":3078},[715,2835],[],{"type":16,"tag":103,"props":3081,"children":3083},{"className":3082,"style":2820},[715,2819],[3084],{"type":16,"tag":103,"props":3085,"children":3087},{"className":3086},[2825,228],[3088],{"type":22,"value":541},{"type":16,"tag":103,"props":3090,"children":3092},{"className":3091,"style":633},[632],[],{"type":16,"tag":103,"props":3094,"children":3096},{"className":3095,"style":1497},[183,188],[3097],{"type":22,"value":2531},{"type":16,"tag":103,"props":3099,"children":3102},{"className":3100},[3101],"tag",[3103,3107],{"type":16,"tag":103,"props":3104,"children":3106},{"className":3105,"style":2797},[177],[],{"type":16,"tag":103,"props":3108,"children":3110},{"className":3109},[183,22],[3111,3116,3125],{"type":16,"tag":103,"props":3112,"children":3114},{"className":3113},[183],[3115],{"type":22,"value":498},{"type":16,"tag":103,"props":3117,"children":3119},{"className":3118},[183],[3120],{"type":16,"tag":103,"props":3121,"children":3123},{"className":3122},[183],[3124],{"type":22,"value":155},{"type":16,"tag":103,"props":3126,"children":3128},{"className":3127},[183],[3129],{"type":22,"value":541},{"type":16,"tag":25,"props":3131,"children":3132},{},[3133],{"type":22,"value":3134},"最常用的两种注意力函数是加性注意力 [2] 和点积（乘性）注意力。除缩放因子外，点积注意力与我们的算法相同。加性注意力使用一个带单隐藏层的前馈网络来计算兼容性函数。两者在理论复杂度上相似，但实践中点积注意力更快、更节省空间，因为它可以使用高度优化的矩阵乘法代码实现。",{"type":16,"tag":25,"props":3136,"children":3137},{},[3138,3140,3241,3243,3344,3346,3447,3449,3631],{"type":22,"value":3139},"当 ",{"type":16,"tag":103,"props":3141,"children":3143},{"className":3142},[106],[3144,3172],{"type":16,"tag":103,"props":3145,"children":3147},{"className":3146},[111],[3148],{"type":16,"tag":114,"props":3149,"children":3150},{"xmlns":116},[3151],{"type":16,"tag":119,"props":3152,"children":3153},{},[3154,3168],{"type":16,"tag":123,"props":3155,"children":3156},{},[3157],{"type":16,"tag":127,"props":3158,"children":3159},{},[3160,3164],{"type":16,"tag":131,"props":3161,"children":3162},{},[3163],{"type":22,"value":1650},{"type":16,"tag":131,"props":3165,"children":3166},{},[3167],{"type":22,"value":2032},{"type":16,"tag":157,"props":3169,"children":3170},{"encoding":159},[3171],{"type":22,"value":2037},{"type":16,"tag":103,"props":3173,"children":3175},{"className":3174,"ariaHidden":167},[166],[3176],{"type":16,"tag":103,"props":3177,"children":3179},{"className":3178},[172],[3180,3184],{"type":16,"tag":103,"props":3181,"children":3183},{"className":3182,"style":361},[177],[],{"type":16,"tag":103,"props":3185,"children":3187},{"className":3186},[183],[3188,3193],{"type":16,"tag":103,"props":3189,"children":3191},{"className":3190},[183,188],[3192],{"type":22,"value":1650},{"type":16,"tag":103,"props":3194,"children":3196},{"className":3195},[194],[3197],{"type":16,"tag":103,"props":3198,"children":3200},{"className":3199},[199,200],[3201,3230],{"type":16,"tag":103,"props":3202,"children":3204},{"className":3203},[205],[3205,3225],{"type":16,"tag":103,"props":3206,"children":3208},{"className":3207,"style":1727},[210],[3209],{"type":16,"tag":103,"props":3210,"children":3211},{"style":215},[3212,3216],{"type":16,"tag":103,"props":3213,"children":3215},{"className":3214,"style":221},[220],[],{"type":16,"tag":103,"props":3217,"children":3219},{"className":3218},[226,227,228,229],[3220],{"type":16,"tag":103,"props":3221,"children":3223},{"className":3222,"style":2089},[183,188,229],[3224],{"type":22,"value":2032},{"type":16,"tag":103,"props":3226,"children":3228},{"className":3227},[254],[3229],{"type":22,"value":257},{"type":16,"tag":103,"props":3231,"children":3233},{"className":3232},[205],[3234],{"type":16,"tag":103,"props":3235,"children":3237},{"className":3236,"style":417},[210],[3238],{"type":16,"tag":103,"props":3239,"children":3240},{},[],{"type":22,"value":3242}," 较小时，两种机制表现相近；但当 ",{"type":16,"tag":103,"props":3244,"children":3246},{"className":3245},[106],[3247,3275],{"type":16,"tag":103,"props":3248,"children":3250},{"className":3249},[111],[3251],{"type":16,"tag":114,"props":3252,"children":3253},{"xmlns":116},[3254],{"type":16,"tag":119,"props":3255,"children":3256},{},[3257,3271],{"type":16,"tag":123,"props":3258,"children":3259},{},[3260],{"type":16,"tag":127,"props":3261,"children":3262},{},[3263,3267],{"type":16,"tag":131,"props":3264,"children":3265},{},[3266],{"type":22,"value":1650},{"type":16,"tag":131,"props":3268,"children":3269},{},[3270],{"type":22,"value":2032},{"type":16,"tag":157,"props":3272,"children":3273},{"encoding":159},[3274],{"type":22,"value":2037},{"type":16,"tag":103,"props":3276,"children":3278},{"className":3277,"ariaHidden":167},[166],[3279],{"type":16,"tag":103,"props":3280,"children":3282},{"className":3281},[172],[3283,3287],{"type":16,"tag":103,"props":3284,"children":3286},{"className":3285,"style":361},[177],[],{"type":16,"tag":103,"props":3288,"children":3290},{"className":3289},[183],[3291,3296],{"type":16,"tag":103,"props":3292,"children":3294},{"className":3293},[183,188],[3295],{"type":22,"value":1650},{"type":16,"tag":103,"props":3297,"children":3299},{"className":3298},[194],[3300],{"type":16,"tag":103,"props":3301,"children":3303},{"className":3302},[199,200],[3304,3333],{"type":16,"tag":103,"props":3305,"children":3307},{"className":3306},[205],[3308,3328],{"type":16,"tag":103,"props":3309,"children":3311},{"className":3310,"style":1727},[210],[3312],{"type":16,"tag":103,"props":3313,"children":3314},{"style":215},[3315,3319],{"type":16,"tag":103,"props":3316,"children":3318},{"className":3317,"style":221},[220],[],{"type":16,"tag":103,"props":3320,"children":3322},{"className":3321},[226,227,228,229],[3323],{"type":16,"tag":103,"props":3324,"children":3326},{"className":3325,"style":2089},[183,188,229],[3327],{"type":22,"value":2032},{"type":16,"tag":103,"props":3329,"children":3331},{"className":3330},[254],[3332],{"type":22,"value":257},{"type":16,"tag":103,"props":3334,"children":3336},{"className":3335},[205],[3337],{"type":16,"tag":103,"props":3338,"children":3340},{"className":3339,"style":417},[210],[3341],{"type":16,"tag":103,"props":3342,"children":3343},{},[],{"type":22,"value":3345}," 较大时，不带缩放的加性注意力优于点积注意力 [3]。我们推测，当 ",{"type":16,"tag":103,"props":3347,"children":3349},{"className":3348},[106],[3350,3378],{"type":16,"tag":103,"props":3351,"children":3353},{"className":3352},[111],[3354],{"type":16,"tag":114,"props":3355,"children":3356},{"xmlns":116},[3357],{"type":16,"tag":119,"props":3358,"children":3359},{},[3360,3374],{"type":16,"tag":123,"props":3361,"children":3362},{},[3363],{"type":16,"tag":127,"props":3364,"children":3365},{},[3366,3370],{"type":16,"tag":131,"props":3367,"children":3368},{},[3369],{"type":22,"value":1650},{"type":16,"tag":131,"props":3371,"children":3372},{},[3373],{"type":22,"value":2032},{"type":16,"tag":157,"props":3375,"children":3376},{"encoding":159},[3377],{"type":22,"value":2037},{"type":16,"tag":103,"props":3379,"children":3381},{"className":3380,"ariaHidden":167},[166],[3382],{"type":16,"tag":103,"props":3383,"children":3385},{"className":3384},[172],[3386,3390],{"type":16,"tag":103,"props":3387,"children":3389},{"className":3388,"style":361},[177],[],{"type":16,"tag":103,"props":3391,"children":3393},{"className":3392},[183],[3394,3399],{"type":16,"tag":103,"props":3395,"children":3397},{"className":3396},[183,188],[3398],{"type":22,"value":1650},{"type":16,"tag":103,"props":3400,"children":3402},{"className":3401},[194],[3403],{"type":16,"tag":103,"props":3404,"children":3406},{"className":3405},[199,200],[3407,3436],{"type":16,"tag":103,"props":3408,"children":3410},{"className":3409},[205],[3411,3431],{"type":16,"tag":103,"props":3412,"children":3414},{"className":3413,"style":1727},[210],[3415],{"type":16,"tag":103,"props":3416,"children":3417},{"style":215},[3418,3422],{"type":16,"tag":103,"props":3419,"children":3421},{"className":3420,"style":221},[220],[],{"type":16,"tag":103,"props":3423,"children":3425},{"className":3424},[226,227,228,229],[3426],{"type":16,"tag":103,"props":3427,"children":3429},{"className":3428,"style":2089},[183,188,229],[3430],{"type":22,"value":2032},{"type":16,"tag":103,"props":3432,"children":3434},{"className":3433},[254],[3435],{"type":22,"value":257},{"type":16,"tag":103,"props":3437,"children":3439},{"className":3438},[205],[3440],{"type":16,"tag":103,"props":3441,"children":3443},{"className":3442,"style":417},[210],[3444],{"type":16,"tag":103,"props":3445,"children":3446},{},[],{"type":22,"value":3448}," 较大时，点积的数值幅度会变大，使 softmax 进入梯度极小的区域。为了抵消这种影响，我们用 ",{"type":16,"tag":103,"props":3450,"children":3452},{"className":3451},[106],[3453,3494],{"type":16,"tag":103,"props":3454,"children":3456},{"className":3455},[111],[3457],{"type":16,"tag":114,"props":3458,"children":3459},{"xmlns":116},[3460],{"type":16,"tag":119,"props":3461,"children":3462},{},[3463,3489],{"type":16,"tag":123,"props":3464,"children":3465},{},[3466,3470,3475],{"type":16,"tag":151,"props":3467,"children":3468},{},[3469],{"type":22,"value":155},{"type":16,"tag":131,"props":3471,"children":3472},{"mathvariant":1409},[3473],{"type":22,"value":3474},"\u002F",{"type":16,"tag":2233,"props":3476,"children":3477},{},[3478],{"type":16,"tag":127,"props":3479,"children":3480},{},[3481,3485],{"type":16,"tag":131,"props":3482,"children":3483},{},[3484],{"type":22,"value":1650},{"type":16,"tag":131,"props":3486,"children":3487},{},[3488],{"type":22,"value":2032},{"type":16,"tag":157,"props":3490,"children":3491},{"encoding":159},[3492],{"type":22,"value":3493},"1\u002F\\sqrt{d_k}",{"type":16,"tag":103,"props":3495,"children":3497},{"className":3496,"ariaHidden":167},[166],[3498],{"type":16,"tag":103,"props":3499,"children":3501},{"className":3500},[172],[3502,3507,3513],{"type":16,"tag":103,"props":3503,"children":3506},{"className":3504,"style":3505},[177],"height:1.1072em;vertical-align:-0.25em;",[],{"type":16,"tag":103,"props":3508,"children":3510},{"className":3509},[183],[3511],{"type":22,"value":3512},"1\u002F",{"type":16,"tag":103,"props":3514,"children":3516},{"className":3515},[183,2268],[3517],{"type":16,"tag":103,"props":3518,"children":3520},{"className":3519},[199,200],[3521,3620],{"type":16,"tag":103,"props":3522,"children":3524},{"className":3523},[205],[3525,3615],{"type":16,"tag":103,"props":3526,"children":3528},{"className":3527,"style":2281},[210],[3529,3598],{"type":16,"tag":103,"props":3530,"children":3532},{"className":3531,"style":2287},[2286],[3533,3537],{"type":16,"tag":103,"props":3534,"children":3536},{"className":3535,"style":2292},[220],[],{"type":16,"tag":103,"props":3538,"children":3540},{"className":3539,"style":2297},[183],[3541],{"type":16,"tag":103,"props":3542,"children":3544},{"className":3543},[183],[3545,3550],{"type":16,"tag":103,"props":3546,"children":3548},{"className":3547},[183,188],[3549],{"type":22,"value":1650},{"type":16,"tag":103,"props":3551,"children":3553},{"className":3552},[194],[3554],{"type":16,"tag":103,"props":3555,"children":3557},{"className":3556},[199,200],[3558,3587],{"type":16,"tag":103,"props":3559,"children":3561},{"className":3560},[205],[3562,3582],{"type":16,"tag":103,"props":3563,"children":3565},{"className":3564,"style":1727},[210],[3566],{"type":16,"tag":103,"props":3567,"children":3568},{"style":215},[3569,3573],{"type":16,"tag":103,"props":3570,"children":3572},{"className":3571,"style":221},[220],[],{"type":16,"tag":103,"props":3574,"children":3576},{"className":3575},[226,227,228,229],[3577],{"type":16,"tag":103,"props":3578,"children":3580},{"className":3579,"style":2089},[183,188,229],[3581],{"type":22,"value":2032},{"type":16,"tag":103,"props":3583,"children":3585},{"className":3584},[254],[3586],{"type":22,"value":257},{"type":16,"tag":103,"props":3588,"children":3590},{"className":3589},[205],[3591],{"type":16,"tag":103,"props":3592,"children":3594},{"className":3593,"style":417},[210],[3595],{"type":16,"tag":103,"props":3596,"children":3597},{},[],{"type":16,"tag":103,"props":3599,"children":3600},{"style":2358},[3601,3605],{"type":16,"tag":103,"props":3602,"children":3604},{"className":3603,"style":2292},[220],[],{"type":16,"tag":103,"props":3606,"children":3608},{"className":3607,"style":2368},[2367],[3609],{"type":16,"tag":2371,"props":3610,"children":3611},{"xmlns":2373,"width":2374,"height":2375,"viewBox":2376,"preserveAspectRatio":2377},[3612],{"type":16,"tag":2380,"props":3613,"children":3614},{"d":2382},[],{"type":16,"tag":103,"props":3616,"children":3618},{"className":3617},[254],[3619],{"type":22,"value":257},{"type":16,"tag":103,"props":3621,"children":3623},{"className":3622},[205],[3624],{"type":16,"tag":103,"props":3625,"children":3627},{"className":3626,"style":2396},[210],[3628],{"type":16,"tag":103,"props":3629,"children":3630},{},[],{"type":22,"value":3632}," 对点积进行缩放。",{"type":16,"tag":25,"props":3634,"children":3635},{},[3636,3638,3682,3683,3726,3728,4050,4052,4153],{"type":22,"value":3637},"为了说明点积为什么会变大，假设 ",{"type":16,"tag":103,"props":3639,"children":3641},{"className":3640},[106],[3642,3664],{"type":16,"tag":103,"props":3643,"children":3645},{"className":3644},[111],[3646],{"type":16,"tag":114,"props":3647,"children":3648},{"xmlns":116},[3649],{"type":16,"tag":119,"props":3650,"children":3651},{},[3652,3660],{"type":16,"tag":123,"props":3653,"children":3654},{},[3655],{"type":16,"tag":131,"props":3656,"children":3657},{},[3658],{"type":22,"value":3659},"q",{"type":16,"tag":157,"props":3661,"children":3662},{"encoding":159},[3663],{"type":22,"value":3659},{"type":16,"tag":103,"props":3665,"children":3667},{"className":3666,"ariaHidden":167},[166],[3668],{"type":16,"tag":103,"props":3669,"children":3671},{"className":3670},[172],[3672,3677],{"type":16,"tag":103,"props":3673,"children":3676},{"className":3674,"style":3675},[177],"height:0.625em;vertical-align:-0.1944em;",[],{"type":16,"tag":103,"props":3678,"children":3680},{"className":3679,"style":1124},[183,188],[3681],{"type":22,"value":3659},{"type":22,"value":2509},{"type":16,"tag":103,"props":3684,"children":3686},{"className":3685},[106],[3687,3708],{"type":16,"tag":103,"props":3688,"children":3690},{"className":3689},[111],[3691],{"type":16,"tag":114,"props":3692,"children":3693},{"xmlns":116},[3694],{"type":16,"tag":119,"props":3695,"children":3696},{},[3697,3704],{"type":16,"tag":123,"props":3698,"children":3699},{},[3700],{"type":16,"tag":131,"props":3701,"children":3702},{},[3703],{"type":22,"value":2032},{"type":16,"tag":157,"props":3705,"children":3706},{"encoding":159},[3707],{"type":22,"value":2032},{"type":16,"tag":103,"props":3709,"children":3711},{"className":3710,"ariaHidden":167},[166],[3712],{"type":16,"tag":103,"props":3713,"children":3715},{"className":3714},[172],[3716,3721],{"type":16,"tag":103,"props":3717,"children":3720},{"className":3718,"style":3719},[177],"height:0.6944em;",[],{"type":16,"tag":103,"props":3722,"children":3724},{"className":3723,"style":2089},[183,188],[3725],{"type":22,"value":2032},{"type":22,"value":3727}," 的各个分量相互独立，均值为 0、方差为 1。那么点积 ",{"type":16,"tag":103,"props":3729,"children":3731},{"className":3730},[106],[3732,3801],{"type":16,"tag":103,"props":3733,"children":3735},{"className":3734},[111],[3736],{"type":16,"tag":114,"props":3737,"children":3738},{"xmlns":116},[3739],{"type":16,"tag":119,"props":3740,"children":3741},{},[3742,3796],{"type":16,"tag":123,"props":3743,"children":3744},{},[3745,3749,3754,3758,3762,3774,3785],{"type":16,"tag":131,"props":3746,"children":3747},{},[3748],{"type":22,"value":3659},{"type":16,"tag":145,"props":3750,"children":3751},{},[3752],{"type":22,"value":3753},"⋅",{"type":16,"tag":131,"props":3755,"children":3756},{},[3757],{"type":22,"value":2032},{"type":16,"tag":145,"props":3759,"children":3760},{},[3761],{"type":22,"value":746},{"type":16,"tag":127,"props":3763,"children":3764},{},[3765,3770],{"type":16,"tag":145,"props":3766,"children":3767},{},[3768],{"type":22,"value":3769},"∑",{"type":16,"tag":131,"props":3771,"children":3772},{},[3773],{"type":22,"value":1912},{"type":16,"tag":127,"props":3775,"children":3776},{},[3777,3781],{"type":16,"tag":131,"props":3778,"children":3779},{},[3780],{"type":22,"value":3659},{"type":16,"tag":131,"props":3782,"children":3783},{},[3784],{"type":22,"value":1912},{"type":16,"tag":127,"props":3786,"children":3787},{},[3788,3792],{"type":16,"tag":131,"props":3789,"children":3790},{},[3791],{"type":22,"value":2032},{"type":16,"tag":131,"props":3793,"children":3794},{},[3795],{"type":22,"value":1912},{"type":16,"tag":157,"props":3797,"children":3798},{"encoding":159},[3799],{"type":22,"value":3800},"q \\cdot k = \\sum_i q_i k_i",{"type":16,"tag":103,"props":3802,"children":3804},{"className":3803,"ariaHidden":167},[166],[3805,3832,3858],{"type":16,"tag":103,"props":3806,"children":3808},{"className":3807},[172],[3809,3814,3819,3823,3828],{"type":16,"tag":103,"props":3810,"children":3813},{"className":3811,"style":3812},[177],"height:0.6389em;vertical-align:-0.1944em;",[],{"type":16,"tag":103,"props":3815,"children":3817},{"className":3816,"style":1124},[183,188],[3818],{"type":22,"value":3659},{"type":16,"tag":103,"props":3820,"children":3822},{"className":3821,"style":1497},[632],[],{"type":16,"tag":103,"props":3824,"children":3826},{"className":3825},[243],[3827],{"type":22,"value":3753},{"type":16,"tag":103,"props":3829,"children":3831},{"className":3830,"style":1497},[632],[],{"type":16,"tag":103,"props":3833,"children":3835},{"className":3834},[172],[3836,3840,3845,3849,3854],{"type":16,"tag":103,"props":3837,"children":3839},{"className":3838,"style":3719},[177],[],{"type":16,"tag":103,"props":3841,"children":3843},{"className":3842,"style":2089},[183,188],[3844],{"type":22,"value":2032},{"type":16,"tag":103,"props":3846,"children":3848},{"className":3847,"style":816},[632],[],{"type":16,"tag":103,"props":3850,"children":3852},{"className":3851},[821],[3853],{"type":22,"value":746},{"type":16,"tag":103,"props":3855,"children":3857},{"className":3856,"style":816},[632],[],{"type":16,"tag":103,"props":3859,"children":3861},{"className":3860},[172],[3862,3867,3930,3934,3992],{"type":16,"tag":103,"props":3863,"children":3866},{"className":3864,"style":3865},[177],"height:1.0497em;vertical-align:-0.2997em;",[],{"type":16,"tag":103,"props":3868,"children":3870},{"className":3869},[1476],[3871,3879],{"type":16,"tag":103,"props":3872,"children":3877},{"className":3873,"style":3876},[1476,3874,3875],"op-symbol","small-op","position:relative;top:0em;",[3878],{"type":22,"value":3769},{"type":16,"tag":103,"props":3880,"children":3882},{"className":3881},[194],[3883],{"type":16,"tag":103,"props":3884,"children":3886},{"className":3885},[199,200],[3887,3918],{"type":16,"tag":103,"props":3888,"children":3890},{"className":3889},[205],[3891,3913],{"type":16,"tag":103,"props":3892,"children":3895},{"className":3893,"style":3894},[210],"height:0.162em;",[3896],{"type":16,"tag":103,"props":3897,"children":3899},{"style":3898},"top:-2.4003em;margin-left:0em;margin-right:0.05em;",[3900,3904],{"type":16,"tag":103,"props":3901,"children":3903},{"className":3902,"style":221},[220],[],{"type":16,"tag":103,"props":3905,"children":3907},{"className":3906},[226,227,228,229],[3908],{"type":16,"tag":103,"props":3909,"children":3911},{"className":3910},[183,188,229],[3912],{"type":22,"value":1912},{"type":16,"tag":103,"props":3914,"children":3916},{"className":3915},[254],[3917],{"type":22,"value":257},{"type":16,"tag":103,"props":3919,"children":3921},{"className":3920},[205],[3922],{"type":16,"tag":103,"props":3923,"children":3926},{"className":3924,"style":3925},[210],"height:0.2997em;",[3927],{"type":16,"tag":103,"props":3928,"children":3929},{},[],{"type":16,"tag":103,"props":3931,"children":3933},{"className":3932,"style":633},[632],[],{"type":16,"tag":103,"props":3935,"children":3937},{"className":3936},[183],[3938,3943],{"type":16,"tag":103,"props":3939,"children":3941},{"className":3940,"style":1124},[183,188],[3942],{"type":22,"value":3659},{"type":16,"tag":103,"props":3944,"children":3946},{"className":3945},[194],[3947],{"type":16,"tag":103,"props":3948,"children":3950},{"className":3949},[199,200],[3951,3981],{"type":16,"tag":103,"props":3952,"children":3954},{"className":3953},[205],[3955,3976],{"type":16,"tag":103,"props":3956,"children":3959},{"className":3957,"style":3958},[210],"height:0.3117em;",[3960],{"type":16,"tag":103,"props":3961,"children":3962},{"style":1145},[3963,3967],{"type":16,"tag":103,"props":3964,"children":3966},{"className":3965,"style":221},[220],[],{"type":16,"tag":103,"props":3968,"children":3970},{"className":3969},[226,227,228,229],[3971],{"type":16,"tag":103,"props":3972,"children":3974},{"className":3973},[183,188,229],[3975],{"type":22,"value":1912},{"type":16,"tag":103,"props":3977,"children":3979},{"className":3978},[254],[3980],{"type":22,"value":257},{"type":16,"tag":103,"props":3982,"children":3984},{"className":3983},[205],[3985],{"type":16,"tag":103,"props":3986,"children":3988},{"className":3987,"style":417},[210],[3989],{"type":16,"tag":103,"props":3990,"children":3991},{},[],{"type":16,"tag":103,"props":3993,"children":3995},{"className":3994},[183],[3996,4001],{"type":16,"tag":103,"props":3997,"children":3999},{"className":3998,"style":2089},[183,188],[4000],{"type":22,"value":2032},{"type":16,"tag":103,"props":4002,"children":4004},{"className":4003},[194],[4005],{"type":16,"tag":103,"props":4006,"children":4008},{"className":4007},[199,200],[4009,4039],{"type":16,"tag":103,"props":4010,"children":4012},{"className":4011},[205],[4013,4034],{"type":16,"tag":103,"props":4014,"children":4016},{"className":4015,"style":3958},[210],[4017],{"type":16,"tag":103,"props":4018,"children":4020},{"style":4019},"top:-2.55em;margin-left:-0.0315em;margin-right:0.05em;",[4021,4025],{"type":16,"tag":103,"props":4022,"children":4024},{"className":4023,"style":221},[220],[],{"type":16,"tag":103,"props":4026,"children":4028},{"className":4027},[226,227,228,229],[4029],{"type":16,"tag":103,"props":4030,"children":4032},{"className":4031},[183,188,229],[4033],{"type":22,"value":1912},{"type":16,"tag":103,"props":4035,"children":4037},{"className":4036},[254],[4038],{"type":22,"value":257},{"type":16,"tag":103,"props":4040,"children":4042},{"className":4041},[205],[4043],{"type":16,"tag":103,"props":4044,"children":4046},{"className":4045,"style":417},[210],[4047],{"type":16,"tag":103,"props":4048,"children":4049},{},[],{"type":22,"value":4051}," 的均值为 0，方差为 ",{"type":16,"tag":103,"props":4053,"children":4055},{"className":4054},[106],[4056,4084],{"type":16,"tag":103,"props":4057,"children":4059},{"className":4058},[111],[4060],{"type":16,"tag":114,"props":4061,"children":4062},{"xmlns":116},[4063],{"type":16,"tag":119,"props":4064,"children":4065},{},[4066,4080],{"type":16,"tag":123,"props":4067,"children":4068},{},[4069],{"type":16,"tag":127,"props":4070,"children":4071},{},[4072,4076],{"type":16,"tag":131,"props":4073,"children":4074},{},[4075],{"type":22,"value":1650},{"type":16,"tag":131,"props":4077,"children":4078},{},[4079],{"type":22,"value":2032},{"type":16,"tag":157,"props":4081,"children":4082},{"encoding":159},[4083],{"type":22,"value":2037},{"type":16,"tag":103,"props":4085,"children":4087},{"className":4086,"ariaHidden":167},[166],[4088],{"type":16,"tag":103,"props":4089,"children":4091},{"className":4090},[172],[4092,4096],{"type":16,"tag":103,"props":4093,"children":4095},{"className":4094,"style":361},[177],[],{"type":16,"tag":103,"props":4097,"children":4099},{"className":4098},[183],[4100,4105],{"type":16,"tag":103,"props":4101,"children":4103},{"className":4102},[183,188],[4104],{"type":22,"value":1650},{"type":16,"tag":103,"props":4106,"children":4108},{"className":4107},[194],[4109],{"type":16,"tag":103,"props":4110,"children":4112},{"className":4111},[199,200],[4113,4142],{"type":16,"tag":103,"props":4114,"children":4116},{"className":4115},[205],[4117,4137],{"type":16,"tag":103,"props":4118,"children":4120},{"className":4119,"style":1727},[210],[4121],{"type":16,"tag":103,"props":4122,"children":4123},{"style":215},[4124,4128],{"type":16,"tag":103,"props":4125,"children":4127},{"className":4126,"style":221},[220],[],{"type":16,"tag":103,"props":4129,"children":4131},{"className":4130},[226,227,228,229],[4132],{"type":16,"tag":103,"props":4133,"children":4135},{"className":4134,"style":2089},[183,188,229],[4136],{"type":22,"value":2032},{"type":16,"tag":103,"props":4138,"children":4140},{"className":4139},[254],[4141],{"type":22,"value":257},{"type":16,"tag":103,"props":4143,"children":4145},{"className":4144},[205],[4146],{"type":16,"tag":103,"props":4147,"children":4149},{"className":4148,"style":417},[210],[4150],{"type":16,"tag":103,"props":4151,"children":4152},{},[],{"type":22,"value":1266},{"type":16,"tag":1993,"props":4155,"children":4157},{"id":4156},"_322-多头注意力",[4158],{"type":22,"value":4159},"3.2.2 多头注意力",{"type":16,"tag":25,"props":4161,"children":4162},{},[4163,4165,4294,4296,4338,4340,4441,4443,4544,4545,4646,4648,4749],{"type":22,"value":4164},"我们发现，与其对维度为 ",{"type":16,"tag":103,"props":4166,"children":4168},{"className":4167},[106],[4169,4217],{"type":16,"tag":103,"props":4170,"children":4172},{"className":4171},[111],[4173],{"type":16,"tag":114,"props":4174,"children":4175},{"xmlns":116},[4176],{"type":16,"tag":119,"props":4177,"children":4178},{},[4179,4212],{"type":16,"tag":123,"props":4180,"children":4181},{},[4182],{"type":16,"tag":127,"props":4183,"children":4184},{},[4185,4189],{"type":16,"tag":131,"props":4186,"children":4187},{},[4188],{"type":22,"value":1650},{"type":16,"tag":123,"props":4190,"children":4191},{},[4192,4196,4200,4204,4208],{"type":16,"tag":131,"props":4193,"children":4194},{"mathvariant":1409},[4195],{"type":22,"value":1090},{"type":16,"tag":131,"props":4197,"children":4198},{"mathvariant":1409},[4199],{"type":22,"value":1662},{"type":16,"tag":131,"props":4201,"children":4202},{"mathvariant":1409},[4203],{"type":22,"value":1650},{"type":16,"tag":131,"props":4205,"children":4206},{"mathvariant":1409},[4207],{"type":22,"value":1671},{"type":16,"tag":131,"props":4209,"children":4210},{"mathvariant":1409},[4211],{"type":22,"value":1676},{"type":16,"tag":157,"props":4213,"children":4214},{"encoding":159},[4215],{"type":22,"value":4216},"d_{\\mathrm{model}}",{"type":16,"tag":103,"props":4218,"children":4220},{"className":4219,"ariaHidden":167},[166],[4221],{"type":16,"tag":103,"props":4222,"children":4224},{"className":4223},[172],[4225,4229],{"type":16,"tag":103,"props":4226,"children":4228},{"className":4227,"style":361},[177],[],{"type":16,"tag":103,"props":4230,"children":4232},{"className":4231},[183],[4233,4238],{"type":16,"tag":103,"props":4234,"children":4236},{"className":4235},[183,188],[4237],{"type":22,"value":1650},{"type":16,"tag":103,"props":4239,"children":4241},{"className":4240},[194],[4242],{"type":16,"tag":103,"props":4243,"children":4245},{"className":4244},[199,200],[4246,4283],{"type":16,"tag":103,"props":4247,"children":4249},{"className":4248},[205],[4250,4278],{"type":16,"tag":103,"props":4251,"children":4253},{"className":4252,"style":1727},[210],[4254],{"type":16,"tag":103,"props":4255,"children":4256},{"style":215},[4257,4261],{"type":16,"tag":103,"props":4258,"children":4260},{"className":4259,"style":221},[220],[],{"type":16,"tag":103,"props":4262,"children":4264},{"className":4263},[226,227,228,229],[4265],{"type":16,"tag":103,"props":4266,"children":4268},{"className":4267},[183,229],[4269],{"type":16,"tag":103,"props":4270,"children":4272},{"className":4271},[183,229],[4273],{"type":16,"tag":103,"props":4274,"children":4276},{"className":4275},[183,1481,229],[4277],{"type":22,"value":1753},{"type":16,"tag":103,"props":4279,"children":4281},{"className":4280},[254],[4282],{"type":22,"value":257},{"type":16,"tag":103,"props":4284,"children":4286},{"className":4285},[205],[4287],{"type":16,"tag":103,"props":4288,"children":4290},{"className":4289,"style":417},[210],[4291],{"type":16,"tag":103,"props":4292,"children":4293},{},[],{"type":22,"value":4295}," 的键、值和查询执行一次注意力函数，不如先使用不同的、可学习的线性投影，把查询、键和值分别投影 ",{"type":16,"tag":103,"props":4297,"children":4299},{"className":4298},[106],[4300,4321],{"type":16,"tag":103,"props":4301,"children":4303},{"className":4302},[111],[4304],{"type":16,"tag":114,"props":4305,"children":4306},{"xmlns":116},[4307],{"type":16,"tag":119,"props":4308,"children":4309},{},[4310,4317],{"type":16,"tag":123,"props":4311,"children":4312},{},[4313],{"type":16,"tag":131,"props":4314,"children":4315},{},[4316],{"type":22,"value":135},{"type":16,"tag":157,"props":4318,"children":4319},{"encoding":159},[4320],{"type":22,"value":135},{"type":16,"tag":103,"props":4322,"children":4324},{"className":4323,"ariaHidden":167},[166],[4325],{"type":16,"tag":103,"props":4326,"children":4328},{"className":4327},[172],[4329,4333],{"type":16,"tag":103,"props":4330,"children":4332},{"className":4331,"style":3719},[177],[],{"type":16,"tag":103,"props":4334,"children":4336},{"className":4335},[183,188],[4337],{"type":22,"value":135},{"type":22,"value":4339}," 次，映射到维度 ",{"type":16,"tag":103,"props":4341,"children":4343},{"className":4342},[106],[4344,4372],{"type":16,"tag":103,"props":4345,"children":4347},{"className":4346},[111],[4348],{"type":16,"tag":114,"props":4349,"children":4350},{"xmlns":116},[4351],{"type":16,"tag":119,"props":4352,"children":4353},{},[4354,4368],{"type":16,"tag":123,"props":4355,"children":4356},{},[4357],{"type":16,"tag":127,"props":4358,"children":4359},{},[4360,4364],{"type":16,"tag":131,"props":4361,"children":4362},{},[4363],{"type":22,"value":1650},{"type":16,"tag":131,"props":4365,"children":4366},{},[4367],{"type":22,"value":2032},{"type":16,"tag":157,"props":4369,"children":4370},{"encoding":159},[4371],{"type":22,"value":2037},{"type":16,"tag":103,"props":4373,"children":4375},{"className":4374,"ariaHidden":167},[166],[4376],{"type":16,"tag":103,"props":4377,"children":4379},{"className":4378},[172],[4380,4384],{"type":16,"tag":103,"props":4381,"children":4383},{"className":4382,"style":361},[177],[],{"type":16,"tag":103,"props":4385,"children":4387},{"className":4386},[183],[4388,4393],{"type":16,"tag":103,"props":4389,"children":4391},{"className":4390},[183,188],[4392],{"type":22,"value":1650},{"type":16,"tag":103,"props":4394,"children":4396},{"className":4395},[194],[4397],{"type":16,"tag":103,"props":4398,"children":4400},{"className":4399},[199,200],[4401,4430],{"type":16,"tag":103,"props":4402,"children":4404},{"className":4403},[205],[4405,4425],{"type":16,"tag":103,"props":4406,"children":4408},{"className":4407,"style":1727},[210],[4409],{"type":16,"tag":103,"props":4410,"children":4411},{"style":215},[4412,4416],{"type":16,"tag":103,"props":4413,"children":4415},{"className":4414,"style":221},[220],[],{"type":16,"tag":103,"props":4417,"children":4419},{"className":4418},[226,227,228,229],[4420],{"type":16,"tag":103,"props":4421,"children":4423},{"className":4422,"style":2089},[183,188,229],[4424],{"type":22,"value":2032},{"type":16,"tag":103,"props":4426,"children":4428},{"className":4427},[254],[4429],{"type":22,"value":257},{"type":16,"tag":103,"props":4431,"children":4433},{"className":4432},[205],[4434],{"type":16,"tag":103,"props":4435,"children":4437},{"className":4436,"style":417},[210],[4438],{"type":16,"tag":103,"props":4439,"children":4440},{},[],{"type":22,"value":4442},"、",{"type":16,"tag":103,"props":4444,"children":4446},{"className":4445},[106],[4447,4475],{"type":16,"tag":103,"props":4448,"children":4450},{"className":4449},[111],[4451],{"type":16,"tag":114,"props":4452,"children":4453},{"xmlns":116},[4454],{"type":16,"tag":119,"props":4455,"children":4456},{},[4457,4471],{"type":16,"tag":123,"props":4458,"children":4459},{},[4460],{"type":16,"tag":127,"props":4461,"children":4462},{},[4463,4467],{"type":16,"tag":131,"props":4464,"children":4465},{},[4466],{"type":22,"value":1650},{"type":16,"tag":131,"props":4468,"children":4469},{},[4470],{"type":22,"value":2032},{"type":16,"tag":157,"props":4472,"children":4473},{"encoding":159},[4474],{"type":22,"value":2037},{"type":16,"tag":103,"props":4476,"children":4478},{"className":4477,"ariaHidden":167},[166],[4479],{"type":16,"tag":103,"props":4480,"children":4482},{"className":4481},[172],[4483,4487],{"type":16,"tag":103,"props":4484,"children":4486},{"className":4485,"style":361},[177],[],{"type":16,"tag":103,"props":4488,"children":4490},{"className":4489},[183],[4491,4496],{"type":16,"tag":103,"props":4492,"children":4494},{"className":4493},[183,188],[4495],{"type":22,"value":1650},{"type":16,"tag":103,"props":4497,"children":4499},{"className":4498},[194],[4500],{"type":16,"tag":103,"props":4501,"children":4503},{"className":4502},[199,200],[4504,4533],{"type":16,"tag":103,"props":4505,"children":4507},{"className":4506},[205],[4508,4528],{"type":16,"tag":103,"props":4509,"children":4511},{"className":4510,"style":1727},[210],[4512],{"type":16,"tag":103,"props":4513,"children":4514},{"style":215},[4515,4519],{"type":16,"tag":103,"props":4516,"children":4518},{"className":4517,"style":221},[220],[],{"type":16,"tag":103,"props":4520,"children":4522},{"className":4521},[226,227,228,229],[4523],{"type":16,"tag":103,"props":4524,"children":4526},{"className":4525,"style":2089},[183,188,229],[4527],{"type":22,"value":2032},{"type":16,"tag":103,"props":4529,"children":4531},{"className":4530},[254],[4532],{"type":22,"value":257},{"type":16,"tag":103,"props":4534,"children":4536},{"className":4535},[205],[4537],{"type":16,"tag":103,"props":4538,"children":4540},{"className":4539,"style":417},[210],[4541],{"type":16,"tag":103,"props":4542,"children":4543},{},[],{"type":22,"value":2509},{"type":16,"tag":103,"props":4546,"children":4548},{"className":4547},[106],[4549,4577],{"type":16,"tag":103,"props":4550,"children":4552},{"className":4551},[111],[4553],{"type":16,"tag":114,"props":4554,"children":4555},{"xmlns":116},[4556],{"type":16,"tag":119,"props":4557,"children":4558},{},[4559,4573],{"type":16,"tag":123,"props":4560,"children":4561},{},[4562],{"type":16,"tag":127,"props":4563,"children":4564},{},[4565,4569],{"type":16,"tag":131,"props":4566,"children":4567},{},[4568],{"type":22,"value":1650},{"type":16,"tag":131,"props":4570,"children":4571},{},[4572],{"type":22,"value":2138},{"type":16,"tag":157,"props":4574,"children":4575},{"encoding":159},[4576],{"type":22,"value":2143},{"type":16,"tag":103,"props":4578,"children":4580},{"className":4579,"ariaHidden":167},[166],[4581],{"type":16,"tag":103,"props":4582,"children":4584},{"className":4583},[172],[4585,4589],{"type":16,"tag":103,"props":4586,"children":4588},{"className":4587,"style":361},[177],[],{"type":16,"tag":103,"props":4590,"children":4592},{"className":4591},[183],[4593,4598],{"type":16,"tag":103,"props":4594,"children":4596},{"className":4595},[183,188],[4597],{"type":22,"value":1650},{"type":16,"tag":103,"props":4599,"children":4601},{"className":4600},[194],[4602],{"type":16,"tag":103,"props":4603,"children":4605},{"className":4604},[199,200],[4606,4635],{"type":16,"tag":103,"props":4607,"children":4609},{"className":4608},[205],[4610,4630],{"type":16,"tag":103,"props":4611,"children":4613},{"className":4612,"style":678},[210],[4614],{"type":16,"tag":103,"props":4615,"children":4616},{"style":215},[4617,4621],{"type":16,"tag":103,"props":4618,"children":4620},{"className":4619,"style":221},[220],[],{"type":16,"tag":103,"props":4622,"children":4624},{"className":4623},[226,227,228,229],[4625],{"type":16,"tag":103,"props":4626,"children":4628},{"className":4627,"style":1124},[183,188,229],[4629],{"type":22,"value":2138},{"type":16,"tag":103,"props":4631,"children":4633},{"className":4632},[254],[4634],{"type":22,"value":257},{"type":16,"tag":103,"props":4636,"children":4638},{"className":4637},[205],[4639],{"type":16,"tag":103,"props":4640,"children":4642},{"className":4641,"style":417},[210],[4643],{"type":16,"tag":103,"props":4644,"children":4645},{},[],{"type":22,"value":4647},"，然后在每组投影后的查询、键和值上并行执行注意力函数，得到维度为 ",{"type":16,"tag":103,"props":4649,"children":4651},{"className":4650},[106],[4652,4680],{"type":16,"tag":103,"props":4653,"children":4655},{"className":4654},[111],[4656],{"type":16,"tag":114,"props":4657,"children":4658},{"xmlns":116},[4659],{"type":16,"tag":119,"props":4660,"children":4661},{},[4662,4676],{"type":16,"tag":123,"props":4663,"children":4664},{},[4665],{"type":16,"tag":127,"props":4666,"children":4667},{},[4668,4672],{"type":16,"tag":131,"props":4669,"children":4670},{},[4671],{"type":22,"value":1650},{"type":16,"tag":131,"props":4673,"children":4674},{},[4675],{"type":22,"value":2138},{"type":16,"tag":157,"props":4677,"children":4678},{"encoding":159},[4679],{"type":22,"value":2143},{"type":16,"tag":103,"props":4681,"children":4683},{"className":4682,"ariaHidden":167},[166],[4684],{"type":16,"tag":103,"props":4685,"children":4687},{"className":4686},[172],[4688,4692],{"type":16,"tag":103,"props":4689,"children":4691},{"className":4690,"style":361},[177],[],{"type":16,"tag":103,"props":4693,"children":4695},{"className":4694},[183],[4696,4701],{"type":16,"tag":103,"props":4697,"children":4699},{"className":4698},[183,188],[4700],{"type":22,"value":1650},{"type":16,"tag":103,"props":4702,"children":4704},{"className":4703},[194],[4705],{"type":16,"tag":103,"props":4706,"children":4708},{"className":4707},[199,200],[4709,4738],{"type":16,"tag":103,"props":4710,"children":4712},{"className":4711},[205],[4713,4733],{"type":16,"tag":103,"props":4714,"children":4716},{"className":4715,"style":678},[210],[4717],{"type":16,"tag":103,"props":4718,"children":4719},{"style":215},[4720,4724],{"type":16,"tag":103,"props":4721,"children":4723},{"className":4722,"style":221},[220],[],{"type":16,"tag":103,"props":4725,"children":4727},{"className":4726},[226,227,228,229],[4728],{"type":16,"tag":103,"props":4729,"children":4731},{"className":4730,"style":1124},[183,188,229],[4732],{"type":22,"value":2138},{"type":16,"tag":103,"props":4734,"children":4736},{"className":4735},[254],[4737],{"type":22,"value":257},{"type":16,"tag":103,"props":4739,"children":4741},{"className":4740},[205],[4742],{"type":16,"tag":103,"props":4743,"children":4745},{"className":4744,"style":417},[210],[4746],{"type":16,"tag":103,"props":4747,"children":4748},{},[],{"type":22,"value":4750}," 的输出。最后把这些输出拼接起来，再进行一次线性投影，得到最终结果，如图 2 所示。",{"type":16,"tag":25,"props":4752,"children":4753},{},[4754],{"type":22,"value":4755},"多头注意力使模型可以在不同位置、联合关注不同表示子空间中的信息。只有一个注意力头时，平均操作会抑制这种能力。",{"type":16,"tag":103,"props":4757,"children":4759},{"className":4758},[1387],[4760],{"type":16,"tag":103,"props":4761,"children":4763},{"className":4762},[106],[4764,4902],{"type":16,"tag":103,"props":4765,"children":4767},{"className":4766},[111],[4768],{"type":16,"tag":114,"props":4769,"children":4770},{"xmlns":116,"display":1399},[4771],{"type":16,"tag":119,"props":4772,"children":4773},{},[4774,4897],{"type":16,"tag":123,"props":4775,"children":4776},{},[4777,4782,4786,4790,4794,4798,4802,4806,4810,4814,4818,4823,4827,4831,4850,4854,4858,4862,4880,4884],{"type":16,"tag":131,"props":4778,"children":4779},{"mathvariant":1409},[4780],{"type":22,"value":4781},"MultiHead",{"type":16,"tag":145,"props":4783,"children":4784},{},[4785],{"type":22,"value":1417},{"type":16,"tag":145,"props":4787,"children":4788},{"stretchy":495},[4789],{"type":22,"value":498},{"type":16,"tag":131,"props":4791,"children":4792},{},[4793],{"type":22,"value":2439},{"type":16,"tag":145,"props":4795,"children":4796},{"separator":167},[4797],{"type":22,"value":515},{"type":16,"tag":131,"props":4799,"children":4800},{},[4801],{"type":22,"value":2485},{"type":16,"tag":145,"props":4803,"children":4804},{"separator":167},[4805],{"type":22,"value":515},{"type":16,"tag":131,"props":4807,"children":4808},{},[4809],{"type":22,"value":2531},{"type":16,"tag":145,"props":4811,"children":4812},{"stretchy":495},[4813],{"type":22,"value":541},{"type":16,"tag":145,"props":4815,"children":4816},{},[4817],{"type":22,"value":746},{"type":16,"tag":131,"props":4819,"children":4820},{"mathvariant":1409},[4821],{"type":22,"value":4822},"Concat",{"type":16,"tag":145,"props":4824,"children":4825},{},[4826],{"type":22,"value":1417},{"type":16,"tag":145,"props":4828,"children":4829},{"stretchy":495},[4830],{"type":22,"value":498},{"type":16,"tag":127,"props":4832,"children":4833},{},[4834,4846],{"type":16,"tag":123,"props":4835,"children":4836},{},[4837,4842],{"type":16,"tag":131,"props":4838,"children":4839},{"mathvariant":1409},[4840],{"type":22,"value":4841},"head",{"type":16,"tag":145,"props":4843,"children":4844},{},[4845],{"type":22,"value":1417},{"type":16,"tag":151,"props":4847,"children":4848},{},[4849],{"type":22,"value":155},{"type":16,"tag":145,"props":4851,"children":4852},{"separator":167},[4853],{"type":22,"value":515},{"type":16,"tag":145,"props":4855,"children":4856},{},[4857],{"type":22,"value":520},{"type":16,"tag":145,"props":4859,"children":4860},{"separator":167},[4861],{"type":22,"value":515},{"type":16,"tag":127,"props":4863,"children":4864},{},[4865,4876],{"type":16,"tag":123,"props":4866,"children":4867},{},[4868,4872],{"type":16,"tag":131,"props":4869,"children":4870},{"mathvariant":1409},[4871],{"type":22,"value":4841},{"type":16,"tag":145,"props":4873,"children":4874},{},[4875],{"type":22,"value":1417},{"type":16,"tag":131,"props":4877,"children":4878},{},[4879],{"type":22,"value":135},{"type":16,"tag":145,"props":4881,"children":4882},{"stretchy":495},[4883],{"type":22,"value":541},{"type":16,"tag":2662,"props":4885,"children":4886},{},[4887,4892],{"type":16,"tag":131,"props":4888,"children":4889},{},[4890],{"type":22,"value":4891},"W",{"type":16,"tag":131,"props":4893,"children":4894},{},[4895],{"type":22,"value":4896},"O",{"type":16,"tag":157,"props":4898,"children":4899},{"encoding":159},[4900],{"type":22,"value":4901},"\\operatorname{MultiHead}(Q, K, V) = \\operatorname{Concat}(\\operatorname{head}_1, \\ldots, \\operatorname{head}_h)W^O",{"type":16,"tag":103,"props":4903,"children":4905},{"className":4904,"ariaHidden":167},[166],[4906,4979],{"type":16,"tag":103,"props":4907,"children":4909},{"className":4908},[172],[4910,4914,4923,4928,4933,4938,4942,4947,4952,4956,4961,4966,4970,4975],{"type":16,"tag":103,"props":4911,"children":4913},{"className":4912,"style":558},[177],[],{"type":16,"tag":103,"props":4915,"children":4917},{"className":4916},[1476],[4918],{"type":16,"tag":103,"props":4919,"children":4921},{"className":4920},[183,1481],[4922],{"type":22,"value":4781},{"type":16,"tag":103,"props":4924,"children":4926},{"className":4925},[563],[4927],{"type":22,"value":498},{"type":16,"tag":103,"props":4929,"children":4931},{"className":4930},[183,188],[4932],{"type":22,"value":2439},{"type":16,"tag":103,"props":4934,"children":4936},{"className":4935},[626],[4937],{"type":22,"value":515},{"type":16,"tag":103,"props":4939,"children":4941},{"className":4940,"style":633},[632],[],{"type":16,"tag":103,"props":4943,"children":4945},{"className":4944,"style":2505},[183,188],[4946],{"type":22,"value":2485},{"type":16,"tag":103,"props":4948,"children":4950},{"className":4949},[626],[4951],{"type":22,"value":515},{"type":16,"tag":103,"props":4953,"children":4955},{"className":4954,"style":633},[632],[],{"type":16,"tag":103,"props":4957,"children":4959},{"className":4958,"style":1497},[183,188],[4960],{"type":22,"value":2531},{"type":16,"tag":103,"props":4962,"children":4964},{"className":4963},[715],[4965],{"type":22,"value":541},{"type":16,"tag":103,"props":4967,"children":4969},{"className":4968,"style":816},[632],[],{"type":16,"tag":103,"props":4971,"children":4973},{"className":4972},[821],[4974],{"type":22,"value":746},{"type":16,"tag":103,"props":4976,"children":4978},{"className":4977,"style":816},[632],[],{"type":16,"tag":103,"props":4980,"children":4982},{"className":4981},[172],[4983,4988,4997,5002,5064,5069,5073,5078,5082,5087,5091,5152,5157],{"type":16,"tag":103,"props":4984,"children":4987},{"className":4985,"style":4986},[177],"height:1.1413em;vertical-align:-0.25em;",[],{"type":16,"tag":103,"props":4989,"children":4991},{"className":4990},[1476],[4992],{"type":16,"tag":103,"props":4993,"children":4995},{"className":4994},[183,1481],[4996],{"type":22,"value":4822},{"type":16,"tag":103,"props":4998,"children":5000},{"className":4999},[563],[5001],{"type":22,"value":498},{"type":16,"tag":103,"props":5003,"children":5005},{"className":5004},[1476],[5006,5015],{"type":16,"tag":103,"props":5007,"children":5009},{"className":5008},[1476],[5010],{"type":16,"tag":103,"props":5011,"children":5013},{"className":5012},[183,1481],[5014],{"type":22,"value":4841},{"type":16,"tag":103,"props":5016,"children":5018},{"className":5017},[194],[5019],{"type":16,"tag":103,"props":5020,"children":5022},{"className":5021},[199,200],[5023,5053],{"type":16,"tag":103,"props":5024,"children":5026},{"className":5025},[205],[5027,5048],{"type":16,"tag":103,"props":5028,"children":5030},{"className":5029,"style":211},[210],[5031],{"type":16,"tag":103,"props":5032,"children":5034},{"style":5033},"top:-2.55em;margin-right:0.05em;",[5035,5039],{"type":16,"tag":103,"props":5036,"children":5038},{"className":5037,"style":221},[220],[],{"type":16,"tag":103,"props":5040,"children":5042},{"className":5041},[226,227,228,229],[5043],{"type":16,"tag":103,"props":5044,"children":5046},{"className":5045},[183,229],[5047],{"type":22,"value":155},{"type":16,"tag":103,"props":5049,"children":5051},{"className":5050},[254],[5052],{"type":22,"value":257},{"type":16,"tag":103,"props":5054,"children":5056},{"className":5055},[205],[5057],{"type":16,"tag":103,"props":5058,"children":5060},{"className":5059,"style":417},[210],[5061],{"type":16,"tag":103,"props":5062,"children":5063},{},[],{"type":16,"tag":103,"props":5065,"children":5067},{"className":5066},[626],[5068],{"type":22,"value":515},{"type":16,"tag":103,"props":5070,"children":5072},{"className":5071,"style":633},[632],[],{"type":16,"tag":103,"props":5074,"children":5076},{"className":5075},[638],[5077],{"type":22,"value":520},{"type":16,"tag":103,"props":5079,"children":5081},{"className":5080,"style":633},[632],[],{"type":16,"tag":103,"props":5083,"children":5085},{"className":5084},[626],[5086],{"type":22,"value":515},{"type":16,"tag":103,"props":5088,"children":5090},{"className":5089,"style":633},[632],[],{"type":16,"tag":103,"props":5092,"children":5094},{"className":5093},[1476],[5095,5104],{"type":16,"tag":103,"props":5096,"children":5098},{"className":5097},[1476],[5099],{"type":16,"tag":103,"props":5100,"children":5102},{"className":5101},[183,1481],[5103],{"type":22,"value":4841},{"type":16,"tag":103,"props":5105,"children":5107},{"className":5106},[194],[5108],{"type":16,"tag":103,"props":5109,"children":5111},{"className":5110},[199,200],[5112,5141],{"type":16,"tag":103,"props":5113,"children":5115},{"className":5114},[205],[5116,5136],{"type":16,"tag":103,"props":5117,"children":5119},{"className":5118,"style":1727},[210],[5120],{"type":16,"tag":103,"props":5121,"children":5122},{"style":5033},[5123,5127],{"type":16,"tag":103,"props":5124,"children":5126},{"className":5125,"style":221},[220],[],{"type":16,"tag":103,"props":5128,"children":5130},{"className":5129},[226,227,228,229],[5131],{"type":16,"tag":103,"props":5132,"children":5134},{"className":5133},[183,188,229],[5135],{"type":22,"value":135},{"type":16,"tag":103,"props":5137,"children":5139},{"className":5138},[254],[5140],{"type":22,"value":257},{"type":16,"tag":103,"props":5142,"children":5144},{"className":5143},[205],[5145],{"type":16,"tag":103,"props":5146,"children":5148},{"className":5147,"style":417},[210],[5149],{"type":16,"tag":103,"props":5150,"children":5151},{},[],{"type":16,"tag":103,"props":5153,"children":5155},{"className":5154},[715],[5156],{"type":22,"value":541},{"type":16,"tag":103,"props":5158,"children":5160},{"className":5159},[183],[5161,5166],{"type":16,"tag":103,"props":5162,"children":5164},{"className":5163,"style":3056},[183,188],[5165],{"type":22,"value":4891},{"type":16,"tag":103,"props":5167,"children":5169},{"className":5168},[194],[5170],{"type":16,"tag":103,"props":5171,"children":5173},{"className":5172},[199],[5174],{"type":16,"tag":103,"props":5175,"children":5177},{"className":5176},[205],[5178],{"type":16,"tag":103,"props":5179,"children":5182},{"className":5180,"style":5181},[210],"height:0.8913em;",[5183],{"type":16,"tag":103,"props":5184,"children":5186},{"style":5185},"top:-3.113em;margin-right:0.05em;",[5187,5191],{"type":16,"tag":103,"props":5188,"children":5190},{"className":5189,"style":221},[220],[],{"type":16,"tag":103,"props":5192,"children":5194},{"className":5193},[226,227,228,229],[5195],{"type":16,"tag":103,"props":5196,"children":5199},{"className":5197,"style":5198},[183,188,229],"margin-right:0.0278em;",[5200],{"type":22,"value":4896},{"type":16,"tag":25,"props":5202,"children":5203},{},[5204],{"type":22,"value":5205},"其中：",{"type":16,"tag":103,"props":5207,"children":5209},{"className":5208},[1387],[5210],{"type":16,"tag":103,"props":5211,"children":5213},{"className":5212},[106],[5214,5336],{"type":16,"tag":103,"props":5215,"children":5217},{"className":5216},[111],[5218],{"type":16,"tag":114,"props":5219,"children":5220},{"xmlns":116,"display":1399},[5221],{"type":16,"tag":119,"props":5222,"children":5223},{},[5224,5331],{"type":16,"tag":123,"props":5225,"children":5226},{},[5227,5245,5249,5253,5257,5261,5265,5281,5285,5289,5304,5308,5312,5327],{"type":16,"tag":127,"props":5228,"children":5229},{},[5230,5241],{"type":16,"tag":123,"props":5231,"children":5232},{},[5233,5237],{"type":16,"tag":131,"props":5234,"children":5235},{"mathvariant":1409},[5236],{"type":22,"value":4841},{"type":16,"tag":145,"props":5238,"children":5239},{},[5240],{"type":22,"value":1417},{"type":16,"tag":131,"props":5242,"children":5243},{},[5244],{"type":22,"value":1912},{"type":16,"tag":145,"props":5246,"children":5247},{},[5248],{"type":22,"value":746},{"type":16,"tag":131,"props":5250,"children":5251},{"mathvariant":1409},[5252],{"type":22,"value":2597},{"type":16,"tag":145,"props":5254,"children":5255},{},[5256],{"type":22,"value":1417},{"type":16,"tag":145,"props":5258,"children":5259},{"stretchy":495},[5260],{"type":22,"value":498},{"type":16,"tag":131,"props":5262,"children":5263},{},[5264],{"type":22,"value":2439},{"type":16,"tag":5266,"props":5267,"children":5268},"msubsup",{},[5269,5273,5277],{"type":16,"tag":131,"props":5270,"children":5271},{},[5272],{"type":22,"value":4891},{"type":16,"tag":131,"props":5274,"children":5275},{},[5276],{"type":22,"value":1912},{"type":16,"tag":131,"props":5278,"children":5279},{},[5280],{"type":22,"value":2439},{"type":16,"tag":145,"props":5282,"children":5283},{"separator":167},[5284],{"type":22,"value":515},{"type":16,"tag":131,"props":5286,"children":5287},{},[5288],{"type":22,"value":2485},{"type":16,"tag":5266,"props":5290,"children":5291},{},[5292,5296,5300],{"type":16,"tag":131,"props":5293,"children":5294},{},[5295],{"type":22,"value":4891},{"type":16,"tag":131,"props":5297,"children":5298},{},[5299],{"type":22,"value":1912},{"type":16,"tag":131,"props":5301,"children":5302},{},[5303],{"type":22,"value":2485},{"type":16,"tag":145,"props":5305,"children":5306},{"separator":167},[5307],{"type":22,"value":515},{"type":16,"tag":131,"props":5309,"children":5310},{},[5311],{"type":22,"value":2531},{"type":16,"tag":5266,"props":5313,"children":5314},{},[5315,5319,5323],{"type":16,"tag":131,"props":5316,"children":5317},{},[5318],{"type":22,"value":4891},{"type":16,"tag":131,"props":5320,"children":5321},{},[5322],{"type":22,"value":1912},{"type":16,"tag":131,"props":5324,"children":5325},{},[5326],{"type":22,"value":2531},{"type":16,"tag":145,"props":5328,"children":5329},{"stretchy":495},[5330],{"type":22,"value":541},{"type":16,"tag":157,"props":5332,"children":5333},{"encoding":159},[5334],{"type":22,"value":5335},"\\operatorname{head}_i = \\operatorname{Attention}(QW_i^Q, KW_i^K, VW_i^V)",{"type":16,"tag":103,"props":5337,"children":5339},{"className":5338,"ariaHidden":167},[166],[5340,5422],{"type":16,"tag":103,"props":5341,"children":5343},{"className":5342},[172],[5344,5348,5409,5413,5418],{"type":16,"tag":103,"props":5345,"children":5347},{"className":5346,"style":361},[177],[],{"type":16,"tag":103,"props":5349,"children":5351},{"className":5350},[1476],[5352,5361],{"type":16,"tag":103,"props":5353,"children":5355},{"className":5354},[1476],[5356],{"type":16,"tag":103,"props":5357,"children":5359},{"className":5358},[183,1481],[5360],{"type":22,"value":4841},{"type":16,"tag":103,"props":5362,"children":5364},{"className":5363},[194],[5365],{"type":16,"tag":103,"props":5366,"children":5368},{"className":5367},[199,200],[5369,5398],{"type":16,"tag":103,"props":5370,"children":5372},{"className":5371},[205],[5373,5393],{"type":16,"tag":103,"props":5374,"children":5376},{"className":5375,"style":3958},[210],[5377],{"type":16,"tag":103,"props":5378,"children":5379},{"style":5033},[5380,5384],{"type":16,"tag":103,"props":5381,"children":5383},{"className":5382,"style":221},[220],[],{"type":16,"tag":103,"props":5385,"children":5387},{"className":5386},[226,227,228,229],[5388],{"type":16,"tag":103,"props":5389,"children":5391},{"className":5390},[183,188,229],[5392],{"type":22,"value":1912},{"type":16,"tag":103,"props":5394,"children":5396},{"className":5395},[254],[5397],{"type":22,"value":257},{"type":16,"tag":103,"props":5399,"children":5401},{"className":5400},[205],[5402],{"type":16,"tag":103,"props":5403,"children":5405},{"className":5404,"style":417},[210],[5406],{"type":16,"tag":103,"props":5407,"children":5408},{},[],{"type":16,"tag":103,"props":5410,"children":5412},{"className":5411,"style":816},[632],[],{"type":16,"tag":103,"props":5414,"children":5416},{"className":5415},[821],[5417],{"type":22,"value":746},{"type":16,"tag":103,"props":5419,"children":5421},{"className":5420,"style":816},[632],[],{"type":16,"tag":103,"props":5423,"children":5425},{"className":5424},[172],[5426,5431,5440,5445,5450,5527,5532,5536,5541,5616,5621,5625,5630,5703],{"type":16,"tag":103,"props":5427,"children":5430},{"className":5428,"style":5429},[177],"height:1.2361em;vertical-align:-0.2769em;",[],{"type":16,"tag":103,"props":5432,"children":5434},{"className":5433},[1476],[5435],{"type":16,"tag":103,"props":5436,"children":5438},{"className":5437},[183,1481],[5439],{"type":22,"value":2597},{"type":16,"tag":103,"props":5441,"children":5443},{"className":5442},[563],[5444],{"type":22,"value":498},{"type":16,"tag":103,"props":5446,"children":5448},{"className":5447},[183,188],[5449],{"type":22,"value":2439},{"type":16,"tag":103,"props":5451,"children":5453},{"className":5452},[183],[5454,5459],{"type":16,"tag":103,"props":5455,"children":5457},{"className":5456,"style":3056},[183,188],[5458],{"type":22,"value":4891},{"type":16,"tag":103,"props":5460,"children":5462},{"className":5461},[194],[5463],{"type":16,"tag":103,"props":5464,"children":5466},{"className":5465},[199,200],[5467,5515],{"type":16,"tag":103,"props":5468,"children":5470},{"className":5469},[205],[5471,5510],{"type":16,"tag":103,"props":5472,"children":5475},{"className":5473,"style":5474},[210],"height:0.9592em;",[5476,5493],{"type":16,"tag":103,"props":5477,"children":5479},{"style":5478},"top:-2.4231em;margin-left:-0.1389em;margin-right:0.05em;",[5480,5484],{"type":16,"tag":103,"props":5481,"children":5483},{"className":5482,"style":221},[220],[],{"type":16,"tag":103,"props":5485,"children":5487},{"className":5486},[226,227,228,229],[5488],{"type":16,"tag":103,"props":5489,"children":5491},{"className":5490},[183,188,229],[5492],{"type":22,"value":1912},{"type":16,"tag":103,"props":5494,"children":5496},{"style":5495},"top:-3.1809em;margin-right:0.05em;",[5497,5501],{"type":16,"tag":103,"props":5498,"children":5500},{"className":5499,"style":221},[220],[],{"type":16,"tag":103,"props":5502,"children":5504},{"className":5503},[226,227,228,229],[5505],{"type":16,"tag":103,"props":5506,"children":5508},{"className":5507},[183,188,229],[5509],{"type":22,"value":2439},{"type":16,"tag":103,"props":5511,"children":5513},{"className":5512},[254],[5514],{"type":22,"value":257},{"type":16,"tag":103,"props":5516,"children":5518},{"className":5517},[205],[5519],{"type":16,"tag":103,"props":5520,"children":5523},{"className":5521,"style":5522},[210],"height:0.2769em;",[5524],{"type":16,"tag":103,"props":5525,"children":5526},{},[],{"type":16,"tag":103,"props":5528,"children":5530},{"className":5529},[626],[5531],{"type":22,"value":515},{"type":16,"tag":103,"props":5533,"children":5535},{"className":5534,"style":633},[632],[],{"type":16,"tag":103,"props":5537,"children":5539},{"className":5538,"style":2505},[183,188],[5540],{"type":22,"value":2485},{"type":16,"tag":103,"props":5542,"children":5544},{"className":5543},[183],[5545,5550],{"type":16,"tag":103,"props":5546,"children":5548},{"className":5547,"style":3056},[183,188],[5549],{"type":22,"value":4891},{"type":16,"tag":103,"props":5551,"children":5553},{"className":5552},[194],[5554],{"type":16,"tag":103,"props":5555,"children":5557},{"className":5556},[199,200],[5558,5604],{"type":16,"tag":103,"props":5559,"children":5561},{"className":5560},[205],[5562,5599],{"type":16,"tag":103,"props":5563,"children":5565},{"className":5564,"style":5181},[210],[5566,5583],{"type":16,"tag":103,"props":5567,"children":5569},{"style":5568},"top:-2.453em;margin-left:-0.1389em;margin-right:0.05em;",[5570,5574],{"type":16,"tag":103,"props":5571,"children":5573},{"className":5572,"style":221},[220],[],{"type":16,"tag":103,"props":5575,"children":5577},{"className":5576},[226,227,228,229],[5578],{"type":16,"tag":103,"props":5579,"children":5581},{"className":5580},[183,188,229],[5582],{"type":22,"value":1912},{"type":16,"tag":103,"props":5584,"children":5585},{"style":5185},[5586,5590],{"type":16,"tag":103,"props":5587,"children":5589},{"className":5588,"style":221},[220],[],{"type":16,"tag":103,"props":5591,"children":5593},{"className":5592},[226,227,228,229],[5594],{"type":16,"tag":103,"props":5595,"children":5597},{"className":5596,"style":2505},[183,188,229],[5598],{"type":22,"value":2485},{"type":16,"tag":103,"props":5600,"children":5602},{"className":5601},[254],[5603],{"type":22,"value":257},{"type":16,"tag":103,"props":5605,"children":5607},{"className":5606},[205],[5608],{"type":16,"tag":103,"props":5609,"children":5612},{"className":5610,"style":5611},[210],"height:0.247em;",[5613],{"type":16,"tag":103,"props":5614,"children":5615},{},[],{"type":16,"tag":103,"props":5617,"children":5619},{"className":5618},[626],[5620],{"type":22,"value":515},{"type":16,"tag":103,"props":5622,"children":5624},{"className":5623,"style":633},[632],[],{"type":16,"tag":103,"props":5626,"children":5628},{"className":5627,"style":1497},[183,188],[5629],{"type":22,"value":2531},{"type":16,"tag":103,"props":5631,"children":5633},{"className":5632},[183],[5634,5639],{"type":16,"tag":103,"props":5635,"children":5637},{"className":5636,"style":3056},[183,188],[5638],{"type":22,"value":4891},{"type":16,"tag":103,"props":5640,"children":5642},{"className":5641},[194],[5643],{"type":16,"tag":103,"props":5644,"children":5646},{"className":5645},[199,200],[5647,5692],{"type":16,"tag":103,"props":5648,"children":5650},{"className":5649},[205],[5651,5687],{"type":16,"tag":103,"props":5652,"children":5654},{"className":5653,"style":5181},[210],[5655,5671],{"type":16,"tag":103,"props":5656,"children":5657},{"style":5568},[5658,5662],{"type":16,"tag":103,"props":5659,"children":5661},{"className":5660,"style":221},[220],[],{"type":16,"tag":103,"props":5663,"children":5665},{"className":5664},[226,227,228,229],[5666],{"type":16,"tag":103,"props":5667,"children":5669},{"className":5668},[183,188,229],[5670],{"type":22,"value":1912},{"type":16,"tag":103,"props":5672,"children":5673},{"style":5185},[5674,5678],{"type":16,"tag":103,"props":5675,"children":5677},{"className":5676,"style":221},[220],[],{"type":16,"tag":103,"props":5679,"children":5681},{"className":5680},[226,227,228,229],[5682],{"type":16,"tag":103,"props":5683,"children":5685},{"className":5684,"style":1497},[183,188,229],[5686],{"type":22,"value":2531},{"type":16,"tag":103,"props":5688,"children":5690},{"className":5689},[254],[5691],{"type":22,"value":257},{"type":16,"tag":103,"props":5693,"children":5695},{"className":5694},[205],[5696],{"type":16,"tag":103,"props":5697,"children":5699},{"className":5698,"style":5611},[210],[5700],{"type":16,"tag":103,"props":5701,"children":5702},{},[],{"type":16,"tag":103,"props":5704,"children":5706},{"className":5705},[715],[5707],{"type":22,"value":541},{"type":16,"tag":25,"props":5709,"children":5710},{},[5711],{"type":22,"value":5712},"投影矩阵为：",{"type":16,"tag":103,"props":5714,"children":5716},{"className":5715},[1387],[5717],{"type":16,"tag":103,"props":5718,"children":5720},{"className":5719},[106],[5721,6145],{"type":16,"tag":103,"props":5722,"children":5724},{"className":5723},[111],[5725],{"type":16,"tag":114,"props":5726,"children":5727},{"xmlns":116,"display":1399},[5728],{"type":16,"tag":119,"props":5729,"children":5730},{},[5731,6140],{"type":16,"tag":2574,"props":5732,"children":5736},{"rowspacing":5733,"columnalign":5734,"columnspacing":5735},"0.25em","right left","0em",[5737,5842,5941,6040],{"type":16,"tag":2579,"props":5738,"children":5739},{},[5740,5763],{"type":16,"tag":2583,"props":5741,"children":5742},{},[5743],{"type":16,"tag":5744,"props":5745,"children":5747},"mstyle",{"scriptlevel":5746,"displaystyle":167},"0",[5748],{"type":16,"tag":5266,"props":5749,"children":5750},{},[5751,5755,5759],{"type":16,"tag":131,"props":5752,"children":5753},{},[5754],{"type":22,"value":4891},{"type":16,"tag":131,"props":5756,"children":5757},{},[5758],{"type":22,"value":1912},{"type":16,"tag":131,"props":5760,"children":5761},{},[5762],{"type":22,"value":2439},{"type":16,"tag":2583,"props":5764,"children":5765},{},[5766],{"type":16,"tag":5744,"props":5767,"children":5768},{"scriptlevel":5746,"displaystyle":167},[5769],{"type":16,"tag":123,"props":5770,"children":5771},{},[5772,5775,5780,5838],{"type":16,"tag":123,"props":5773,"children":5774},{},[],{"type":16,"tag":145,"props":5776,"children":5777},{},[5778],{"type":22,"value":5779},"∈",{"type":16,"tag":2662,"props":5781,"children":5782},{},[5783,5789],{"type":16,"tag":131,"props":5784,"children":5786},{"mathvariant":5785},"double-struck",[5787],{"type":22,"value":5788},"R",{"type":16,"tag":123,"props":5790,"children":5791},{},[5792,5822,5827],{"type":16,"tag":127,"props":5793,"children":5794},{},[5795,5799],{"type":16,"tag":131,"props":5796,"children":5797},{},[5798],{"type":22,"value":1650},{"type":16,"tag":123,"props":5800,"children":5801},{},[5802,5806,5810,5814,5818],{"type":16,"tag":131,"props":5803,"children":5804},{"mathvariant":1409},[5805],{"type":22,"value":1090},{"type":16,"tag":131,"props":5807,"children":5808},{"mathvariant":1409},[5809],{"type":22,"value":1662},{"type":16,"tag":131,"props":5811,"children":5812},{"mathvariant":1409},[5813],{"type":22,"value":1650},{"type":16,"tag":131,"props":5815,"children":5816},{"mathvariant":1409},[5817],{"type":22,"value":1671},{"type":16,"tag":131,"props":5819,"children":5820},{"mathvariant":1409},[5821],{"type":22,"value":1676},{"type":16,"tag":145,"props":5823,"children":5824},{},[5825],{"type":22,"value":5826},"×",{"type":16,"tag":127,"props":5828,"children":5829},{},[5830,5834],{"type":16,"tag":131,"props":5831,"children":5832},{},[5833],{"type":22,"value":1650},{"type":16,"tag":131,"props":5835,"children":5836},{},[5837],{"type":22,"value":2032},{"type":16,"tag":145,"props":5839,"children":5840},{"separator":167},[5841],{"type":22,"value":515},{"type":16,"tag":2579,"props":5843,"children":5844},{},[5845,5866],{"type":16,"tag":2583,"props":5846,"children":5847},{},[5848],{"type":16,"tag":5744,"props":5849,"children":5850},{"scriptlevel":5746,"displaystyle":167},[5851],{"type":16,"tag":5266,"props":5852,"children":5853},{},[5854,5858,5862],{"type":16,"tag":131,"props":5855,"children":5856},{},[5857],{"type":22,"value":4891},{"type":16,"tag":131,"props":5859,"children":5860},{},[5861],{"type":22,"value":1912},{"type":16,"tag":131,"props":5863,"children":5864},{},[5865],{"type":22,"value":2485},{"type":16,"tag":2583,"props":5867,"children":5868},{},[5869],{"type":16,"tag":5744,"props":5870,"children":5871},{"scriptlevel":5746,"displaystyle":167},[5872],{"type":16,"tag":123,"props":5873,"children":5874},{},[5875,5878,5882,5937],{"type":16,"tag":123,"props":5876,"children":5877},{},[],{"type":16,"tag":145,"props":5879,"children":5880},{},[5881],{"type":22,"value":5779},{"type":16,"tag":2662,"props":5883,"children":5884},{},[5885,5889],{"type":16,"tag":131,"props":5886,"children":5887},{"mathvariant":5785},[5888],{"type":22,"value":5788},{"type":16,"tag":123,"props":5890,"children":5891},{},[5892,5922,5926],{"type":16,"tag":127,"props":5893,"children":5894},{},[5895,5899],{"type":16,"tag":131,"props":5896,"children":5897},{},[5898],{"type":22,"value":1650},{"type":16,"tag":123,"props":5900,"children":5901},{},[5902,5906,5910,5914,5918],{"type":16,"tag":131,"props":5903,"children":5904},{"mathvariant":1409},[5905],{"type":22,"value":1090},{"type":16,"tag":131,"props":5907,"children":5908},{"mathvariant":1409},[5909],{"type":22,"value":1662},{"type":16,"tag":131,"props":5911,"children":5912},{"mathvariant":1409},[5913],{"type":22,"value":1650},{"type":16,"tag":131,"props":5915,"children":5916},{"mathvariant":1409},[5917],{"type":22,"value":1671},{"type":16,"tag":131,"props":5919,"children":5920},{"mathvariant":1409},[5921],{"type":22,"value":1676},{"type":16,"tag":145,"props":5923,"children":5924},{},[5925],{"type":22,"value":5826},{"type":16,"tag":127,"props":5927,"children":5928},{},[5929,5933],{"type":16,"tag":131,"props":5930,"children":5931},{},[5932],{"type":22,"value":1650},{"type":16,"tag":131,"props":5934,"children":5935},{},[5936],{"type":22,"value":2032},{"type":16,"tag":145,"props":5938,"children":5939},{"separator":167},[5940],{"type":22,"value":515},{"type":16,"tag":2579,"props":5942,"children":5943},{},[5944,5965],{"type":16,"tag":2583,"props":5945,"children":5946},{},[5947],{"type":16,"tag":5744,"props":5948,"children":5949},{"scriptlevel":5746,"displaystyle":167},[5950],{"type":16,"tag":5266,"props":5951,"children":5952},{},[5953,5957,5961],{"type":16,"tag":131,"props":5954,"children":5955},{},[5956],{"type":22,"value":4891},{"type":16,"tag":131,"props":5958,"children":5959},{},[5960],{"type":22,"value":1912},{"type":16,"tag":131,"props":5962,"children":5963},{},[5964],{"type":22,"value":2531},{"type":16,"tag":2583,"props":5966,"children":5967},{},[5968],{"type":16,"tag":5744,"props":5969,"children":5970},{"scriptlevel":5746,"displaystyle":167},[5971],{"type":16,"tag":123,"props":5972,"children":5973},{},[5974,5977,5981,6036],{"type":16,"tag":123,"props":5975,"children":5976},{},[],{"type":16,"tag":145,"props":5978,"children":5979},{},[5980],{"type":22,"value":5779},{"type":16,"tag":2662,"props":5982,"children":5983},{},[5984,5988],{"type":16,"tag":131,"props":5985,"children":5986},{"mathvariant":5785},[5987],{"type":22,"value":5788},{"type":16,"tag":123,"props":5989,"children":5990},{},[5991,6021,6025],{"type":16,"tag":127,"props":5992,"children":5993},{},[5994,5998],{"type":16,"tag":131,"props":5995,"children":5996},{},[5997],{"type":22,"value":1650},{"type":16,"tag":123,"props":5999,"children":6000},{},[6001,6005,6009,6013,6017],{"type":16,"tag":131,"props":6002,"children":6003},{"mathvariant":1409},[6004],{"type":22,"value":1090},{"type":16,"tag":131,"props":6006,"children":6007},{"mathvariant":1409},[6008],{"type":22,"value":1662},{"type":16,"tag":131,"props":6010,"children":6011},{"mathvariant":1409},[6012],{"type":22,"value":1650},{"type":16,"tag":131,"props":6014,"children":6015},{"mathvariant":1409},[6016],{"type":22,"value":1671},{"type":16,"tag":131,"props":6018,"children":6019},{"mathvariant":1409},[6020],{"type":22,"value":1676},{"type":16,"tag":145,"props":6022,"children":6023},{},[6024],{"type":22,"value":5826},{"type":16,"tag":127,"props":6026,"children":6027},{},[6028,6032],{"type":16,"tag":131,"props":6029,"children":6030},{},[6031],{"type":22,"value":1650},{"type":16,"tag":131,"props":6033,"children":6034},{},[6035],{"type":22,"value":2138},{"type":16,"tag":145,"props":6037,"children":6038},{"separator":167},[6039],{"type":22,"value":515},{"type":16,"tag":2579,"props":6041,"children":6042},{},[6043,6060],{"type":16,"tag":2583,"props":6044,"children":6045},{},[6046],{"type":16,"tag":5744,"props":6047,"children":6048},{"scriptlevel":5746,"displaystyle":167},[6049],{"type":16,"tag":2662,"props":6050,"children":6051},{},[6052,6056],{"type":16,"tag":131,"props":6053,"children":6054},{},[6055],{"type":22,"value":4891},{"type":16,"tag":131,"props":6057,"children":6058},{},[6059],{"type":22,"value":4896},{"type":16,"tag":2583,"props":6061,"children":6062},{},[6063],{"type":16,"tag":5744,"props":6064,"children":6065},{"scriptlevel":5746,"displaystyle":167},[6066],{"type":16,"tag":123,"props":6067,"children":6068},{},[6069,6072,6076,6135],{"type":16,"tag":123,"props":6070,"children":6071},{},[],{"type":16,"tag":145,"props":6073,"children":6074},{},[6075],{"type":22,"value":5779},{"type":16,"tag":2662,"props":6077,"children":6078},{},[6079,6083],{"type":16,"tag":131,"props":6080,"children":6081},{"mathvariant":5785},[6082],{"type":22,"value":5788},{"type":16,"tag":123,"props":6084,"children":6085},{},[6086,6090,6101,6105],{"type":16,"tag":131,"props":6087,"children":6088},{},[6089],{"type":22,"value":135},{"type":16,"tag":127,"props":6091,"children":6092},{},[6093,6097],{"type":16,"tag":131,"props":6094,"children":6095},{},[6096],{"type":22,"value":1650},{"type":16,"tag":131,"props":6098,"children":6099},{},[6100],{"type":22,"value":2138},{"type":16,"tag":145,"props":6102,"children":6103},{},[6104],{"type":22,"value":5826},{"type":16,"tag":127,"props":6106,"children":6107},{},[6108,6112],{"type":16,"tag":131,"props":6109,"children":6110},{},[6111],{"type":22,"value":1650},{"type":16,"tag":123,"props":6113,"children":6114},{},[6115,6119,6123,6127,6131],{"type":16,"tag":131,"props":6116,"children":6117},{"mathvariant":1409},[6118],{"type":22,"value":1090},{"type":16,"tag":131,"props":6120,"children":6121},{"mathvariant":1409},[6122],{"type":22,"value":1662},{"type":16,"tag":131,"props":6124,"children":6125},{"mathvariant":1409},[6126],{"type":22,"value":1650},{"type":16,"tag":131,"props":6128,"children":6129},{"mathvariant":1409},[6130],{"type":22,"value":1671},{"type":16,"tag":131,"props":6132,"children":6133},{"mathvariant":1409},[6134],{"type":22,"value":1676},{"type":16,"tag":131,"props":6136,"children":6137},{"mathvariant":1409},[6138],{"type":22,"value":6139},".",{"type":16,"tag":157,"props":6141,"children":6142},{"encoding":159},[6143],{"type":22,"value":6144},"\\begin{aligned}\nW_i^Q &\\in \\mathbb{R}^{d_{\\mathrm{model}} \\times d_k}, \\\\\nW_i^K &\\in \\mathbb{R}^{d_{\\mathrm{model}} \\times d_k}, \\\\\nW_i^V &\\in \\mathbb{R}^{d_{\\mathrm{model}} \\times d_v}, \\\\\nW^O &\\in \\mathbb{R}^{h d_v \\times d_{\\mathrm{model}}}.\n\\end{aligned}",{"type":16,"tag":103,"props":6146,"children":6148},{"className":6147,"ariaHidden":167},[166],[6149],{"type":16,"tag":103,"props":6150,"children":6152},{"className":6151},[172],[6153,6158],{"type":16,"tag":103,"props":6154,"children":6157},{"className":6155,"style":6156},[177],"height:5.9966em;vertical-align:-2.7483em;",[],{"type":16,"tag":103,"props":6159,"children":6161},{"className":6160},[183],[6162],{"type":16,"tag":103,"props":6163,"children":6165},{"className":6164},[2574],[6166,6509],{"type":16,"tag":103,"props":6167,"children":6170},{"className":6168},[6169],"col-align-r",[6171],{"type":16,"tag":103,"props":6172,"children":6174},{"className":6173},[199,200],[6175,6497],{"type":16,"tag":103,"props":6176,"children":6178},{"className":6177},[205],[6179,6492],{"type":16,"tag":103,"props":6180,"children":6183},{"className":6181,"style":6182},[210],"height:3.2483em;",[6184,6269,6354,6439],{"type":16,"tag":103,"props":6185,"children":6187},{"style":6186},"top:-5.289em;",[6188,6192],{"type":16,"tag":103,"props":6189,"children":6191},{"className":6190,"style":2292},[220],[],{"type":16,"tag":103,"props":6193,"children":6195},{"className":6194},[183],[6196],{"type":16,"tag":103,"props":6197,"children":6199},{"className":6198},[183],[6200,6205],{"type":16,"tag":103,"props":6201,"children":6203},{"className":6202,"style":3056},[183,188],[6204],{"type":22,"value":4891},{"type":16,"tag":103,"props":6206,"children":6208},{"className":6207},[194],[6209],{"type":16,"tag":103,"props":6210,"children":6212},{"className":6211},[199,200],[6213,6258],{"type":16,"tag":103,"props":6214,"children":6216},{"className":6215},[205],[6217,6253],{"type":16,"tag":103,"props":6218,"children":6220},{"className":6219,"style":5474},[210],[6221,6237],{"type":16,"tag":103,"props":6222,"children":6223},{"style":5478},[6224,6228],{"type":16,"tag":103,"props":6225,"children":6227},{"className":6226,"style":221},[220],[],{"type":16,"tag":103,"props":6229,"children":6231},{"className":6230},[226,227,228,229],[6232],{"type":16,"tag":103,"props":6233,"children":6235},{"className":6234},[183,188,229],[6236],{"type":22,"value":1912},{"type":16,"tag":103,"props":6238,"children":6239},{"style":5495},[6240,6244],{"type":16,"tag":103,"props":6241,"children":6243},{"className":6242,"style":221},[220],[],{"type":16,"tag":103,"props":6245,"children":6247},{"className":6246},[226,227,228,229],[6248],{"type":16,"tag":103,"props":6249,"children":6251},{"className":6250},[183,188,229],[6252],{"type":22,"value":2439},{"type":16,"tag":103,"props":6254,"children":6256},{"className":6255},[254],[6257],{"type":22,"value":257},{"type":16,"tag":103,"props":6259,"children":6261},{"className":6260},[205],[6262],{"type":16,"tag":103,"props":6263,"children":6265},{"className":6264,"style":5522},[210],[6266],{"type":16,"tag":103,"props":6267,"children":6268},{},[],{"type":16,"tag":103,"props":6270,"children":6272},{"style":6271},"top:-3.7299em;",[6273,6277],{"type":16,"tag":103,"props":6274,"children":6276},{"className":6275,"style":2292},[220],[],{"type":16,"tag":103,"props":6278,"children":6280},{"className":6279},[183],[6281],{"type":16,"tag":103,"props":6282,"children":6284},{"className":6283},[183],[6285,6290],{"type":16,"tag":103,"props":6286,"children":6288},{"className":6287,"style":3056},[183,188],[6289],{"type":22,"value":4891},{"type":16,"tag":103,"props":6291,"children":6293},{"className":6292},[194],[6294],{"type":16,"tag":103,"props":6295,"children":6297},{"className":6296},[199,200],[6298,6343],{"type":16,"tag":103,"props":6299,"children":6301},{"className":6300},[205],[6302,6338],{"type":16,"tag":103,"props":6303,"children":6305},{"className":6304,"style":5181},[210],[6306,6322],{"type":16,"tag":103,"props":6307,"children":6308},{"style":5568},[6309,6313],{"type":16,"tag":103,"props":6310,"children":6312},{"className":6311,"style":221},[220],[],{"type":16,"tag":103,"props":6314,"children":6316},{"className":6315},[226,227,228,229],[6317],{"type":16,"tag":103,"props":6318,"children":6320},{"className":6319},[183,188,229],[6321],{"type":22,"value":1912},{"type":16,"tag":103,"props":6323,"children":6324},{"style":5185},[6325,6329],{"type":16,"tag":103,"props":6326,"children":6328},{"className":6327,"style":221},[220],[],{"type":16,"tag":103,"props":6330,"children":6332},{"className":6331},[226,227,228,229],[6333],{"type":16,"tag":103,"props":6334,"children":6336},{"className":6335,"style":2505},[183,188,229],[6337],{"type":22,"value":2485},{"type":16,"tag":103,"props":6339,"children":6341},{"className":6340},[254],[6342],{"type":22,"value":257},{"type":16,"tag":103,"props":6344,"children":6346},{"className":6345},[205],[6347],{"type":16,"tag":103,"props":6348,"children":6350},{"className":6349,"style":5611},[210],[6351],{"type":16,"tag":103,"props":6352,"children":6353},{},[],{"type":16,"tag":103,"props":6355,"children":6357},{"style":6356},"top:-2.1708em;",[6358,6362],{"type":16,"tag":103,"props":6359,"children":6361},{"className":6360,"style":2292},[220],[],{"type":16,"tag":103,"props":6363,"children":6365},{"className":6364},[183],[6366],{"type":16,"tag":103,"props":6367,"children":6369},{"className":6368},[183],[6370,6375],{"type":16,"tag":103,"props":6371,"children":6373},{"className":6372,"style":3056},[183,188],[6374],{"type":22,"value":4891},{"type":16,"tag":103,"props":6376,"children":6378},{"className":6377},[194],[6379],{"type":16,"tag":103,"props":6380,"children":6382},{"className":6381},[199,200],[6383,6428],{"type":16,"tag":103,"props":6384,"children":6386},{"className":6385},[205],[6387,6423],{"type":16,"tag":103,"props":6388,"children":6390},{"className":6389,"style":5181},[210],[6391,6407],{"type":16,"tag":103,"props":6392,"children":6393},{"style":5568},[6394,6398],{"type":16,"tag":103,"props":6395,"children":6397},{"className":6396,"style":221},[220],[],{"type":16,"tag":103,"props":6399,"children":6401},{"className":6400},[226,227,228,229],[6402],{"type":16,"tag":103,"props":6403,"children":6405},{"className":6404},[183,188,229],[6406],{"type":22,"value":1912},{"type":16,"tag":103,"props":6408,"children":6409},{"style":5185},[6410,6414],{"type":16,"tag":103,"props":6411,"children":6413},{"className":6412,"style":221},[220],[],{"type":16,"tag":103,"props":6415,"children":6417},{"className":6416},[226,227,228,229],[6418],{"type":16,"tag":103,"props":6419,"children":6421},{"className":6420,"style":1497},[183,188,229],[6422],{"type":22,"value":2531},{"type":16,"tag":103,"props":6424,"children":6426},{"className":6425},[254],[6427],{"type":22,"value":257},{"type":16,"tag":103,"props":6429,"children":6431},{"className":6430},[205],[6432],{"type":16,"tag":103,"props":6433,"children":6435},{"className":6434,"style":5611},[210],[6436],{"type":16,"tag":103,"props":6437,"children":6438},{},[],{"type":16,"tag":103,"props":6440,"children":6442},{"style":6441},"top:-0.6117em;",[6443,6447],{"type":16,"tag":103,"props":6444,"children":6446},{"className":6445,"style":2292},[220],[],{"type":16,"tag":103,"props":6448,"children":6450},{"className":6449},[183],[6451],{"type":16,"tag":103,"props":6452,"children":6454},{"className":6453},[183],[6455,6460],{"type":16,"tag":103,"props":6456,"children":6458},{"className":6457,"style":3056},[183,188],[6459],{"type":22,"value":4891},{"type":16,"tag":103,"props":6461,"children":6463},{"className":6462},[194],[6464],{"type":16,"tag":103,"props":6465,"children":6467},{"className":6466},[199],[6468],{"type":16,"tag":103,"props":6469,"children":6471},{"className":6470},[205],[6472],{"type":16,"tag":103,"props":6473,"children":6475},{"className":6474,"style":5181},[210],[6476],{"type":16,"tag":103,"props":6477,"children":6478},{"style":5185},[6479,6483],{"type":16,"tag":103,"props":6480,"children":6482},{"className":6481,"style":221},[220],[],{"type":16,"tag":103,"props":6484,"children":6486},{"className":6485},[226,227,228,229],[6487],{"type":16,"tag":103,"props":6488,"children":6490},{"className":6489,"style":5198},[183,188,229],[6491],{"type":22,"value":4896},{"type":16,"tag":103,"props":6493,"children":6495},{"className":6494},[254],[6496],{"type":22,"value":257},{"type":16,"tag":103,"props":6498,"children":6500},{"className":6499},[205],[6501],{"type":16,"tag":103,"props":6502,"children":6505},{"className":6503,"style":6504},[210],"height:2.7483em;",[6506],{"type":16,"tag":103,"props":6507,"children":6508},{},[],{"type":16,"tag":103,"props":6510,"children":6513},{"className":6511},[6512],"col-align-l",[6514],{"type":16,"tag":103,"props":6515,"children":6517},{"className":6516},[199,200],[6518,7347],{"type":16,"tag":103,"props":6519,"children":6521},{"className":6520},[205],[6522,7342],{"type":16,"tag":103,"props":6523,"children":6525},{"className":6524,"style":6182},[210],[6526,6734,6934,7137],{"type":16,"tag":103,"props":6527,"children":6528},{"style":6186},[6529,6533],{"type":16,"tag":103,"props":6530,"children":6532},{"className":6531,"style":2292},[220],[],{"type":16,"tag":103,"props":6534,"children":6536},{"className":6535},[183],[6537,6541,6545,6550,6554,6729],{"type":16,"tag":103,"props":6538,"children":6540},{"className":6539},[183],[],{"type":16,"tag":103,"props":6542,"children":6544},{"className":6543,"style":816},[632],[],{"type":16,"tag":103,"props":6546,"children":6548},{"className":6547},[821],[6549],{"type":22,"value":5779},{"type":16,"tag":103,"props":6551,"children":6553},{"className":6552,"style":816},[632],[],{"type":16,"tag":103,"props":6555,"children":6557},{"className":6556},[183],[6558,6564],{"type":16,"tag":103,"props":6559,"children":6562},{"className":6560},[183,6561],"mathbb",[6563],{"type":22,"value":5788},{"type":16,"tag":103,"props":6565,"children":6567},{"className":6566},[194],[6568],{"type":16,"tag":103,"props":6569,"children":6571},{"className":6570},[199],[6572],{"type":16,"tag":103,"props":6573,"children":6575},{"className":6574},[205],[6576],{"type":16,"tag":103,"props":6577,"children":6580},{"className":6578,"style":6579},[210],"height:0.8991em;",[6581],{"type":16,"tag":103,"props":6582,"children":6583},{"style":5185},[6584,6588],{"type":16,"tag":103,"props":6585,"children":6587},{"className":6586,"style":221},[220],[],{"type":16,"tag":103,"props":6589,"children":6591},{"className":6590},[226,227,228,229],[6592],{"type":16,"tag":103,"props":6593,"children":6595},{"className":6594},[183,229],[6596,6667,6672],{"type":16,"tag":103,"props":6597,"children":6599},{"className":6598},[183,229],[6600,6605],{"type":16,"tag":103,"props":6601,"children":6603},{"className":6602},[183,188,229],[6604],{"type":22,"value":1650},{"type":16,"tag":103,"props":6606,"children":6608},{"className":6607},[194],[6609],{"type":16,"tag":103,"props":6610,"children":6612},{"className":6611},[199,200],[6613,6655],{"type":16,"tag":103,"props":6614,"children":6616},{"className":6615},[205],[6617,6650],{"type":16,"tag":103,"props":6618,"children":6621},{"className":6619,"style":6620},[210],"height:0.3448em;",[6622],{"type":16,"tag":103,"props":6623,"children":6625},{"style":6624},"top:-2.3488em;margin-left:0em;margin-right:0.0714em;",[6626,6631],{"type":16,"tag":103,"props":6627,"children":6630},{"className":6628,"style":6629},[220],"height:2.5em;",[],{"type":16,"tag":103,"props":6632,"children":6636},{"className":6633},[226,6634,6635,229],"reset-size3","size1",[6637],{"type":16,"tag":103,"props":6638,"children":6640},{"className":6639},[183,229],[6641],{"type":16,"tag":103,"props":6642,"children":6644},{"className":6643},[183,229],[6645],{"type":16,"tag":103,"props":6646,"children":6648},{"className":6647},[183,1481,229],[6649],{"type":22,"value":1753},{"type":16,"tag":103,"props":6651,"children":6653},{"className":6652},[254],[6654],{"type":22,"value":257},{"type":16,"tag":103,"props":6656,"children":6658},{"className":6657},[205],[6659],{"type":16,"tag":103,"props":6660,"children":6663},{"className":6661,"style":6662},[210],"height:0.1512em;",[6664],{"type":16,"tag":103,"props":6665,"children":6666},{},[],{"type":16,"tag":103,"props":6668,"children":6670},{"className":6669},[243,229],[6671],{"type":22,"value":5826},{"type":16,"tag":103,"props":6673,"children":6675},{"className":6674},[183,229],[6676,6681],{"type":16,"tag":103,"props":6677,"children":6679},{"className":6678},[183,188,229],[6680],{"type":22,"value":1650},{"type":16,"tag":103,"props":6682,"children":6684},{"className":6683},[194],[6685],{"type":16,"tag":103,"props":6686,"children":6688},{"className":6687},[199,200],[6689,6718],{"type":16,"tag":103,"props":6690,"children":6692},{"className":6691},[205],[6693,6713],{"type":16,"tag":103,"props":6694,"children":6696},{"className":6695,"style":6620},[210],[6697],{"type":16,"tag":103,"props":6698,"children":6699},{"style":6624},[6700,6704],{"type":16,"tag":103,"props":6701,"children":6703},{"className":6702,"style":6629},[220],[],{"type":16,"tag":103,"props":6705,"children":6707},{"className":6706},[226,6634,6635,229],[6708],{"type":16,"tag":103,"props":6709,"children":6711},{"className":6710,"style":2089},[183,188,229],[6712],{"type":22,"value":2032},{"type":16,"tag":103,"props":6714,"children":6716},{"className":6715},[254],[6717],{"type":22,"value":257},{"type":16,"tag":103,"props":6719,"children":6721},{"className":6720},[205],[6722],{"type":16,"tag":103,"props":6723,"children":6725},{"className":6724,"style":6662},[210],[6726],{"type":16,"tag":103,"props":6727,"children":6728},{},[],{"type":16,"tag":103,"props":6730,"children":6732},{"className":6731},[626],[6733],{"type":22,"value":515},{"type":16,"tag":103,"props":6735,"children":6736},{"style":6271},[6737,6741],{"type":16,"tag":103,"props":6738,"children":6740},{"className":6739,"style":2292},[220],[],{"type":16,"tag":103,"props":6742,"children":6744},{"className":6743},[183],[6745,6749,6753,6758,6762,6929],{"type":16,"tag":103,"props":6746,"children":6748},{"className":6747},[183],[],{"type":16,"tag":103,"props":6750,"children":6752},{"className":6751,"style":816},[632],[],{"type":16,"tag":103,"props":6754,"children":6756},{"className":6755},[821],[6757],{"type":22,"value":5779},{"type":16,"tag":103,"props":6759,"children":6761},{"className":6760,"style":816},[632],[],{"type":16,"tag":103,"props":6763,"children":6765},{"className":6764},[183],[6766,6771],{"type":16,"tag":103,"props":6767,"children":6769},{"className":6768},[183,6561],[6770],{"type":22,"value":5788},{"type":16,"tag":103,"props":6772,"children":6774},{"className":6773},[194],[6775],{"type":16,"tag":103,"props":6776,"children":6778},{"className":6777},[199],[6779],{"type":16,"tag":103,"props":6780,"children":6782},{"className":6781},[205],[6783],{"type":16,"tag":103,"props":6784,"children":6786},{"className":6785,"style":6579},[210],[6787],{"type":16,"tag":103,"props":6788,"children":6789},{"style":5185},[6790,6794],{"type":16,"tag":103,"props":6791,"children":6793},{"className":6792,"style":221},[220],[],{"type":16,"tag":103,"props":6795,"children":6797},{"className":6796},[226,227,228,229],[6798],{"type":16,"tag":103,"props":6799,"children":6801},{"className":6800},[183,229],[6802,6867,6872],{"type":16,"tag":103,"props":6803,"children":6805},{"className":6804},[183,229],[6806,6811],{"type":16,"tag":103,"props":6807,"children":6809},{"className":6808},[183,188,229],[6810],{"type":22,"value":1650},{"type":16,"tag":103,"props":6812,"children":6814},{"className":6813},[194],[6815],{"type":16,"tag":103,"props":6816,"children":6818},{"className":6817},[199,200],[6819,6856],{"type":16,"tag":103,"props":6820,"children":6822},{"className":6821},[205],[6823,6851],{"type":16,"tag":103,"props":6824,"children":6826},{"className":6825,"style":6620},[210],[6827],{"type":16,"tag":103,"props":6828,"children":6829},{"style":6624},[6830,6834],{"type":16,"tag":103,"props":6831,"children":6833},{"className":6832,"style":6629},[220],[],{"type":16,"tag":103,"props":6835,"children":6837},{"className":6836},[226,6634,6635,229],[6838],{"type":16,"tag":103,"props":6839,"children":6841},{"className":6840},[183,229],[6842],{"type":16,"tag":103,"props":6843,"children":6845},{"className":6844},[183,229],[6846],{"type":16,"tag":103,"props":6847,"children":6849},{"className":6848},[183,1481,229],[6850],{"type":22,"value":1753},{"type":16,"tag":103,"props":6852,"children":6854},{"className":6853},[254],[6855],{"type":22,"value":257},{"type":16,"tag":103,"props":6857,"children":6859},{"className":6858},[205],[6860],{"type":16,"tag":103,"props":6861,"children":6863},{"className":6862,"style":6662},[210],[6864],{"type":16,"tag":103,"props":6865,"children":6866},{},[],{"type":16,"tag":103,"props":6868,"children":6870},{"className":6869},[243,229],[6871],{"type":22,"value":5826},{"type":16,"tag":103,"props":6873,"children":6875},{"className":6874},[183,229],[6876,6881],{"type":16,"tag":103,"props":6877,"children":6879},{"className":6878},[183,188,229],[6880],{"type":22,"value":1650},{"type":16,"tag":103,"props":6882,"children":6884},{"className":6883},[194],[6885],{"type":16,"tag":103,"props":6886,"children":6888},{"className":6887},[199,200],[6889,6918],{"type":16,"tag":103,"props":6890,"children":6892},{"className":6891},[205],[6893,6913],{"type":16,"tag":103,"props":6894,"children":6896},{"className":6895,"style":6620},[210],[6897],{"type":16,"tag":103,"props":6898,"children":6899},{"style":6624},[6900,6904],{"type":16,"tag":103,"props":6901,"children":6903},{"className":6902,"style":6629},[220],[],{"type":16,"tag":103,"props":6905,"children":6907},{"className":6906},[226,6634,6635,229],[6908],{"type":16,"tag":103,"props":6909,"children":6911},{"className":6910,"style":2089},[183,188,229],[6912],{"type":22,"value":2032},{"type":16,"tag":103,"props":6914,"children":6916},{"className":6915},[254],[6917],{"type":22,"value":257},{"type":16,"tag":103,"props":6919,"children":6921},{"className":6920},[205],[6922],{"type":16,"tag":103,"props":6923,"children":6925},{"className":6924,"style":6662},[210],[6926],{"type":16,"tag":103,"props":6927,"children":6928},{},[],{"type":16,"tag":103,"props":6930,"children":6932},{"className":6931},[626],[6933],{"type":22,"value":515},{"type":16,"tag":103,"props":6935,"children":6936},{"style":6356},[6937,6941],{"type":16,"tag":103,"props":6938,"children":6940},{"className":6939,"style":2292},[220],[],{"type":16,"tag":103,"props":6942,"children":6944},{"className":6943},[183],[6945,6949,6953,6958,6962,7132],{"type":16,"tag":103,"props":6946,"children":6948},{"className":6947},[183],[],{"type":16,"tag":103,"props":6950,"children":6952},{"className":6951,"style":816},[632],[],{"type":16,"tag":103,"props":6954,"children":6956},{"className":6955},[821],[6957],{"type":22,"value":5779},{"type":16,"tag":103,"props":6959,"children":6961},{"className":6960,"style":816},[632],[],{"type":16,"tag":103,"props":6963,"children":6965},{"className":6964},[183],[6966,6971],{"type":16,"tag":103,"props":6967,"children":6969},{"className":6968},[183,6561],[6970],{"type":22,"value":5788},{"type":16,"tag":103,"props":6972,"children":6974},{"className":6973},[194],[6975],{"type":16,"tag":103,"props":6976,"children":6978},{"className":6977},[199],[6979],{"type":16,"tag":103,"props":6980,"children":6982},{"className":6981},[205],[6983],{"type":16,"tag":103,"props":6984,"children":6986},{"className":6985,"style":6579},[210],[6987],{"type":16,"tag":103,"props":6988,"children":6989},{"style":5185},[6990,6994],{"type":16,"tag":103,"props":6991,"children":6993},{"className":6992,"style":221},[220],[],{"type":16,"tag":103,"props":6995,"children":6997},{"className":6996},[226,227,228,229],[6998],{"type":16,"tag":103,"props":6999,"children":7001},{"className":7000},[183,229],[7002,7067,7072],{"type":16,"tag":103,"props":7003,"children":7005},{"className":7004},[183,229],[7006,7011],{"type":16,"tag":103,"props":7007,"children":7009},{"className":7008},[183,188,229],[7010],{"type":22,"value":1650},{"type":16,"tag":103,"props":7012,"children":7014},{"className":7013},[194],[7015],{"type":16,"tag":103,"props":7016,"children":7018},{"className":7017},[199,200],[7019,7056],{"type":16,"tag":103,"props":7020,"children":7022},{"className":7021},[205],[7023,7051],{"type":16,"tag":103,"props":7024,"children":7026},{"className":7025,"style":6620},[210],[7027],{"type":16,"tag":103,"props":7028,"children":7029},{"style":6624},[7030,7034],{"type":16,"tag":103,"props":7031,"children":7033},{"className":7032,"style":6629},[220],[],{"type":16,"tag":103,"props":7035,"children":7037},{"className":7036},[226,6634,6635,229],[7038],{"type":16,"tag":103,"props":7039,"children":7041},{"className":7040},[183,229],[7042],{"type":16,"tag":103,"props":7043,"children":7045},{"className":7044},[183,229],[7046],{"type":16,"tag":103,"props":7047,"children":7049},{"className":7048},[183,1481,229],[7050],{"type":22,"value":1753},{"type":16,"tag":103,"props":7052,"children":7054},{"className":7053},[254],[7055],{"type":22,"value":257},{"type":16,"tag":103,"props":7057,"children":7059},{"className":7058},[205],[7060],{"type":16,"tag":103,"props":7061,"children":7063},{"className":7062,"style":6662},[210],[7064],{"type":16,"tag":103,"props":7065,"children":7066},{},[],{"type":16,"tag":103,"props":7068,"children":7070},{"className":7069},[243,229],[7071],{"type":22,"value":5826},{"type":16,"tag":103,"props":7073,"children":7075},{"className":7074},[183,229],[7076,7081],{"type":16,"tag":103,"props":7077,"children":7079},{"className":7078},[183,188,229],[7080],{"type":22,"value":1650},{"type":16,"tag":103,"props":7082,"children":7084},{"className":7083},[194],[7085],{"type":16,"tag":103,"props":7086,"children":7088},{"className":7087},[199,200],[7089,7120],{"type":16,"tag":103,"props":7090,"children":7092},{"className":7091},[205],[7093,7115],{"type":16,"tag":103,"props":7094,"children":7097},{"className":7095,"style":7096},[210],"height:0.1645em;",[7098],{"type":16,"tag":103,"props":7099,"children":7101},{"style":7100},"top:-2.357em;margin-left:0em;margin-right:0.0714em;",[7102,7106],{"type":16,"tag":103,"props":7103,"children":7105},{"className":7104,"style":6629},[220],[],{"type":16,"tag":103,"props":7107,"children":7109},{"className":7108},[226,6634,6635,229],[7110],{"type":16,"tag":103,"props":7111,"children":7113},{"className":7112,"style":1124},[183,188,229],[7114],{"type":22,"value":2138},{"type":16,"tag":103,"props":7116,"children":7118},{"className":7117},[254],[7119],{"type":22,"value":257},{"type":16,"tag":103,"props":7121,"children":7123},{"className":7122},[205],[7124],{"type":16,"tag":103,"props":7125,"children":7128},{"className":7126,"style":7127},[210],"height:0.143em;",[7129],{"type":16,"tag":103,"props":7130,"children":7131},{},[],{"type":16,"tag":103,"props":7133,"children":7135},{"className":7134},[626],[7136],{"type":22,"value":515},{"type":16,"tag":103,"props":7138,"children":7139},{"style":6441},[7140,7144],{"type":16,"tag":103,"props":7141,"children":7143},{"className":7142,"style":2292},[220],[],{"type":16,"tag":103,"props":7145,"children":7147},{"className":7146},[183],[7148,7152,7156,7161,7165,7337],{"type":16,"tag":103,"props":7149,"children":7151},{"className":7150},[183],[],{"type":16,"tag":103,"props":7153,"children":7155},{"className":7154,"style":816},[632],[],{"type":16,"tag":103,"props":7157,"children":7159},{"className":7158},[821],[7160],{"type":22,"value":5779},{"type":16,"tag":103,"props":7162,"children":7164},{"className":7163,"style":816},[632],[],{"type":16,"tag":103,"props":7166,"children":7168},{"className":7167},[183],[7169,7174],{"type":16,"tag":103,"props":7170,"children":7172},{"className":7171},[183,6561],[7173],{"type":22,"value":5788},{"type":16,"tag":103,"props":7175,"children":7177},{"className":7176},[194],[7178],{"type":16,"tag":103,"props":7179,"children":7181},{"className":7180},[199],[7182],{"type":16,"tag":103,"props":7183,"children":7185},{"className":7184},[205],[7186],{"type":16,"tag":103,"props":7187,"children":7189},{"className":7188,"style":6579},[210],[7190],{"type":16,"tag":103,"props":7191,"children":7192},{"style":5185},[7193,7197],{"type":16,"tag":103,"props":7194,"children":7196},{"className":7195,"style":221},[220],[],{"type":16,"tag":103,"props":7198,"children":7200},{"className":7199},[226,227,228,229],[7201],{"type":16,"tag":103,"props":7202,"children":7204},{"className":7203},[183,229],[7205,7210,7267,7272],{"type":16,"tag":103,"props":7206,"children":7208},{"className":7207},[183,188,229],[7209],{"type":22,"value":135},{"type":16,"tag":103,"props":7211,"children":7213},{"className":7212},[183,229],[7214,7219],{"type":16,"tag":103,"props":7215,"children":7217},{"className":7216},[183,188,229],[7218],{"type":22,"value":1650},{"type":16,"tag":103,"props":7220,"children":7222},{"className":7221},[194],[7223],{"type":16,"tag":103,"props":7224,"children":7226},{"className":7225},[199,200],[7227,7256],{"type":16,"tag":103,"props":7228,"children":7230},{"className":7229},[205],[7231,7251],{"type":16,"tag":103,"props":7232,"children":7234},{"className":7233,"style":7096},[210],[7235],{"type":16,"tag":103,"props":7236,"children":7237},{"style":7100},[7238,7242],{"type":16,"tag":103,"props":7239,"children":7241},{"className":7240,"style":6629},[220],[],{"type":16,"tag":103,"props":7243,"children":7245},{"className":7244},[226,6634,6635,229],[7246],{"type":16,"tag":103,"props":7247,"children":7249},{"className":7248,"style":1124},[183,188,229],[7250],{"type":22,"value":2138},{"type":16,"tag":103,"props":7252,"children":7254},{"className":7253},[254],[7255],{"type":22,"value":257},{"type":16,"tag":103,"props":7257,"children":7259},{"className":7258},[205],[7260],{"type":16,"tag":103,"props":7261,"children":7263},{"className":7262,"style":7127},[210],[7264],{"type":16,"tag":103,"props":7265,"children":7266},{},[],{"type":16,"tag":103,"props":7268,"children":7270},{"className":7269},[243,229],[7271],{"type":22,"value":5826},{"type":16,"tag":103,"props":7273,"children":7275},{"className":7274},[183,229],[7276,7281],{"type":16,"tag":103,"props":7277,"children":7279},{"className":7278},[183,188,229],[7280],{"type":22,"value":1650},{"type":16,"tag":103,"props":7282,"children":7284},{"className":7283},[194],[7285],{"type":16,"tag":103,"props":7286,"children":7288},{"className":7287},[199,200],[7289,7326],{"type":16,"tag":103,"props":7290,"children":7292},{"className":7291},[205],[7293,7321],{"type":16,"tag":103,"props":7294,"children":7296},{"className":7295,"style":6620},[210],[7297],{"type":16,"tag":103,"props":7298,"children":7299},{"style":6624},[7300,7304],{"type":16,"tag":103,"props":7301,"children":7303},{"className":7302,"style":6629},[220],[],{"type":16,"tag":103,"props":7305,"children":7307},{"className":7306},[226,6634,6635,229],[7308],{"type":16,"tag":103,"props":7309,"children":7311},{"className":7310},[183,229],[7312],{"type":16,"tag":103,"props":7313,"children":7315},{"className":7314},[183,229],[7316],{"type":16,"tag":103,"props":7317,"children":7319},{"className":7318},[183,1481,229],[7320],{"type":22,"value":1753},{"type":16,"tag":103,"props":7322,"children":7324},{"className":7323},[254],[7325],{"type":22,"value":257},{"type":16,"tag":103,"props":7327,"children":7329},{"className":7328},[205],[7330],{"type":16,"tag":103,"props":7331,"children":7333},{"className":7332,"style":6662},[210],[7334],{"type":16,"tag":103,"props":7335,"children":7336},{},[],{"type":16,"tag":103,"props":7338,"children":7340},{"className":7339},[183],[7341],{"type":22,"value":6139},{"type":16,"tag":103,"props":7343,"children":7345},{"className":7344},[254],[7346],{"type":22,"value":257},{"type":16,"tag":103,"props":7348,"children":7350},{"className":7349},[205],[7351],{"type":16,"tag":103,"props":7352,"children":7354},{"className":7353,"style":6504},[210],[7355],{"type":16,"tag":103,"props":7356,"children":7357},{},[],{"type":16,"tag":25,"props":7359,"children":7360},{},[7361,7363,7441,7443,7811],{"type":22,"value":7362},"本文使用 ",{"type":16,"tag":103,"props":7364,"children":7366},{"className":7365},[106],[7367,7398],{"type":16,"tag":103,"props":7368,"children":7370},{"className":7369},[111],[7371],{"type":16,"tag":114,"props":7372,"children":7373},{"xmlns":116},[7374],{"type":16,"tag":119,"props":7375,"children":7376},{},[7377,7393],{"type":16,"tag":123,"props":7378,"children":7379},{},[7380,7384,7388],{"type":16,"tag":131,"props":7381,"children":7382},{},[7383],{"type":22,"value":135},{"type":16,"tag":145,"props":7385,"children":7386},{},[7387],{"type":22,"value":746},{"type":16,"tag":151,"props":7389,"children":7390},{},[7391],{"type":22,"value":7392},"8",{"type":16,"tag":157,"props":7394,"children":7395},{"encoding":159},[7396],{"type":22,"value":7397},"h = 8",{"type":16,"tag":103,"props":7399,"children":7401},{"className":7400,"ariaHidden":167},[166],[7402,7428],{"type":16,"tag":103,"props":7403,"children":7405},{"className":7404},[172],[7406,7410,7415,7419,7424],{"type":16,"tag":103,"props":7407,"children":7409},{"className":7408,"style":3719},[177],[],{"type":16,"tag":103,"props":7411,"children":7413},{"className":7412},[183,188],[7414],{"type":22,"value":135},{"type":16,"tag":103,"props":7416,"children":7418},{"className":7417,"style":816},[632],[],{"type":16,"tag":103,"props":7420,"children":7422},{"className":7421},[821],[7423],{"type":22,"value":746},{"type":16,"tag":103,"props":7425,"children":7427},{"className":7426,"style":816},[632],[],{"type":16,"tag":103,"props":7429,"children":7431},{"className":7430},[172],[7432,7436],{"type":16,"tag":103,"props":7433,"children":7435},{"className":7434,"style":1375},[177],[],{"type":16,"tag":103,"props":7437,"children":7439},{"className":7438},[183],[7440],{"type":22,"value":7392},{"type":22,"value":7442}," 个并行注意力层，即 8 个头。每个头使用 ",{"type":16,"tag":103,"props":7444,"children":7446},{"className":7445},[106],[7447,7542],{"type":16,"tag":103,"props":7448,"children":7450},{"className":7449},[111],[7451],{"type":16,"tag":114,"props":7452,"children":7453},{"xmlns":116},[7454],{"type":16,"tag":119,"props":7455,"children":7456},{},[7457,7537],{"type":16,"tag":123,"props":7458,"children":7459},{},[7460,7471,7475,7486,7490,7520,7524,7528,7532],{"type":16,"tag":127,"props":7461,"children":7462},{},[7463,7467],{"type":16,"tag":131,"props":7464,"children":7465},{},[7466],{"type":22,"value":1650},{"type":16,"tag":131,"props":7468,"children":7469},{},[7470],{"type":22,"value":2032},{"type":16,"tag":145,"props":7472,"children":7473},{},[7474],{"type":22,"value":746},{"type":16,"tag":127,"props":7476,"children":7477},{},[7478,7482],{"type":16,"tag":131,"props":7479,"children":7480},{},[7481],{"type":22,"value":1650},{"type":16,"tag":131,"props":7483,"children":7484},{},[7485],{"type":22,"value":2138},{"type":16,"tag":145,"props":7487,"children":7488},{},[7489],{"type":22,"value":746},{"type":16,"tag":127,"props":7491,"children":7492},{},[7493,7497],{"type":16,"tag":131,"props":7494,"children":7495},{},[7496],{"type":22,"value":1650},{"type":16,"tag":123,"props":7498,"children":7499},{},[7500,7504,7508,7512,7516],{"type":16,"tag":131,"props":7501,"children":7502},{"mathvariant":1409},[7503],{"type":22,"value":1090},{"type":16,"tag":131,"props":7505,"children":7506},{"mathvariant":1409},[7507],{"type":22,"value":1662},{"type":16,"tag":131,"props":7509,"children":7510},{"mathvariant":1409},[7511],{"type":22,"value":1650},{"type":16,"tag":131,"props":7513,"children":7514},{"mathvariant":1409},[7515],{"type":22,"value":1671},{"type":16,"tag":131,"props":7517,"children":7518},{"mathvariant":1409},[7519],{"type":22,"value":1676},{"type":16,"tag":131,"props":7521,"children":7522},{"mathvariant":1409},[7523],{"type":22,"value":3474},{"type":16,"tag":131,"props":7525,"children":7526},{},[7527],{"type":22,"value":135},{"type":16,"tag":145,"props":7529,"children":7530},{},[7531],{"type":22,"value":746},{"type":16,"tag":151,"props":7533,"children":7534},{},[7535],{"type":22,"value":7536},"64",{"type":16,"tag":157,"props":7538,"children":7539},{"encoding":159},[7540],{"type":22,"value":7541},"d_k = d_v = d_{\\mathrm{model}} \u002F h = 64",{"type":16,"tag":103,"props":7543,"children":7545},{"className":7544,"ariaHidden":167},[166],[7546,7624,7702,7798],{"type":16,"tag":103,"props":7547,"children":7549},{"className":7548},[172],[7550,7554,7611,7615,7620],{"type":16,"tag":103,"props":7551,"children":7553},{"className":7552,"style":361},[177],[],{"type":16,"tag":103,"props":7555,"children":7557},{"className":7556},[183],[7558,7563],{"type":16,"tag":103,"props":7559,"children":7561},{"className":7560},[183,188],[7562],{"type":22,"value":1650},{"type":16,"tag":103,"props":7564,"children":7566},{"className":7565},[194],[7567],{"type":16,"tag":103,"props":7568,"children":7570},{"className":7569},[199,200],[7571,7600],{"type":16,"tag":103,"props":7572,"children":7574},{"className":7573},[205],[7575,7595],{"type":16,"tag":103,"props":7576,"children":7578},{"className":7577,"style":1727},[210],[7579],{"type":16,"tag":103,"props":7580,"children":7581},{"style":215},[7582,7586],{"type":16,"tag":103,"props":7583,"children":7585},{"className":7584,"style":221},[220],[],{"type":16,"tag":103,"props":7587,"children":7589},{"className":7588},[226,227,228,229],[7590],{"type":16,"tag":103,"props":7591,"children":7593},{"className":7592,"style":2089},[183,188,229],[7594],{"type":22,"value":2032},{"type":16,"tag":103,"props":7596,"children":7598},{"className":7597},[254],[7599],{"type":22,"value":257},{"type":16,"tag":103,"props":7601,"children":7603},{"className":7602},[205],[7604],{"type":16,"tag":103,"props":7605,"children":7607},{"className":7606,"style":417},[210],[7608],{"type":16,"tag":103,"props":7609,"children":7610},{},[],{"type":16,"tag":103,"props":7612,"children":7614},{"className":7613,"style":816},[632],[],{"type":16,"tag":103,"props":7616,"children":7618},{"className":7617},[821],[7619],{"type":22,"value":746},{"type":16,"tag":103,"props":7621,"children":7623},{"className":7622,"style":816},[632],[],{"type":16,"tag":103,"props":7625,"children":7627},{"className":7626},[172],[7628,7632,7689,7693,7698],{"type":16,"tag":103,"props":7629,"children":7631},{"className":7630,"style":361},[177],[],{"type":16,"tag":103,"props":7633,"children":7635},{"className":7634},[183],[7636,7641],{"type":16,"tag":103,"props":7637,"children":7639},{"className":7638},[183,188],[7640],{"type":22,"value":1650},{"type":16,"tag":103,"props":7642,"children":7644},{"className":7643},[194],[7645],{"type":16,"tag":103,"props":7646,"children":7648},{"className":7647},[199,200],[7649,7678],{"type":16,"tag":103,"props":7650,"children":7652},{"className":7651},[205],[7653,7673],{"type":16,"tag":103,"props":7654,"children":7656},{"className":7655,"style":678},[210],[7657],{"type":16,"tag":103,"props":7658,"children":7659},{"style":215},[7660,7664],{"type":16,"tag":103,"props":7661,"children":7663},{"className":7662,"style":221},[220],[],{"type":16,"tag":103,"props":7665,"children":7667},{"className":7666},[226,227,228,229],[7668],{"type":16,"tag":103,"props":7669,"children":7671},{"className":7670,"style":1124},[183,188,229],[7672],{"type":22,"value":2138},{"type":16,"tag":103,"props":7674,"children":7676},{"className":7675},[254],[7677],{"type":22,"value":257},{"type":16,"tag":103,"props":7679,"children":7681},{"className":7680},[205],[7682],{"type":16,"tag":103,"props":7683,"children":7685},{"className":7684,"style":417},[210],[7686],{"type":16,"tag":103,"props":7687,"children":7688},{},[],{"type":16,"tag":103,"props":7690,"children":7692},{"className":7691,"style":816},[632],[],{"type":16,"tag":103,"props":7694,"children":7696},{"className":7695},[821],[7697],{"type":22,"value":746},{"type":16,"tag":103,"props":7699,"children":7701},{"className":7700,"style":816},[632],[],{"type":16,"tag":103,"props":7703,"children":7705},{"className":7704},[172],[7706,7710,7775,7780,7785,7789,7794],{"type":16,"tag":103,"props":7707,"children":7709},{"className":7708,"style":558},[177],[],{"type":16,"tag":103,"props":7711,"children":7713},{"className":7712},[183],[7714,7719],{"type":16,"tag":103,"props":7715,"children":7717},{"className":7716},[183,188],[7718],{"type":22,"value":1650},{"type":16,"tag":103,"props":7720,"children":7722},{"className":7721},[194],[7723],{"type":16,"tag":103,"props":7724,"children":7726},{"className":7725},[199,200],[7727,7764],{"type":16,"tag":103,"props":7728,"children":7730},{"className":7729},[205],[7731,7759],{"type":16,"tag":103,"props":7732,"children":7734},{"className":7733,"style":1727},[210],[7735],{"type":16,"tag":103,"props":7736,"children":7737},{"style":215},[7738,7742],{"type":16,"tag":103,"props":7739,"children":7741},{"className":7740,"style":221},[220],[],{"type":16,"tag":103,"props":7743,"children":7745},{"className":7744},[226,227,228,229],[7746],{"type":16,"tag":103,"props":7747,"children":7749},{"className":7748},[183,229],[7750],{"type":16,"tag":103,"props":7751,"children":7753},{"className":7752},[183,229],[7754],{"type":16,"tag":103,"props":7755,"children":7757},{"className":7756},[183,1481,229],[7758],{"type":22,"value":1753},{"type":16,"tag":103,"props":7760,"children":7762},{"className":7761},[254],[7763],{"type":22,"value":257},{"type":16,"tag":103,"props":7765,"children":7767},{"className":7766},[205],[7768],{"type":16,"tag":103,"props":7769,"children":7771},{"className":7770,"style":417},[210],[7772],{"type":16,"tag":103,"props":7773,"children":7774},{},[],{"type":16,"tag":103,"props":7776,"children":7778},{"className":7777},[183],[7779],{"type":22,"value":3474},{"type":16,"tag":103,"props":7781,"children":7783},{"className":7782},[183,188],[7784],{"type":22,"value":135},{"type":16,"tag":103,"props":7786,"children":7788},{"className":7787,"style":816},[632],[],{"type":16,"tag":103,"props":7790,"children":7792},{"className":7791},[821],[7793],{"type":22,"value":746},{"type":16,"tag":103,"props":7795,"children":7797},{"className":7796,"style":816},[632],[],{"type":16,"tag":103,"props":7799,"children":7801},{"className":7800},[172],[7802,7806],{"type":16,"tag":103,"props":7803,"children":7805},{"className":7804,"style":1375},[177],[],{"type":16,"tag":103,"props":7807,"children":7809},{"className":7808},[183],[7810],{"type":22,"value":7536},{"type":22,"value":7812},"。由于每个头的维度降低，总计算成本与完整维度的单头注意力相近。",{"type":16,"tag":1993,"props":7814,"children":7816},{"id":7815},"_323-注意力在本文模型中的应用",[7817],{"type":22,"value":7818},"3.2.3 注意力在本文模型中的应用",{"type":16,"tag":25,"props":7820,"children":7821},{},[7822],{"type":22,"value":7823},"Transformer 以三种不同方式使用多头注意力：",{"type":16,"tag":7825,"props":7826,"children":7827},"ul",{},[7828,7839,7849],{"type":16,"tag":7829,"props":7830,"children":7831},"li",{},[7832,7837],{"type":16,"tag":29,"props":7833,"children":7834},{},[7835],{"type":22,"value":7836},"编码器-解码器注意力：",{"type":22,"value":7838}," 查询来自前一个解码器层，记忆键和值来自编码器输出。因此，解码器的每个位置都可以关注输入序列的所有位置。这类似于 [31, 2, 8] 等序列到序列模型中的典型编码器-解码器注意力机制。",{"type":16,"tag":7829,"props":7840,"children":7841},{},[7842,7847],{"type":16,"tag":29,"props":7843,"children":7844},{},[7845],{"type":22,"value":7846},"编码器自注意力：",{"type":22,"value":7848}," 编码器中的键、值和查询都来自同一个位置，即编码器前一层的输出。因此，编码器中的每个位置都可以关注编码器前一层的所有位置。",{"type":16,"tag":7829,"props":7850,"children":7851},{},[7852,7857,7859,7913],{"type":16,"tag":29,"props":7853,"children":7854},{},[7855],{"type":22,"value":7856},"解码器自注意力：",{"type":22,"value":7858}," 解码器中的每个位置都可以关注解码器中截至该位置的所有位置。为了保持自回归性质，必须阻止信息向左之外的未来位置流动。我们在缩放点积注意力中通过掩码实现这一点：把 softmax 输入中对应非法连接的值设置为 ",{"type":16,"tag":103,"props":7860,"children":7862},{"className":7861},[106],[7863,7890],{"type":16,"tag":103,"props":7864,"children":7866},{"className":7865},[111],[7867],{"type":16,"tag":114,"props":7868,"children":7869},{"xmlns":116},[7870],{"type":16,"tag":119,"props":7871,"children":7872},{},[7873,7885],{"type":16,"tag":123,"props":7874,"children":7875},{},[7876,7880],{"type":16,"tag":145,"props":7877,"children":7878},{},[7879],{"type":22,"value":149},{"type":16,"tag":131,"props":7881,"children":7882},{"mathvariant":1409},[7883],{"type":22,"value":7884},"∞",{"type":16,"tag":157,"props":7886,"children":7887},{"encoding":159},[7888],{"type":22,"value":7889},"-\\infty",{"type":16,"tag":103,"props":7891,"children":7893},{"className":7892,"ariaHidden":167},[166],[7894],{"type":16,"tag":103,"props":7895,"children":7897},{"className":7896},[172],[7898,7903,7908],{"type":16,"tag":103,"props":7899,"children":7902},{"className":7900,"style":7901},[177],"height:0.6667em;vertical-align:-0.0833em;",[],{"type":16,"tag":103,"props":7904,"children":7906},{"className":7905},[183],[7907],{"type":22,"value":149},{"type":16,"tag":103,"props":7909,"children":7911},{"className":7910},[183],[7912],{"type":22,"value":7884},{"type":22,"value":1266},{"type":16,"tag":1284,"props":7915,"children":7917},{"id":7916},"_33-逐位置前馈网络",[7918],{"type":22,"value":7919},"3.3 逐位置前馈网络",{"type":16,"tag":25,"props":7921,"children":7922},{},[7923],{"type":22,"value":7924},"除了注意力子层外，编码器和解码器的每一层还包含一个逐位置、参数共享的全连接前馈网络。它由两个线性变换组成，中间使用 ReLU 激活：",{"type":16,"tag":103,"props":7926,"children":7928},{"className":7927},[1387],[7929],{"type":16,"tag":103,"props":7930,"children":7932},{"className":7931},[106],[7933,8082],{"type":16,"tag":103,"props":7934,"children":7936},{"className":7935},[111],[7937],{"type":16,"tag":114,"props":7938,"children":7939},{"xmlns":116,"display":1399},[7940],{"type":16,"tag":119,"props":7941,"children":7942},{},[7943,8077],{"type":16,"tag":2574,"props":7944,"children":7945},{"width":2576},[7946],{"type":16,"tag":2579,"props":7947,"children":7948},{},[7949,7952,8066,8069],{"type":16,"tag":2583,"props":7950,"children":7951},{"width":2585},[],{"type":16,"tag":2583,"props":7953,"children":7954},{},[7955],{"type":16,"tag":123,"props":7956,"children":7957},{},[7958,7963,7967,7971,7975,7979,7983,7988,7992,7996,8000,8004,8008,8019,8023,8035,8039,8051,8055],{"type":16,"tag":131,"props":7959,"children":7960},{"mathvariant":1409},[7961],{"type":22,"value":7962},"FFN",{"type":16,"tag":145,"props":7964,"children":7965},{},[7966],{"type":22,"value":1417},{"type":16,"tag":145,"props":7968,"children":7969},{"stretchy":495},[7970],{"type":22,"value":498},{"type":16,"tag":131,"props":7972,"children":7973},{},[7974],{"type":22,"value":506},{"type":16,"tag":145,"props":7976,"children":7977},{"stretchy":495},[7978],{"type":22,"value":541},{"type":16,"tag":145,"props":7980,"children":7981},{},[7982],{"type":22,"value":746},{"type":16,"tag":131,"props":7984,"children":7985},{},[7986],{"type":22,"value":7987},"max",{"type":16,"tag":145,"props":7989,"children":7990},{},[7991],{"type":22,"value":1417},{"type":16,"tag":145,"props":7993,"children":7994},{"stretchy":495},[7995],{"type":22,"value":498},{"type":16,"tag":151,"props":7997,"children":7998},{},[7999],{"type":22,"value":5746},{"type":16,"tag":145,"props":8001,"children":8002},{"separator":167},[8003],{"type":22,"value":515},{"type":16,"tag":131,"props":8005,"children":8006},{},[8007],{"type":22,"value":506},{"type":16,"tag":127,"props":8009,"children":8010},{},[8011,8015],{"type":16,"tag":131,"props":8012,"children":8013},{},[8014],{"type":22,"value":4891},{"type":16,"tag":151,"props":8016,"children":8017},{},[8018],{"type":22,"value":155},{"type":16,"tag":145,"props":8020,"children":8021},{},[8022],{"type":22,"value":1430},{"type":16,"tag":127,"props":8024,"children":8025},{},[8026,8031],{"type":16,"tag":131,"props":8027,"children":8028},{},[8029],{"type":22,"value":8030},"b",{"type":16,"tag":151,"props":8032,"children":8033},{},[8034],{"type":22,"value":155},{"type":16,"tag":145,"props":8036,"children":8037},{"stretchy":495},[8038],{"type":22,"value":541},{"type":16,"tag":127,"props":8040,"children":8041},{},[8042,8046],{"type":16,"tag":131,"props":8043,"children":8044},{},[8045],{"type":22,"value":4891},{"type":16,"tag":151,"props":8047,"children":8048},{},[8049],{"type":22,"value":8050},"2",{"type":16,"tag":145,"props":8052,"children":8053},{},[8054],{"type":22,"value":1430},{"type":16,"tag":127,"props":8056,"children":8057},{},[8058,8062],{"type":16,"tag":131,"props":8059,"children":8060},{},[8061],{"type":22,"value":8030},{"type":16,"tag":151,"props":8063,"children":8064},{},[8065],{"type":22,"value":8050},{"type":16,"tag":2583,"props":8067,"children":8068},{"width":2585},[],{"type":16,"tag":2583,"props":8070,"children":8071},{},[8072],{"type":16,"tag":2703,"props":8073,"children":8074},{},[8075],{"type":22,"value":8076},"(2)",{"type":16,"tag":157,"props":8078,"children":8079},{"encoding":159},[8080],{"type":22,"value":8081},"\\operatorname{FFN}(x) = \\max(0, xW_1 + b_1)W_2 + b_2\\tag{2}",{"type":16,"tag":103,"props":8083,"children":8085},{"className":8084,"ariaHidden":167},[166],[8086,8131,8239,8379,8444],{"type":16,"tag":103,"props":8087,"children":8089},{"className":8088},[172],[8090,8094,8103,8108,8113,8118,8122,8127],{"type":16,"tag":103,"props":8091,"children":8093},{"className":8092,"style":558},[177],[],{"type":16,"tag":103,"props":8095,"children":8097},{"className":8096},[1476],[8098],{"type":16,"tag":103,"props":8099,"children":8101},{"className":8100},[183,1481],[8102],{"type":22,"value":7962},{"type":16,"tag":103,"props":8104,"children":8106},{"className":8105},[563],[8107],{"type":22,"value":498},{"type":16,"tag":103,"props":8109,"children":8111},{"className":8110},[183,188],[8112],{"type":22,"value":506},{"type":16,"tag":103,"props":8114,"children":8116},{"className":8115},[715],[8117],{"type":22,"value":541},{"type":16,"tag":103,"props":8119,"children":8121},{"className":8120,"style":816},[632],[],{"type":16,"tag":103,"props":8123,"children":8125},{"className":8124},[821],[8126],{"type":22,"value":746},{"type":16,"tag":103,"props":8128,"children":8130},{"className":8129,"style":816},[632],[],{"type":16,"tag":103,"props":8132,"children":8134},{"className":8133},[172],[8135,8139,8144,8149,8154,8159,8163,8168,8226,8230,8235],{"type":16,"tag":103,"props":8136,"children":8138},{"className":8137,"style":558},[177],[],{"type":16,"tag":103,"props":8140,"children":8142},{"className":8141},[1476],[8143],{"type":22,"value":7987},{"type":16,"tag":103,"props":8145,"children":8147},{"className":8146},[563],[8148],{"type":22,"value":498},{"type":16,"tag":103,"props":8150,"children":8152},{"className":8151},[183],[8153],{"type":22,"value":5746},{"type":16,"tag":103,"props":8155,"children":8157},{"className":8156},[626],[8158],{"type":22,"value":515},{"type":16,"tag":103,"props":8160,"children":8162},{"className":8161,"style":633},[632],[],{"type":16,"tag":103,"props":8164,"children":8166},{"className":8165},[183,188],[8167],{"type":22,"value":506},{"type":16,"tag":103,"props":8169,"children":8171},{"className":8170},[183],[8172,8177],{"type":16,"tag":103,"props":8173,"children":8175},{"className":8174,"style":3056},[183,188],[8176],{"type":22,"value":4891},{"type":16,"tag":103,"props":8178,"children":8180},{"className":8179},[194],[8181],{"type":16,"tag":103,"props":8182,"children":8184},{"className":8183},[199,200],[8185,8215],{"type":16,"tag":103,"props":8186,"children":8188},{"className":8187},[205],[8189,8210],{"type":16,"tag":103,"props":8190,"children":8192},{"className":8191,"style":211},[210],[8193],{"type":16,"tag":103,"props":8194,"children":8196},{"style":8195},"top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;",[8197,8201],{"type":16,"tag":103,"props":8198,"children":8200},{"className":8199,"style":221},[220],[],{"type":16,"tag":103,"props":8202,"children":8204},{"className":8203},[226,227,228,229],[8205],{"type":16,"tag":103,"props":8206,"children":8208},{"className":8207},[183,229],[8209],{"type":22,"value":155},{"type":16,"tag":103,"props":8211,"children":8213},{"className":8212},[254],[8214],{"type":22,"value":257},{"type":16,"tag":103,"props":8216,"children":8218},{"className":8217},[205],[8219],{"type":16,"tag":103,"props":8220,"children":8222},{"className":8221,"style":417},[210],[8223],{"type":16,"tag":103,"props":8224,"children":8225},{},[],{"type":16,"tag":103,"props":8227,"children":8229},{"className":8228,"style":1497},[632],[],{"type":16,"tag":103,"props":8231,"children":8233},{"className":8232},[243],[8234],{"type":22,"value":1430},{"type":16,"tag":103,"props":8236,"children":8238},{"className":8237,"style":1497},[632],[],{"type":16,"tag":103,"props":8240,"children":8242},{"className":8241},[172],[8243,8247,8304,8309,8366,8370,8375],{"type":16,"tag":103,"props":8244,"children":8246},{"className":8245,"style":558},[177],[],{"type":16,"tag":103,"props":8248,"children":8250},{"className":8249},[183],[8251,8256],{"type":16,"tag":103,"props":8252,"children":8254},{"className":8253},[183,188],[8255],{"type":22,"value":8030},{"type":16,"tag":103,"props":8257,"children":8259},{"className":8258},[194],[8260],{"type":16,"tag":103,"props":8261,"children":8263},{"className":8262},[199,200],[8264,8293],{"type":16,"tag":103,"props":8265,"children":8267},{"className":8266},[205],[8268,8288],{"type":16,"tag":103,"props":8269,"children":8271},{"className":8270,"style":211},[210],[8272],{"type":16,"tag":103,"props":8273,"children":8274},{"style":215},[8275,8279],{"type":16,"tag":103,"props":8276,"children":8278},{"className":8277,"style":221},[220],[],{"type":16,"tag":103,"props":8280,"children":8282},{"className":8281},[226,227,228,229],[8283],{"type":16,"tag":103,"props":8284,"children":8286},{"className":8285},[183,229],[8287],{"type":22,"value":155},{"type":16,"tag":103,"props":8289,"children":8291},{"className":8290},[254],[8292],{"type":22,"value":257},{"type":16,"tag":103,"props":8294,"children":8296},{"className":8295},[205],[8297],{"type":16,"tag":103,"props":8298,"children":8300},{"className":8299,"style":417},[210],[8301],{"type":16,"tag":103,"props":8302,"children":8303},{},[],{"type":16,"tag":103,"props":8305,"children":8307},{"className":8306},[715],[8308],{"type":22,"value":541},{"type":16,"tag":103,"props":8310,"children":8312},{"className":8311},[183],[8313,8318],{"type":16,"tag":103,"props":8314,"children":8316},{"className":8315,"style":3056},[183,188],[8317],{"type":22,"value":4891},{"type":16,"tag":103,"props":8319,"children":8321},{"className":8320},[194],[8322],{"type":16,"tag":103,"props":8323,"children":8325},{"className":8324},[199,200],[8326,8355],{"type":16,"tag":103,"props":8327,"children":8329},{"className":8328},[205],[8330,8350],{"type":16,"tag":103,"props":8331,"children":8333},{"className":8332,"style":211},[210],[8334],{"type":16,"tag":103,"props":8335,"children":8336},{"style":8195},[8337,8341],{"type":16,"tag":103,"props":8338,"children":8340},{"className":8339,"style":221},[220],[],{"type":16,"tag":103,"props":8342,"children":8344},{"className":8343},[226,227,228,229],[8345],{"type":16,"tag":103,"props":8346,"children":8348},{"className":8347},[183,229],[8349],{"type":22,"value":8050},{"type":16,"tag":103,"props":8351,"children":8353},{"className":8352},[254],[8354],{"type":22,"value":257},{"type":16,"tag":103,"props":8356,"children":8358},{"className":8357},[205],[8359],{"type":16,"tag":103,"props":8360,"children":8362},{"className":8361,"style":417},[210],[8363],{"type":16,"tag":103,"props":8364,"children":8365},{},[],{"type":16,"tag":103,"props":8367,"children":8369},{"className":8368,"style":1497},[632],[],{"type":16,"tag":103,"props":8371,"children":8373},{"className":8372},[243],[8374],{"type":22,"value":1430},{"type":16,"tag":103,"props":8376,"children":8378},{"className":8377,"style":1497},[632],[],{"type":16,"tag":103,"props":8380,"children":8382},{"className":8381},[172],[8383,8387],{"type":16,"tag":103,"props":8384,"children":8386},{"className":8385,"style":361},[177],[],{"type":16,"tag":103,"props":8388,"children":8390},{"className":8389},[183],[8391,8396],{"type":16,"tag":103,"props":8392,"children":8394},{"className":8393},[183,188],[8395],{"type":22,"value":8030},{"type":16,"tag":103,"props":8397,"children":8399},{"className":8398},[194],[8400],{"type":16,"tag":103,"props":8401,"children":8403},{"className":8402},[199,200],[8404,8433],{"type":16,"tag":103,"props":8405,"children":8407},{"className":8406},[205],[8408,8428],{"type":16,"tag":103,"props":8409,"children":8411},{"className":8410,"style":211},[210],[8412],{"type":16,"tag":103,"props":8413,"children":8414},{"style":215},[8415,8419],{"type":16,"tag":103,"props":8416,"children":8418},{"className":8417,"style":221},[220],[],{"type":16,"tag":103,"props":8420,"children":8422},{"className":8421},[226,227,228,229],[8423],{"type":16,"tag":103,"props":8424,"children":8426},{"className":8425},[183,229],[8427],{"type":22,"value":8050},{"type":16,"tag":103,"props":8429,"children":8431},{"className":8430},[254],[8432],{"type":22,"value":257},{"type":16,"tag":103,"props":8434,"children":8436},{"className":8435},[205],[8437],{"type":16,"tag":103,"props":8438,"children":8440},{"className":8439,"style":417},[210],[8441],{"type":16,"tag":103,"props":8442,"children":8443},{},[],{"type":16,"tag":103,"props":8445,"children":8447},{"className":8446},[3101],[8448,8452],{"type":16,"tag":103,"props":8449,"children":8451},{"className":8450,"style":558},[177],[],{"type":16,"tag":103,"props":8453,"children":8455},{"className":8454},[183,22],[8456,8461,8470],{"type":16,"tag":103,"props":8457,"children":8459},{"className":8458},[183],[8460],{"type":22,"value":498},{"type":16,"tag":103,"props":8462,"children":8464},{"className":8463},[183],[8465],{"type":16,"tag":103,"props":8466,"children":8468},{"className":8467},[183],[8469],{"type":22,"value":8050},{"type":16,"tag":103,"props":8471,"children":8473},{"className":8472},[183],[8474],{"type":22,"value":541},{"type":16,"tag":25,"props":8476,"children":8477},{},[8478],{"type":22,"value":8479},"不同位置使用相同的线性变换，但不同层使用不同参数。换一种描述方式，它相当于两个 kernel size 为 1 的卷积。",{"type":16,"tag":25,"props":8481,"children":8482},{},[8483,8485,8647,8649,8804],{"type":22,"value":8484},"输入和输出维度为 ",{"type":16,"tag":103,"props":8486,"children":8488},{"className":8487},[106],[8489,8544],{"type":16,"tag":103,"props":8490,"children":8492},{"className":8491},[111],[8493],{"type":16,"tag":114,"props":8494,"children":8495},{"xmlns":116},[8496],{"type":16,"tag":119,"props":8497,"children":8498},{},[8499,8540],{"type":16,"tag":123,"props":8500,"children":8501},{},[8502,8532,8536],{"type":16,"tag":127,"props":8503,"children":8504},{},[8505,8509],{"type":16,"tag":131,"props":8506,"children":8507},{},[8508],{"type":22,"value":1650},{"type":16,"tag":123,"props":8510,"children":8511},{},[8512,8516,8520,8524,8528],{"type":16,"tag":131,"props":8513,"children":8514},{"mathvariant":1409},[8515],{"type":22,"value":1090},{"type":16,"tag":131,"props":8517,"children":8518},{"mathvariant":1409},[8519],{"type":22,"value":1662},{"type":16,"tag":131,"props":8521,"children":8522},{"mathvariant":1409},[8523],{"type":22,"value":1650},{"type":16,"tag":131,"props":8525,"children":8526},{"mathvariant":1409},[8527],{"type":22,"value":1671},{"type":16,"tag":131,"props":8529,"children":8530},{"mathvariant":1409},[8531],{"type":22,"value":1676},{"type":16,"tag":145,"props":8533,"children":8534},{},[8535],{"type":22,"value":746},{"type":16,"tag":151,"props":8537,"children":8538},{},[8539],{"type":22,"value":1685},{"type":16,"tag":157,"props":8541,"children":8542},{"encoding":159},[8543],{"type":22,"value":1690},{"type":16,"tag":103,"props":8545,"children":8547},{"className":8546,"ariaHidden":167},[166],[8548,8634],{"type":16,"tag":103,"props":8549,"children":8551},{"className":8550},[172],[8552,8556,8621,8625,8630],{"type":16,"tag":103,"props":8553,"children":8555},{"className":8554,"style":361},[177],[],{"type":16,"tag":103,"props":8557,"children":8559},{"className":8558},[183],[8560,8565],{"type":16,"tag":103,"props":8561,"children":8563},{"className":8562},[183,188],[8564],{"type":22,"value":1650},{"type":16,"tag":103,"props":8566,"children":8568},{"className":8567},[194],[8569],{"type":16,"tag":103,"props":8570,"children":8572},{"className":8571},[199,200],[8573,8610],{"type":16,"tag":103,"props":8574,"children":8576},{"className":8575},[205],[8577,8605],{"type":16,"tag":103,"props":8578,"children":8580},{"className":8579,"style":1727},[210],[8581],{"type":16,"tag":103,"props":8582,"children":8583},{"style":215},[8584,8588],{"type":16,"tag":103,"props":8585,"children":8587},{"className":8586,"style":221},[220],[],{"type":16,"tag":103,"props":8589,"children":8591},{"className":8590},[226,227,228,229],[8592],{"type":16,"tag":103,"props":8593,"children":8595},{"className":8594},[183,229],[8596],{"type":16,"tag":103,"props":8597,"children":8599},{"className":8598},[183,229],[8600],{"type":16,"tag":103,"props":8601,"children":8603},{"className":8602},[183,1481,229],[8604],{"type":22,"value":1753},{"type":16,"tag":103,"props":8606,"children":8608},{"className":8607},[254],[8609],{"type":22,"value":257},{"type":16,"tag":103,"props":8611,"children":8613},{"className":8612},[205],[8614],{"type":16,"tag":103,"props":8615,"children":8617},{"className":8616,"style":417},[210],[8618],{"type":16,"tag":103,"props":8619,"children":8620},{},[],{"type":16,"tag":103,"props":8622,"children":8624},{"className":8623,"style":816},[632],[],{"type":16,"tag":103,"props":8626,"children":8628},{"className":8627},[821],[8629],{"type":22,"value":746},{"type":16,"tag":103,"props":8631,"children":8633},{"className":8632,"style":816},[632],[],{"type":16,"tag":103,"props":8635,"children":8637},{"className":8636},[172],[8638,8642],{"type":16,"tag":103,"props":8639,"children":8641},{"className":8640,"style":1375},[177],[],{"type":16,"tag":103,"props":8643,"children":8645},{"className":8644},[183],[8646],{"type":22,"value":1685},{"type":22,"value":8648},"，内层维度为 ",{"type":16,"tag":103,"props":8650,"children":8652},{"className":8651},[106],[8653,8699],{"type":16,"tag":103,"props":8654,"children":8656},{"className":8655},[111],[8657],{"type":16,"tag":114,"props":8658,"children":8659},{"xmlns":116},[8660],{"type":16,"tag":119,"props":8661,"children":8662},{},[8663,8694],{"type":16,"tag":123,"props":8664,"children":8665},{},[8666,8685,8689],{"type":16,"tag":127,"props":8667,"children":8668},{},[8669,8673],{"type":16,"tag":131,"props":8670,"children":8671},{},[8672],{"type":22,"value":1650},{"type":16,"tag":123,"props":8674,"children":8675},{},[8676,8681],{"type":16,"tag":131,"props":8677,"children":8678},{"mathvariant":1409},[8679],{"type":22,"value":8680},"f",{"type":16,"tag":131,"props":8682,"children":8683},{"mathvariant":1409},[8684],{"type":22,"value":8680},{"type":16,"tag":145,"props":8686,"children":8687},{},[8688],{"type":22,"value":746},{"type":16,"tag":151,"props":8690,"children":8691},{},[8692],{"type":22,"value":8693},"2048",{"type":16,"tag":157,"props":8695,"children":8696},{"encoding":159},[8697],{"type":22,"value":8698},"d_{\\mathrm{ff}} = 2048",{"type":16,"tag":103,"props":8700,"children":8702},{"className":8701,"ariaHidden":167},[166],[8703,8791],{"type":16,"tag":103,"props":8704,"children":8706},{"className":8705},[172],[8707,8711,8778,8782,8787],{"type":16,"tag":103,"props":8708,"children":8710},{"className":8709,"style":361},[177],[],{"type":16,"tag":103,"props":8712,"children":8714},{"className":8713},[183],[8715,8720],{"type":16,"tag":103,"props":8716,"children":8718},{"className":8717},[183,188],[8719],{"type":22,"value":1650},{"type":16,"tag":103,"props":8721,"children":8723},{"className":8722},[194],[8724],{"type":16,"tag":103,"props":8725,"children":8727},{"className":8726},[199,200],[8728,8767],{"type":16,"tag":103,"props":8729,"children":8731},{"className":8730},[205],[8732,8762],{"type":16,"tag":103,"props":8733,"children":8735},{"className":8734,"style":1727},[210],[8736],{"type":16,"tag":103,"props":8737,"children":8738},{"style":215},[8739,8743],{"type":16,"tag":103,"props":8740,"children":8742},{"className":8741,"style":221},[220],[],{"type":16,"tag":103,"props":8744,"children":8746},{"className":8745},[226,227,228,229],[8747],{"type":16,"tag":103,"props":8748,"children":8750},{"className":8749},[183,229],[8751],{"type":16,"tag":103,"props":8752,"children":8754},{"className":8753},[183,229],[8755],{"type":16,"tag":103,"props":8756,"children":8759},{"className":8757,"style":8758},[183,1481,229],"margin-right:0.0778em;",[8760],{"type":22,"value":8761},"ff",{"type":16,"tag":103,"props":8763,"children":8765},{"className":8764},[254],[8766],{"type":22,"value":257},{"type":16,"tag":103,"props":8768,"children":8770},{"className":8769},[205],[8771],{"type":16,"tag":103,"props":8772,"children":8774},{"className":8773,"style":417},[210],[8775],{"type":16,"tag":103,"props":8776,"children":8777},{},[],{"type":16,"tag":103,"props":8779,"children":8781},{"className":8780,"style":816},[632],[],{"type":16,"tag":103,"props":8783,"children":8785},{"className":8784},[821],[8786],{"type":22,"value":746},{"type":16,"tag":103,"props":8788,"children":8790},{"className":8789,"style":816},[632],[],{"type":16,"tag":103,"props":8792,"children":8794},{"className":8793},[172],[8795,8799],{"type":16,"tag":103,"props":8796,"children":8798},{"className":8797,"style":1375},[177],[],{"type":16,"tag":103,"props":8800,"children":8802},{"className":8801},[183],[8803],{"type":22,"value":8693},{"type":22,"value":1266},{"type":16,"tag":1284,"props":8806,"children":8808},{"id":8807},"_34-嵌入与-softmax",[8809],{"type":22,"value":8810},"3.4 嵌入与 Softmax",{"type":16,"tag":25,"props":8812,"children":8813},{},[8814,8816,8944],{"type":22,"value":8815},"与其他序列转换模型类似，我们使用可学习的嵌入，把输入 token 和输出 token 转换为维度为 ",{"type":16,"tag":103,"props":8817,"children":8819},{"className":8818},[106],[8820,8867],{"type":16,"tag":103,"props":8821,"children":8823},{"className":8822},[111],[8824],{"type":16,"tag":114,"props":8825,"children":8826},{"xmlns":116},[8827],{"type":16,"tag":119,"props":8828,"children":8829},{},[8830,8863],{"type":16,"tag":123,"props":8831,"children":8832},{},[8833],{"type":16,"tag":127,"props":8834,"children":8835},{},[8836,8840],{"type":16,"tag":131,"props":8837,"children":8838},{},[8839],{"type":22,"value":1650},{"type":16,"tag":123,"props":8841,"children":8842},{},[8843,8847,8851,8855,8859],{"type":16,"tag":131,"props":8844,"children":8845},{"mathvariant":1409},[8846],{"type":22,"value":1090},{"type":16,"tag":131,"props":8848,"children":8849},{"mathvariant":1409},[8850],{"type":22,"value":1662},{"type":16,"tag":131,"props":8852,"children":8853},{"mathvariant":1409},[8854],{"type":22,"value":1650},{"type":16,"tag":131,"props":8856,"children":8857},{"mathvariant":1409},[8858],{"type":22,"value":1671},{"type":16,"tag":131,"props":8860,"children":8861},{"mathvariant":1409},[8862],{"type":22,"value":1676},{"type":16,"tag":157,"props":8864,"children":8865},{"encoding":159},[8866],{"type":22,"value":4216},{"type":16,"tag":103,"props":8868,"children":8870},{"className":8869,"ariaHidden":167},[166],[8871],{"type":16,"tag":103,"props":8872,"children":8874},{"className":8873},[172],[8875,8879],{"type":16,"tag":103,"props":8876,"children":8878},{"className":8877,"style":361},[177],[],{"type":16,"tag":103,"props":8880,"children":8882},{"className":8881},[183],[8883,8888],{"type":16,"tag":103,"props":8884,"children":8886},{"className":8885},[183,188],[8887],{"type":22,"value":1650},{"type":16,"tag":103,"props":8889,"children":8891},{"className":8890},[194],[8892],{"type":16,"tag":103,"props":8893,"children":8895},{"className":8894},[199,200],[8896,8933],{"type":16,"tag":103,"props":8897,"children":8899},{"className":8898},[205],[8900,8928],{"type":16,"tag":103,"props":8901,"children":8903},{"className":8902,"style":1727},[210],[8904],{"type":16,"tag":103,"props":8905,"children":8906},{"style":215},[8907,8911],{"type":16,"tag":103,"props":8908,"children":8910},{"className":8909,"style":221},[220],[],{"type":16,"tag":103,"props":8912,"children":8914},{"className":8913},[226,227,228,229],[8915],{"type":16,"tag":103,"props":8916,"children":8918},{"className":8917},[183,229],[8919],{"type":16,"tag":103,"props":8920,"children":8922},{"className":8921},[183,229],[8923],{"type":16,"tag":103,"props":8924,"children":8926},{"className":8925},[183,1481,229],[8927],{"type":22,"value":1753},{"type":16,"tag":103,"props":8929,"children":8931},{"className":8930},[254],[8932],{"type":22,"value":257},{"type":16,"tag":103,"props":8934,"children":8936},{"className":8935},[205],[8937],{"type":16,"tag":103,"props":8938,"children":8940},{"className":8939,"style":417},[210],[8941],{"type":16,"tag":103,"props":8942,"children":8943},{},[],{"type":22,"value":8945}," 的向量。我们还使用通常的可学习线性变换和 softmax 函数，把解码器输出转换为下一个 token 的预测概率。",{"type":16,"tag":25,"props":8947,"children":8948},{},[8949,8951,9144],{"type":22,"value":8950},"与 [24] 类似，我们在两个嵌入层和 softmax 前的线性变换之间共享同一个权重矩阵。在嵌入层中，我们把这些权重乘以 ",{"type":16,"tag":103,"props":8952,"children":8954},{"className":8953},[106],[8955,9006],{"type":16,"tag":103,"props":8956,"children":8958},{"className":8957},[111],[8959],{"type":16,"tag":114,"props":8960,"children":8961},{"xmlns":116},[8962],{"type":16,"tag":119,"props":8963,"children":8964},{},[8965,9001],{"type":16,"tag":123,"props":8966,"children":8967},{},[8968],{"type":16,"tag":2233,"props":8969,"children":8970},{},[8971],{"type":16,"tag":127,"props":8972,"children":8973},{},[8974,8978],{"type":16,"tag":131,"props":8975,"children":8976},{},[8977],{"type":22,"value":1650},{"type":16,"tag":123,"props":8979,"children":8980},{},[8981,8985,8989,8993,8997],{"type":16,"tag":131,"props":8982,"children":8983},{"mathvariant":1409},[8984],{"type":22,"value":1090},{"type":16,"tag":131,"props":8986,"children":8987},{"mathvariant":1409},[8988],{"type":22,"value":1662},{"type":16,"tag":131,"props":8990,"children":8991},{"mathvariant":1409},[8992],{"type":22,"value":1650},{"type":16,"tag":131,"props":8994,"children":8995},{"mathvariant":1409},[8996],{"type":22,"value":1671},{"type":16,"tag":131,"props":8998,"children":8999},{"mathvariant":1409},[9000],{"type":22,"value":1676},{"type":16,"tag":157,"props":9002,"children":9003},{"encoding":159},[9004],{"type":22,"value":9005},"\\sqrt{d_{\\mathrm{model}}}",{"type":16,"tag":103,"props":9007,"children":9009},{"className":9008,"ariaHidden":167},[166],[9010],{"type":16,"tag":103,"props":9011,"children":9013},{"className":9012},[172],[9014,9018],{"type":16,"tag":103,"props":9015,"children":9017},{"className":9016,"style":2263},[177],[],{"type":16,"tag":103,"props":9019,"children":9021},{"className":9020},[183,2268],[9022],{"type":16,"tag":103,"props":9023,"children":9025},{"className":9024},[199,200],[9026,9133],{"type":16,"tag":103,"props":9027,"children":9029},{"className":9028},[205],[9030,9128],{"type":16,"tag":103,"props":9031,"children":9033},{"className":9032,"style":2281},[210],[9034,9111],{"type":16,"tag":103,"props":9035,"children":9037},{"className":9036,"style":2287},[2286],[9038,9042],{"type":16,"tag":103,"props":9039,"children":9041},{"className":9040,"style":2292},[220],[],{"type":16,"tag":103,"props":9043,"children":9045},{"className":9044,"style":2297},[183],[9046],{"type":16,"tag":103,"props":9047,"children":9049},{"className":9048},[183],[9050,9055],{"type":16,"tag":103,"props":9051,"children":9053},{"className":9052},[183,188],[9054],{"type":22,"value":1650},{"type":16,"tag":103,"props":9056,"children":9058},{"className":9057},[194],[9059],{"type":16,"tag":103,"props":9060,"children":9062},{"className":9061},[199,200],[9063,9100],{"type":16,"tag":103,"props":9064,"children":9066},{"className":9065},[205],[9067,9095],{"type":16,"tag":103,"props":9068,"children":9070},{"className":9069,"style":1727},[210],[9071],{"type":16,"tag":103,"props":9072,"children":9073},{"style":215},[9074,9078],{"type":16,"tag":103,"props":9075,"children":9077},{"className":9076,"style":221},[220],[],{"type":16,"tag":103,"props":9079,"children":9081},{"className":9080},[226,227,228,229],[9082],{"type":16,"tag":103,"props":9083,"children":9085},{"className":9084},[183,229],[9086],{"type":16,"tag":103,"props":9087,"children":9089},{"className":9088},[183,229],[9090],{"type":16,"tag":103,"props":9091,"children":9093},{"className":9092},[183,1481,229],[9094],{"type":22,"value":1753},{"type":16,"tag":103,"props":9096,"children":9098},{"className":9097},[254],[9099],{"type":22,"value":257},{"type":16,"tag":103,"props":9101,"children":9103},{"className":9102},[205],[9104],{"type":16,"tag":103,"props":9105,"children":9107},{"className":9106,"style":417},[210],[9108],{"type":16,"tag":103,"props":9109,"children":9110},{},[],{"type":16,"tag":103,"props":9112,"children":9113},{"style":2358},[9114,9118],{"type":16,"tag":103,"props":9115,"children":9117},{"className":9116,"style":2292},[220],[],{"type":16,"tag":103,"props":9119,"children":9121},{"className":9120,"style":2368},[2367],[9122],{"type":16,"tag":2371,"props":9123,"children":9124},{"xmlns":2373,"width":2374,"height":2375,"viewBox":2376,"preserveAspectRatio":2377},[9125],{"type":16,"tag":2380,"props":9126,"children":9127},{"d":2382},[],{"type":16,"tag":103,"props":9129,"children":9131},{"className":9130},[254],[9132],{"type":22,"value":257},{"type":16,"tag":103,"props":9134,"children":9136},{"className":9135},[205],[9137],{"type":16,"tag":103,"props":9138,"children":9140},{"className":9139,"style":2396},[210],[9141],{"type":16,"tag":103,"props":9142,"children":9143},{},[],{"type":22,"value":1266},{"type":16,"tag":1284,"props":9146,"children":9148},{"id":9147},"_35-位置编码",[9149],{"type":22,"value":9150},"3.5 位置编码",{"type":16,"tag":25,"props":9152,"children":9153},{},[9154,9156,9284],{"type":22,"value":9155},"由于我们的模型不包含循环和卷积，为了让模型利用序列顺序，必须把关于 token 相对位置或绝对位置的信息注入模型。为此，我们在编码器和解码器堆栈底部的输入嵌入上添加“位置编码”。位置编码与嵌入具有相同维度 ",{"type":16,"tag":103,"props":9157,"children":9159},{"className":9158},[106],[9160,9207],{"type":16,"tag":103,"props":9161,"children":9163},{"className":9162},[111],[9164],{"type":16,"tag":114,"props":9165,"children":9166},{"xmlns":116},[9167],{"type":16,"tag":119,"props":9168,"children":9169},{},[9170,9203],{"type":16,"tag":123,"props":9171,"children":9172},{},[9173],{"type":16,"tag":127,"props":9174,"children":9175},{},[9176,9180],{"type":16,"tag":131,"props":9177,"children":9178},{},[9179],{"type":22,"value":1650},{"type":16,"tag":123,"props":9181,"children":9182},{},[9183,9187,9191,9195,9199],{"type":16,"tag":131,"props":9184,"children":9185},{"mathvariant":1409},[9186],{"type":22,"value":1090},{"type":16,"tag":131,"props":9188,"children":9189},{"mathvariant":1409},[9190],{"type":22,"value":1662},{"type":16,"tag":131,"props":9192,"children":9193},{"mathvariant":1409},[9194],{"type":22,"value":1650},{"type":16,"tag":131,"props":9196,"children":9197},{"mathvariant":1409},[9198],{"type":22,"value":1671},{"type":16,"tag":131,"props":9200,"children":9201},{"mathvariant":1409},[9202],{"type":22,"value":1676},{"type":16,"tag":157,"props":9204,"children":9205},{"encoding":159},[9206],{"type":22,"value":4216},{"type":16,"tag":103,"props":9208,"children":9210},{"className":9209,"ariaHidden":167},[166],[9211],{"type":16,"tag":103,"props":9212,"children":9214},{"className":9213},[172],[9215,9219],{"type":16,"tag":103,"props":9216,"children":9218},{"className":9217,"style":361},[177],[],{"type":16,"tag":103,"props":9220,"children":9222},{"className":9221},[183],[9223,9228],{"type":16,"tag":103,"props":9224,"children":9226},{"className":9225},[183,188],[9227],{"type":22,"value":1650},{"type":16,"tag":103,"props":9229,"children":9231},{"className":9230},[194],[9232],{"type":16,"tag":103,"props":9233,"children":9235},{"className":9234},[199,200],[9236,9273],{"type":16,"tag":103,"props":9237,"children":9239},{"className":9238},[205],[9240,9268],{"type":16,"tag":103,"props":9241,"children":9243},{"className":9242,"style":1727},[210],[9244],{"type":16,"tag":103,"props":9245,"children":9246},{"style":215},[9247,9251],{"type":16,"tag":103,"props":9248,"children":9250},{"className":9249,"style":221},[220],[],{"type":16,"tag":103,"props":9252,"children":9254},{"className":9253},[226,227,228,229],[9255],{"type":16,"tag":103,"props":9256,"children":9258},{"className":9257},[183,229],[9259],{"type":16,"tag":103,"props":9260,"children":9262},{"className":9261},[183,229],[9263],{"type":16,"tag":103,"props":9264,"children":9266},{"className":9265},[183,1481,229],[9267],{"type":22,"value":1753},{"type":16,"tag":103,"props":9269,"children":9271},{"className":9270},[254],[9272],{"type":22,"value":257},{"type":16,"tag":103,"props":9274,"children":9276},{"className":9275},[205],[9277],{"type":16,"tag":103,"props":9278,"children":9280},{"className":9279,"style":417},[210],[9281],{"type":16,"tag":103,"props":9282,"children":9283},{},[],{"type":22,"value":9285},"，因此二者可以直接相加。位置编码有许多可选方案，包括可学习和固定形式 [8]。",{"type":16,"tag":25,"props":9287,"children":9288},{},[9289],{"type":22,"value":9290},"本文使用不同频率的正弦和余弦函数：",{"type":16,"tag":103,"props":9292,"children":9294},{"className":9293},[1387],[9295],{"type":16,"tag":103,"props":9296,"children":9298},{"className":9297},[106],[9299,9457],{"type":16,"tag":103,"props":9300,"children":9302},{"className":9301},[111],[9303],{"type":16,"tag":114,"props":9304,"children":9305},{"xmlns":116,"display":1399},[9306],{"type":16,"tag":119,"props":9307,"children":9308},{},[9309,9452],{"type":16,"tag":123,"props":9310,"children":9311},{},[9312,9317,9321,9325,9341,9345,9349,9353,9357,9361,9366,9370],{"type":16,"tag":131,"props":9313,"children":9314},{"mathvariant":1409},[9315],{"type":22,"value":9316},"PE",{"type":16,"tag":145,"props":9318,"children":9319},{},[9320],{"type":22,"value":1417},{"type":16,"tag":145,"props":9322,"children":9323},{"stretchy":495},[9324],{"type":22,"value":498},{"type":16,"tag":123,"props":9326,"children":9327},{},[9328,9332,9336],{"type":16,"tag":131,"props":9329,"children":9330},{"mathvariant":1409},[9331],{"type":22,"value":25},{"type":16,"tag":131,"props":9333,"children":9334},{"mathvariant":1409},[9335],{"type":22,"value":1662},{"type":16,"tag":131,"props":9337,"children":9338},{"mathvariant":1409},[9339],{"type":22,"value":9340},"s",{"type":16,"tag":145,"props":9342,"children":9343},{"separator":167},[9344],{"type":22,"value":515},{"type":16,"tag":151,"props":9346,"children":9347},{},[9348],{"type":22,"value":8050},{"type":16,"tag":131,"props":9350,"children":9351},{},[9352],{"type":22,"value":1912},{"type":16,"tag":145,"props":9354,"children":9355},{"stretchy":495},[9356],{"type":22,"value":541},{"type":16,"tag":145,"props":9358,"children":9359},{},[9360],{"type":22,"value":746},{"type":16,"tag":131,"props":9362,"children":9363},{},[9364],{"type":22,"value":9365},"sin",{"type":16,"tag":145,"props":9367,"children":9368},{},[9369],{"type":22,"value":1417},{"type":16,"tag":123,"props":9371,"children":9372},{},[9373,9377,9448],{"type":16,"tag":145,"props":9374,"children":9375},{"fence":167},[9376],{"type":22,"value":498},{"type":16,"tag":2651,"props":9378,"children":9379},{},[9380,9395],{"type":16,"tag":123,"props":9381,"children":9382},{},[9383,9387,9391],{"type":16,"tag":131,"props":9384,"children":9385},{"mathvariant":1409},[9386],{"type":22,"value":25},{"type":16,"tag":131,"props":9388,"children":9389},{"mathvariant":1409},[9390],{"type":22,"value":1662},{"type":16,"tag":131,"props":9392,"children":9393},{"mathvariant":1409},[9394],{"type":22,"value":9340},{"type":16,"tag":2662,"props":9396,"children":9397},{},[9398,9403],{"type":16,"tag":151,"props":9399,"children":9400},{},[9401],{"type":22,"value":9402},"10000",{"type":16,"tag":123,"props":9404,"children":9405},{},[9406,9410,9414,9418],{"type":16,"tag":151,"props":9407,"children":9408},{},[9409],{"type":22,"value":8050},{"type":16,"tag":131,"props":9411,"children":9412},{},[9413],{"type":22,"value":1912},{"type":16,"tag":131,"props":9415,"children":9416},{"mathvariant":1409},[9417],{"type":22,"value":3474},{"type":16,"tag":127,"props":9419,"children":9420},{},[9421,9425],{"type":16,"tag":131,"props":9422,"children":9423},{},[9424],{"type":22,"value":1650},{"type":16,"tag":123,"props":9426,"children":9427},{},[9428,9432,9436,9440,9444],{"type":16,"tag":131,"props":9429,"children":9430},{"mathvariant":1409},[9431],{"type":22,"value":1090},{"type":16,"tag":131,"props":9433,"children":9434},{"mathvariant":1409},[9435],{"type":22,"value":1662},{"type":16,"tag":131,"props":9437,"children":9438},{"mathvariant":1409},[9439],{"type":22,"value":1650},{"type":16,"tag":131,"props":9441,"children":9442},{"mathvariant":1409},[9443],{"type":22,"value":1671},{"type":16,"tag":131,"props":9445,"children":9446},{"mathvariant":1409},[9447],{"type":22,"value":1676},{"type":16,"tag":145,"props":9449,"children":9450},{"fence":167},[9451],{"type":22,"value":541},{"type":16,"tag":157,"props":9453,"children":9454},{"encoding":159},[9455],{"type":22,"value":9456},"\\operatorname{PE}(\\mathrm{pos}, 2i) = \\sin\\left(\\frac{\\mathrm{pos}}{10000^{2i\u002Fd_{\\mathrm{model}}}}\\right)",{"type":16,"tag":103,"props":9458,"children":9460},{"className":9459,"ariaHidden":167},[166],[9461,9530],{"type":16,"tag":103,"props":9462,"children":9464},{"className":9463},[172],[9465,9469,9478,9483,9493,9498,9502,9507,9512,9517,9521,9526],{"type":16,"tag":103,"props":9466,"children":9468},{"className":9467,"style":558},[177],[],{"type":16,"tag":103,"props":9470,"children":9472},{"className":9471},[1476],[9473],{"type":16,"tag":103,"props":9474,"children":9476},{"className":9475},[183,1481],[9477],{"type":22,"value":9316},{"type":16,"tag":103,"props":9479,"children":9481},{"className":9480},[563],[9482],{"type":22,"value":498},{"type":16,"tag":103,"props":9484,"children":9486},{"className":9485},[183],[9487],{"type":16,"tag":103,"props":9488,"children":9490},{"className":9489},[183,1481],[9491],{"type":22,"value":9492},"pos",{"type":16,"tag":103,"props":9494,"children":9496},{"className":9495},[626],[9497],{"type":22,"value":515},{"type":16,"tag":103,"props":9499,"children":9501},{"className":9500,"style":633},[632],[],{"type":16,"tag":103,"props":9503,"children":9505},{"className":9504},[183],[9506],{"type":22,"value":8050},{"type":16,"tag":103,"props":9508,"children":9510},{"className":9509},[183,188],[9511],{"type":22,"value":1912},{"type":16,"tag":103,"props":9513,"children":9515},{"className":9514},[715],[9516],{"type":22,"value":541},{"type":16,"tag":103,"props":9518,"children":9520},{"className":9519,"style":816},[632],[],{"type":16,"tag":103,"props":9522,"children":9524},{"className":9523},[821],[9525],{"type":22,"value":746},{"type":16,"tag":103,"props":9527,"children":9529},{"className":9528,"style":816},[632],[],{"type":16,"tag":103,"props":9531,"children":9533},{"className":9532},[172],[9534,9539,9544,9548],{"type":16,"tag":103,"props":9535,"children":9538},{"className":9536,"style":9537},[177],"height:1.854em;vertical-align:-0.704em;",[],{"type":16,"tag":103,"props":9540,"children":9542},{"className":9541},[1476],[9543],{"type":22,"value":9365},{"type":16,"tag":103,"props":9545,"children":9547},{"className":9546,"style":633},[632],[],{"type":16,"tag":103,"props":9549,"children":9551},{"className":9550},[638],[9552,9562,9779],{"type":16,"tag":103,"props":9553,"children":9555},{"className":9554,"style":2820},[563,2819],[9556],{"type":16,"tag":103,"props":9557,"children":9560},{"className":9558},[2825,9559],"size2",[9561],{"type":22,"value":498},{"type":16,"tag":103,"props":9563,"children":9565},{"className":9564},[183],[9566,9570,9775],{"type":16,"tag":103,"props":9567,"children":9569},{"className":9568},[563,2835],[],{"type":16,"tag":103,"props":9571,"children":9573},{"className":9572},[2651],[9574],{"type":16,"tag":103,"props":9575,"children":9577},{"className":9576},[199,200],[9578,9763],{"type":16,"tag":103,"props":9579,"children":9581},{"className":9580},[205],[9582,9758],{"type":16,"tag":103,"props":9583,"children":9586},{"className":9584,"style":9585},[210],"height:1.1076em;",[9587,9727,9738],{"type":16,"tag":103,"props":9588,"children":9590},{"style":9589},"top:-2.296em;",[9591,9595],{"type":16,"tag":103,"props":9592,"children":9594},{"className":9593,"style":2292},[220],[],{"type":16,"tag":103,"props":9596,"children":9598},{"className":9597},[183],[9599,9605],{"type":16,"tag":103,"props":9600,"children":9602},{"className":9601},[183],[9603],{"type":22,"value":9604},"1000",{"type":16,"tag":103,"props":9606,"children":9608},{"className":9607},[183],[9609,9614],{"type":16,"tag":103,"props":9610,"children":9612},{"className":9611},[183],[9613],{"type":22,"value":5746},{"type":16,"tag":103,"props":9615,"children":9617},{"className":9616},[194],[9618],{"type":16,"tag":103,"props":9619,"children":9621},{"className":9620},[199],[9622],{"type":16,"tag":103,"props":9623,"children":9625},{"className":9624},[205],[9626],{"type":16,"tag":103,"props":9627,"children":9630},{"className":9628,"style":9629},[210],"height:0.814em;",[9631],{"type":16,"tag":103,"props":9632,"children":9634},{"style":9633},"top:-2.989em;margin-right:0.05em;",[9635,9639],{"type":16,"tag":103,"props":9636,"children":9638},{"className":9637,"style":221},[220],[],{"type":16,"tag":103,"props":9640,"children":9642},{"className":9641},[226,227,228,229],[9643],{"type":16,"tag":103,"props":9644,"children":9646},{"className":9645},[183,229],[9647,9652,9657,9662],{"type":16,"tag":103,"props":9648,"children":9650},{"className":9649},[183,229],[9651],{"type":22,"value":8050},{"type":16,"tag":103,"props":9653,"children":9655},{"className":9654},[183,188,229],[9656],{"type":22,"value":1912},{"type":16,"tag":103,"props":9658,"children":9660},{"className":9659},[183,229],[9661],{"type":22,"value":3474},{"type":16,"tag":103,"props":9663,"children":9665},{"className":9664},[183,229],[9666,9671],{"type":16,"tag":103,"props":9667,"children":9669},{"className":9668},[183,188,229],[9670],{"type":22,"value":1650},{"type":16,"tag":103,"props":9672,"children":9674},{"className":9673},[194],[9675],{"type":16,"tag":103,"props":9676,"children":9678},{"className":9677},[199,200],[9679,9716],{"type":16,"tag":103,"props":9680,"children":9682},{"className":9681},[205],[9683,9711],{"type":16,"tag":103,"props":9684,"children":9686},{"className":9685,"style":6620},[210],[9687],{"type":16,"tag":103,"props":9688,"children":9689},{"style":6624},[9690,9694],{"type":16,"tag":103,"props":9691,"children":9693},{"className":9692,"style":6629},[220],[],{"type":16,"tag":103,"props":9695,"children":9697},{"className":9696},[226,6634,6635,229],[9698],{"type":16,"tag":103,"props":9699,"children":9701},{"className":9700},[183,229],[9702],{"type":16,"tag":103,"props":9703,"children":9705},{"className":9704},[183,229],[9706],{"type":16,"tag":103,"props":9707,"children":9709},{"className":9708},[183,1481,229],[9710],{"type":22,"value":1753},{"type":16,"tag":103,"props":9712,"children":9714},{"className":9713},[254],[9715],{"type":22,"value":257},{"type":16,"tag":103,"props":9717,"children":9719},{"className":9718},[205],[9720],{"type":16,"tag":103,"props":9721,"children":9723},{"className":9722,"style":6662},[210],[9724],{"type":16,"tag":103,"props":9725,"children":9726},{},[],{"type":16,"tag":103,"props":9728,"children":9729},{"style":2986},[9730,9734],{"type":16,"tag":103,"props":9731,"children":9733},{"className":9732,"style":2292},[220],[],{"type":16,"tag":103,"props":9735,"children":9737},{"className":9736,"style":2996},[2995],[],{"type":16,"tag":103,"props":9739,"children":9740},{"style":3000},[9741,9745],{"type":16,"tag":103,"props":9742,"children":9744},{"className":9743,"style":2292},[220],[],{"type":16,"tag":103,"props":9746,"children":9748},{"className":9747},[183],[9749],{"type":16,"tag":103,"props":9750,"children":9752},{"className":9751},[183],[9753],{"type":16,"tag":103,"props":9754,"children":9756},{"className":9755},[183,1481],[9757],{"type":22,"value":9492},{"type":16,"tag":103,"props":9759,"children":9761},{"className":9760},[254],[9762],{"type":22,"value":257},{"type":16,"tag":103,"props":9764,"children":9766},{"className":9765},[205],[9767],{"type":16,"tag":103,"props":9768,"children":9771},{"className":9769,"style":9770},[210],"height:0.704em;",[9772],{"type":16,"tag":103,"props":9773,"children":9774},{},[],{"type":16,"tag":103,"props":9776,"children":9778},{"className":9777},[715,2835],[],{"type":16,"tag":103,"props":9780,"children":9782},{"className":9781,"style":2820},[715,2819],[9783],{"type":16,"tag":103,"props":9784,"children":9786},{"className":9785},[2825,9559],[9787],{"type":22,"value":541},{"type":16,"tag":103,"props":9789,"children":9791},{"className":9790},[1387],[9792],{"type":16,"tag":103,"props":9793,"children":9795},{"className":9794},[106],[9796,9959],{"type":16,"tag":103,"props":9797,"children":9799},{"className":9798},[111],[9800],{"type":16,"tag":114,"props":9801,"children":9802},{"xmlns":116,"display":1399},[9803],{"type":16,"tag":119,"props":9804,"children":9805},{},[9806,9954],{"type":16,"tag":123,"props":9807,"children":9808},{},[9809,9813,9817,9821,9836,9840,9844,9848,9852,9856,9860,9864,9869,9873],{"type":16,"tag":131,"props":9810,"children":9811},{"mathvariant":1409},[9812],{"type":22,"value":9316},{"type":16,"tag":145,"props":9814,"children":9815},{},[9816],{"type":22,"value":1417},{"type":16,"tag":145,"props":9818,"children":9819},{"stretchy":495},[9820],{"type":22,"value":498},{"type":16,"tag":123,"props":9822,"children":9823},{},[9824,9828,9832],{"type":16,"tag":131,"props":9825,"children":9826},{"mathvariant":1409},[9827],{"type":22,"value":25},{"type":16,"tag":131,"props":9829,"children":9830},{"mathvariant":1409},[9831],{"type":22,"value":1662},{"type":16,"tag":131,"props":9833,"children":9834},{"mathvariant":1409},[9835],{"type":22,"value":9340},{"type":16,"tag":145,"props":9837,"children":9838},{"separator":167},[9839],{"type":22,"value":515},{"type":16,"tag":151,"props":9841,"children":9842},{},[9843],{"type":22,"value":8050},{"type":16,"tag":131,"props":9845,"children":9846},{},[9847],{"type":22,"value":1912},{"type":16,"tag":145,"props":9849,"children":9850},{},[9851],{"type":22,"value":1430},{"type":16,"tag":151,"props":9853,"children":9854},{},[9855],{"type":22,"value":155},{"type":16,"tag":145,"props":9857,"children":9858},{"stretchy":495},[9859],{"type":22,"value":541},{"type":16,"tag":145,"props":9861,"children":9862},{},[9863],{"type":22,"value":746},{"type":16,"tag":131,"props":9865,"children":9866},{},[9867],{"type":22,"value":9868},"cos",{"type":16,"tag":145,"props":9870,"children":9871},{},[9872],{"type":22,"value":1417},{"type":16,"tag":123,"props":9874,"children":9875},{},[9876,9880,9950],{"type":16,"tag":145,"props":9877,"children":9878},{"fence":167},[9879],{"type":22,"value":498},{"type":16,"tag":2651,"props":9881,"children":9882},{},[9883,9898],{"type":16,"tag":123,"props":9884,"children":9885},{},[9886,9890,9894],{"type":16,"tag":131,"props":9887,"children":9888},{"mathvariant":1409},[9889],{"type":22,"value":25},{"type":16,"tag":131,"props":9891,"children":9892},{"mathvariant":1409},[9893],{"type":22,"value":1662},{"type":16,"tag":131,"props":9895,"children":9896},{"mathvariant":1409},[9897],{"type":22,"value":9340},{"type":16,"tag":2662,"props":9899,"children":9900},{},[9901,9905],{"type":16,"tag":151,"props":9902,"children":9903},{},[9904],{"type":22,"value":9402},{"type":16,"tag":123,"props":9906,"children":9907},{},[9908,9912,9916,9920],{"type":16,"tag":151,"props":9909,"children":9910},{},[9911],{"type":22,"value":8050},{"type":16,"tag":131,"props":9913,"children":9914},{},[9915],{"type":22,"value":1912},{"type":16,"tag":131,"props":9917,"children":9918},{"mathvariant":1409},[9919],{"type":22,"value":3474},{"type":16,"tag":127,"props":9921,"children":9922},{},[9923,9927],{"type":16,"tag":131,"props":9924,"children":9925},{},[9926],{"type":22,"value":1650},{"type":16,"tag":123,"props":9928,"children":9929},{},[9930,9934,9938,9942,9946],{"type":16,"tag":131,"props":9931,"children":9932},{"mathvariant":1409},[9933],{"type":22,"value":1090},{"type":16,"tag":131,"props":9935,"children":9936},{"mathvariant":1409},[9937],{"type":22,"value":1662},{"type":16,"tag":131,"props":9939,"children":9940},{"mathvariant":1409},[9941],{"type":22,"value":1650},{"type":16,"tag":131,"props":9943,"children":9944},{"mathvariant":1409},[9945],{"type":22,"value":1671},{"type":16,"tag":131,"props":9947,"children":9948},{"mathvariant":1409},[9949],{"type":22,"value":1676},{"type":16,"tag":145,"props":9951,"children":9952},{"fence":167},[9953],{"type":22,"value":541},{"type":16,"tag":157,"props":9955,"children":9956},{"encoding":159},[9957],{"type":22,"value":9958},"\\operatorname{PE}(\\mathrm{pos}, 2i+1) = \\cos\\left(\\frac{\\mathrm{pos}}{10000^{2i\u002Fd_{\\mathrm{model}}}}\\right)",{"type":16,"tag":103,"props":9960,"children":9962},{"className":9961,"ariaHidden":167},[166],[9963,10026,10057],{"type":16,"tag":103,"props":9964,"children":9966},{"className":9965},[172],[9967,9971,9980,9985,9994,9999,10003,10008,10013,10017,10022],{"type":16,"tag":103,"props":9968,"children":9970},{"className":9969,"style":558},[177],[],{"type":16,"tag":103,"props":9972,"children":9974},{"className":9973},[1476],[9975],{"type":16,"tag":103,"props":9976,"children":9978},{"className":9977},[183,1481],[9979],{"type":22,"value":9316},{"type":16,"tag":103,"props":9981,"children":9983},{"className":9982},[563],[9984],{"type":22,"value":498},{"type":16,"tag":103,"props":9986,"children":9988},{"className":9987},[183],[9989],{"type":16,"tag":103,"props":9990,"children":9992},{"className":9991},[183,1481],[9993],{"type":22,"value":9492},{"type":16,"tag":103,"props":9995,"children":9997},{"className":9996},[626],[9998],{"type":22,"value":515},{"type":16,"tag":103,"props":10000,"children":10002},{"className":10001,"style":633},[632],[],{"type":16,"tag":103,"props":10004,"children":10006},{"className":10005},[183],[10007],{"type":22,"value":8050},{"type":16,"tag":103,"props":10009,"children":10011},{"className":10010},[183,188],[10012],{"type":22,"value":1912},{"type":16,"tag":103,"props":10014,"children":10016},{"className":10015,"style":1497},[632],[],{"type":16,"tag":103,"props":10018,"children":10020},{"className":10019},[243],[10021],{"type":22,"value":1430},{"type":16,"tag":103,"props":10023,"children":10025},{"className":10024,"style":1497},[632],[],{"type":16,"tag":103,"props":10027,"children":10029},{"className":10028},[172],[10030,10034,10039,10044,10048,10053],{"type":16,"tag":103,"props":10031,"children":10033},{"className":10032,"style":558},[177],[],{"type":16,"tag":103,"props":10035,"children":10037},{"className":10036},[183],[10038],{"type":22,"value":155},{"type":16,"tag":103,"props":10040,"children":10042},{"className":10041},[715],[10043],{"type":22,"value":541},{"type":16,"tag":103,"props":10045,"children":10047},{"className":10046,"style":816},[632],[],{"type":16,"tag":103,"props":10049,"children":10051},{"className":10050},[821],[10052],{"type":22,"value":746},{"type":16,"tag":103,"props":10054,"children":10056},{"className":10055,"style":816},[632],[],{"type":16,"tag":103,"props":10058,"children":10060},{"className":10059},[172],[10061,10065,10070,10074],{"type":16,"tag":103,"props":10062,"children":10064},{"className":10063,"style":9537},[177],[],{"type":16,"tag":103,"props":10066,"children":10068},{"className":10067},[1476],[10069],{"type":22,"value":9868},{"type":16,"tag":103,"props":10071,"children":10073},{"className":10072,"style":633},[632],[],{"type":16,"tag":103,"props":10075,"children":10077},{"className":10076},[638],[10078,10087,10298],{"type":16,"tag":103,"props":10079,"children":10081},{"className":10080,"style":2820},[563,2819],[10082],{"type":16,"tag":103,"props":10083,"children":10085},{"className":10084},[2825,9559],[10086],{"type":22,"value":498},{"type":16,"tag":103,"props":10088,"children":10090},{"className":10089},[183],[10091,10095,10294],{"type":16,"tag":103,"props":10092,"children":10094},{"className":10093},[563,2835],[],{"type":16,"tag":103,"props":10096,"children":10098},{"className":10097},[2651],[10099],{"type":16,"tag":103,"props":10100,"children":10102},{"className":10101},[199,200],[10103,10283],{"type":16,"tag":103,"props":10104,"children":10106},{"className":10105},[205],[10107,10278],{"type":16,"tag":103,"props":10108,"children":10110},{"className":10109,"style":9585},[210],[10111,10247,10258],{"type":16,"tag":103,"props":10112,"children":10113},{"style":9589},[10114,10118],{"type":16,"tag":103,"props":10115,"children":10117},{"className":10116,"style":2292},[220],[],{"type":16,"tag":103,"props":10119,"children":10121},{"className":10120},[183],[10122,10127],{"type":16,"tag":103,"props":10123,"children":10125},{"className":10124},[183],[10126],{"type":22,"value":9604},{"type":16,"tag":103,"props":10128,"children":10130},{"className":10129},[183],[10131,10136],{"type":16,"tag":103,"props":10132,"children":10134},{"className":10133},[183],[10135],{"type":22,"value":5746},{"type":16,"tag":103,"props":10137,"children":10139},{"className":10138},[194],[10140],{"type":16,"tag":103,"props":10141,"children":10143},{"className":10142},[199],[10144],{"type":16,"tag":103,"props":10145,"children":10147},{"className":10146},[205],[10148],{"type":16,"tag":103,"props":10149,"children":10151},{"className":10150,"style":9629},[210],[10152],{"type":16,"tag":103,"props":10153,"children":10154},{"style":9633},[10155,10159],{"type":16,"tag":103,"props":10156,"children":10158},{"className":10157,"style":221},[220],[],{"type":16,"tag":103,"props":10160,"children":10162},{"className":10161},[226,227,228,229],[10163],{"type":16,"tag":103,"props":10164,"children":10166},{"className":10165},[183,229],[10167,10172,10177,10182],{"type":16,"tag":103,"props":10168,"children":10170},{"className":10169},[183,229],[10171],{"type":22,"value":8050},{"type":16,"tag":103,"props":10173,"children":10175},{"className":10174},[183,188,229],[10176],{"type":22,"value":1912},{"type":16,"tag":103,"props":10178,"children":10180},{"className":10179},[183,229],[10181],{"type":22,"value":3474},{"type":16,"tag":103,"props":10183,"children":10185},{"className":10184},[183,229],[10186,10191],{"type":16,"tag":103,"props":10187,"children":10189},{"className":10188},[183,188,229],[10190],{"type":22,"value":1650},{"type":16,"tag":103,"props":10192,"children":10194},{"className":10193},[194],[10195],{"type":16,"tag":103,"props":10196,"children":10198},{"className":10197},[199,200],[10199,10236],{"type":16,"tag":103,"props":10200,"children":10202},{"className":10201},[205],[10203,10231],{"type":16,"tag":103,"props":10204,"children":10206},{"className":10205,"style":6620},[210],[10207],{"type":16,"tag":103,"props":10208,"children":10209},{"style":6624},[10210,10214],{"type":16,"tag":103,"props":10211,"children":10213},{"className":10212,"style":6629},[220],[],{"type":16,"tag":103,"props":10215,"children":10217},{"className":10216},[226,6634,6635,229],[10218],{"type":16,"tag":103,"props":10219,"children":10221},{"className":10220},[183,229],[10222],{"type":16,"tag":103,"props":10223,"children":10225},{"className":10224},[183,229],[10226],{"type":16,"tag":103,"props":10227,"children":10229},{"className":10228},[183,1481,229],[10230],{"type":22,"value":1753},{"type":16,"tag":103,"props":10232,"children":10234},{"className":10233},[254],[10235],{"type":22,"value":257},{"type":16,"tag":103,"props":10237,"children":10239},{"className":10238},[205],[10240],{"type":16,"tag":103,"props":10241,"children":10243},{"className":10242,"style":6662},[210],[10244],{"type":16,"tag":103,"props":10245,"children":10246},{},[],{"type":16,"tag":103,"props":10248,"children":10249},{"style":2986},[10250,10254],{"type":16,"tag":103,"props":10251,"children":10253},{"className":10252,"style":2292},[220],[],{"type":16,"tag":103,"props":10255,"children":10257},{"className":10256,"style":2996},[2995],[],{"type":16,"tag":103,"props":10259,"children":10260},{"style":3000},[10261,10265],{"type":16,"tag":103,"props":10262,"children":10264},{"className":10263,"style":2292},[220],[],{"type":16,"tag":103,"props":10266,"children":10268},{"className":10267},[183],[10269],{"type":16,"tag":103,"props":10270,"children":10272},{"className":10271},[183],[10273],{"type":16,"tag":103,"props":10274,"children":10276},{"className":10275},[183,1481],[10277],{"type":22,"value":9492},{"type":16,"tag":103,"props":10279,"children":10281},{"className":10280},[254],[10282],{"type":22,"value":257},{"type":16,"tag":103,"props":10284,"children":10286},{"className":10285},[205],[10287],{"type":16,"tag":103,"props":10288,"children":10290},{"className":10289,"style":9770},[210],[10291],{"type":16,"tag":103,"props":10292,"children":10293},{},[],{"type":16,"tag":103,"props":10295,"children":10297},{"className":10296},[715,2835],[],{"type":16,"tag":103,"props":10299,"children":10301},{"className":10300,"style":2820},[715,2819],[10302],{"type":16,"tag":103,"props":10303,"children":10305},{"className":10304},[2825,9559],[10306],{"type":22,"value":541},{"type":16,"tag":25,"props":10308,"children":10309},{},[10310,10311,10366,10368,10410,10412,10465,10467,10553],{"type":22,"value":1545},{"type":16,"tag":103,"props":10312,"children":10314},{"className":10313},[106],[10315,10345],{"type":16,"tag":103,"props":10316,"children":10318},{"className":10317},[111],[10319],{"type":16,"tag":114,"props":10320,"children":10321},{"xmlns":116},[10322],{"type":16,"tag":119,"props":10323,"children":10324},{},[10325,10340],{"type":16,"tag":123,"props":10326,"children":10327},{},[10328,10332,10336],{"type":16,"tag":131,"props":10329,"children":10330},{"mathvariant":1409},[10331],{"type":22,"value":25},{"type":16,"tag":131,"props":10333,"children":10334},{"mathvariant":1409},[10335],{"type":22,"value":1662},{"type":16,"tag":131,"props":10337,"children":10338},{"mathvariant":1409},[10339],{"type":22,"value":9340},{"type":16,"tag":157,"props":10341,"children":10342},{"encoding":159},[10343],{"type":22,"value":10344},"\\mathrm{pos}",{"type":16,"tag":103,"props":10346,"children":10348},{"className":10347,"ariaHidden":167},[166],[10349],{"type":16,"tag":103,"props":10350,"children":10352},{"className":10351},[172],[10353,10357],{"type":16,"tag":103,"props":10354,"children":10356},{"className":10355,"style":3675},[177],[],{"type":16,"tag":103,"props":10358,"children":10360},{"className":10359},[183],[10361],{"type":16,"tag":103,"props":10362,"children":10364},{"className":10363},[183,1481],[10365],{"type":22,"value":9492},{"type":22,"value":10367}," 表示位置，",{"type":16,"tag":103,"props":10369,"children":10371},{"className":10370},[106],[10372,10393],{"type":16,"tag":103,"props":10373,"children":10375},{"className":10374},[111],[10376],{"type":16,"tag":114,"props":10377,"children":10378},{"xmlns":116},[10379],{"type":16,"tag":119,"props":10380,"children":10381},{},[10382,10389],{"type":16,"tag":123,"props":10383,"children":10384},{},[10385],{"type":16,"tag":131,"props":10386,"children":10387},{},[10388],{"type":22,"value":1912},{"type":16,"tag":157,"props":10390,"children":10391},{"encoding":159},[10392],{"type":22,"value":1912},{"type":16,"tag":103,"props":10394,"children":10396},{"className":10395,"ariaHidden":167},[166],[10397],{"type":16,"tag":103,"props":10398,"children":10400},{"className":10399},[172],[10401,10405],{"type":16,"tag":103,"props":10402,"children":10404},{"className":10403,"style":1928},[177],[],{"type":16,"tag":103,"props":10406,"children":10408},{"className":10407},[183,188],[10409],{"type":22,"value":1912},{"type":22,"value":10411}," 表示维度。也就是说，位置编码的每个维度都对应一个正弦曲线。波长构成从 ",{"type":16,"tag":103,"props":10413,"children":10415},{"className":10414},[106],[10416,10443],{"type":16,"tag":103,"props":10417,"children":10419},{"className":10418},[111],[10420],{"type":16,"tag":114,"props":10421,"children":10422},{"xmlns":116},[10423],{"type":16,"tag":119,"props":10424,"children":10425},{},[10426,10438],{"type":16,"tag":123,"props":10427,"children":10428},{},[10429,10433],{"type":16,"tag":151,"props":10430,"children":10431},{},[10432],{"type":22,"value":8050},{"type":16,"tag":131,"props":10434,"children":10435},{},[10436],{"type":22,"value":10437},"π",{"type":16,"tag":157,"props":10439,"children":10440},{"encoding":159},[10441],{"type":22,"value":10442},"2\\pi",{"type":16,"tag":103,"props":10444,"children":10446},{"className":10445,"ariaHidden":167},[166],[10447],{"type":16,"tag":103,"props":10448,"children":10450},{"className":10449},[172],[10451,10455,10460],{"type":16,"tag":103,"props":10452,"children":10454},{"className":10453,"style":1375},[177],[],{"type":16,"tag":103,"props":10456,"children":10458},{"className":10457},[183],[10459],{"type":22,"value":8050},{"type":16,"tag":103,"props":10461,"children":10463},{"className":10462,"style":1124},[183,188],[10464],{"type":22,"value":10437},{"type":22,"value":10466}," 到 ",{"type":16,"tag":103,"props":10468,"children":10470},{"className":10469},[106],[10471,10505],{"type":16,"tag":103,"props":10472,"children":10474},{"className":10473},[111],[10475],{"type":16,"tag":114,"props":10476,"children":10477},{"xmlns":116},[10478],{"type":16,"tag":119,"props":10479,"children":10480},{},[10481,10500],{"type":16,"tag":123,"props":10482,"children":10483},{},[10484,10488,10492,10496],{"type":16,"tag":151,"props":10485,"children":10486},{},[10487],{"type":22,"value":9402},{"type":16,"tag":145,"props":10489,"children":10490},{},[10491],{"type":22,"value":3753},{"type":16,"tag":151,"props":10493,"children":10494},{},[10495],{"type":22,"value":8050},{"type":16,"tag":131,"props":10497,"children":10498},{},[10499],{"type":22,"value":10437},{"type":16,"tag":157,"props":10501,"children":10502},{"encoding":159},[10503],{"type":22,"value":10504},"10000 \\cdot 2\\pi",{"type":16,"tag":103,"props":10506,"children":10508},{"className":10507,"ariaHidden":167},[166],[10509,10535],{"type":16,"tag":103,"props":10510,"children":10512},{"className":10511},[172],[10513,10517,10522,10526,10531],{"type":16,"tag":103,"props":10514,"children":10516},{"className":10515,"style":1375},[177],[],{"type":16,"tag":103,"props":10518,"children":10520},{"className":10519},[183],[10521],{"type":22,"value":9402},{"type":16,"tag":103,"props":10523,"children":10525},{"className":10524,"style":1497},[632],[],{"type":16,"tag":103,"props":10527,"children":10529},{"className":10528},[243],[10530],{"type":22,"value":3753},{"type":16,"tag":103,"props":10532,"children":10534},{"className":10533,"style":1497},[632],[],{"type":16,"tag":103,"props":10536,"children":10538},{"className":10537},[172],[10539,10543,10548],{"type":16,"tag":103,"props":10540,"children":10542},{"className":10541,"style":1375},[177],[],{"type":16,"tag":103,"props":10544,"children":10546},{"className":10545},[183],[10547],{"type":22,"value":8050},{"type":16,"tag":103,"props":10549,"children":10551},{"className":10550,"style":1124},[183,188],[10552],{"type":22,"value":10437},{"type":22,"value":10554}," 的几何级数。",{"type":16,"tag":25,"props":10556,"children":10557},{},[10558,10560,10602,10604,10759,10761,10893],{"type":22,"value":10559},"我们选择这个函数，是因为假设它可以让模型更容易学习按相对位置进行注意：对于任意固定偏移量 ",{"type":16,"tag":103,"props":10561,"children":10563},{"className":10562},[106],[10564,10585],{"type":16,"tag":103,"props":10565,"children":10567},{"className":10566},[111],[10568],{"type":16,"tag":114,"props":10569,"children":10570},{"xmlns":116},[10571],{"type":16,"tag":119,"props":10572,"children":10573},{},[10574,10581],{"type":16,"tag":123,"props":10575,"children":10576},{},[10577],{"type":16,"tag":131,"props":10578,"children":10579},{},[10580],{"type":22,"value":2032},{"type":16,"tag":157,"props":10582,"children":10583},{"encoding":159},[10584],{"type":22,"value":2032},{"type":16,"tag":103,"props":10586,"children":10588},{"className":10587,"ariaHidden":167},[166],[10589],{"type":16,"tag":103,"props":10590,"children":10592},{"className":10591},[172],[10593,10597],{"type":16,"tag":103,"props":10594,"children":10596},{"className":10595,"style":3719},[177],[],{"type":16,"tag":103,"props":10598,"children":10600},{"className":10599,"style":2089},[183,188],[10601],{"type":22,"value":2032},{"type":22,"value":10603},"，",{"type":16,"tag":103,"props":10605,"children":10607},{"className":10606},[106],[10608,10666],{"type":16,"tag":103,"props":10609,"children":10611},{"className":10610},[111],[10612],{"type":16,"tag":114,"props":10613,"children":10614},{"xmlns":116},[10615],{"type":16,"tag":119,"props":10616,"children":10617},{},[10618,10661],{"type":16,"tag":123,"props":10619,"children":10620},{},[10621],{"type":16,"tag":127,"props":10622,"children":10623},{},[10624,10635],{"type":16,"tag":123,"props":10625,"children":10626},{},[10627,10631],{"type":16,"tag":131,"props":10628,"children":10629},{"mathvariant":1409},[10630],{"type":22,"value":9316},{"type":16,"tag":145,"props":10632,"children":10633},{},[10634],{"type":22,"value":1417},{"type":16,"tag":123,"props":10636,"children":10637},{},[10638,10653,10657],{"type":16,"tag":123,"props":10639,"children":10640},{},[10641,10645,10649],{"type":16,"tag":131,"props":10642,"children":10643},{"mathvariant":1409},[10644],{"type":22,"value":25},{"type":16,"tag":131,"props":10646,"children":10647},{"mathvariant":1409},[10648],{"type":22,"value":1662},{"type":16,"tag":131,"props":10650,"children":10651},{"mathvariant":1409},[10652],{"type":22,"value":9340},{"type":16,"tag":145,"props":10654,"children":10655},{},[10656],{"type":22,"value":1430},{"type":16,"tag":131,"props":10658,"children":10659},{},[10660],{"type":22,"value":2032},{"type":16,"tag":157,"props":10662,"children":10663},{"encoding":159},[10664],{"type":22,"value":10665},"\\operatorname{PE}_{\\mathrm{pos}+k}",{"type":16,"tag":103,"props":10667,"children":10669},{"className":10668,"ariaHidden":167},[166],[10670],{"type":16,"tag":103,"props":10671,"children":10673},{"className":10672},[172],[10674,10679],{"type":16,"tag":103,"props":10675,"children":10678},{"className":10676,"style":10677},[177],"height:0.9694em;vertical-align:-0.2861em;",[],{"type":16,"tag":103,"props":10680,"children":10682},{"className":10681},[1476],[10683,10692],{"type":16,"tag":103,"props":10684,"children":10686},{"className":10685},[1476],[10687],{"type":16,"tag":103,"props":10688,"children":10690},{"className":10689},[183,1481],[10691],{"type":22,"value":9316},{"type":16,"tag":103,"props":10693,"children":10695},{"className":10694},[194],[10696],{"type":16,"tag":103,"props":10697,"children":10699},{"className":10698},[199,200],[10700,10747],{"type":16,"tag":103,"props":10701,"children":10703},{"className":10702},[205],[10704,10742],{"type":16,"tag":103,"props":10705,"children":10707},{"className":10706,"style":1727},[210],[10708],{"type":16,"tag":103,"props":10709,"children":10710},{"style":5033},[10711,10715],{"type":16,"tag":103,"props":10712,"children":10714},{"className":10713,"style":221},[220],[],{"type":16,"tag":103,"props":10716,"children":10718},{"className":10717},[226,227,228,229],[10719],{"type":16,"tag":103,"props":10720,"children":10722},{"className":10721},[183,229],[10723,10732,10737],{"type":16,"tag":103,"props":10724,"children":10726},{"className":10725},[183,229],[10727],{"type":16,"tag":103,"props":10728,"children":10730},{"className":10729},[183,1481,229],[10731],{"type":22,"value":9492},{"type":16,"tag":103,"props":10733,"children":10735},{"className":10734},[243,229],[10736],{"type":22,"value":1430},{"type":16,"tag":103,"props":10738,"children":10740},{"className":10739,"style":2089},[183,188,229],[10741],{"type":22,"value":2032},{"type":16,"tag":103,"props":10743,"children":10745},{"className":10744},[254],[10746],{"type":22,"value":257},{"type":16,"tag":103,"props":10748,"children":10750},{"className":10749},[205],[10751],{"type":16,"tag":103,"props":10752,"children":10755},{"className":10753,"style":10754},[210],"height:0.2861em;",[10756],{"type":16,"tag":103,"props":10757,"children":10758},{},[],{"type":22,"value":10760}," 都可以表示为 ",{"type":16,"tag":103,"props":10762,"children":10764},{"className":10763},[106],[10765,10812],{"type":16,"tag":103,"props":10766,"children":10768},{"className":10767},[111],[10769],{"type":16,"tag":114,"props":10770,"children":10771},{"xmlns":116},[10772],{"type":16,"tag":119,"props":10773,"children":10774},{},[10775,10807],{"type":16,"tag":123,"props":10776,"children":10777},{},[10778],{"type":16,"tag":127,"props":10779,"children":10780},{},[10781,10792],{"type":16,"tag":123,"props":10782,"children":10783},{},[10784,10788],{"type":16,"tag":131,"props":10785,"children":10786},{"mathvariant":1409},[10787],{"type":22,"value":9316},{"type":16,"tag":145,"props":10789,"children":10790},{},[10791],{"type":22,"value":1417},{"type":16,"tag":123,"props":10793,"children":10794},{},[10795,10799,10803],{"type":16,"tag":131,"props":10796,"children":10797},{"mathvariant":1409},[10798],{"type":22,"value":25},{"type":16,"tag":131,"props":10800,"children":10801},{"mathvariant":1409},[10802],{"type":22,"value":1662},{"type":16,"tag":131,"props":10804,"children":10805},{"mathvariant":1409},[10806],{"type":22,"value":9340},{"type":16,"tag":157,"props":10808,"children":10809},{"encoding":159},[10810],{"type":22,"value":10811},"\\operatorname{PE}_{\\mathrm{pos}}",{"type":16,"tag":103,"props":10813,"children":10815},{"className":10814,"ariaHidden":167},[166],[10816],{"type":16,"tag":103,"props":10817,"children":10819},{"className":10818},[172],[10820,10824],{"type":16,"tag":103,"props":10821,"children":10823},{"className":10822,"style":10677},[177],[],{"type":16,"tag":103,"props":10825,"children":10827},{"className":10826},[1476],[10828,10837],{"type":16,"tag":103,"props":10829,"children":10831},{"className":10830},[1476],[10832],{"type":16,"tag":103,"props":10833,"children":10835},{"className":10834},[183,1481],[10836],{"type":22,"value":9316},{"type":16,"tag":103,"props":10838,"children":10840},{"className":10839},[194],[10841],{"type":16,"tag":103,"props":10842,"children":10844},{"className":10843},[199,200],[10845,10882],{"type":16,"tag":103,"props":10846,"children":10848},{"className":10847},[205],[10849,10877],{"type":16,"tag":103,"props":10850,"children":10852},{"className":10851,"style":678},[210],[10853],{"type":16,"tag":103,"props":10854,"children":10855},{"style":5033},[10856,10860],{"type":16,"tag":103,"props":10857,"children":10859},{"className":10858,"style":221},[220],[],{"type":16,"tag":103,"props":10861,"children":10863},{"className":10862},[226,227,228,229],[10864],{"type":16,"tag":103,"props":10865,"children":10867},{"className":10866},[183,229],[10868],{"type":16,"tag":103,"props":10869,"children":10871},{"className":10870},[183,229],[10872],{"type":16,"tag":103,"props":10873,"children":10875},{"className":10874},[183,1481,229],[10876],{"type":22,"value":9492},{"type":16,"tag":103,"props":10878,"children":10880},{"className":10879},[254],[10881],{"type":22,"value":257},{"type":16,"tag":103,"props":10883,"children":10885},{"className":10884},[205],[10886],{"type":16,"tag":103,"props":10887,"children":10889},{"className":10888,"style":10754},[210],[10890],{"type":16,"tag":103,"props":10891,"children":10892},{},[],{"type":22,"value":10894}," 的线性函数。",{"type":16,"tag":25,"props":10896,"children":10897},{},[10898],{"type":22,"value":10899},"我们还尝试了可学习的位置嵌入 [8]，发现两种版本产生的结果几乎相同（表 3 的 E 行）。我们选择正弦版本，是因为它可能允许模型外推到训练时没有遇到过的更长序列。",{"type":16,"tag":1284,"props":10901,"children":10903},{"id":10902},"表-1不同层类型的最大路径长度逐层复杂度和最少顺序操作数",[10904],{"type":22,"value":10905},"表 1：不同层类型的最大路径长度、逐层复杂度和最少顺序操作数",{"type":16,"tag":25,"props":10907,"children":10908},{},[10909,10951,10953,10995,10997,11039,11041,11084],{"type":16,"tag":103,"props":10910,"children":10912},{"className":10911},[106],[10913,10934],{"type":16,"tag":103,"props":10914,"children":10916},{"className":10915},[111],[10917],{"type":16,"tag":114,"props":10918,"children":10919},{"xmlns":116},[10920],{"type":16,"tag":119,"props":10921,"children":10922},{},[10923,10930],{"type":16,"tag":123,"props":10924,"children":10925},{},[10926],{"type":16,"tag":131,"props":10927,"children":10928},{},[10929],{"type":22,"value":536},{"type":16,"tag":157,"props":10931,"children":10932},{"encoding":159},[10933],{"type":22,"value":536},{"type":16,"tag":103,"props":10935,"children":10937},{"className":10936,"ariaHidden":167},[166],[10938],{"type":16,"tag":103,"props":10939,"children":10941},{"className":10940},[172],[10942,10946],{"type":16,"tag":103,"props":10943,"children":10945},{"className":10944,"style":805},[177],[],{"type":16,"tag":103,"props":10947,"children":10949},{"className":10948},[183,188],[10950],{"type":22,"value":536},{"type":22,"value":10952}," 是序列长度，",{"type":16,"tag":103,"props":10954,"children":10956},{"className":10955},[106],[10957,10978],{"type":16,"tag":103,"props":10958,"children":10960},{"className":10959},[111],[10961],{"type":16,"tag":114,"props":10962,"children":10963},{"xmlns":116},[10964],{"type":16,"tag":119,"props":10965,"children":10966},{},[10967,10974],{"type":16,"tag":123,"props":10968,"children":10969},{},[10970],{"type":16,"tag":131,"props":10971,"children":10972},{},[10973],{"type":22,"value":1650},{"type":16,"tag":157,"props":10975,"children":10976},{"encoding":159},[10977],{"type":22,"value":1650},{"type":16,"tag":103,"props":10979,"children":10981},{"className":10980,"ariaHidden":167},[166],[10982],{"type":16,"tag":103,"props":10983,"children":10985},{"className":10984},[172],[10986,10990],{"type":16,"tag":103,"props":10987,"children":10989},{"className":10988,"style":3719},[177],[],{"type":16,"tag":103,"props":10991,"children":10993},{"className":10992},[183,188],[10994],{"type":22,"value":1650},{"type":22,"value":10996}," 是表示维度，",{"type":16,"tag":103,"props":10998,"children":11000},{"className":10999},[106],[11001,11022],{"type":16,"tag":103,"props":11002,"children":11004},{"className":11003},[111],[11005],{"type":16,"tag":114,"props":11006,"children":11007},{"xmlns":116},[11008],{"type":16,"tag":119,"props":11009,"children":11010},{},[11011,11018],{"type":16,"tag":123,"props":11012,"children":11013},{},[11014],{"type":16,"tag":131,"props":11015,"children":11016},{},[11017],{"type":22,"value":2032},{"type":16,"tag":157,"props":11019,"children":11020},{"encoding":159},[11021],{"type":22,"value":2032},{"type":16,"tag":103,"props":11023,"children":11025},{"className":11024,"ariaHidden":167},[166],[11026],{"type":16,"tag":103,"props":11027,"children":11029},{"className":11028},[172],[11030,11034],{"type":16,"tag":103,"props":11031,"children":11033},{"className":11032,"style":3719},[177],[],{"type":16,"tag":103,"props":11035,"children":11037},{"className":11036,"style":2089},[183,188],[11038],{"type":22,"value":2032},{"type":22,"value":11040}," 是卷积 kernel size，",{"type":16,"tag":103,"props":11042,"children":11044},{"className":11043},[106],[11045,11067],{"type":16,"tag":103,"props":11046,"children":11048},{"className":11047},[111],[11049],{"type":16,"tag":114,"props":11050,"children":11051},{"xmlns":116},[11052],{"type":16,"tag":119,"props":11053,"children":11054},{},[11055,11063],{"type":16,"tag":123,"props":11056,"children":11057},{},[11058],{"type":16,"tag":131,"props":11059,"children":11060},{},[11061],{"type":22,"value":11062},"r",{"type":16,"tag":157,"props":11064,"children":11065},{"encoding":159},[11066],{"type":22,"value":11062},{"type":16,"tag":103,"props":11068,"children":11070},{"className":11069,"ariaHidden":167},[166],[11071],{"type":16,"tag":103,"props":11072,"children":11074},{"className":11073},[172],[11075,11079],{"type":16,"tag":103,"props":11076,"children":11078},{"className":11077,"style":805},[177],[],{"type":16,"tag":103,"props":11080,"children":11082},{"className":11081,"style":5198},[183,188],[11083],{"type":22,"value":11062},{"type":22,"value":11085}," 是受限自注意力中的邻域大小。",{"type":16,"tag":11087,"props":11088,"children":11089},"table",{},[11090,11119],{"type":16,"tag":11091,"props":11092,"children":11093},"thead",{},[11094],{"type":16,"tag":11095,"props":11096,"children":11097},"tr",{},[11098,11104,11109,11114],{"type":16,"tag":11099,"props":11100,"children":11101},"th",{},[11102],{"type":22,"value":11103},"层类型",{"type":16,"tag":11099,"props":11105,"children":11106},{},[11107],{"type":22,"value":11108},"每层复杂度",{"type":16,"tag":11099,"props":11110,"children":11111},{},[11112],{"type":22,"value":11113},"顺序操作数",{"type":16,"tag":11099,"props":11115,"children":11116},{},[11117],{"type":22,"value":11118},"最大路径长度",{"type":16,"tag":11120,"props":11121,"children":11122},"tbody",{},[11123,11429,11732,12169],{"type":16,"tag":11095,"props":11124,"children":11125},{},[11126,11132,11284,11357],{"type":16,"tag":11127,"props":11128,"children":11129},"td",{},[11130],{"type":22,"value":11131},"自注意力",{"type":16,"tag":11127,"props":11133,"children":11134},{},[11135],{"type":16,"tag":103,"props":11136,"children":11138},{"className":11137},[106],[11139,11188],{"type":16,"tag":103,"props":11140,"children":11142},{"className":11141},[111],[11143],{"type":16,"tag":114,"props":11144,"children":11145},{"xmlns":116},[11146],{"type":16,"tag":119,"props":11147,"children":11148},{},[11149,11183],{"type":16,"tag":123,"props":11150,"children":11151},{},[11152,11156,11160,11171,11175,11179],{"type":16,"tag":131,"props":11153,"children":11154},{},[11155],{"type":22,"value":4896},{"type":16,"tag":145,"props":11157,"children":11158},{"stretchy":495},[11159],{"type":22,"value":498},{"type":16,"tag":2662,"props":11161,"children":11162},{},[11163,11167],{"type":16,"tag":131,"props":11164,"children":11165},{},[11166],{"type":22,"value":536},{"type":16,"tag":151,"props":11168,"children":11169},{},[11170],{"type":22,"value":8050},{"type":16,"tag":145,"props":11172,"children":11173},{},[11174],{"type":22,"value":3753},{"type":16,"tag":131,"props":11176,"children":11177},{},[11178],{"type":22,"value":1650},{"type":16,"tag":145,"props":11180,"children":11181},{"stretchy":495},[11182],{"type":22,"value":541},{"type":16,"tag":157,"props":11184,"children":11185},{"encoding":159},[11186],{"type":22,"value":11187},"O(n^2 \\cdot d)",{"type":16,"tag":103,"props":11189,"children":11191},{"className":11190,"ariaHidden":167},[166],[11192,11266],{"type":16,"tag":103,"props":11193,"children":11195},{"className":11194},[172],[11196,11201,11206,11211,11253,11257,11262],{"type":16,"tag":103,"props":11197,"children":11200},{"className":11198,"style":11199},[177],"height:1.0641em;vertical-align:-0.25em;",[],{"type":16,"tag":103,"props":11202,"children":11204},{"className":11203,"style":5198},[183,188],[11205],{"type":22,"value":4896},{"type":16,"tag":103,"props":11207,"children":11209},{"className":11208},[563],[11210],{"type":22,"value":498},{"type":16,"tag":103,"props":11212,"children":11214},{"className":11213},[183],[11215,11220],{"type":16,"tag":103,"props":11216,"children":11218},{"className":11217},[183,188],[11219],{"type":22,"value":536},{"type":16,"tag":103,"props":11221,"children":11223},{"className":11222},[194],[11224],{"type":16,"tag":103,"props":11225,"children":11227},{"className":11226},[199],[11228],{"type":16,"tag":103,"props":11229,"children":11231},{"className":11230},[205],[11232],{"type":16,"tag":103,"props":11233,"children":11236},{"className":11234,"style":11235},[210],"height:0.8141em;",[11237],{"type":16,"tag":103,"props":11238,"children":11239},{"style":3043},[11240,11244],{"type":16,"tag":103,"props":11241,"children":11243},{"className":11242,"style":221},[220],[],{"type":16,"tag":103,"props":11245,"children":11247},{"className":11246},[226,227,228,229],[11248],{"type":16,"tag":103,"props":11249,"children":11251},{"className":11250},[183,229],[11252],{"type":22,"value":8050},{"type":16,"tag":103,"props":11254,"children":11256},{"className":11255,"style":1497},[632],[],{"type":16,"tag":103,"props":11258,"children":11260},{"className":11259},[243],[11261],{"type":22,"value":3753},{"type":16,"tag":103,"props":11263,"children":11265},{"className":11264,"style":1497},[632],[],{"type":16,"tag":103,"props":11267,"children":11269},{"className":11268},[172],[11270,11274,11279],{"type":16,"tag":103,"props":11271,"children":11273},{"className":11272,"style":558},[177],[],{"type":16,"tag":103,"props":11275,"children":11277},{"className":11276},[183,188],[11278],{"type":22,"value":1650},{"type":16,"tag":103,"props":11280,"children":11282},{"className":11281},[715],[11283],{"type":22,"value":541},{"type":16,"tag":11127,"props":11285,"children":11286},{},[11287],{"type":16,"tag":103,"props":11288,"children":11290},{"className":11289},[106],[11291,11325],{"type":16,"tag":103,"props":11292,"children":11294},{"className":11293},[111],[11295],{"type":16,"tag":114,"props":11296,"children":11297},{"xmlns":116},[11298],{"type":16,"tag":119,"props":11299,"children":11300},{},[11301,11320],{"type":16,"tag":123,"props":11302,"children":11303},{},[11304,11308,11312,11316],{"type":16,"tag":131,"props":11305,"children":11306},{},[11307],{"type":22,"value":4896},{"type":16,"tag":145,"props":11309,"children":11310},{"stretchy":495},[11311],{"type":22,"value":498},{"type":16,"tag":151,"props":11313,"children":11314},{},[11315],{"type":22,"value":155},{"type":16,"tag":145,"props":11317,"children":11318},{"stretchy":495},[11319],{"type":22,"value":541},{"type":16,"tag":157,"props":11321,"children":11322},{"encoding":159},[11323],{"type":22,"value":11324},"O(1)",{"type":16,"tag":103,"props":11326,"children":11328},{"className":11327,"ariaHidden":167},[166],[11329],{"type":16,"tag":103,"props":11330,"children":11332},{"className":11331},[172],[11333,11337,11342,11347,11352],{"type":16,"tag":103,"props":11334,"children":11336},{"className":11335,"style":558},[177],[],{"type":16,"tag":103,"props":11338,"children":11340},{"className":11339,"style":5198},[183,188],[11341],{"type":22,"value":4896},{"type":16,"tag":103,"props":11343,"children":11345},{"className":11344},[563],[11346],{"type":22,"value":498},{"type":16,"tag":103,"props":11348,"children":11350},{"className":11349},[183],[11351],{"type":22,"value":155},{"type":16,"tag":103,"props":11353,"children":11355},{"className":11354},[715],[11356],{"type":22,"value":541},{"type":16,"tag":11127,"props":11358,"children":11359},{},[11360],{"type":16,"tag":103,"props":11361,"children":11363},{"className":11362},[106],[11364,11397],{"type":16,"tag":103,"props":11365,"children":11367},{"className":11366},[111],[11368],{"type":16,"tag":114,"props":11369,"children":11370},{"xmlns":116},[11371],{"type":16,"tag":119,"props":11372,"children":11373},{},[11374,11393],{"type":16,"tag":123,"props":11375,"children":11376},{},[11377,11381,11385,11389],{"type":16,"tag":131,"props":11378,"children":11379},{},[11380],{"type":22,"value":4896},{"type":16,"tag":145,"props":11382,"children":11383},{"stretchy":495},[11384],{"type":22,"value":498},{"type":16,"tag":151,"props":11386,"children":11387},{},[11388],{"type":22,"value":155},{"type":16,"tag":145,"props":11390,"children":11391},{"stretchy":495},[11392],{"type":22,"value":541},{"type":16,"tag":157,"props":11394,"children":11395},{"encoding":159},[11396],{"type":22,"value":11324},{"type":16,"tag":103,"props":11398,"children":11400},{"className":11399,"ariaHidden":167},[166],[11401],{"type":16,"tag":103,"props":11402,"children":11404},{"className":11403},[172],[11405,11409,11414,11419,11424],{"type":16,"tag":103,"props":11406,"children":11408},{"className":11407,"style":558},[177],[],{"type":16,"tag":103,"props":11410,"children":11412},{"className":11411,"style":5198},[183,188],[11413],{"type":22,"value":4896},{"type":16,"tag":103,"props":11415,"children":11417},{"className":11416},[563],[11418],{"type":22,"value":498},{"type":16,"tag":103,"props":11420,"children":11422},{"className":11421},[183],[11423],{"type":22,"value":155},{"type":16,"tag":103,"props":11425,"children":11427},{"className":11426},[715],[11428],{"type":22,"value":541},{"type":16,"tag":11095,"props":11430,"children":11431},{},[11432,11437,11587,11660],{"type":16,"tag":11127,"props":11433,"children":11434},{},[11435],{"type":22,"value":11436},"循环层",{"type":16,"tag":11127,"props":11438,"children":11439},{},[11440],{"type":16,"tag":103,"props":11441,"children":11443},{"className":11442},[106],[11444,11493],{"type":16,"tag":103,"props":11445,"children":11447},{"className":11446},[111],[11448],{"type":16,"tag":114,"props":11449,"children":11450},{"xmlns":116},[11451],{"type":16,"tag":119,"props":11452,"children":11453},{},[11454,11488],{"type":16,"tag":123,"props":11455,"children":11456},{},[11457,11461,11465,11469,11473,11484],{"type":16,"tag":131,"props":11458,"children":11459},{},[11460],{"type":22,"value":4896},{"type":16,"tag":145,"props":11462,"children":11463},{"stretchy":495},[11464],{"type":22,"value":498},{"type":16,"tag":131,"props":11466,"children":11467},{},[11468],{"type":22,"value":536},{"type":16,"tag":145,"props":11470,"children":11471},{},[11472],{"type":22,"value":3753},{"type":16,"tag":2662,"props":11474,"children":11475},{},[11476,11480],{"type":16,"tag":131,"props":11477,"children":11478},{},[11479],{"type":22,"value":1650},{"type":16,"tag":151,"props":11481,"children":11482},{},[11483],{"type":22,"value":8050},{"type":16,"tag":145,"props":11485,"children":11486},{"stretchy":495},[11487],{"type":22,"value":541},{"type":16,"tag":157,"props":11489,"children":11490},{"encoding":159},[11491],{"type":22,"value":11492},"O(n \\cdot d^2)",{"type":16,"tag":103,"props":11494,"children":11496},{"className":11495,"ariaHidden":167},[166],[11497,11533],{"type":16,"tag":103,"props":11498,"children":11500},{"className":11499},[172],[11501,11505,11510,11515,11520,11524,11529],{"type":16,"tag":103,"props":11502,"children":11504},{"className":11503,"style":558},[177],[],{"type":16,"tag":103,"props":11506,"children":11508},{"className":11507,"style":5198},[183,188],[11509],{"type":22,"value":4896},{"type":16,"tag":103,"props":11511,"children":11513},{"className":11512},[563],[11514],{"type":22,"value":498},{"type":16,"tag":103,"props":11516,"children":11518},{"className":11517},[183,188],[11519],{"type":22,"value":536},{"type":16,"tag":103,"props":11521,"children":11523},{"className":11522,"style":1497},[632],[],{"type":16,"tag":103,"props":11525,"children":11527},{"className":11526},[243],[11528],{"type":22,"value":3753},{"type":16,"tag":103,"props":11530,"children":11532},{"className":11531,"style":1497},[632],[],{"type":16,"tag":103,"props":11534,"children":11536},{"className":11535},[172],[11537,11541,11582],{"type":16,"tag":103,"props":11538,"children":11540},{"className":11539,"style":11199},[177],[],{"type":16,"tag":103,"props":11542,"children":11544},{"className":11543},[183],[11545,11550],{"type":16,"tag":103,"props":11546,"children":11548},{"className":11547},[183,188],[11549],{"type":22,"value":1650},{"type":16,"tag":103,"props":11551,"children":11553},{"className":11552},[194],[11554],{"type":16,"tag":103,"props":11555,"children":11557},{"className":11556},[199],[11558],{"type":16,"tag":103,"props":11559,"children":11561},{"className":11560},[205],[11562],{"type":16,"tag":103,"props":11563,"children":11565},{"className":11564,"style":11235},[210],[11566],{"type":16,"tag":103,"props":11567,"children":11568},{"style":3043},[11569,11573],{"type":16,"tag":103,"props":11570,"children":11572},{"className":11571,"style":221},[220],[],{"type":16,"tag":103,"props":11574,"children":11576},{"className":11575},[226,227,228,229],[11577],{"type":16,"tag":103,"props":11578,"children":11580},{"className":11579},[183,229],[11581],{"type":22,"value":8050},{"type":16,"tag":103,"props":11583,"children":11585},{"className":11584},[715],[11586],{"type":22,"value":541},{"type":16,"tag":11127,"props":11588,"children":11589},{},[11590],{"type":16,"tag":103,"props":11591,"children":11593},{"className":11592},[106],[11594,11628],{"type":16,"tag":103,"props":11595,"children":11597},{"className":11596},[111],[11598],{"type":16,"tag":114,"props":11599,"children":11600},{"xmlns":116},[11601],{"type":16,"tag":119,"props":11602,"children":11603},{},[11604,11623],{"type":16,"tag":123,"props":11605,"children":11606},{},[11607,11611,11615,11619],{"type":16,"tag":131,"props":11608,"children":11609},{},[11610],{"type":22,"value":4896},{"type":16,"tag":145,"props":11612,"children":11613},{"stretchy":495},[11614],{"type":22,"value":498},{"type":16,"tag":131,"props":11616,"children":11617},{},[11618],{"type":22,"value":536},{"type":16,"tag":145,"props":11620,"children":11621},{"stretchy":495},[11622],{"type":22,"value":541},{"type":16,"tag":157,"props":11624,"children":11625},{"encoding":159},[11626],{"type":22,"value":11627},"O(n)",{"type":16,"tag":103,"props":11629,"children":11631},{"className":11630,"ariaHidden":167},[166],[11632],{"type":16,"tag":103,"props":11633,"children":11635},{"className":11634},[172],[11636,11640,11645,11650,11655],{"type":16,"tag":103,"props":11637,"children":11639},{"className":11638,"style":558},[177],[],{"type":16,"tag":103,"props":11641,"children":11643},{"className":11642,"style":5198},[183,188],[11644],{"type":22,"value":4896},{"type":16,"tag":103,"props":11646,"children":11648},{"className":11647},[563],[11649],{"type":22,"value":498},{"type":16,"tag":103,"props":11651,"children":11653},{"className":11652},[183,188],[11654],{"type":22,"value":536},{"type":16,"tag":103,"props":11656,"children":11658},{"className":11657},[715],[11659],{"type":22,"value":541},{"type":16,"tag":11127,"props":11661,"children":11662},{},[11663],{"type":16,"tag":103,"props":11664,"children":11666},{"className":11665},[106],[11667,11700],{"type":16,"tag":103,"props":11668,"children":11670},{"className":11669},[111],[11671],{"type":16,"tag":114,"props":11672,"children":11673},{"xmlns":116},[11674],{"type":16,"tag":119,"props":11675,"children":11676},{},[11677,11696],{"type":16,"tag":123,"props":11678,"children":11679},{},[11680,11684,11688,11692],{"type":16,"tag":131,"props":11681,"children":11682},{},[11683],{"type":22,"value":4896},{"type":16,"tag":145,"props":11685,"children":11686},{"stretchy":495},[11687],{"type":22,"value":498},{"type":16,"tag":131,"props":11689,"children":11690},{},[11691],{"type":22,"value":536},{"type":16,"tag":145,"props":11693,"children":11694},{"stretchy":495},[11695],{"type":22,"value":541},{"type":16,"tag":157,"props":11697,"children":11698},{"encoding":159},[11699],{"type":22,"value":11627},{"type":16,"tag":103,"props":11701,"children":11703},{"className":11702,"ariaHidden":167},[166],[11704],{"type":16,"tag":103,"props":11705,"children":11707},{"className":11706},[172],[11708,11712,11717,11722,11727],{"type":16,"tag":103,"props":11709,"children":11711},{"className":11710,"style":558},[177],[],{"type":16,"tag":103,"props":11713,"children":11715},{"className":11714,"style":5198},[183,188],[11716],{"type":22,"value":4896},{"type":16,"tag":103,"props":11718,"children":11720},{"className":11719},[563],[11721],{"type":22,"value":498},{"type":16,"tag":103,"props":11723,"children":11725},{"className":11724},[183,188],[11726],{"type":22,"value":536},{"type":16,"tag":103,"props":11728,"children":11730},{"className":11729},[715],[11731],{"type":22,"value":541},{"type":16,"tag":11095,"props":11733,"children":11734},{},[11735,11740,11925,11997],{"type":16,"tag":11127,"props":11736,"children":11737},{},[11738],{"type":22,"value":11739},"卷积层",{"type":16,"tag":11127,"props":11741,"children":11742},{},[11743],{"type":16,"tag":103,"props":11744,"children":11746},{"className":11745},[106],[11747,11804],{"type":16,"tag":103,"props":11748,"children":11750},{"className":11749},[111],[11751],{"type":16,"tag":114,"props":11752,"children":11753},{"xmlns":116},[11754],{"type":16,"tag":119,"props":11755,"children":11756},{},[11757,11799],{"type":16,"tag":123,"props":11758,"children":11759},{},[11760,11764,11768,11772,11776,11780,11784,11795],{"type":16,"tag":131,"props":11761,"children":11762},{},[11763],{"type":22,"value":4896},{"type":16,"tag":145,"props":11765,"children":11766},{"stretchy":495},[11767],{"type":22,"value":498},{"type":16,"tag":131,"props":11769,"children":11770},{},[11771],{"type":22,"value":2032},{"type":16,"tag":145,"props":11773,"children":11774},{},[11775],{"type":22,"value":3753},{"type":16,"tag":131,"props":11777,"children":11778},{},[11779],{"type":22,"value":536},{"type":16,"tag":145,"props":11781,"children":11782},{},[11783],{"type":22,"value":3753},{"type":16,"tag":2662,"props":11785,"children":11786},{},[11787,11791],{"type":16,"tag":131,"props":11788,"children":11789},{},[11790],{"type":22,"value":1650},{"type":16,"tag":151,"props":11792,"children":11793},{},[11794],{"type":22,"value":8050},{"type":16,"tag":145,"props":11796,"children":11797},{"stretchy":495},[11798],{"type":22,"value":541},{"type":16,"tag":157,"props":11800,"children":11801},{"encoding":159},[11802],{"type":22,"value":11803},"O(k \\cdot n \\cdot d^2)",{"type":16,"tag":103,"props":11805,"children":11807},{"className":11806,"ariaHidden":167},[166],[11808,11844,11871],{"type":16,"tag":103,"props":11809,"children":11811},{"className":11810},[172],[11812,11816,11821,11826,11831,11835,11840],{"type":16,"tag":103,"props":11813,"children":11815},{"className":11814,"style":558},[177],[],{"type":16,"tag":103,"props":11817,"children":11819},{"className":11818,"style":5198},[183,188],[11820],{"type":22,"value":4896},{"type":16,"tag":103,"props":11822,"children":11824},{"className":11823},[563],[11825],{"type":22,"value":498},{"type":16,"tag":103,"props":11827,"children":11829},{"className":11828,"style":2089},[183,188],[11830],{"type":22,"value":2032},{"type":16,"tag":103,"props":11832,"children":11834},{"className":11833,"style":1497},[632],[],{"type":16,"tag":103,"props":11836,"children":11838},{"className":11837},[243],[11839],{"type":22,"value":3753},{"type":16,"tag":103,"props":11841,"children":11843},{"className":11842,"style":1497},[632],[],{"type":16,"tag":103,"props":11845,"children":11847},{"className":11846},[172],[11848,11853,11858,11862,11867],{"type":16,"tag":103,"props":11849,"children":11852},{"className":11850,"style":11851},[177],"height:0.4445em;",[],{"type":16,"tag":103,"props":11854,"children":11856},{"className":11855},[183,188],[11857],{"type":22,"value":536},{"type":16,"tag":103,"props":11859,"children":11861},{"className":11860,"style":1497},[632],[],{"type":16,"tag":103,"props":11863,"children":11865},{"className":11864},[243],[11866],{"type":22,"value":3753},{"type":16,"tag":103,"props":11868,"children":11870},{"className":11869,"style":1497},[632],[],{"type":16,"tag":103,"props":11872,"children":11874},{"className":11873},[172],[11875,11879,11920],{"type":16,"tag":103,"props":11876,"children":11878},{"className":11877,"style":11199},[177],[],{"type":16,"tag":103,"props":11880,"children":11882},{"className":11881},[183],[11883,11888],{"type":16,"tag":103,"props":11884,"children":11886},{"className":11885},[183,188],[11887],{"type":22,"value":1650},{"type":16,"tag":103,"props":11889,"children":11891},{"className":11890},[194],[11892],{"type":16,"tag":103,"props":11893,"children":11895},{"className":11894},[199],[11896],{"type":16,"tag":103,"props":11897,"children":11899},{"className":11898},[205],[11900],{"type":16,"tag":103,"props":11901,"children":11903},{"className":11902,"style":11235},[210],[11904],{"type":16,"tag":103,"props":11905,"children":11906},{"style":3043},[11907,11911],{"type":16,"tag":103,"props":11908,"children":11910},{"className":11909,"style":221},[220],[],{"type":16,"tag":103,"props":11912,"children":11914},{"className":11913},[226,227,228,229],[11915],{"type":16,"tag":103,"props":11916,"children":11918},{"className":11917},[183,229],[11919],{"type":22,"value":8050},{"type":16,"tag":103,"props":11921,"children":11923},{"className":11922},[715],[11924],{"type":22,"value":541},{"type":16,"tag":11127,"props":11926,"children":11927},{},[11928],{"type":16,"tag":103,"props":11929,"children":11931},{"className":11930},[106],[11932,11965],{"type":16,"tag":103,"props":11933,"children":11935},{"className":11934},[111],[11936],{"type":16,"tag":114,"props":11937,"children":11938},{"xmlns":116},[11939],{"type":16,"tag":119,"props":11940,"children":11941},{},[11942,11961],{"type":16,"tag":123,"props":11943,"children":11944},{},[11945,11949,11953,11957],{"type":16,"tag":131,"props":11946,"children":11947},{},[11948],{"type":22,"value":4896},{"type":16,"tag":145,"props":11950,"children":11951},{"stretchy":495},[11952],{"type":22,"value":498},{"type":16,"tag":151,"props":11954,"children":11955},{},[11956],{"type":22,"value":155},{"type":16,"tag":145,"props":11958,"children":11959},{"stretchy":495},[11960],{"type":22,"value":541},{"type":16,"tag":157,"props":11962,"children":11963},{"encoding":159},[11964],{"type":22,"value":11324},{"type":16,"tag":103,"props":11966,"children":11968},{"className":11967,"ariaHidden":167},[166],[11969],{"type":16,"tag":103,"props":11970,"children":11972},{"className":11971},[172],[11973,11977,11982,11987,11992],{"type":16,"tag":103,"props":11974,"children":11976},{"className":11975,"style":558},[177],[],{"type":16,"tag":103,"props":11978,"children":11980},{"className":11979,"style":5198},[183,188],[11981],{"type":22,"value":4896},{"type":16,"tag":103,"props":11983,"children":11985},{"className":11984},[563],[11986],{"type":22,"value":498},{"type":16,"tag":103,"props":11988,"children":11990},{"className":11989},[183],[11991],{"type":22,"value":155},{"type":16,"tag":103,"props":11993,"children":11995},{"className":11994},[715],[11996],{"type":22,"value":541},{"type":16,"tag":11127,"props":11998,"children":11999},{},[12000],{"type":16,"tag":103,"props":12001,"children":12003},{"className":12002},[106],[12004,12065],{"type":16,"tag":103,"props":12005,"children":12007},{"className":12006},[111],[12008],{"type":16,"tag":114,"props":12009,"children":12010},{"xmlns":116},[12011],{"type":16,"tag":119,"props":12012,"children":12013},{},[12014,12060],{"type":16,"tag":123,"props":12015,"children":12016},{},[12017,12021,12025,12044,12048,12052,12056],{"type":16,"tag":131,"props":12018,"children":12019},{},[12020],{"type":22,"value":4896},{"type":16,"tag":145,"props":12022,"children":12023},{"stretchy":495},[12024],{"type":22,"value":498},{"type":16,"tag":127,"props":12026,"children":12027},{},[12028,12040],{"type":16,"tag":123,"props":12029,"children":12030},{},[12031,12036],{"type":16,"tag":131,"props":12032,"children":12033},{},[12034],{"type":22,"value":12035},"log",{"type":16,"tag":145,"props":12037,"children":12038},{},[12039],{"type":22,"value":1417},{"type":16,"tag":131,"props":12041,"children":12042},{},[12043],{"type":22,"value":2032},{"type":16,"tag":145,"props":12045,"children":12046},{"stretchy":495},[12047],{"type":22,"value":498},{"type":16,"tag":131,"props":12049,"children":12050},{},[12051],{"type":22,"value":536},{"type":16,"tag":145,"props":12053,"children":12054},{"stretchy":495},[12055],{"type":22,"value":541},{"type":16,"tag":145,"props":12057,"children":12058},{"stretchy":495},[12059],{"type":22,"value":541},{"type":16,"tag":157,"props":12061,"children":12062},{"encoding":159},[12063],{"type":22,"value":12064},"O(\\log_k(n))",{"type":16,"tag":103,"props":12066,"children":12068},{"className":12067,"ariaHidden":167},[166],[12069],{"type":16,"tag":103,"props":12070,"children":12072},{"className":12071},[172],[12073,12077,12082,12087,12154,12159,12164],{"type":16,"tag":103,"props":12074,"children":12076},{"className":12075,"style":558},[177],[],{"type":16,"tag":103,"props":12078,"children":12080},{"className":12079,"style":5198},[183,188],[12081],{"type":22,"value":4896},{"type":16,"tag":103,"props":12083,"children":12085},{"className":12084},[563],[12086],{"type":22,"value":498},{"type":16,"tag":103,"props":12088,"children":12090},{"className":12089},[1476],[12091,12103],{"type":16,"tag":103,"props":12092,"children":12094},{"className":12093},[1476],[12095,12097],{"type":22,"value":12096},"lo",{"type":16,"tag":103,"props":12098,"children":12100},{"style":12099},"margin-right:0.0139em;",[12101],{"type":22,"value":12102},"g",{"type":16,"tag":103,"props":12104,"children":12106},{"className":12105},[194],[12107],{"type":16,"tag":103,"props":12108,"children":12110},{"className":12109},[199,200],[12111,12142],{"type":16,"tag":103,"props":12112,"children":12114},{"className":12113},[205],[12115,12137],{"type":16,"tag":103,"props":12116,"children":12119},{"className":12117,"style":12118},[210],"height:0.242em;",[12120],{"type":16,"tag":103,"props":12121,"children":12123},{"style":12122},"top:-2.4559em;margin-right:0.05em;",[12124,12128],{"type":16,"tag":103,"props":12125,"children":12127},{"className":12126,"style":221},[220],[],{"type":16,"tag":103,"props":12129,"children":12131},{"className":12130},[226,227,228,229],[12132],{"type":16,"tag":103,"props":12133,"children":12135},{"className":12134,"style":2089},[183,188,229],[12136],{"type":22,"value":2032},{"type":16,"tag":103,"props":12138,"children":12140},{"className":12139},[254],[12141],{"type":22,"value":257},{"type":16,"tag":103,"props":12143,"children":12145},{"className":12144},[205],[12146],{"type":16,"tag":103,"props":12147,"children":12150},{"className":12148,"style":12149},[210],"height:0.2441em;",[12151],{"type":16,"tag":103,"props":12152,"children":12153},{},[],{"type":16,"tag":103,"props":12155,"children":12157},{"className":12156},[563],[12158],{"type":22,"value":498},{"type":16,"tag":103,"props":12160,"children":12162},{"className":12161},[183,188],[12163],{"type":22,"value":536},{"type":16,"tag":103,"props":12165,"children":12167},{"className":12166},[715],[12168],{"type":22,"value":1540},{"type":16,"tag":11095,"props":12170,"children":12171},{},[12172,12177,12318,12390],{"type":16,"tag":11127,"props":12173,"children":12174},{},[12175],{"type":22,"value":12176},"受限自注意力",{"type":16,"tag":11127,"props":12178,"children":12179},{},[12180],{"type":16,"tag":103,"props":12181,"children":12183},{"className":12182},[106],[12184,12234],{"type":16,"tag":103,"props":12185,"children":12187},{"className":12186},[111],[12188],{"type":16,"tag":114,"props":12189,"children":12190},{"xmlns":116},[12191],{"type":16,"tag":119,"props":12192,"children":12193},{},[12194,12229],{"type":16,"tag":123,"props":12195,"children":12196},{},[12197,12201,12205,12209,12213,12217,12221,12225],{"type":16,"tag":131,"props":12198,"children":12199},{},[12200],{"type":22,"value":4896},{"type":16,"tag":145,"props":12202,"children":12203},{"stretchy":495},[12204],{"type":22,"value":498},{"type":16,"tag":131,"props":12206,"children":12207},{},[12208],{"type":22,"value":11062},{"type":16,"tag":145,"props":12210,"children":12211},{},[12212],{"type":22,"value":3753},{"type":16,"tag":131,"props":12214,"children":12215},{},[12216],{"type":22,"value":536},{"type":16,"tag":145,"props":12218,"children":12219},{},[12220],{"type":22,"value":3753},{"type":16,"tag":131,"props":12222,"children":12223},{},[12224],{"type":22,"value":1650},{"type":16,"tag":145,"props":12226,"children":12227},{"stretchy":495},[12228],{"type":22,"value":541},{"type":16,"tag":157,"props":12230,"children":12231},{"encoding":159},[12232],{"type":22,"value":12233},"O(r \\cdot n \\cdot d)",{"type":16,"tag":103,"props":12235,"children":12237},{"className":12236,"ariaHidden":167},[166],[12238,12274,12300],{"type":16,"tag":103,"props":12239,"children":12241},{"className":12240},[172],[12242,12246,12251,12256,12261,12265,12270],{"type":16,"tag":103,"props":12243,"children":12245},{"className":12244,"style":558},[177],[],{"type":16,"tag":103,"props":12247,"children":12249},{"className":12248,"style":5198},[183,188],[12250],{"type":22,"value":4896},{"type":16,"tag":103,"props":12252,"children":12254},{"className":12253},[563],[12255],{"type":22,"value":498},{"type":16,"tag":103,"props":12257,"children":12259},{"className":12258,"style":5198},[183,188],[12260],{"type":22,"value":11062},{"type":16,"tag":103,"props":12262,"children":12264},{"className":12263,"style":1497},[632],[],{"type":16,"tag":103,"props":12266,"children":12268},{"className":12267},[243],[12269],{"type":22,"value":3753},{"type":16,"tag":103,"props":12271,"children":12273},{"className":12272,"style":1497},[632],[],{"type":16,"tag":103,"props":12275,"children":12277},{"className":12276},[172],[12278,12282,12287,12291,12296],{"type":16,"tag":103,"props":12279,"children":12281},{"className":12280,"style":11851},[177],[],{"type":16,"tag":103,"props":12283,"children":12285},{"className":12284},[183,188],[12286],{"type":22,"value":536},{"type":16,"tag":103,"props":12288,"children":12290},{"className":12289,"style":1497},[632],[],{"type":16,"tag":103,"props":12292,"children":12294},{"className":12293},[243],[12295],{"type":22,"value":3753},{"type":16,"tag":103,"props":12297,"children":12299},{"className":12298,"style":1497},[632],[],{"type":16,"tag":103,"props":12301,"children":12303},{"className":12302},[172],[12304,12308,12313],{"type":16,"tag":103,"props":12305,"children":12307},{"className":12306,"style":558},[177],[],{"type":16,"tag":103,"props":12309,"children":12311},{"className":12310},[183,188],[12312],{"type":22,"value":1650},{"type":16,"tag":103,"props":12314,"children":12316},{"className":12315},[715],[12317],{"type":22,"value":541},{"type":16,"tag":11127,"props":12319,"children":12320},{},[12321],{"type":16,"tag":103,"props":12322,"children":12324},{"className":12323},[106],[12325,12358],{"type":16,"tag":103,"props":12326,"children":12328},{"className":12327},[111],[12329],{"type":16,"tag":114,"props":12330,"children":12331},{"xmlns":116},[12332],{"type":16,"tag":119,"props":12333,"children":12334},{},[12335,12354],{"type":16,"tag":123,"props":12336,"children":12337},{},[12338,12342,12346,12350],{"type":16,"tag":131,"props":12339,"children":12340},{},[12341],{"type":22,"value":4896},{"type":16,"tag":145,"props":12343,"children":12344},{"stretchy":495},[12345],{"type":22,"value":498},{"type":16,"tag":151,"props":12347,"children":12348},{},[12349],{"type":22,"value":155},{"type":16,"tag":145,"props":12351,"children":12352},{"stretchy":495},[12353],{"type":22,"value":541},{"type":16,"tag":157,"props":12355,"children":12356},{"encoding":159},[12357],{"type":22,"value":11324},{"type":16,"tag":103,"props":12359,"children":12361},{"className":12360,"ariaHidden":167},[166],[12362],{"type":16,"tag":103,"props":12363,"children":12365},{"className":12364},[172],[12366,12370,12375,12380,12385],{"type":16,"tag":103,"props":12367,"children":12369},{"className":12368,"style":558},[177],[],{"type":16,"tag":103,"props":12371,"children":12373},{"className":12372,"style":5198},[183,188],[12374],{"type":22,"value":4896},{"type":16,"tag":103,"props":12376,"children":12378},{"className":12377},[563],[12379],{"type":22,"value":498},{"type":16,"tag":103,"props":12381,"children":12383},{"className":12382},[183],[12384],{"type":22,"value":155},{"type":16,"tag":103,"props":12386,"children":12388},{"className":12387},[715],[12389],{"type":22,"value":541},{"type":16,"tag":11127,"props":12391,"children":12392},{},[12393],{"type":16,"tag":103,"props":12394,"children":12396},{"className":12395},[106],[12397,12439],{"type":16,"tag":103,"props":12398,"children":12400},{"className":12399},[111],[12401],{"type":16,"tag":114,"props":12402,"children":12403},{"xmlns":116},[12404],{"type":16,"tag":119,"props":12405,"children":12406},{},[12407,12434],{"type":16,"tag":123,"props":12408,"children":12409},{},[12410,12414,12418,12422,12426,12430],{"type":16,"tag":131,"props":12411,"children":12412},{},[12413],{"type":22,"value":4896},{"type":16,"tag":145,"props":12415,"children":12416},{"stretchy":495},[12417],{"type":22,"value":498},{"type":16,"tag":131,"props":12419,"children":12420},{},[12421],{"type":22,"value":536},{"type":16,"tag":131,"props":12423,"children":12424},{"mathvariant":1409},[12425],{"type":22,"value":3474},{"type":16,"tag":131,"props":12427,"children":12428},{},[12429],{"type":22,"value":11062},{"type":16,"tag":145,"props":12431,"children":12432},{"stretchy":495},[12433],{"type":22,"value":541},{"type":16,"tag":157,"props":12435,"children":12436},{"encoding":159},[12437],{"type":22,"value":12438},"O(n\u002Fr)",{"type":16,"tag":103,"props":12440,"children":12442},{"className":12441,"ariaHidden":167},[166],[12443],{"type":16,"tag":103,"props":12444,"children":12446},{"className":12445},[172],[12447,12451,12456,12461,12466,12471,12476],{"type":16,"tag":103,"props":12448,"children":12450},{"className":12449,"style":558},[177],[],{"type":16,"tag":103,"props":12452,"children":12454},{"className":12453,"style":5198},[183,188],[12455],{"type":22,"value":4896},{"type":16,"tag":103,"props":12457,"children":12459},{"className":12458},[563],[12460],{"type":22,"value":498},{"type":16,"tag":103,"props":12462,"children":12464},{"className":12463},[183,188],[12465],{"type":22,"value":536},{"type":16,"tag":103,"props":12467,"children":12469},{"className":12468},[183],[12470],{"type":22,"value":3474},{"type":16,"tag":103,"props":12472,"children":12474},{"className":12473,"style":5198},[183,188],[12475],{"type":22,"value":11062},{"type":16,"tag":103,"props":12477,"children":12479},{"className":12478},[715],[12480],{"type":22,"value":541},{"type":16,"tag":66,"props":12482,"children":12484},{"id":12483},"_4-为什么使用自注意力",[12485],{"type":22,"value":12486},"4 为什么使用自注意力",{"type":16,"tag":25,"props":12488,"children":12489},{},[12490,12492,12718,12720,12947,12949,13211],{"type":22,"value":12491},"本节比较自注意力层与通常用于把一个变长符号表示序列 ",{"type":16,"tag":103,"props":12493,"children":12495},{"className":12494},[106],[12496,12555],{"type":16,"tag":103,"props":12497,"children":12499},{"className":12498},[111],[12500],{"type":16,"tag":114,"props":12501,"children":12502},{"xmlns":116},[12503],{"type":16,"tag":119,"props":12504,"children":12505},{},[12506,12551],{"type":16,"tag":123,"props":12507,"children":12508},{},[12509,12513,12524,12528,12532,12536,12547],{"type":16,"tag":145,"props":12510,"children":12511},{"stretchy":495},[12512],{"type":22,"value":498},{"type":16,"tag":127,"props":12514,"children":12515},{},[12516,12520],{"type":16,"tag":131,"props":12517,"children":12518},{},[12519],{"type":22,"value":506},{"type":16,"tag":151,"props":12521,"children":12522},{},[12523],{"type":22,"value":155},{"type":16,"tag":145,"props":12525,"children":12526},{"separator":167},[12527],{"type":22,"value":515},{"type":16,"tag":145,"props":12529,"children":12530},{},[12531],{"type":22,"value":520},{"type":16,"tag":145,"props":12533,"children":12534},{"separator":167},[12535],{"type":22,"value":515},{"type":16,"tag":127,"props":12537,"children":12538},{},[12539,12543],{"type":16,"tag":131,"props":12540,"children":12541},{},[12542],{"type":22,"value":506},{"type":16,"tag":131,"props":12544,"children":12545},{},[12546],{"type":22,"value":536},{"type":16,"tag":145,"props":12548,"children":12549},{"stretchy":495},[12550],{"type":22,"value":541},{"type":16,"tag":157,"props":12552,"children":12553},{"encoding":159},[12554],{"type":22,"value":546},{"type":16,"tag":103,"props":12556,"children":12558},{"className":12557,"ariaHidden":167},[166],[12559],{"type":16,"tag":103,"props":12560,"children":12562},{"className":12561},[172],[12563,12567,12572,12629,12634,12638,12643,12647,12652,12656,12713],{"type":16,"tag":103,"props":12564,"children":12566},{"className":12565,"style":558},[177],[],{"type":16,"tag":103,"props":12568,"children":12570},{"className":12569},[563],[12571],{"type":22,"value":498},{"type":16,"tag":103,"props":12573,"children":12575},{"className":12574},[183],[12576,12581],{"type":16,"tag":103,"props":12577,"children":12579},{"className":12578},[183,188],[12580],{"type":22,"value":506},{"type":16,"tag":103,"props":12582,"children":12584},{"className":12583},[194],[12585],{"type":16,"tag":103,"props":12586,"children":12588},{"className":12587},[199,200],[12589,12618],{"type":16,"tag":103,"props":12590,"children":12592},{"className":12591},[205],[12593,12613],{"type":16,"tag":103,"props":12594,"children":12596},{"className":12595,"style":211},[210],[12597],{"type":16,"tag":103,"props":12598,"children":12599},{"style":215},[12600,12604],{"type":16,"tag":103,"props":12601,"children":12603},{"className":12602,"style":221},[220],[],{"type":16,"tag":103,"props":12605,"children":12607},{"className":12606},[226,227,228,229],[12608],{"type":16,"tag":103,"props":12609,"children":12611},{"className":12610},[183,229],[12612],{"type":22,"value":155},{"type":16,"tag":103,"props":12614,"children":12616},{"className":12615},[254],[12617],{"type":22,"value":257},{"type":16,"tag":103,"props":12619,"children":12621},{"className":12620},[205],[12622],{"type":16,"tag":103,"props":12623,"children":12625},{"className":12624,"style":417},[210],[12626],{"type":16,"tag":103,"props":12627,"children":12628},{},[],{"type":16,"tag":103,"props":12630,"children":12632},{"className":12631},[626],[12633],{"type":22,"value":515},{"type":16,"tag":103,"props":12635,"children":12637},{"className":12636,"style":633},[632],[],{"type":16,"tag":103,"props":12639,"children":12641},{"className":12640},[638],[12642],{"type":22,"value":520},{"type":16,"tag":103,"props":12644,"children":12646},{"className":12645,"style":633},[632],[],{"type":16,"tag":103,"props":12648,"children":12650},{"className":12649},[626],[12651],{"type":22,"value":515},{"type":16,"tag":103,"props":12653,"children":12655},{"className":12654,"style":633},[632],[],{"type":16,"tag":103,"props":12657,"children":12659},{"className":12658},[183],[12660,12665],{"type":16,"tag":103,"props":12661,"children":12663},{"className":12662},[183,188],[12664],{"type":22,"value":506},{"type":16,"tag":103,"props":12666,"children":12668},{"className":12667},[194],[12669],{"type":16,"tag":103,"props":12670,"children":12672},{"className":12671},[199,200],[12673,12702],{"type":16,"tag":103,"props":12674,"children":12676},{"className":12675},[205],[12677,12697],{"type":16,"tag":103,"props":12678,"children":12680},{"className":12679,"style":678},[210],[12681],{"type":16,"tag":103,"props":12682,"children":12683},{"style":215},[12684,12688],{"type":16,"tag":103,"props":12685,"children":12687},{"className":12686,"style":221},[220],[],{"type":16,"tag":103,"props":12689,"children":12691},{"className":12690},[226,227,228,229],[12692],{"type":16,"tag":103,"props":12693,"children":12695},{"className":12694},[183,188,229],[12696],{"type":22,"value":536},{"type":16,"tag":103,"props":12698,"children":12700},{"className":12699},[254],[12701],{"type":22,"value":257},{"type":16,"tag":103,"props":12703,"children":12705},{"className":12704},[205],[12706],{"type":16,"tag":103,"props":12707,"children":12709},{"className":12708,"style":417},[210],[12710],{"type":16,"tag":103,"props":12711,"children":12712},{},[],{"type":16,"tag":103,"props":12714,"children":12716},{"className":12715},[715],[12717],{"type":22,"value":541},{"type":22,"value":12719}," 映射到另一个等长序列 ",{"type":16,"tag":103,"props":12721,"children":12723},{"className":12722},[106],[12724,12784],{"type":16,"tag":103,"props":12725,"children":12727},{"className":12726},[111],[12728],{"type":16,"tag":114,"props":12729,"children":12730},{"xmlns":116},[12731],{"type":16,"tag":119,"props":12732,"children":12733},{},[12734,12779],{"type":16,"tag":123,"props":12735,"children":12736},{},[12737,12741,12752,12756,12760,12764,12775],{"type":16,"tag":145,"props":12738,"children":12739},{"stretchy":495},[12740],{"type":22,"value":498},{"type":16,"tag":127,"props":12742,"children":12743},{},[12744,12748],{"type":16,"tag":131,"props":12745,"children":12746},{},[12747],{"type":22,"value":741},{"type":16,"tag":151,"props":12749,"children":12750},{},[12751],{"type":22,"value":155},{"type":16,"tag":145,"props":12753,"children":12754},{"separator":167},[12755],{"type":22,"value":515},{"type":16,"tag":145,"props":12757,"children":12758},{},[12759],{"type":22,"value":520},{"type":16,"tag":145,"props":12761,"children":12762},{"separator":167},[12763],{"type":22,"value":515},{"type":16,"tag":127,"props":12765,"children":12766},{},[12767,12771],{"type":16,"tag":131,"props":12768,"children":12769},{},[12770],{"type":22,"value":741},{"type":16,"tag":131,"props":12772,"children":12773},{},[12774],{"type":22,"value":536},{"type":16,"tag":145,"props":12776,"children":12777},{"stretchy":495},[12778],{"type":22,"value":541},{"type":16,"tag":157,"props":12780,"children":12781},{"encoding":159},[12782],{"type":22,"value":12783},"(z_1, \\ldots, z_n)",{"type":16,"tag":103,"props":12785,"children":12787},{"className":12786,"ariaHidden":167},[166],[12788],{"type":16,"tag":103,"props":12789,"children":12791},{"className":12790},[172],[12792,12796,12801,12858,12863,12867,12872,12876,12881,12885,12942],{"type":16,"tag":103,"props":12793,"children":12795},{"className":12794,"style":558},[177],[],{"type":16,"tag":103,"props":12797,"children":12799},{"className":12798},[563],[12800],{"type":22,"value":498},{"type":16,"tag":103,"props":12802,"children":12804},{"className":12803},[183],[12805,12810],{"type":16,"tag":103,"props":12806,"children":12808},{"className":12807,"style":810},[183,188],[12809],{"type":22,"value":741},{"type":16,"tag":103,"props":12811,"children":12813},{"className":12812},[194],[12814],{"type":16,"tag":103,"props":12815,"children":12817},{"className":12816},[199,200],[12818,12847],{"type":16,"tag":103,"props":12819,"children":12821},{"className":12820},[205],[12822,12842],{"type":16,"tag":103,"props":12823,"children":12825},{"className":12824,"style":211},[210],[12826],{"type":16,"tag":103,"props":12827,"children":12828},{"style":868},[12829,12833],{"type":16,"tag":103,"props":12830,"children":12832},{"className":12831,"style":221},[220],[],{"type":16,"tag":103,"props":12834,"children":12836},{"className":12835},[226,227,228,229],[12837],{"type":16,"tag":103,"props":12838,"children":12840},{"className":12839},[183,229],[12841],{"type":22,"value":155},{"type":16,"tag":103,"props":12843,"children":12845},{"className":12844},[254],[12846],{"type":22,"value":257},{"type":16,"tag":103,"props":12848,"children":12850},{"className":12849},[205],[12851],{"type":16,"tag":103,"props":12852,"children":12854},{"className":12853,"style":417},[210],[12855],{"type":16,"tag":103,"props":12856,"children":12857},{},[],{"type":16,"tag":103,"props":12859,"children":12861},{"className":12860},[626],[12862],{"type":22,"value":515},{"type":16,"tag":103,"props":12864,"children":12866},{"className":12865,"style":633},[632],[],{"type":16,"tag":103,"props":12868,"children":12870},{"className":12869},[638],[12871],{"type":22,"value":520},{"type":16,"tag":103,"props":12873,"children":12875},{"className":12874,"style":633},[632],[],{"type":16,"tag":103,"props":12877,"children":12879},{"className":12878},[626],[12880],{"type":22,"value":515},{"type":16,"tag":103,"props":12882,"children":12884},{"className":12883,"style":633},[632],[],{"type":16,"tag":103,"props":12886,"children":12888},{"className":12887},[183],[12889,12894],{"type":16,"tag":103,"props":12890,"children":12892},{"className":12891,"style":810},[183,188],[12893],{"type":22,"value":741},{"type":16,"tag":103,"props":12895,"children":12897},{"className":12896},[194],[12898],{"type":16,"tag":103,"props":12899,"children":12901},{"className":12900},[199,200],[12902,12931],{"type":16,"tag":103,"props":12903,"children":12905},{"className":12904},[205],[12906,12926],{"type":16,"tag":103,"props":12907,"children":12909},{"className":12908,"style":678},[210],[12910],{"type":16,"tag":103,"props":12911,"children":12912},{"style":868},[12913,12917],{"type":16,"tag":103,"props":12914,"children":12916},{"className":12915,"style":221},[220],[],{"type":16,"tag":103,"props":12918,"children":12920},{"className":12919},[226,227,228,229],[12921],{"type":16,"tag":103,"props":12922,"children":12924},{"className":12923},[183,188,229],[12925],{"type":22,"value":536},{"type":16,"tag":103,"props":12927,"children":12929},{"className":12928},[254],[12930],{"type":22,"value":257},{"type":16,"tag":103,"props":12932,"children":12934},{"className":12933},[205],[12935],{"type":16,"tag":103,"props":12936,"children":12938},{"className":12937,"style":417},[210],[12939],{"type":16,"tag":103,"props":12940,"children":12941},{},[],{"type":16,"tag":103,"props":12943,"children":12945},{"className":12944},[715],[12946],{"type":22,"value":541},{"type":22,"value":12948}," 的循环层和卷积层。这里 ",{"type":16,"tag":103,"props":12950,"children":12952},{"className":12951},[106],[12953,13012],{"type":16,"tag":103,"props":12954,"children":12956},{"className":12955},[111],[12957],{"type":16,"tag":114,"props":12958,"children":12959},{"xmlns":116},[12960],{"type":16,"tag":119,"props":12961,"children":12962},{},[12963,13007],{"type":16,"tag":123,"props":12964,"children":12965},{},[12966,12977,12981,12992,12996],{"type":16,"tag":127,"props":12967,"children":12968},{},[12969,12973],{"type":16,"tag":131,"props":12970,"children":12971},{},[12972],{"type":22,"value":506},{"type":16,"tag":131,"props":12974,"children":12975},{},[12976],{"type":22,"value":1912},{"type":16,"tag":145,"props":12978,"children":12979},{"separator":167},[12980],{"type":22,"value":515},{"type":16,"tag":127,"props":12982,"children":12983},{},[12984,12988],{"type":16,"tag":131,"props":12985,"children":12986},{},[12987],{"type":22,"value":741},{"type":16,"tag":131,"props":12989,"children":12990},{},[12991],{"type":22,"value":1912},{"type":16,"tag":145,"props":12993,"children":12994},{},[12995],{"type":22,"value":5779},{"type":16,"tag":2662,"props":12997,"children":12998},{},[12999,13003],{"type":16,"tag":131,"props":13000,"children":13001},{"mathvariant":5785},[13002],{"type":22,"value":5788},{"type":16,"tag":131,"props":13004,"children":13005},{},[13006],{"type":22,"value":1650},{"type":16,"tag":157,"props":13008,"children":13009},{"encoding":159},[13010],{"type":22,"value":13011},"x_i, z_i \\in \\mathbb{R}^d",{"type":16,"tag":103,"props":13013,"children":13015},{"className":13014,"ariaHidden":167},[166],[13016,13161],{"type":16,"tag":103,"props":13017,"children":13019},{"className":13018},[172],[13020,13025,13082,13087,13091,13148,13152,13157],{"type":16,"tag":103,"props":13021,"children":13024},{"className":13022,"style":13023},[177],"height:0.7335em;vertical-align:-0.1944em;",[],{"type":16,"tag":103,"props":13026,"children":13028},{"className":13027},[183],[13029,13034],{"type":16,"tag":103,"props":13030,"children":13032},{"className":13031},[183,188],[13033],{"type":22,"value":506},{"type":16,"tag":103,"props":13035,"children":13037},{"className":13036},[194],[13038],{"type":16,"tag":103,"props":13039,"children":13041},{"className":13040},[199,200],[13042,13071],{"type":16,"tag":103,"props":13043,"children":13045},{"className":13044},[205],[13046,13066],{"type":16,"tag":103,"props":13047,"children":13049},{"className":13048,"style":3958},[210],[13050],{"type":16,"tag":103,"props":13051,"children":13052},{"style":215},[13053,13057],{"type":16,"tag":103,"props":13054,"children":13056},{"className":13055,"style":221},[220],[],{"type":16,"tag":103,"props":13058,"children":13060},{"className":13059},[226,227,228,229],[13061],{"type":16,"tag":103,"props":13062,"children":13064},{"className":13063},[183,188,229],[13065],{"type":22,"value":1912},{"type":16,"tag":103,"props":13067,"children":13069},{"className":13068},[254],[13070],{"type":22,"value":257},{"type":16,"tag":103,"props":13072,"children":13074},{"className":13073},[205],[13075],{"type":16,"tag":103,"props":13076,"children":13078},{"className":13077,"style":417},[210],[13079],{"type":16,"tag":103,"props":13080,"children":13081},{},[],{"type":16,"tag":103,"props":13083,"children":13085},{"className":13084},[626],[13086],{"type":22,"value":515},{"type":16,"tag":103,"props":13088,"children":13090},{"className":13089,"style":633},[632],[],{"type":16,"tag":103,"props":13092,"children":13094},{"className":13093},[183],[13095,13100],{"type":16,"tag":103,"props":13096,"children":13098},{"className":13097,"style":810},[183,188],[13099],{"type":22,"value":741},{"type":16,"tag":103,"props":13101,"children":13103},{"className":13102},[194],[13104],{"type":16,"tag":103,"props":13105,"children":13107},{"className":13106},[199,200],[13108,13137],{"type":16,"tag":103,"props":13109,"children":13111},{"className":13110},[205],[13112,13132],{"type":16,"tag":103,"props":13113,"children":13115},{"className":13114,"style":3958},[210],[13116],{"type":16,"tag":103,"props":13117,"children":13118},{"style":868},[13119,13123],{"type":16,"tag":103,"props":13120,"children":13122},{"className":13121,"style":221},[220],[],{"type":16,"tag":103,"props":13124,"children":13126},{"className":13125},[226,227,228,229],[13127],{"type":16,"tag":103,"props":13128,"children":13130},{"className":13129},[183,188,229],[13131],{"type":22,"value":1912},{"type":16,"tag":103,"props":13133,"children":13135},{"className":13134},[254],[13136],{"type":22,"value":257},{"type":16,"tag":103,"props":13138,"children":13140},{"className":13139},[205],[13141],{"type":16,"tag":103,"props":13142,"children":13144},{"className":13143,"style":417},[210],[13145],{"type":16,"tag":103,"props":13146,"children":13147},{},[],{"type":16,"tag":103,"props":13149,"children":13151},{"className":13150,"style":816},[632],[],{"type":16,"tag":103,"props":13153,"children":13155},{"className":13154},[821],[13156],{"type":22,"value":5779},{"type":16,"tag":103,"props":13158,"children":13160},{"className":13159,"style":816},[632],[],{"type":16,"tag":103,"props":13162,"children":13164},{"className":13163},[172],[13165,13170],{"type":16,"tag":103,"props":13166,"children":13169},{"className":13167,"style":13168},[177],"height:0.8491em;",[],{"type":16,"tag":103,"props":13171,"children":13173},{"className":13172},[183],[13174,13179],{"type":16,"tag":103,"props":13175,"children":13177},{"className":13176},[183,6561],[13178],{"type":22,"value":5788},{"type":16,"tag":103,"props":13180,"children":13182},{"className":13181},[194],[13183],{"type":16,"tag":103,"props":13184,"children":13186},{"className":13185},[199],[13187],{"type":16,"tag":103,"props":13188,"children":13190},{"className":13189},[205],[13191],{"type":16,"tag":103,"props":13192,"children":13194},{"className":13193,"style":13168},[210],[13195],{"type":16,"tag":103,"props":13196,"children":13197},{"style":3043},[13198,13202],{"type":16,"tag":103,"props":13199,"children":13201},{"className":13200,"style":221},[220],[],{"type":16,"tag":103,"props":13203,"children":13205},{"className":13204},[226,227,228,229],[13206],{"type":16,"tag":103,"props":13207,"children":13209},{"className":13208},[183,188,229],[13210],{"type":22,"value":1650},{"type":22,"value":13212},"；这种映射可以看作典型序列转换编码器或解码器中的一个隐藏层。我们从三个要求出发来说明采用自注意力的动机。",{"type":16,"tag":25,"props":13214,"children":13215},{},[13216],{"type":22,"value":13217},"第一个要求是每层的总计算复杂度。第二个要求是可并行化的计算量，用执行所需的最少顺序操作数衡量。第三个要求是网络中长距离依赖之间的路径长度。",{"type":16,"tag":25,"props":13219,"children":13220},{},[13221],{"type":22,"value":13222},"学习长距离依赖是许多序列转换任务中的关键挑战。影响模型学习这些依赖能力的一个关键因素，是前向和反向信号必须在网络中经过的路径长度。输入和输出位置任意组合之间的路径越短，模型就越容易学习长距离依赖 [11]。因此，我们也比较了由不同层类型组成的网络中，任意输入位置与输出位置之间的最大路径长度。",{"type":16,"tag":25,"props":13224,"children":13225},{},[13226,13228,13297,13299,13341,13343,13385],{"type":22,"value":13227},"如表 1 所示，自注意力层用常数个顺序执行的操作连接所有位置，而循环层需要 ",{"type":16,"tag":103,"props":13229,"children":13231},{"className":13230},[106],[13232,13265],{"type":16,"tag":103,"props":13233,"children":13235},{"className":13234},[111],[13236],{"type":16,"tag":114,"props":13237,"children":13238},{"xmlns":116},[13239],{"type":16,"tag":119,"props":13240,"children":13241},{},[13242,13261],{"type":16,"tag":123,"props":13243,"children":13244},{},[13245,13249,13253,13257],{"type":16,"tag":131,"props":13246,"children":13247},{},[13248],{"type":22,"value":4896},{"type":16,"tag":145,"props":13250,"children":13251},{"stretchy":495},[13252],{"type":22,"value":498},{"type":16,"tag":131,"props":13254,"children":13255},{},[13256],{"type":22,"value":536},{"type":16,"tag":145,"props":13258,"children":13259},{"stretchy":495},[13260],{"type":22,"value":541},{"type":16,"tag":157,"props":13262,"children":13263},{"encoding":159},[13264],{"type":22,"value":11627},{"type":16,"tag":103,"props":13266,"children":13268},{"className":13267,"ariaHidden":167},[166],[13269],{"type":16,"tag":103,"props":13270,"children":13272},{"className":13271},[172],[13273,13277,13282,13287,13292],{"type":16,"tag":103,"props":13274,"children":13276},{"className":13275,"style":558},[177],[],{"type":16,"tag":103,"props":13278,"children":13280},{"className":13279,"style":5198},[183,188],[13281],{"type":22,"value":4896},{"type":16,"tag":103,"props":13283,"children":13285},{"className":13284},[563],[13286],{"type":22,"value":498},{"type":16,"tag":103,"props":13288,"children":13290},{"className":13289},[183,188],[13291],{"type":22,"value":536},{"type":16,"tag":103,"props":13293,"children":13295},{"className":13294},[715],[13296],{"type":22,"value":541},{"type":22,"value":13298}," 个顺序操作。在计算复杂度方面，当序列长度 ",{"type":16,"tag":103,"props":13300,"children":13302},{"className":13301},[106],[13303,13324],{"type":16,"tag":103,"props":13304,"children":13306},{"className":13305},[111],[13307],{"type":16,"tag":114,"props":13308,"children":13309},{"xmlns":116},[13310],{"type":16,"tag":119,"props":13311,"children":13312},{},[13313,13320],{"type":16,"tag":123,"props":13314,"children":13315},{},[13316],{"type":16,"tag":131,"props":13317,"children":13318},{},[13319],{"type":22,"value":536},{"type":16,"tag":157,"props":13321,"children":13322},{"encoding":159},[13323],{"type":22,"value":536},{"type":16,"tag":103,"props":13325,"children":13327},{"className":13326,"ariaHidden":167},[166],[13328],{"type":16,"tag":103,"props":13329,"children":13331},{"className":13330},[172],[13332,13336],{"type":16,"tag":103,"props":13333,"children":13335},{"className":13334,"style":805},[177],[],{"type":16,"tag":103,"props":13337,"children":13339},{"className":13338},[183,188],[13340],{"type":22,"value":536},{"type":22,"value":13342}," 小于表示维度 ",{"type":16,"tag":103,"props":13344,"children":13346},{"className":13345},[106],[13347,13368],{"type":16,"tag":103,"props":13348,"children":13350},{"className":13349},[111],[13351],{"type":16,"tag":114,"props":13352,"children":13353},{"xmlns":116},[13354],{"type":16,"tag":119,"props":13355,"children":13356},{},[13357,13364],{"type":16,"tag":123,"props":13358,"children":13359},{},[13360],{"type":16,"tag":131,"props":13361,"children":13362},{},[13363],{"type":22,"value":1650},{"type":16,"tag":157,"props":13365,"children":13366},{"encoding":159},[13367],{"type":22,"value":1650},{"type":16,"tag":103,"props":13369,"children":13371},{"className":13370,"ariaHidden":167},[166],[13372],{"type":16,"tag":103,"props":13373,"children":13375},{"className":13374},[172],[13376,13380],{"type":16,"tag":103,"props":13377,"children":13379},{"className":13378,"style":3719},[177],[],{"type":16,"tag":103,"props":13381,"children":13383},{"className":13382},[183,188],[13384],{"type":22,"value":1650},{"type":22,"value":13386}," 时，自注意力层比循环层更快；在机器翻译最新模型使用的词片段 [31] 和字节对 [25] 表示中，通常就是这种情况。",{"type":16,"tag":25,"props":13388,"children":13389},{},[13390,13392,13434,13436,13523],{"type":22,"value":13391},"为了提高超长序列任务的计算性能，可以把自注意力限制在输入序列中、以相应输出位置为中心的大小为 ",{"type":16,"tag":103,"props":13393,"children":13395},{"className":13394},[106],[13396,13417],{"type":16,"tag":103,"props":13397,"children":13399},{"className":13398},[111],[13400],{"type":16,"tag":114,"props":13401,"children":13402},{"xmlns":116},[13403],{"type":16,"tag":119,"props":13404,"children":13405},{},[13406,13413],{"type":16,"tag":123,"props":13407,"children":13408},{},[13409],{"type":16,"tag":131,"props":13410,"children":13411},{},[13412],{"type":22,"value":11062},{"type":16,"tag":157,"props":13414,"children":13415},{"encoding":159},[13416],{"type":22,"value":11062},{"type":16,"tag":103,"props":13418,"children":13420},{"className":13419,"ariaHidden":167},[166],[13421],{"type":16,"tag":103,"props":13422,"children":13424},{"className":13423},[172],[13425,13429],{"type":16,"tag":103,"props":13426,"children":13428},{"className":13427,"style":805},[177],[],{"type":16,"tag":103,"props":13430,"children":13432},{"className":13431,"style":5198},[183,188],[13433],{"type":22,"value":11062},{"type":22,"value":13435}," 的邻域内。这样会把最大路径长度增加到 ",{"type":16,"tag":103,"props":13437,"children":13439},{"className":13438},[106],[13440,13481],{"type":16,"tag":103,"props":13441,"children":13443},{"className":13442},[111],[13444],{"type":16,"tag":114,"props":13445,"children":13446},{"xmlns":116},[13447],{"type":16,"tag":119,"props":13448,"children":13449},{},[13450,13477],{"type":16,"tag":123,"props":13451,"children":13452},{},[13453,13457,13461,13465,13469,13473],{"type":16,"tag":131,"props":13454,"children":13455},{},[13456],{"type":22,"value":4896},{"type":16,"tag":145,"props":13458,"children":13459},{"stretchy":495},[13460],{"type":22,"value":498},{"type":16,"tag":131,"props":13462,"children":13463},{},[13464],{"type":22,"value":536},{"type":16,"tag":131,"props":13466,"children":13467},{"mathvariant":1409},[13468],{"type":22,"value":3474},{"type":16,"tag":131,"props":13470,"children":13471},{},[13472],{"type":22,"value":11062},{"type":16,"tag":145,"props":13474,"children":13475},{"stretchy":495},[13476],{"type":22,"value":541},{"type":16,"tag":157,"props":13478,"children":13479},{"encoding":159},[13480],{"type":22,"value":12438},{"type":16,"tag":103,"props":13482,"children":13484},{"className":13483,"ariaHidden":167},[166],[13485],{"type":16,"tag":103,"props":13486,"children":13488},{"className":13487},[172],[13489,13493,13498,13503,13508,13513,13518],{"type":16,"tag":103,"props":13490,"children":13492},{"className":13491,"style":558},[177],[],{"type":16,"tag":103,"props":13494,"children":13496},{"className":13495,"style":5198},[183,188],[13497],{"type":22,"value":4896},{"type":16,"tag":103,"props":13499,"children":13501},{"className":13500},[563],[13502],{"type":22,"value":498},{"type":16,"tag":103,"props":13504,"children":13506},{"className":13505},[183,188],[13507],{"type":22,"value":536},{"type":16,"tag":103,"props":13509,"children":13511},{"className":13510},[183],[13512],{"type":22,"value":3474},{"type":16,"tag":103,"props":13514,"children":13516},{"className":13515,"style":5198},[183,188],[13517],{"type":22,"value":11062},{"type":16,"tag":103,"props":13519,"children":13521},{"className":13520},[715],[13522],{"type":22,"value":541},{"type":22,"value":13524},"。我们计划在未来工作中进一步研究这种方法。",{"type":16,"tag":25,"props":13526,"children":13527},{},[13528,13530,13609,13611,13699,13701,13862],{"type":22,"value":13529},"当 kernel 宽度 ",{"type":16,"tag":103,"props":13531,"children":13533},{"className":13532},[106],[13534,13565],{"type":16,"tag":103,"props":13535,"children":13537},{"className":13536},[111],[13538],{"type":16,"tag":114,"props":13539,"children":13540},{"xmlns":116},[13541],{"type":16,"tag":119,"props":13542,"children":13543},{},[13544,13560],{"type":16,"tag":123,"props":13545,"children":13546},{},[13547,13551,13556],{"type":16,"tag":131,"props":13548,"children":13549},{},[13550],{"type":22,"value":2032},{"type":16,"tag":145,"props":13552,"children":13553},{},[13554],{"type":22,"value":13555},"\u003C",{"type":16,"tag":131,"props":13557,"children":13558},{},[13559],{"type":22,"value":536},{"type":16,"tag":157,"props":13561,"children":13562},{"encoding":159},[13563],{"type":22,"value":13564},"k \u003C n",{"type":16,"tag":103,"props":13566,"children":13568},{"className":13567,"ariaHidden":167},[166],[13569,13596],{"type":16,"tag":103,"props":13570,"children":13572},{"className":13571},[172],[13573,13578,13583,13587,13592],{"type":16,"tag":103,"props":13574,"children":13577},{"className":13575,"style":13576},[177],"height:0.7335em;vertical-align:-0.0391em;",[],{"type":16,"tag":103,"props":13579,"children":13581},{"className":13580,"style":2089},[183,188],[13582],{"type":22,"value":2032},{"type":16,"tag":103,"props":13584,"children":13586},{"className":13585,"style":816},[632],[],{"type":16,"tag":103,"props":13588,"children":13590},{"className":13589},[821],[13591],{"type":22,"value":13555},{"type":16,"tag":103,"props":13593,"children":13595},{"className":13594,"style":816},[632],[],{"type":16,"tag":103,"props":13597,"children":13599},{"className":13598},[172],[13600,13604],{"type":16,"tag":103,"props":13601,"children":13603},{"className":13602,"style":805},[177],[],{"type":16,"tag":103,"props":13605,"children":13607},{"className":13606},[183,188],[13608],{"type":22,"value":536},{"type":22,"value":13610}," 时，单个卷积层并不能连接所有输入位置和输出位置。对于连续 kernel，需要堆叠 ",{"type":16,"tag":103,"props":13612,"children":13614},{"className":13613},[106],[13615,13657],{"type":16,"tag":103,"props":13616,"children":13618},{"className":13617},[111],[13619],{"type":16,"tag":114,"props":13620,"children":13621},{"xmlns":116},[13622],{"type":16,"tag":119,"props":13623,"children":13624},{},[13625,13652],{"type":16,"tag":123,"props":13626,"children":13627},{},[13628,13632,13636,13640,13644,13648],{"type":16,"tag":131,"props":13629,"children":13630},{},[13631],{"type":22,"value":4896},{"type":16,"tag":145,"props":13633,"children":13634},{"stretchy":495},[13635],{"type":22,"value":498},{"type":16,"tag":131,"props":13637,"children":13638},{},[13639],{"type":22,"value":536},{"type":16,"tag":131,"props":13641,"children":13642},{"mathvariant":1409},[13643],{"type":22,"value":3474},{"type":16,"tag":131,"props":13645,"children":13646},{},[13647],{"type":22,"value":2032},{"type":16,"tag":145,"props":13649,"children":13650},{"stretchy":495},[13651],{"type":22,"value":541},{"type":16,"tag":157,"props":13653,"children":13654},{"encoding":159},[13655],{"type":22,"value":13656},"O(n\u002Fk)",{"type":16,"tag":103,"props":13658,"children":13660},{"className":13659,"ariaHidden":167},[166],[13661],{"type":16,"tag":103,"props":13662,"children":13664},{"className":13663},[172],[13665,13669,13674,13679,13684,13689,13694],{"type":16,"tag":103,"props":13666,"children":13668},{"className":13667,"style":558},[177],[],{"type":16,"tag":103,"props":13670,"children":13672},{"className":13671,"style":5198},[183,188],[13673],{"type":22,"value":4896},{"type":16,"tag":103,"props":13675,"children":13677},{"className":13676},[563],[13678],{"type":22,"value":498},{"type":16,"tag":103,"props":13680,"children":13682},{"className":13681},[183,188],[13683],{"type":22,"value":536},{"type":16,"tag":103,"props":13685,"children":13687},{"className":13686},[183],[13688],{"type":22,"value":3474},{"type":16,"tag":103,"props":13690,"children":13692},{"className":13691,"style":2089},[183,188],[13693],{"type":22,"value":2032},{"type":16,"tag":103,"props":13695,"children":13697},{"className":13696},[715],[13698],{"type":22,"value":541},{"type":22,"value":13700}," 个卷积层；对于扩张卷积 [15]，则需要 ",{"type":16,"tag":103,"props":13702,"children":13704},{"className":13703},[106],[13705,13764],{"type":16,"tag":103,"props":13706,"children":13708},{"className":13707},[111],[13709],{"type":16,"tag":114,"props":13710,"children":13711},{"xmlns":116},[13712],{"type":16,"tag":119,"props":13713,"children":13714},{},[13715,13760],{"type":16,"tag":123,"props":13716,"children":13717},{},[13718,13722,13726,13744,13748,13752,13756],{"type":16,"tag":131,"props":13719,"children":13720},{},[13721],{"type":22,"value":4896},{"type":16,"tag":145,"props":13723,"children":13724},{"stretchy":495},[13725],{"type":22,"value":498},{"type":16,"tag":127,"props":13727,"children":13728},{},[13729,13740],{"type":16,"tag":123,"props":13730,"children":13731},{},[13732,13736],{"type":16,"tag":131,"props":13733,"children":13734},{},[13735],{"type":22,"value":12035},{"type":16,"tag":145,"props":13737,"children":13738},{},[13739],{"type":22,"value":1417},{"type":16,"tag":131,"props":13741,"children":13742},{},[13743],{"type":22,"value":2032},{"type":16,"tag":145,"props":13745,"children":13746},{"stretchy":495},[13747],{"type":22,"value":498},{"type":16,"tag":131,"props":13749,"children":13750},{},[13751],{"type":22,"value":536},{"type":16,"tag":145,"props":13753,"children":13754},{"stretchy":495},[13755],{"type":22,"value":541},{"type":16,"tag":145,"props":13757,"children":13758},{"stretchy":495},[13759],{"type":22,"value":541},{"type":16,"tag":157,"props":13761,"children":13762},{"encoding":159},[13763],{"type":22,"value":12064},{"type":16,"tag":103,"props":13765,"children":13767},{"className":13766,"ariaHidden":167},[166],[13768],{"type":16,"tag":103,"props":13769,"children":13771},{"className":13770},[172],[13772,13776,13781,13786,13847,13852,13857],{"type":16,"tag":103,"props":13773,"children":13775},{"className":13774,"style":558},[177],[],{"type":16,"tag":103,"props":13777,"children":13779},{"className":13778,"style":5198},[183,188],[13780],{"type":22,"value":4896},{"type":16,"tag":103,"props":13782,"children":13784},{"className":13783},[563],[13785],{"type":22,"value":498},{"type":16,"tag":103,"props":13787,"children":13789},{"className":13788},[1476],[13790,13799],{"type":16,"tag":103,"props":13791,"children":13793},{"className":13792},[1476],[13794,13795],{"type":22,"value":12096},{"type":16,"tag":103,"props":13796,"children":13797},{"style":12099},[13798],{"type":22,"value":12102},{"type":16,"tag":103,"props":13800,"children":13802},{"className":13801},[194],[13803],{"type":16,"tag":103,"props":13804,"children":13806},{"className":13805},[199,200],[13807,13836],{"type":16,"tag":103,"props":13808,"children":13810},{"className":13809},[205],[13811,13831],{"type":16,"tag":103,"props":13812,"children":13814},{"className":13813,"style":12118},[210],[13815],{"type":16,"tag":103,"props":13816,"children":13817},{"style":12122},[13818,13822],{"type":16,"tag":103,"props":13819,"children":13821},{"className":13820,"style":221},[220],[],{"type":16,"tag":103,"props":13823,"children":13825},{"className":13824},[226,227,228,229],[13826],{"type":16,"tag":103,"props":13827,"children":13829},{"className":13828,"style":2089},[183,188,229],[13830],{"type":22,"value":2032},{"type":16,"tag":103,"props":13832,"children":13834},{"className":13833},[254],[13835],{"type":22,"value":257},{"type":16,"tag":103,"props":13837,"children":13839},{"className":13838},[205],[13840],{"type":16,"tag":103,"props":13841,"children":13843},{"className":13842,"style":12149},[210],[13844],{"type":16,"tag":103,"props":13845,"children":13846},{},[],{"type":16,"tag":103,"props":13848,"children":13850},{"className":13849},[563],[13851],{"type":22,"value":498},{"type":16,"tag":103,"props":13853,"children":13855},{"className":13854},[183,188],[13856],{"type":22,"value":536},{"type":16,"tag":103,"props":13858,"children":13860},{"className":13859},[715],[13861],{"type":22,"value":1540},{"type":22,"value":13863}," 个卷积层。这会增加网络中任意两个位置之间最长路径的长度。",{"type":16,"tag":25,"props":13865,"children":13866},{},[13867,13869,13911,13913,14163,14165,14242],{"type":22,"value":13868},"卷积层通常比循环层更昂贵，复杂度多出一个 ",{"type":16,"tag":103,"props":13870,"children":13872},{"className":13871},[106],[13873,13894],{"type":16,"tag":103,"props":13874,"children":13876},{"className":13875},[111],[13877],{"type":16,"tag":114,"props":13878,"children":13879},{"xmlns":116},[13880],{"type":16,"tag":119,"props":13881,"children":13882},{},[13883,13890],{"type":16,"tag":123,"props":13884,"children":13885},{},[13886],{"type":16,"tag":131,"props":13887,"children":13888},{},[13889],{"type":22,"value":2032},{"type":16,"tag":157,"props":13891,"children":13892},{"encoding":159},[13893],{"type":22,"value":2032},{"type":16,"tag":103,"props":13895,"children":13897},{"className":13896,"ariaHidden":167},[166],[13898],{"type":16,"tag":103,"props":13899,"children":13901},{"className":13900},[172],[13902,13906],{"type":16,"tag":103,"props":13903,"children":13905},{"className":13904,"style":3719},[177],[],{"type":16,"tag":103,"props":13907,"children":13909},{"className":13908,"style":2089},[183,188],[13910],{"type":22,"value":2032},{"type":22,"value":13912}," 的因子。不过，可分离卷积 [6] 可以显著降低复杂度，使其达到 ",{"type":16,"tag":103,"props":13914,"children":13916},{"className":13915},[106],[13917,13990],{"type":16,"tag":103,"props":13918,"children":13920},{"className":13919},[111],[13921],{"type":16,"tag":114,"props":13922,"children":13923},{"xmlns":116},[13924],{"type":16,"tag":119,"props":13925,"children":13926},{},[13927,13985],{"type":16,"tag":123,"props":13928,"children":13929},{},[13930,13934,13938,13942,13946,13950,13954,13958,13962,13966,13970,13981],{"type":16,"tag":131,"props":13931,"children":13932},{},[13933],{"type":22,"value":4896},{"type":16,"tag":145,"props":13935,"children":13936},{"stretchy":495},[13937],{"type":22,"value":498},{"type":16,"tag":131,"props":13939,"children":13940},{},[13941],{"type":22,"value":2032},{"type":16,"tag":145,"props":13943,"children":13944},{},[13945],{"type":22,"value":3753},{"type":16,"tag":131,"props":13947,"children":13948},{},[13949],{"type":22,"value":536},{"type":16,"tag":145,"props":13951,"children":13952},{},[13953],{"type":22,"value":3753},{"type":16,"tag":131,"props":13955,"children":13956},{},[13957],{"type":22,"value":1650},{"type":16,"tag":145,"props":13959,"children":13960},{},[13961],{"type":22,"value":1430},{"type":16,"tag":131,"props":13963,"children":13964},{},[13965],{"type":22,"value":536},{"type":16,"tag":145,"props":13967,"children":13968},{},[13969],{"type":22,"value":3753},{"type":16,"tag":2662,"props":13971,"children":13972},{},[13973,13977],{"type":16,"tag":131,"props":13974,"children":13975},{},[13976],{"type":22,"value":1650},{"type":16,"tag":151,"props":13978,"children":13979},{},[13980],{"type":22,"value":8050},{"type":16,"tag":145,"props":13982,"children":13983},{"stretchy":495},[13984],{"type":22,"value":541},{"type":16,"tag":157,"props":13986,"children":13987},{"encoding":159},[13988],{"type":22,"value":13989},"O(k \\cdot n \\cdot d + n \\cdot d^2)",{"type":16,"tag":103,"props":13991,"children":13993},{"className":13992,"ariaHidden":167},[166],[13994,14030,14056,14083,14109],{"type":16,"tag":103,"props":13995,"children":13997},{"className":13996},[172],[13998,14002,14007,14012,14017,14021,14026],{"type":16,"tag":103,"props":13999,"children":14001},{"className":14000,"style":558},[177],[],{"type":16,"tag":103,"props":14003,"children":14005},{"className":14004,"style":5198},[183,188],[14006],{"type":22,"value":4896},{"type":16,"tag":103,"props":14008,"children":14010},{"className":14009},[563],[14011],{"type":22,"value":498},{"type":16,"tag":103,"props":14013,"children":14015},{"className":14014,"style":2089},[183,188],[14016],{"type":22,"value":2032},{"type":16,"tag":103,"props":14018,"children":14020},{"className":14019,"style":1497},[632],[],{"type":16,"tag":103,"props":14022,"children":14024},{"className":14023},[243],[14025],{"type":22,"value":3753},{"type":16,"tag":103,"props":14027,"children":14029},{"className":14028,"style":1497},[632],[],{"type":16,"tag":103,"props":14031,"children":14033},{"className":14032},[172],[14034,14038,14043,14047,14052],{"type":16,"tag":103,"props":14035,"children":14037},{"className":14036,"style":11851},[177],[],{"type":16,"tag":103,"props":14039,"children":14041},{"className":14040},[183,188],[14042],{"type":22,"value":536},{"type":16,"tag":103,"props":14044,"children":14046},{"className":14045,"style":1497},[632],[],{"type":16,"tag":103,"props":14048,"children":14050},{"className":14049},[243],[14051],{"type":22,"value":3753},{"type":16,"tag":103,"props":14053,"children":14055},{"className":14054,"style":1497},[632],[],{"type":16,"tag":103,"props":14057,"children":14059},{"className":14058},[172],[14060,14065,14070,14074,14079],{"type":16,"tag":103,"props":14061,"children":14064},{"className":14062,"style":14063},[177],"height:0.7778em;vertical-align:-0.0833em;",[],{"type":16,"tag":103,"props":14066,"children":14068},{"className":14067},[183,188],[14069],{"type":22,"value":1650},{"type":16,"tag":103,"props":14071,"children":14073},{"className":14072,"style":1497},[632],[],{"type":16,"tag":103,"props":14075,"children":14077},{"className":14076},[243],[14078],{"type":22,"value":1430},{"type":16,"tag":103,"props":14080,"children":14082},{"className":14081,"style":1497},[632],[],{"type":16,"tag":103,"props":14084,"children":14086},{"className":14085},[172],[14087,14091,14096,14100,14105],{"type":16,"tag":103,"props":14088,"children":14090},{"className":14089,"style":11851},[177],[],{"type":16,"tag":103,"props":14092,"children":14094},{"className":14093},[183,188],[14095],{"type":22,"value":536},{"type":16,"tag":103,"props":14097,"children":14099},{"className":14098,"style":1497},[632],[],{"type":16,"tag":103,"props":14101,"children":14103},{"className":14102},[243],[14104],{"type":22,"value":3753},{"type":16,"tag":103,"props":14106,"children":14108},{"className":14107,"style":1497},[632],[],{"type":16,"tag":103,"props":14110,"children":14112},{"className":14111},[172],[14113,14117,14158],{"type":16,"tag":103,"props":14114,"children":14116},{"className":14115,"style":11199},[177],[],{"type":16,"tag":103,"props":14118,"children":14120},{"className":14119},[183],[14121,14126],{"type":16,"tag":103,"props":14122,"children":14124},{"className":14123},[183,188],[14125],{"type":22,"value":1650},{"type":16,"tag":103,"props":14127,"children":14129},{"className":14128},[194],[14130],{"type":16,"tag":103,"props":14131,"children":14133},{"className":14132},[199],[14134],{"type":16,"tag":103,"props":14135,"children":14137},{"className":14136},[205],[14138],{"type":16,"tag":103,"props":14139,"children":14141},{"className":14140,"style":11235},[210],[14142],{"type":16,"tag":103,"props":14143,"children":14144},{"style":3043},[14145,14149],{"type":16,"tag":103,"props":14146,"children":14148},{"className":14147,"style":221},[220],[],{"type":16,"tag":103,"props":14150,"children":14152},{"className":14151},[226,227,228,229],[14153],{"type":16,"tag":103,"props":14154,"children":14156},{"className":14155},[183,229],[14157],{"type":22,"value":8050},{"type":16,"tag":103,"props":14159,"children":14161},{"className":14160},[715],[14162],{"type":22,"value":541},{"type":22,"value":14164},"。即使 ",{"type":16,"tag":103,"props":14166,"children":14168},{"className":14167},[106],[14169,14199],{"type":16,"tag":103,"props":14170,"children":14172},{"className":14171},[111],[14173],{"type":16,"tag":114,"props":14174,"children":14175},{"xmlns":116},[14176],{"type":16,"tag":119,"props":14177,"children":14178},{},[14179,14194],{"type":16,"tag":123,"props":14180,"children":14181},{},[14182,14186,14190],{"type":16,"tag":131,"props":14183,"children":14184},{},[14185],{"type":22,"value":2032},{"type":16,"tag":145,"props":14187,"children":14188},{},[14189],{"type":22,"value":746},{"type":16,"tag":131,"props":14191,"children":14192},{},[14193],{"type":22,"value":536},{"type":16,"tag":157,"props":14195,"children":14196},{"encoding":159},[14197],{"type":22,"value":14198},"k = n",{"type":16,"tag":103,"props":14200,"children":14202},{"className":14201,"ariaHidden":167},[166],[14203,14229],{"type":16,"tag":103,"props":14204,"children":14206},{"className":14205},[172],[14207,14211,14216,14220,14225],{"type":16,"tag":103,"props":14208,"children":14210},{"className":14209,"style":3719},[177],[],{"type":16,"tag":103,"props":14212,"children":14214},{"className":14213,"style":2089},[183,188],[14215],{"type":22,"value":2032},{"type":16,"tag":103,"props":14217,"children":14219},{"className":14218,"style":816},[632],[],{"type":16,"tag":103,"props":14221,"children":14223},{"className":14222},[821],[14224],{"type":22,"value":746},{"type":16,"tag":103,"props":14226,"children":14228},{"className":14227,"style":816},[632],[],{"type":16,"tag":103,"props":14230,"children":14232},{"className":14231},[172],[14233,14237],{"type":16,"tag":103,"props":14234,"children":14236},{"className":14235,"style":805},[177],[],{"type":16,"tag":103,"props":14238,"children":14240},{"className":14239},[183,188],[14241],{"type":22,"value":536},{"type":22,"value":14243},"，可分离卷积的复杂度也等于一个自注意力层加一个逐位置前馈层，这正是我们模型采用的方法。",{"type":16,"tag":25,"props":14245,"children":14246},{},[14247],{"type":22,"value":14248},"自注意力还有一个附带好处：可能产生更容易解释的模型。我们检查了模型中的注意力分布，并在附录中展示和讨论了一些例子。不同注意力头不仅明显学会了不同任务，许多头还表现出与句子的句法和语义结构相关的行为。",{"type":16,"tag":66,"props":14250,"children":14252},{"id":14251},"_5-训练",[14253],{"type":22,"value":14254},"5 训练",{"type":16,"tag":25,"props":14256,"children":14257},{},[14258],{"type":22,"value":14259},"本节介绍模型的训练方案。",{"type":16,"tag":1284,"props":14261,"children":14263},{"id":14262},"_51-训练数据和批处理",[14264],{"type":22,"value":14265},"5.1 训练数据和批处理",{"type":16,"tag":25,"props":14267,"children":14268},{},[14269],{"type":22,"value":14270},"我们在标准 WMT 2014 英德数据集上训练，该数据集包含约 450 万个句子对。句子使用字节对编码 [3]，源语言和目标语言共享约 37,000 个 token 的词表。",{"type":16,"tag":25,"props":14272,"children":14273},{},[14274],{"type":22,"value":14275},"对于英法任务，我们使用规模明显更大的 WMT 2014 英法数据集，包含 3,600 万个句子，并按照 [31] 把 token 划分为 32,000 个词片段词表。",{"type":16,"tag":25,"props":14277,"children":14278},{},[14279],{"type":22,"value":14280},"我们按照近似序列长度把句子对组成 batch。每个训练 batch 包含一组句子对，约有 25,000 个源 token 和 25,000 个目标 token。",{"type":16,"tag":1284,"props":14282,"children":14284},{"id":14283},"_52-硬件和训练计划",[14285],{"type":22,"value":14286},"5.2 硬件和训练计划",{"type":16,"tag":25,"props":14288,"children":14289},{},[14290],{"type":22,"value":14291},"我们在一台配备 8 块 NVIDIA P100 GPU 的机器上训练模型。对于全文中描述超参数的 base 模型，每个训练步骤约耗时 0.4 秒。base 模型总共训练 100,000 步，即 12 小时。",{"type":16,"tag":25,"props":14293,"children":14294},{},[14295],{"type":22,"value":14296},"对于 big 模型（表 3 最后一行），每步耗时 1.0 秒。big 模型训练 300,000 步，即 3.5 天。",{"type":16,"tag":1284,"props":14298,"children":14300},{"id":14299},"_53-优化器",[14301],{"type":22,"value":14302},"5.3 优化器",{"type":16,"tag":25,"props":14304,"children":14305},{},[14306,14308,14449,14450,14587,14588,14732],{"type":22,"value":14307},"我们使用 Adam 优化器 [17]，参数为 ",{"type":16,"tag":103,"props":14309,"children":14311},{"className":14310},[106],[14312,14351],{"type":16,"tag":103,"props":14313,"children":14315},{"className":14314},[111],[14316],{"type":16,"tag":114,"props":14317,"children":14318},{"xmlns":116},[14319],{"type":16,"tag":119,"props":14320,"children":14321},{},[14322,14346],{"type":16,"tag":123,"props":14323,"children":14324},{},[14325,14337,14341],{"type":16,"tag":127,"props":14326,"children":14327},{},[14328,14333],{"type":16,"tag":131,"props":14329,"children":14330},{},[14331],{"type":22,"value":14332},"β",{"type":16,"tag":151,"props":14334,"children":14335},{},[14336],{"type":22,"value":155},{"type":16,"tag":145,"props":14338,"children":14339},{},[14340],{"type":22,"value":746},{"type":16,"tag":151,"props":14342,"children":14343},{},[14344],{"type":22,"value":14345},"0.9",{"type":16,"tag":157,"props":14347,"children":14348},{"encoding":159},[14349],{"type":22,"value":14350},"\\beta_1 = 0.9",{"type":16,"tag":103,"props":14352,"children":14354},{"className":14353,"ariaHidden":167},[166],[14355,14436],{"type":16,"tag":103,"props":14356,"children":14358},{"className":14357},[172],[14359,14364,14423,14427,14432],{"type":16,"tag":103,"props":14360,"children":14363},{"className":14361,"style":14362},[177],"height:0.8889em;vertical-align:-0.1944em;",[],{"type":16,"tag":103,"props":14365,"children":14367},{"className":14366},[183],[14368,14374],{"type":16,"tag":103,"props":14369,"children":14372},{"className":14370,"style":14371},[183,188],"margin-right:0.0528em;",[14373],{"type":22,"value":14332},{"type":16,"tag":103,"props":14375,"children":14377},{"className":14376},[194],[14378],{"type":16,"tag":103,"props":14379,"children":14381},{"className":14380},[199,200],[14382,14412],{"type":16,"tag":103,"props":14383,"children":14385},{"className":14384},[205],[14386,14407],{"type":16,"tag":103,"props":14387,"children":14389},{"className":14388,"style":211},[210],[14390],{"type":16,"tag":103,"props":14391,"children":14393},{"style":14392},"top:-2.55em;margin-left:-0.0528em;margin-right:0.05em;",[14394,14398],{"type":16,"tag":103,"props":14395,"children":14397},{"className":14396,"style":221},[220],[],{"type":16,"tag":103,"props":14399,"children":14401},{"className":14400},[226,227,228,229],[14402],{"type":16,"tag":103,"props":14403,"children":14405},{"className":14404},[183,229],[14406],{"type":22,"value":155},{"type":16,"tag":103,"props":14408,"children":14410},{"className":14409},[254],[14411],{"type":22,"value":257},{"type":16,"tag":103,"props":14413,"children":14415},{"className":14414},[205],[14416],{"type":16,"tag":103,"props":14417,"children":14419},{"className":14418,"style":417},[210],[14420],{"type":16,"tag":103,"props":14421,"children":14422},{},[],{"type":16,"tag":103,"props":14424,"children":14426},{"className":14425,"style":816},[632],[],{"type":16,"tag":103,"props":14428,"children":14430},{"className":14429},[821],[14431],{"type":22,"value":746},{"type":16,"tag":103,"props":14433,"children":14435},{"className":14434,"style":816},[632],[],{"type":16,"tag":103,"props":14437,"children":14439},{"className":14438},[172],[14440,14444],{"type":16,"tag":103,"props":14441,"children":14443},{"className":14442,"style":1375},[177],[],{"type":16,"tag":103,"props":14445,"children":14447},{"className":14446},[183],[14448],{"type":22,"value":14345},{"type":22,"value":4442},{"type":16,"tag":103,"props":14451,"children":14453},{"className":14452},[106],[14454,14492],{"type":16,"tag":103,"props":14455,"children":14457},{"className":14456},[111],[14458],{"type":16,"tag":114,"props":14459,"children":14460},{"xmlns":116},[14461],{"type":16,"tag":119,"props":14462,"children":14463},{},[14464,14487],{"type":16,"tag":123,"props":14465,"children":14466},{},[14467,14478,14482],{"type":16,"tag":127,"props":14468,"children":14469},{},[14470,14474],{"type":16,"tag":131,"props":14471,"children":14472},{},[14473],{"type":22,"value":14332},{"type":16,"tag":151,"props":14475,"children":14476},{},[14477],{"type":22,"value":8050},{"type":16,"tag":145,"props":14479,"children":14480},{},[14481],{"type":22,"value":746},{"type":16,"tag":151,"props":14483,"children":14484},{},[14485],{"type":22,"value":14486},"0.98",{"type":16,"tag":157,"props":14488,"children":14489},{"encoding":159},[14490],{"type":22,"value":14491},"\\beta_2 = 0.98",{"type":16,"tag":103,"props":14493,"children":14495},{"className":14494,"ariaHidden":167},[166],[14496,14574],{"type":16,"tag":103,"props":14497,"children":14499},{"className":14498},[172],[14500,14504,14561,14565,14570],{"type":16,"tag":103,"props":14501,"children":14503},{"className":14502,"style":14362},[177],[],{"type":16,"tag":103,"props":14505,"children":14507},{"className":14506},[183],[14508,14513],{"type":16,"tag":103,"props":14509,"children":14511},{"className":14510,"style":14371},[183,188],[14512],{"type":22,"value":14332},{"type":16,"tag":103,"props":14514,"children":14516},{"className":14515},[194],[14517],{"type":16,"tag":103,"props":14518,"children":14520},{"className":14519},[199,200],[14521,14550],{"type":16,"tag":103,"props":14522,"children":14524},{"className":14523},[205],[14525,14545],{"type":16,"tag":103,"props":14526,"children":14528},{"className":14527,"style":211},[210],[14529],{"type":16,"tag":103,"props":14530,"children":14531},{"style":14392},[14532,14536],{"type":16,"tag":103,"props":14533,"children":14535},{"className":14534,"style":221},[220],[],{"type":16,"tag":103,"props":14537,"children":14539},{"className":14538},[226,227,228,229],[14540],{"type":16,"tag":103,"props":14541,"children":14543},{"className":14542},[183,229],[14544],{"type":22,"value":8050},{"type":16,"tag":103,"props":14546,"children":14548},{"className":14547},[254],[14549],{"type":22,"value":257},{"type":16,"tag":103,"props":14551,"children":14553},{"className":14552},[205],[14554],{"type":16,"tag":103,"props":14555,"children":14557},{"className":14556,"style":417},[210],[14558],{"type":16,"tag":103,"props":14559,"children":14560},{},[],{"type":16,"tag":103,"props":14562,"children":14564},{"className":14563,"style":816},[632],[],{"type":16,"tag":103,"props":14566,"children":14568},{"className":14567},[821],[14569],{"type":22,"value":746},{"type":16,"tag":103,"props":14571,"children":14573},{"className":14572,"style":816},[632],[],{"type":16,"tag":103,"props":14575,"children":14577},{"className":14576},[172],[14578,14582],{"type":16,"tag":103,"props":14579,"children":14581},{"className":14580,"style":1375},[177],[],{"type":16,"tag":103,"props":14583,"children":14585},{"className":14584},[183],[14586],{"type":22,"value":14486},{"type":22,"value":4442},{"type":16,"tag":103,"props":14589,"children":14591},{"className":14590},[106],[14592,14639],{"type":16,"tag":103,"props":14593,"children":14595},{"className":14594},[111],[14596],{"type":16,"tag":114,"props":14597,"children":14598},{"xmlns":116},[14599],{"type":16,"tag":119,"props":14600,"children":14601},{},[14602,14634],{"type":16,"tag":123,"props":14603,"children":14604},{},[14605,14610,14614],{"type":16,"tag":131,"props":14606,"children":14607},{},[14608],{"type":22,"value":14609},"ϵ",{"type":16,"tag":145,"props":14611,"children":14612},{},[14613],{"type":22,"value":746},{"type":16,"tag":2662,"props":14615,"children":14616},{},[14617,14622],{"type":16,"tag":151,"props":14618,"children":14619},{},[14620],{"type":22,"value":14621},"10",{"type":16,"tag":123,"props":14623,"children":14624},{},[14625,14629],{"type":16,"tag":145,"props":14626,"children":14627},{},[14628],{"type":22,"value":149},{"type":16,"tag":151,"props":14630,"children":14631},{},[14632],{"type":22,"value":14633},"9",{"type":16,"tag":157,"props":14635,"children":14636},{"encoding":159},[14637],{"type":22,"value":14638},"\\epsilon = 10^{-9}",{"type":16,"tag":103,"props":14640,"children":14642},{"className":14641,"ariaHidden":167},[166],[14643,14669],{"type":16,"tag":103,"props":14644,"children":14646},{"className":14645},[172],[14647,14651,14656,14660,14665],{"type":16,"tag":103,"props":14648,"children":14650},{"className":14649,"style":805},[177],[],{"type":16,"tag":103,"props":14652,"children":14654},{"className":14653},[183,188],[14655],{"type":22,"value":14609},{"type":16,"tag":103,"props":14657,"children":14659},{"className":14658,"style":816},[632],[],{"type":16,"tag":103,"props":14661,"children":14663},{"className":14662},[821],[14664],{"type":22,"value":746},{"type":16,"tag":103,"props":14666,"children":14668},{"className":14667,"style":816},[632],[],{"type":16,"tag":103,"props":14670,"children":14672},{"className":14671},[172],[14673,14677,14682],{"type":16,"tag":103,"props":14674,"children":14676},{"className":14675,"style":11235},[177],[],{"type":16,"tag":103,"props":14678,"children":14680},{"className":14679},[183],[14681],{"type":22,"value":155},{"type":16,"tag":103,"props":14683,"children":14685},{"className":14684},[183],[14686,14691],{"type":16,"tag":103,"props":14687,"children":14689},{"className":14688},[183],[14690],{"type":22,"value":5746},{"type":16,"tag":103,"props":14692,"children":14694},{"className":14693},[194],[14695],{"type":16,"tag":103,"props":14696,"children":14698},{"className":14697},[199],[14699],{"type":16,"tag":103,"props":14700,"children":14702},{"className":14701},[205],[14703],{"type":16,"tag":103,"props":14704,"children":14706},{"className":14705,"style":11235},[210],[14707],{"type":16,"tag":103,"props":14708,"children":14709},{"style":3043},[14710,14714],{"type":16,"tag":103,"props":14711,"children":14713},{"className":14712,"style":221},[220],[],{"type":16,"tag":103,"props":14715,"children":14717},{"className":14716},[226,227,228,229],[14718],{"type":16,"tag":103,"props":14719,"children":14721},{"className":14720},[183,229],[14722,14727],{"type":16,"tag":103,"props":14723,"children":14725},{"className":14724},[183,229],[14726],{"type":22,"value":149},{"type":16,"tag":103,"props":14728,"children":14730},{"className":14729},[183,229],[14731],{"type":22,"value":14633},{"type":22,"value":14733},"。训练期间，学习率按照下式变化：",{"type":16,"tag":103,"props":14735,"children":14737},{"className":14736},[1387],[14738],{"type":16,"tag":103,"props":14739,"children":14741},{"className":14740},[106],[14742,15092],{"type":16,"tag":103,"props":14743,"children":14745},{"className":14744},[111],[14746],{"type":16,"tag":114,"props":14747,"children":14748},{"xmlns":116,"display":1399},[14749],{"type":16,"tag":119,"props":14750,"children":14751},{},[14752,15087],{"type":16,"tag":2574,"props":14753,"children":14754},{"width":2576},[14755],{"type":16,"tag":2579,"props":14756,"children":14757},{},[14758,14761,15076,15079],{"type":16,"tag":2583,"props":14759,"children":14760},{"width":2585},[],{"type":16,"tag":2583,"props":14762,"children":14763},{},[14764],{"type":16,"tag":2574,"props":14765,"children":14766},{"rowspacing":5733,"columnalign":5734,"columnspacing":5735},[14767,14858],{"type":16,"tag":2579,"props":14768,"children":14769},{},[14770,14800],{"type":16,"tag":2583,"props":14771,"children":14772},{},[14773],{"type":16,"tag":5744,"props":14774,"children":14775},{"scriptlevel":5746,"displaystyle":167},[14776],{"type":16,"tag":123,"props":14777,"children":14778},{},[14779,14783,14787,14792,14796],{"type":16,"tag":131,"props":14780,"children":14781},{"mathvariant":1409},[14782],{"type":22,"value":1676},{"type":16,"tag":131,"props":14784,"children":14785},{"mathvariant":1409},[14786],{"type":22,"value":11062},{"type":16,"tag":131,"props":14788,"children":14789},{"mathvariant":1409},[14790],{"type":22,"value":14791},"a",{"type":16,"tag":131,"props":14793,"children":14794},{"mathvariant":1409},[14795],{"type":22,"value":143},{"type":16,"tag":131,"props":14797,"children":14798},{"mathvariant":1409},[14799],{"type":22,"value":1671},{"type":16,"tag":2583,"props":14801,"children":14802},{},[14803],{"type":16,"tag":5744,"props":14804,"children":14805},{"scriptlevel":5746,"displaystyle":167},[14806],{"type":16,"tag":123,"props":14807,"children":14808},{},[14809,14812,14816],{"type":16,"tag":123,"props":14810,"children":14811},{},[],{"type":16,"tag":145,"props":14813,"children":14814},{},[14815],{"type":22,"value":746},{"type":16,"tag":5266,"props":14817,"children":14818},{},[14819,14823,14846],{"type":16,"tag":131,"props":14820,"children":14821},{},[14822],{"type":22,"value":1650},{"type":16,"tag":123,"props":14824,"children":14825},{},[14826,14830,14834,14838,14842],{"type":16,"tag":131,"props":14827,"children":14828},{"mathvariant":1409},[14829],{"type":22,"value":1090},{"type":16,"tag":131,"props":14831,"children":14832},{"mathvariant":1409},[14833],{"type":22,"value":1662},{"type":16,"tag":131,"props":14835,"children":14836},{"mathvariant":1409},[14837],{"type":22,"value":1650},{"type":16,"tag":131,"props":14839,"children":14840},{"mathvariant":1409},[14841],{"type":22,"value":1671},{"type":16,"tag":131,"props":14843,"children":14844},{"mathvariant":1409},[14845],{"type":22,"value":1676},{"type":16,"tag":123,"props":14847,"children":14848},{},[14849,14853],{"type":16,"tag":145,"props":14850,"children":14851},{},[14852],{"type":22,"value":149},{"type":16,"tag":151,"props":14854,"children":14855},{},[14856],{"type":22,"value":14857},"0.5",{"type":16,"tag":2579,"props":14859,"children":14860},{},[14861,14870],{"type":16,"tag":2583,"props":14862,"children":14863},{},[14864],{"type":16,"tag":5744,"props":14865,"children":14866},{"scriptlevel":5746,"displaystyle":167},[14867],{"type":16,"tag":123,"props":14868,"children":14869},{},[],{"type":16,"tag":2583,"props":14871,"children":14872},{},[14873],{"type":16,"tag":5744,"props":14874,"children":14875},{"scriptlevel":5746,"displaystyle":167},[14876],{"type":16,"tag":123,"props":14877,"children":14878},{},[14879,14882,14886,14890,14895,14899],{"type":16,"tag":123,"props":14880,"children":14881},{},[],{"type":16,"tag":632,"props":14883,"children":14885},{"width":14884},"1em",[],{"type":16,"tag":145,"props":14887,"children":14888},{},[14889],{"type":22,"value":3753},{"type":16,"tag":131,"props":14891,"children":14892},{},[14893],{"type":22,"value":14894},"min",{"type":16,"tag":145,"props":14896,"children":14897},{},[14898],{"type":22,"value":1417},{"type":16,"tag":123,"props":14900,"children":14901},{},[14902,14906,14957,14961,14966,15001,15005,15072],{"type":16,"tag":145,"props":14903,"children":14904},{"fence":167},[14905],{"type":22,"value":498},{"type":16,"tag":2662,"props":14907,"children":14908},{},[14909,14946],{"type":16,"tag":123,"props":14910,"children":14911},{},[14912,14916,14920,14924,14928,14933,14937,14942],{"type":16,"tag":131,"props":14913,"children":14914},{"mathvariant":1409},[14915],{"type":22,"value":9340},{"type":16,"tag":131,"props":14917,"children":14918},{"mathvariant":1409},[14919],{"type":22,"value":143},{"type":16,"tag":131,"props":14921,"children":14922},{"mathvariant":1409},[14923],{"type":22,"value":1671},{"type":16,"tag":131,"props":14925,"children":14926},{"mathvariant":1409},[14927],{"type":22,"value":25},{"type":16,"tag":131,"props":14929,"children":14930},{"mathvariant":1409},[14931],{"type":22,"value":14932},"_",{"type":16,"tag":131,"props":14934,"children":14935},{"mathvariant":1409},[14936],{"type":22,"value":536},{"type":16,"tag":131,"props":14938,"children":14939},{"mathvariant":1409},[14940],{"type":22,"value":14941},"u",{"type":16,"tag":131,"props":14943,"children":14944},{"mathvariant":1409},[14945],{"type":22,"value":1090},{"type":16,"tag":123,"props":14947,"children":14948},{},[14949,14953],{"type":16,"tag":145,"props":14950,"children":14951},{},[14952],{"type":22,"value":149},{"type":16,"tag":151,"props":14954,"children":14955},{},[14956],{"type":22,"value":14857},{"type":16,"tag":145,"props":14958,"children":14959},{"separator":167},[14960],{"type":22,"value":515},{"type":16,"tag":2703,"props":14962,"children":14963},{},[14964],{"type":22,"value":14965},"  ",{"type":16,"tag":123,"props":14967,"children":14968},{},[14969,14973,14977,14981,14985,14989,14993,14997],{"type":16,"tag":131,"props":14970,"children":14971},{"mathvariant":1409},[14972],{"type":22,"value":9340},{"type":16,"tag":131,"props":14974,"children":14975},{"mathvariant":1409},[14976],{"type":22,"value":143},{"type":16,"tag":131,"props":14978,"children":14979},{"mathvariant":1409},[14980],{"type":22,"value":1671},{"type":16,"tag":131,"props":14982,"children":14983},{"mathvariant":1409},[14984],{"type":22,"value":25},{"type":16,"tag":131,"props":14986,"children":14987},{"mathvariant":1409},[14988],{"type":22,"value":14932},{"type":16,"tag":131,"props":14990,"children":14991},{"mathvariant":1409},[14992],{"type":22,"value":536},{"type":16,"tag":131,"props":14994,"children":14995},{"mathvariant":1409},[14996],{"type":22,"value":14941},{"type":16,"tag":131,"props":14998,"children":14999},{"mathvariant":1409},[15000],{"type":22,"value":1090},{"type":16,"tag":145,"props":15002,"children":15003},{},[15004],{"type":22,"value":3753},{"type":16,"tag":2662,"props":15006,"children":15007},{},[15008,15060],{"type":16,"tag":123,"props":15009,"children":15010},{},[15011,15016,15020,15024,15028,15032,15036,15040,15044,15048,15052,15056],{"type":16,"tag":131,"props":15012,"children":15013},{"mathvariant":1409},[15014],{"type":22,"value":15015},"w",{"type":16,"tag":131,"props":15017,"children":15018},{"mathvariant":1409},[15019],{"type":22,"value":14791},{"type":16,"tag":131,"props":15021,"children":15022},{"mathvariant":1409},[15023],{"type":22,"value":11062},{"type":16,"tag":131,"props":15025,"children":15026},{"mathvariant":1409},[15027],{"type":22,"value":1090},{"type":16,"tag":131,"props":15029,"children":15030},{"mathvariant":1409},[15031],{"type":22,"value":14941},{"type":16,"tag":131,"props":15033,"children":15034},{"mathvariant":1409},[15035],{"type":22,"value":25},{"type":16,"tag":131,"props":15037,"children":15038},{"mathvariant":1409},[15039],{"type":22,"value":14932},{"type":16,"tag":131,"props":15041,"children":15042},{"mathvariant":1409},[15043],{"type":22,"value":9340},{"type":16,"tag":131,"props":15045,"children":15046},{"mathvariant":1409},[15047],{"type":22,"value":143},{"type":16,"tag":131,"props":15049,"children":15050},{"mathvariant":1409},[15051],{"type":22,"value":1671},{"type":16,"tag":131,"props":15053,"children":15054},{"mathvariant":1409},[15055],{"type":22,"value":25},{"type":16,"tag":131,"props":15057,"children":15058},{"mathvariant":1409},[15059],{"type":22,"value":9340},{"type":16,"tag":123,"props":15061,"children":15062},{},[15063,15067],{"type":16,"tag":145,"props":15064,"children":15065},{},[15066],{"type":22,"value":149},{"type":16,"tag":151,"props":15068,"children":15069},{},[15070],{"type":22,"value":15071},"1.5",{"type":16,"tag":145,"props":15073,"children":15074},{"fence":167},[15075],{"type":22,"value":541},{"type":16,"tag":2583,"props":15077,"children":15078},{"width":2585},[],{"type":16,"tag":2583,"props":15080,"children":15081},{},[15082],{"type":16,"tag":2703,"props":15083,"children":15084},{},[15085],{"type":22,"value":15086},"(3)",{"type":16,"tag":157,"props":15088,"children":15089},{"encoding":159},[15090],{"type":22,"value":15091},"\\begin{aligned}\n\\mathrm{lrate} &= d_{\\mathrm{model}}^{-0.5} \\\\\n&\\quad \\cdot \\min\\left(\\mathrm{step\\_num}^{-0.5},\\; \\mathrm{step\\_num} \\cdot \\mathrm{warmup\\_steps}^{-1.5}\\right)\n\\end{aligned}\\tag{3}",{"type":16,"tag":103,"props":15093,"children":15095},{"className":15094,"ariaHidden":167},[166],[15096,15543],{"type":16,"tag":103,"props":15097,"children":15099},{"className":15098},[172],[15100,15105],{"type":16,"tag":103,"props":15101,"children":15104},{"className":15102,"style":15103},[177],"height:2.7482em;vertical-align:-1.1241em;",[],{"type":16,"tag":103,"props":15106,"children":15108},{"className":15107},[183],[15109],{"type":16,"tag":103,"props":15110,"children":15112},{"className":15111},[2574],[15113,15181],{"type":16,"tag":103,"props":15114,"children":15116},{"className":15115},[6169],[15117],{"type":16,"tag":103,"props":15118,"children":15120},{"className":15119},[199,200],[15121,15169],{"type":16,"tag":103,"props":15122,"children":15124},{"className":15123},[205],[15125,15164],{"type":16,"tag":103,"props":15126,"children":15129},{"className":15127,"style":15128},[210],"height:1.6241em;",[15130,15152],{"type":16,"tag":103,"props":15131,"children":15133},{"style":15132},"top:-3.76em;",[15134,15138],{"type":16,"tag":103,"props":15135,"children":15137},{"className":15136,"style":2292},[220],[],{"type":16,"tag":103,"props":15139,"children":15141},{"className":15140},[183],[15142],{"type":16,"tag":103,"props":15143,"children":15145},{"className":15144},[183],[15146],{"type":16,"tag":103,"props":15147,"children":15149},{"className":15148},[183,1481],[15150],{"type":22,"value":15151},"lrate",{"type":16,"tag":103,"props":15153,"children":15155},{"style":15154},"top:-2.2359em;",[15156,15160],{"type":16,"tag":103,"props":15157,"children":15159},{"className":15158,"style":2292},[220],[],{"type":16,"tag":103,"props":15161,"children":15163},{"className":15162},[183],[],{"type":16,"tag":103,"props":15165,"children":15167},{"className":15166},[254],[15168],{"type":22,"value":257},{"type":16,"tag":103,"props":15170,"children":15172},{"className":15171},[205],[15173],{"type":16,"tag":103,"props":15174,"children":15177},{"className":15175,"style":15176},[210],"height:1.1241em;",[15178],{"type":16,"tag":103,"props":15179,"children":15180},{},[],{"type":16,"tag":103,"props":15182,"children":15184},{"className":15183},[6512],[15185],{"type":16,"tag":103,"props":15186,"children":15188},{"className":15187},[199,200],[15189,15532],{"type":16,"tag":103,"props":15190,"children":15192},{"className":15191},[205],[15193,15527],{"type":16,"tag":103,"props":15194,"children":15196},{"className":15195,"style":15128},[210],[15197,15318],{"type":16,"tag":103,"props":15198,"children":15199},{"style":15132},[15200,15204],{"type":16,"tag":103,"props":15201,"children":15203},{"className":15202,"style":2292},[220],[],{"type":16,"tag":103,"props":15205,"children":15207},{"className":15206},[183],[15208,15212,15216,15221,15225],{"type":16,"tag":103,"props":15209,"children":15211},{"className":15210},[183],[],{"type":16,"tag":103,"props":15213,"children":15215},{"className":15214,"style":816},[632],[],{"type":16,"tag":103,"props":15217,"children":15219},{"className":15218},[821],[15220],{"type":22,"value":746},{"type":16,"tag":103,"props":15222,"children":15224},{"className":15223,"style":816},[632],[],{"type":16,"tag":103,"props":15226,"children":15228},{"className":15227},[183],[15229,15234],{"type":16,"tag":103,"props":15230,"children":15232},{"className":15231},[183,188],[15233],{"type":22,"value":1650},{"type":16,"tag":103,"props":15235,"children":15237},{"className":15236},[194],[15238],{"type":16,"tag":103,"props":15239,"children":15241},{"className":15240},[199,200],[15242,15306],{"type":16,"tag":103,"props":15243,"children":15245},{"className":15244},[205],[15246,15301],{"type":16,"tag":103,"props":15247,"children":15250},{"className":15248,"style":15249},[210],"height:0.8641em;",[15251,15276],{"type":16,"tag":103,"props":15252,"children":15254},{"style":15253},"top:-2.4086em;margin-left:0em;margin-right:0.05em;",[15255,15259],{"type":16,"tag":103,"props":15256,"children":15258},{"className":15257,"style":221},[220],[],{"type":16,"tag":103,"props":15260,"children":15262},{"className":15261},[226,227,228,229],[15263],{"type":16,"tag":103,"props":15264,"children":15266},{"className":15265},[183,229],[15267],{"type":16,"tag":103,"props":15268,"children":15270},{"className":15269},[183,229],[15271],{"type":16,"tag":103,"props":15272,"children":15274},{"className":15273},[183,1481,229],[15275],{"type":22,"value":1753},{"type":16,"tag":103,"props":15277,"children":15278},{"style":5185},[15279,15283],{"type":16,"tag":103,"props":15280,"children":15282},{"className":15281,"style":221},[220],[],{"type":16,"tag":103,"props":15284,"children":15286},{"className":15285},[226,227,228,229],[15287],{"type":16,"tag":103,"props":15288,"children":15290},{"className":15289},[183,229],[15291,15296],{"type":16,"tag":103,"props":15292,"children":15294},{"className":15293},[183,229],[15295],{"type":22,"value":149},{"type":16,"tag":103,"props":15297,"children":15299},{"className":15298},[183,229],[15300],{"type":22,"value":14857},{"type":16,"tag":103,"props":15302,"children":15304},{"className":15303},[254],[15305],{"type":22,"value":257},{"type":16,"tag":103,"props":15307,"children":15309},{"className":15308},[205],[15310],{"type":16,"tag":103,"props":15311,"children":15314},{"className":15312,"style":15313},[210],"height:0.2914em;",[15315],{"type":16,"tag":103,"props":15316,"children":15317},{},[],{"type":16,"tag":103,"props":15319,"children":15320},{"style":15154},[15321,15325],{"type":16,"tag":103,"props":15322,"children":15324},{"className":15323,"style":2292},[220],[],{"type":16,"tag":103,"props":15326,"children":15328},{"className":15327},[183],[15329,15333,15338,15342,15347,15351,15356,15360],{"type":16,"tag":103,"props":15330,"children":15332},{"className":15331},[183],[],{"type":16,"tag":103,"props":15334,"children":15337},{"className":15335,"style":15336},[632],"margin-right:1em;",[],{"type":16,"tag":103,"props":15339,"children":15341},{"className":15340,"style":1497},[632],[],{"type":16,"tag":103,"props":15343,"children":15345},{"className":15344},[243],[15346],{"type":22,"value":3753},{"type":16,"tag":103,"props":15348,"children":15350},{"className":15349,"style":1497},[632],[],{"type":16,"tag":103,"props":15352,"children":15354},{"className":15353},[1476],[15355],{"type":22,"value":14894},{"type":16,"tag":103,"props":15357,"children":15359},{"className":15358,"style":633},[632],[],{"type":16,"tag":103,"props":15361,"children":15363},{"className":15362},[638],[15364,15373,15428,15433,15437,15441,15450,15454,15459,15463,15518],{"type":16,"tag":103,"props":15365,"children":15367},{"className":15366,"style":2820},[563,2819],[15368],{"type":16,"tag":103,"props":15369,"children":15371},{"className":15370},[2825,6635],[15372],{"type":22,"value":498},{"type":16,"tag":103,"props":15374,"children":15376},{"className":15375},[183],[15377,15387],{"type":16,"tag":103,"props":15378,"children":15380},{"className":15379},[183],[15381],{"type":16,"tag":103,"props":15382,"children":15384},{"className":15383},[183,1481],[15385],{"type":22,"value":15386},"step_num",{"type":16,"tag":103,"props":15388,"children":15390},{"className":15389},[194],[15391],{"type":16,"tag":103,"props":15392,"children":15394},{"className":15393},[199],[15395],{"type":16,"tag":103,"props":15396,"children":15398},{"className":15397},[205],[15399],{"type":16,"tag":103,"props":15400,"children":15402},{"className":15401,"style":15249},[210],[15403],{"type":16,"tag":103,"props":15404,"children":15405},{"style":5185},[15406,15410],{"type":16,"tag":103,"props":15407,"children":15409},{"className":15408,"style":221},[220],[],{"type":16,"tag":103,"props":15411,"children":15413},{"className":15412},[226,227,228,229],[15414],{"type":16,"tag":103,"props":15415,"children":15417},{"className":15416},[183,229],[15418,15423],{"type":16,"tag":103,"props":15419,"children":15421},{"className":15420},[183,229],[15422],{"type":22,"value":149},{"type":16,"tag":103,"props":15424,"children":15426},{"className":15425},[183,229],[15427],{"type":22,"value":14857},{"type":16,"tag":103,"props":15429,"children":15431},{"className":15430},[626],[15432],{"type":22,"value":515},{"type":16,"tag":103,"props":15434,"children":15436},{"className":15435,"style":816},[632],[],{"type":16,"tag":103,"props":15438,"children":15440},{"className":15439,"style":633},[632],[],{"type":16,"tag":103,"props":15442,"children":15444},{"className":15443},[183],[15445],{"type":16,"tag":103,"props":15446,"children":15448},{"className":15447},[183,1481],[15449],{"type":22,"value":15386},{"type":16,"tag":103,"props":15451,"children":15453},{"className":15452,"style":1497},[632],[],{"type":16,"tag":103,"props":15455,"children":15457},{"className":15456},[243],[15458],{"type":22,"value":3753},{"type":16,"tag":103,"props":15460,"children":15462},{"className":15461,"style":1497},[632],[],{"type":16,"tag":103,"props":15464,"children":15466},{"className":15465},[183],[15467,15477],{"type":16,"tag":103,"props":15468,"children":15470},{"className":15469},[183],[15471],{"type":16,"tag":103,"props":15472,"children":15474},{"className":15473},[183,1481],[15475],{"type":22,"value":15476},"warmup_steps",{"type":16,"tag":103,"props":15478,"children":15480},{"className":15479},[194],[15481],{"type":16,"tag":103,"props":15482,"children":15484},{"className":15483},[199],[15485],{"type":16,"tag":103,"props":15486,"children":15488},{"className":15487},[205],[15489],{"type":16,"tag":103,"props":15490,"children":15492},{"className":15491,"style":15249},[210],[15493],{"type":16,"tag":103,"props":15494,"children":15495},{"style":5185},[15496,15500],{"type":16,"tag":103,"props":15497,"children":15499},{"className":15498,"style":221},[220],[],{"type":16,"tag":103,"props":15501,"children":15503},{"className":15502},[226,227,228,229],[15504],{"type":16,"tag":103,"props":15505,"children":15507},{"className":15506},[183,229],[15508,15513],{"type":16,"tag":103,"props":15509,"children":15511},{"className":15510},[183,229],[15512],{"type":22,"value":149},{"type":16,"tag":103,"props":15514,"children":15516},{"className":15515},[183,229],[15517],{"type":22,"value":15071},{"type":16,"tag":103,"props":15519,"children":15521},{"className":15520,"style":2820},[715,2819],[15522],{"type":16,"tag":103,"props":15523,"children":15525},{"className":15524},[2825,6635],[15526],{"type":22,"value":541},{"type":16,"tag":103,"props":15528,"children":15530},{"className":15529},[254],[15531],{"type":22,"value":257},{"type":16,"tag":103,"props":15533,"children":15535},{"className":15534},[205],[15536],{"type":16,"tag":103,"props":15537,"children":15539},{"className":15538,"style":15176},[210],[15540],{"type":16,"tag":103,"props":15541,"children":15542},{},[],{"type":16,"tag":103,"props":15544,"children":15546},{"className":15545},[3101],[15547,15551],{"type":16,"tag":103,"props":15548,"children":15550},{"className":15549,"style":15103},[177],[],{"type":16,"tag":103,"props":15552,"children":15554},{"className":15553},[183,22],[15555,15560,15570],{"type":16,"tag":103,"props":15556,"children":15558},{"className":15557},[183],[15559],{"type":22,"value":498},{"type":16,"tag":103,"props":15561,"children":15563},{"className":15562},[183],[15564],{"type":16,"tag":103,"props":15565,"children":15567},{"className":15566},[183],[15568],{"type":22,"value":15569},"3",{"type":16,"tag":103,"props":15571,"children":15573},{"className":15572},[183],[15574],{"type":22,"value":541},{"type":16,"tag":25,"props":15576,"children":15577},{},[15578,15580,15672,15674,15803],{"type":22,"value":15579},"这意味着在最初的 ",{"type":16,"tag":103,"props":15581,"children":15583},{"className":15582},[106],[15584,15650],{"type":16,"tag":103,"props":15585,"children":15587},{"className":15586},[111],[15588],{"type":16,"tag":114,"props":15589,"children":15590},{"xmlns":116},[15591],{"type":16,"tag":119,"props":15592,"children":15593},{},[15594,15645],{"type":16,"tag":123,"props":15595,"children":15596},{},[15597,15601,15605,15609,15613,15617,15621,15625,15629,15633,15637,15641],{"type":16,"tag":131,"props":15598,"children":15599},{"mathvariant":1409},[15600],{"type":22,"value":15015},{"type":16,"tag":131,"props":15602,"children":15603},{"mathvariant":1409},[15604],{"type":22,"value":14791},{"type":16,"tag":131,"props":15606,"children":15607},{"mathvariant":1409},[15608],{"type":22,"value":11062},{"type":16,"tag":131,"props":15610,"children":15611},{"mathvariant":1409},[15612],{"type":22,"value":1090},{"type":16,"tag":131,"props":15614,"children":15615},{"mathvariant":1409},[15616],{"type":22,"value":14941},{"type":16,"tag":131,"props":15618,"children":15619},{"mathvariant":1409},[15620],{"type":22,"value":25},{"type":16,"tag":131,"props":15622,"children":15623},{"mathvariant":1409},[15624],{"type":22,"value":14932},{"type":16,"tag":131,"props":15626,"children":15627},{"mathvariant":1409},[15628],{"type":22,"value":9340},{"type":16,"tag":131,"props":15630,"children":15631},{"mathvariant":1409},[15632],{"type":22,"value":143},{"type":16,"tag":131,"props":15634,"children":15635},{"mathvariant":1409},[15636],{"type":22,"value":1671},{"type":16,"tag":131,"props":15638,"children":15639},{"mathvariant":1409},[15640],{"type":22,"value":25},{"type":16,"tag":131,"props":15642,"children":15643},{"mathvariant":1409},[15644],{"type":22,"value":9340},{"type":16,"tag":157,"props":15646,"children":15647},{"encoding":159},[15648],{"type":22,"value":15649},"\\mathrm{warmup\\_steps}",{"type":16,"tag":103,"props":15651,"children":15653},{"className":15652,"ariaHidden":167},[166],[15654],{"type":16,"tag":103,"props":15655,"children":15657},{"className":15656},[172],[15658,15663],{"type":16,"tag":103,"props":15659,"children":15662},{"className":15660,"style":15661},[177],"height:0.9251em;vertical-align:-0.31em;",[],{"type":16,"tag":103,"props":15664,"children":15666},{"className":15665},[183],[15667],{"type":16,"tag":103,"props":15668,"children":15670},{"className":15669},[183,1481],[15671],{"type":22,"value":15476},{"type":22,"value":15673}," 个训练步骤中，学习率线性增加；之后随着步数增加，学习率按步数平方根的倒数下降。我们使用 ",{"type":16,"tag":103,"props":15675,"children":15677},{"className":15676},[106],[15678,15756],{"type":16,"tag":103,"props":15679,"children":15681},{"className":15680},[111],[15682],{"type":16,"tag":114,"props":15683,"children":15684},{"xmlns":116},[15685],{"type":16,"tag":119,"props":15686,"children":15687},{},[15688,15751],{"type":16,"tag":123,"props":15689,"children":15690},{},[15691,15742,15746],{"type":16,"tag":123,"props":15692,"children":15693},{},[15694,15698,15702,15706,15710,15714,15718,15722,15726,15730,15734,15738],{"type":16,"tag":131,"props":15695,"children":15696},{"mathvariant":1409},[15697],{"type":22,"value":15015},{"type":16,"tag":131,"props":15699,"children":15700},{"mathvariant":1409},[15701],{"type":22,"value":14791},{"type":16,"tag":131,"props":15703,"children":15704},{"mathvariant":1409},[15705],{"type":22,"value":11062},{"type":16,"tag":131,"props":15707,"children":15708},{"mathvariant":1409},[15709],{"type":22,"value":1090},{"type":16,"tag":131,"props":15711,"children":15712},{"mathvariant":1409},[15713],{"type":22,"value":14941},{"type":16,"tag":131,"props":15715,"children":15716},{"mathvariant":1409},[15717],{"type":22,"value":25},{"type":16,"tag":131,"props":15719,"children":15720},{"mathvariant":1409},[15721],{"type":22,"value":14932},{"type":16,"tag":131,"props":15723,"children":15724},{"mathvariant":1409},[15725],{"type":22,"value":9340},{"type":16,"tag":131,"props":15727,"children":15728},{"mathvariant":1409},[15729],{"type":22,"value":143},{"type":16,"tag":131,"props":15731,"children":15732},{"mathvariant":1409},[15733],{"type":22,"value":1671},{"type":16,"tag":131,"props":15735,"children":15736},{"mathvariant":1409},[15737],{"type":22,"value":25},{"type":16,"tag":131,"props":15739,"children":15740},{"mathvariant":1409},[15741],{"type":22,"value":9340},{"type":16,"tag":145,"props":15743,"children":15744},{},[15745],{"type":22,"value":746},{"type":16,"tag":151,"props":15747,"children":15748},{},[15749],{"type":22,"value":15750},"4000",{"type":16,"tag":157,"props":15752,"children":15753},{"encoding":159},[15754],{"type":22,"value":15755},"\\mathrm{warmup\\_steps} = 4000",{"type":16,"tag":103,"props":15757,"children":15759},{"className":15758,"ariaHidden":167},[166],[15760,15790],{"type":16,"tag":103,"props":15761,"children":15763},{"className":15762},[172],[15764,15768,15777,15781,15786],{"type":16,"tag":103,"props":15765,"children":15767},{"className":15766,"style":15661},[177],[],{"type":16,"tag":103,"props":15769,"children":15771},{"className":15770},[183],[15772],{"type":16,"tag":103,"props":15773,"children":15775},{"className":15774},[183,1481],[15776],{"type":22,"value":15476},{"type":16,"tag":103,"props":15778,"children":15780},{"className":15779,"style":816},[632],[],{"type":16,"tag":103,"props":15782,"children":15784},{"className":15783},[821],[15785],{"type":22,"value":746},{"type":16,"tag":103,"props":15787,"children":15789},{"className":15788,"style":816},[632],[],{"type":16,"tag":103,"props":15791,"children":15793},{"className":15792},[172],[15794,15798],{"type":16,"tag":103,"props":15795,"children":15797},{"className":15796,"style":1375},[177],[],{"type":16,"tag":103,"props":15799,"children":15801},{"className":15800},[183],[15802],{"type":22,"value":15750},{"type":22,"value":1266},{"type":16,"tag":1284,"props":15805,"children":15807},{"id":15806},"_54-正则化",[15808],{"type":22,"value":15809},"5.4 正则化",{"type":16,"tag":25,"props":15811,"children":15812},{},[15813],{"type":22,"value":15814},"训练中使用三种正则化方法。",{"type":16,"tag":25,"props":15816,"children":15817},{},[15818,15823,15825,15987],{"type":16,"tag":29,"props":15819,"children":15820},{},[15821],{"type":22,"value":15822},"残差 Dropout：",{"type":22,"value":15824}," 我们在每个子层的输出上应用 dropout [27]，位置是在把它加到子层输入并进行归一化之前。此外，我们还对编码器和解码器堆栈中嵌入与位置编码的和应用 dropout。对于 base 模型，我们使用 ",{"type":16,"tag":103,"props":15826,"children":15828},{"className":15827},[106],[15829,15883],{"type":16,"tag":103,"props":15830,"children":15832},{"className":15831},[111],[15833],{"type":16,"tag":114,"props":15834,"children":15835},{"xmlns":116},[15836],{"type":16,"tag":119,"props":15837,"children":15838},{},[15839,15878],{"type":16,"tag":123,"props":15840,"children":15841},{},[15842,15869,15873],{"type":16,"tag":127,"props":15843,"children":15844},{},[15845,15850],{"type":16,"tag":131,"props":15846,"children":15847},{},[15848],{"type":22,"value":15849},"P",{"type":16,"tag":123,"props":15851,"children":15852},{},[15853,15857,15861,15865],{"type":16,"tag":131,"props":15854,"children":15855},{"mathvariant":1409},[15856],{"type":22,"value":1650},{"type":16,"tag":131,"props":15858,"children":15859},{"mathvariant":1409},[15860],{"type":22,"value":11062},{"type":16,"tag":131,"props":15862,"children":15863},{"mathvariant":1409},[15864],{"type":22,"value":1662},{"type":16,"tag":131,"props":15866,"children":15867},{"mathvariant":1409},[15868],{"type":22,"value":25},{"type":16,"tag":145,"props":15870,"children":15871},{},[15872],{"type":22,"value":746},{"type":16,"tag":151,"props":15874,"children":15875},{},[15876],{"type":22,"value":15877},"0.1",{"type":16,"tag":157,"props":15879,"children":15880},{"encoding":159},[15881],{"type":22,"value":15882},"P_{\\mathrm{drop}} = 0.1",{"type":16,"tag":103,"props":15884,"children":15886},{"className":15885,"ariaHidden":167},[166],[15887,15974],{"type":16,"tag":103,"props":15888,"children":15890},{"className":15889},[172],[15891,15895,15961,15965,15970],{"type":16,"tag":103,"props":15892,"children":15894},{"className":15893,"style":10677},[177],[],{"type":16,"tag":103,"props":15896,"children":15898},{"className":15897},[183],[15899,15904],{"type":16,"tag":103,"props":15900,"children":15902},{"className":15901,"style":3056},[183,188],[15903],{"type":22,"value":15849},{"type":16,"tag":103,"props":15905,"children":15907},{"className":15906},[194],[15908],{"type":16,"tag":103,"props":15909,"children":15911},{"className":15910},[199,200],[15912,15950],{"type":16,"tag":103,"props":15913,"children":15915},{"className":15914},[205],[15916,15945],{"type":16,"tag":103,"props":15917,"children":15919},{"className":15918,"style":1727},[210],[15920],{"type":16,"tag":103,"props":15921,"children":15922},{"style":8195},[15923,15927],{"type":16,"tag":103,"props":15924,"children":15926},{"className":15925,"style":221},[220],[],{"type":16,"tag":103,"props":15928,"children":15930},{"className":15929},[226,227,228,229],[15931],{"type":16,"tag":103,"props":15932,"children":15934},{"className":15933},[183,229],[15935],{"type":16,"tag":103,"props":15936,"children":15938},{"className":15937},[183,229],[15939],{"type":16,"tag":103,"props":15940,"children":15942},{"className":15941},[183,1481,229],[15943],{"type":22,"value":15944},"drop",{"type":16,"tag":103,"props":15946,"children":15948},{"className":15947},[254],[15949],{"type":22,"value":257},{"type":16,"tag":103,"props":15951,"children":15953},{"className":15952},[205],[15954],{"type":16,"tag":103,"props":15955,"children":15957},{"className":15956,"style":10754},[210],[15958],{"type":16,"tag":103,"props":15959,"children":15960},{},[],{"type":16,"tag":103,"props":15962,"children":15964},{"className":15963,"style":816},[632],[],{"type":16,"tag":103,"props":15966,"children":15968},{"className":15967},[821],[15969],{"type":22,"value":746},{"type":16,"tag":103,"props":15971,"children":15973},{"className":15972,"style":816},[632],[],{"type":16,"tag":103,"props":15975,"children":15977},{"className":15976},[172],[15978,15982],{"type":16,"tag":103,"props":15979,"children":15981},{"className":15980,"style":1375},[177],[],{"type":16,"tag":103,"props":15983,"children":15985},{"className":15984},[183],[15986],{"type":22,"value":15877},{"type":22,"value":1266},{"type":16,"tag":25,"props":15989,"children":15990},{},[15991,15996,15998,16151],{"type":16,"tag":29,"props":15992,"children":15993},{},[15994],{"type":22,"value":15995},"标签平滑：",{"type":22,"value":15997}," 训练时使用取值 ",{"type":16,"tag":103,"props":15999,"children":16001},{"className":16000},[106],[16002,16046],{"type":16,"tag":103,"props":16003,"children":16005},{"className":16004},[111],[16006],{"type":16,"tag":114,"props":16007,"children":16008},{"xmlns":116},[16009],{"type":16,"tag":119,"props":16010,"children":16011},{},[16012,16041],{"type":16,"tag":123,"props":16013,"children":16014},{},[16015,16033,16037],{"type":16,"tag":127,"props":16016,"children":16017},{},[16018,16022],{"type":16,"tag":131,"props":16019,"children":16020},{},[16021],{"type":22,"value":14609},{"type":16,"tag":123,"props":16023,"children":16024},{},[16025,16029],{"type":16,"tag":131,"props":16026,"children":16027},{"mathvariant":1409},[16028],{"type":22,"value":1676},{"type":16,"tag":131,"props":16030,"children":16031},{"mathvariant":1409},[16032],{"type":22,"value":9340},{"type":16,"tag":145,"props":16034,"children":16035},{},[16036],{"type":22,"value":746},{"type":16,"tag":151,"props":16038,"children":16039},{},[16040],{"type":22,"value":15877},{"type":16,"tag":157,"props":16042,"children":16043},{"encoding":159},[16044],{"type":22,"value":16045},"\\epsilon_{\\mathrm{ls}} = 0.1",{"type":16,"tag":103,"props":16047,"children":16049},{"className":16048,"ariaHidden":167},[166],[16050,16138],{"type":16,"tag":103,"props":16051,"children":16053},{"className":16052},[172],[16054,16059,16125,16129,16134],{"type":16,"tag":103,"props":16055,"children":16058},{"className":16056,"style":16057},[177],"height:0.5806em;vertical-align:-0.15em;",[],{"type":16,"tag":103,"props":16060,"children":16062},{"className":16061},[183],[16063,16068],{"type":16,"tag":103,"props":16064,"children":16066},{"className":16065},[183,188],[16067],{"type":22,"value":14609},{"type":16,"tag":103,"props":16069,"children":16071},{"className":16070},[194],[16072],{"type":16,"tag":103,"props":16073,"children":16075},{"className":16074},[199,200],[16076,16114],{"type":16,"tag":103,"props":16077,"children":16079},{"className":16078},[205],[16080,16109],{"type":16,"tag":103,"props":16081,"children":16083},{"className":16082,"style":1727},[210],[16084],{"type":16,"tag":103,"props":16085,"children":16086},{"style":215},[16087,16091],{"type":16,"tag":103,"props":16088,"children":16090},{"className":16089,"style":221},[220],[],{"type":16,"tag":103,"props":16092,"children":16094},{"className":16093},[226,227,228,229],[16095],{"type":16,"tag":103,"props":16096,"children":16098},{"className":16097},[183,229],[16099],{"type":16,"tag":103,"props":16100,"children":16102},{"className":16101},[183,229],[16103],{"type":16,"tag":103,"props":16104,"children":16106},{"className":16105},[183,1481,229],[16107],{"type":22,"value":16108},"ls",{"type":16,"tag":103,"props":16110,"children":16112},{"className":16111},[254],[16113],{"type":22,"value":257},{"type":16,"tag":103,"props":16115,"children":16117},{"className":16116},[205],[16118],{"type":16,"tag":103,"props":16119,"children":16121},{"className":16120,"style":417},[210],[16122],{"type":16,"tag":103,"props":16123,"children":16124},{},[],{"type":16,"tag":103,"props":16126,"children":16128},{"className":16127,"style":816},[632],[],{"type":16,"tag":103,"props":16130,"children":16132},{"className":16131},[821],[16133],{"type":22,"value":746},{"type":16,"tag":103,"props":16135,"children":16137},{"className":16136,"style":816},[632],[],{"type":16,"tag":103,"props":16139,"children":16141},{"className":16140},[172],[16142,16146],{"type":16,"tag":103,"props":16143,"children":16145},{"className":16144,"style":1375},[177],[],{"type":16,"tag":103,"props":16147,"children":16149},{"className":16148},[183],[16150],{"type":22,"value":15877},{"type":22,"value":16152}," 的标签平滑 [30]。它会损害 perplexity，因为模型会学得更加不确定，但能够提高准确率和 BLEU 分数。",{"type":16,"tag":66,"props":16154,"children":16156},{"id":16155},"_6-结果",[16157],{"type":22,"value":16158},"6 结果",{"type":16,"tag":1284,"props":16160,"children":16162},{"id":16161},"_61-机器翻译",[16163],{"type":22,"value":16164},"6.1 机器翻译",{"type":16,"tag":25,"props":16166,"children":16167},{},[16168],{"type":22,"value":16169},"在 WMT 2014 英德翻译任务上，big Transformer 模型（表 2 中的 Transformer (big)）比此前报告的最佳模型（包括集成模型）高出 2.0 个以上 BLEU，取得 28.4 的新最佳 BLEU。该模型配置列在表 3 的最后一行。训练使用 8 块 P100 GPU，耗时 3.5 天。",{"type":16,"tag":25,"props":16171,"children":16172},{},[16173],{"type":22,"value":16174},"即使是 base 模型也超过了此前发表的所有模型和集成模型，而训练成本只是竞争模型的一小部分。",{"type":16,"tag":25,"props":16176,"children":16177},{},[16178,16180,16338],{"type":22,"value":16179},"在 WMT 2014 英法翻译任务上，我们的 big 模型取得 41.0 BLEU，超过此前发表的所有单模型结果，而训练成本不到此前最佳模型的四分之一。英法 Transformer (big) 模型使用的 dropout 率为 ",{"type":16,"tag":103,"props":16181,"children":16183},{"className":16182},[106],[16184,16235],{"type":16,"tag":103,"props":16185,"children":16187},{"className":16186},[111],[16188],{"type":16,"tag":114,"props":16189,"children":16190},{"xmlns":116},[16191],{"type":16,"tag":119,"props":16192,"children":16193},{},[16194,16231],{"type":16,"tag":123,"props":16195,"children":16196},{},[16197,16223,16227],{"type":16,"tag":127,"props":16198,"children":16199},{},[16200,16204],{"type":16,"tag":131,"props":16201,"children":16202},{},[16203],{"type":22,"value":15849},{"type":16,"tag":123,"props":16205,"children":16206},{},[16207,16211,16215,16219],{"type":16,"tag":131,"props":16208,"children":16209},{"mathvariant":1409},[16210],{"type":22,"value":1650},{"type":16,"tag":131,"props":16212,"children":16213},{"mathvariant":1409},[16214],{"type":22,"value":11062},{"type":16,"tag":131,"props":16216,"children":16217},{"mathvariant":1409},[16218],{"type":22,"value":1662},{"type":16,"tag":131,"props":16220,"children":16221},{"mathvariant":1409},[16222],{"type":22,"value":25},{"type":16,"tag":145,"props":16224,"children":16225},{},[16226],{"type":22,"value":746},{"type":16,"tag":151,"props":16228,"children":16229},{},[16230],{"type":22,"value":15877},{"type":16,"tag":157,"props":16232,"children":16233},{"encoding":159},[16234],{"type":22,"value":15882},{"type":16,"tag":103,"props":16236,"children":16238},{"className":16237,"ariaHidden":167},[166],[16239,16325],{"type":16,"tag":103,"props":16240,"children":16242},{"className":16241},[172],[16243,16247,16312,16316,16321],{"type":16,"tag":103,"props":16244,"children":16246},{"className":16245,"style":10677},[177],[],{"type":16,"tag":103,"props":16248,"children":16250},{"className":16249},[183],[16251,16256],{"type":16,"tag":103,"props":16252,"children":16254},{"className":16253,"style":3056},[183,188],[16255],{"type":22,"value":15849},{"type":16,"tag":103,"props":16257,"children":16259},{"className":16258},[194],[16260],{"type":16,"tag":103,"props":16261,"children":16263},{"className":16262},[199,200],[16264,16301],{"type":16,"tag":103,"props":16265,"children":16267},{"className":16266},[205],[16268,16296],{"type":16,"tag":103,"props":16269,"children":16271},{"className":16270,"style":1727},[210],[16272],{"type":16,"tag":103,"props":16273,"children":16274},{"style":8195},[16275,16279],{"type":16,"tag":103,"props":16276,"children":16278},{"className":16277,"style":221},[220],[],{"type":16,"tag":103,"props":16280,"children":16282},{"className":16281},[226,227,228,229],[16283],{"type":16,"tag":103,"props":16284,"children":16286},{"className":16285},[183,229],[16287],{"type":16,"tag":103,"props":16288,"children":16290},{"className":16289},[183,229],[16291],{"type":16,"tag":103,"props":16292,"children":16294},{"className":16293},[183,1481,229],[16295],{"type":22,"value":15944},{"type":16,"tag":103,"props":16297,"children":16299},{"className":16298},[254],[16300],{"type":22,"value":257},{"type":16,"tag":103,"props":16302,"children":16304},{"className":16303},[205],[16305],{"type":16,"tag":103,"props":16306,"children":16308},{"className":16307,"style":10754},[210],[16309],{"type":16,"tag":103,"props":16310,"children":16311},{},[],{"type":16,"tag":103,"props":16313,"children":16315},{"className":16314,"style":816},[632],[],{"type":16,"tag":103,"props":16317,"children":16319},{"className":16318},[821],[16320],{"type":22,"value":746},{"type":16,"tag":103,"props":16322,"children":16324},{"className":16323,"style":816},[632],[],{"type":16,"tag":103,"props":16326,"children":16328},{"className":16327},[172],[16329,16333],{"type":16,"tag":103,"props":16330,"children":16332},{"className":16331,"style":1375},[177],[],{"type":16,"tag":103,"props":16334,"children":16336},{"className":16335},[183],[16337],{"type":22,"value":15877},{"type":22,"value":16339},"，而不是 0.3。",{"type":16,"tag":25,"props":16341,"children":16342},{},[16343],{"type":22,"value":16344},"对于 base 模型，我们对最后 5 个 checkpoint 求平均；这些 checkpoint 每隔 10 分钟写入一次。对于 big 模型，我们对最后 20 个 checkpoint 求平均。",{"type":16,"tag":25,"props":16346,"children":16347},{},[16348,16350,16430],{"type":22,"value":16349},"我们使用 beam size 为 4、长度惩罚 ",{"type":16,"tag":103,"props":16351,"children":16353},{"className":16352},[106],[16354,16386],{"type":16,"tag":103,"props":16355,"children":16357},{"className":16356},[111],[16358],{"type":16,"tag":114,"props":16359,"children":16360},{"xmlns":116},[16361],{"type":16,"tag":119,"props":16362,"children":16363},{},[16364,16381],{"type":16,"tag":123,"props":16365,"children":16366},{},[16367,16372,16376],{"type":16,"tag":131,"props":16368,"children":16369},{},[16370],{"type":22,"value":16371},"α",{"type":16,"tag":145,"props":16373,"children":16374},{},[16375],{"type":22,"value":746},{"type":16,"tag":151,"props":16377,"children":16378},{},[16379],{"type":22,"value":16380},"0.6",{"type":16,"tag":157,"props":16382,"children":16383},{"encoding":159},[16384],{"type":22,"value":16385},"\\alpha = 0.6",{"type":16,"tag":103,"props":16387,"children":16389},{"className":16388,"ariaHidden":167},[166],[16390,16417],{"type":16,"tag":103,"props":16391,"children":16393},{"className":16392},[172],[16394,16398,16404,16408,16413],{"type":16,"tag":103,"props":16395,"children":16397},{"className":16396,"style":805},[177],[],{"type":16,"tag":103,"props":16399,"children":16402},{"className":16400,"style":16401},[183,188],"margin-right:0.0037em;",[16403],{"type":22,"value":16371},{"type":16,"tag":103,"props":16405,"children":16407},{"className":16406,"style":816},[632],[],{"type":16,"tag":103,"props":16409,"children":16411},{"className":16410},[821],[16412],{"type":22,"value":746},{"type":16,"tag":103,"props":16414,"children":16416},{"className":16415,"style":816},[632],[],{"type":16,"tag":103,"props":16418,"children":16420},{"className":16419},[172],[16421,16425],{"type":16,"tag":103,"props":16422,"children":16424},{"className":16423,"style":1375},[177],[],{"type":16,"tag":103,"props":16426,"children":16428},{"className":16427},[183],[16429],{"type":22,"value":16380},{"type":22,"value":16431}," 的 beam search [31]。这些超参数是在开发集上实验后选择的。推理时，我们把最大输出长度设为输入长度加 50，但在可能时提前终止 [31]。",{"type":16,"tag":25,"props":16433,"children":16434},{},[16435],{"type":22,"value":16436},"表 2 汇总了结果，并将我们的翻译质量和训练成本与文献中的其他模型架构进行比较。我们通过训练时间、使用的 GPU 数量，以及估计的每块 GPU 持续单精度浮点计算能力的乘积，估计训练模型所使用的浮点运算次数。",{"type":16,"tag":1284,"props":16438,"children":16440},{"id":16439},"表-2transformer-在英德和英法-newstest2014-测试集上以很低的训练成本取得更高-bleu",[16441],{"type":22,"value":16442},"表 2：Transformer 在英德和英法 newstest2014 测试集上以很低的训练成本取得更高 BLEU",{"type":16,"tag":11087,"props":16444,"children":16445},{},[16446,16478],{"type":16,"tag":11091,"props":16447,"children":16448},{},[16449],{"type":16,"tag":11095,"props":16450,"children":16451},{},[16452,16457,16463,16468,16473],{"type":16,"tag":11099,"props":16453,"children":16454},{},[16455],{"type":22,"value":16456},"模型",{"type":16,"tag":11099,"props":16458,"children":16460},{"align":16459},"right",[16461],{"type":22,"value":16462},"英德 BLEU",{"type":16,"tag":11099,"props":16464,"children":16465},{"align":16459},[16466],{"type":22,"value":16467},"英法 BLEU",{"type":16,"tag":11099,"props":16469,"children":16470},{"align":16459},[16471],{"type":22,"value":16472},"英德训练成本（FLOPs）",{"type":16,"tag":11099,"props":16474,"children":16475},{"align":16459},[16476],{"type":22,"value":16477},"英法训练成本（FLOPs）",{"type":16,"tag":11120,"props":16479,"children":16480},{},[16481,16507,16662,16947,17231,17515,17669,17954,18237,18392],{"type":16,"tag":11095,"props":16482,"children":16483},{},[16484,16489,16494,16499,16503],{"type":16,"tag":11127,"props":16485,"children":16486},{},[16487],{"type":22,"value":16488},"ByteNet [15]",{"type":16,"tag":11127,"props":16490,"children":16491},{"align":16459},[16492],{"type":22,"value":16493},"23.75",{"type":16,"tag":11127,"props":16495,"children":16496},{"align":16459},[16497],{"type":22,"value":16498},"—",{"type":16,"tag":11127,"props":16500,"children":16501},{"align":16459},[16502],{"type":22,"value":16498},{"type":16,"tag":11127,"props":16504,"children":16505},{"align":16459},[16506],{"type":22,"value":16498},{"type":16,"tag":11095,"props":16508,"children":16509},{},[16510,16515,16519,16524,16658],{"type":16,"tag":11127,"props":16511,"children":16512},{},[16513],{"type":22,"value":16514},"Deep-Att + PosUnk [32]",{"type":16,"tag":11127,"props":16516,"children":16517},{"align":16459},[16518],{"type":22,"value":16498},{"type":16,"tag":11127,"props":16520,"children":16521},{"align":16459},[16522],{"type":22,"value":16523},"39.2",{"type":16,"tag":11127,"props":16525,"children":16526},{"align":16459},[16527],{"type":16,"tag":103,"props":16528,"children":16530},{"className":16529},[106],[16531,16570],{"type":16,"tag":103,"props":16532,"children":16534},{"className":16533},[111],[16535],{"type":16,"tag":114,"props":16536,"children":16537},{"xmlns":116},[16538],{"type":16,"tag":119,"props":16539,"children":16540},{},[16541,16565],{"type":16,"tag":123,"props":16542,"children":16543},{},[16544,16549,16553],{"type":16,"tag":151,"props":16545,"children":16546},{},[16547],{"type":22,"value":16548},"1.0",{"type":16,"tag":145,"props":16550,"children":16551},{},[16552],{"type":22,"value":3753},{"type":16,"tag":2662,"props":16554,"children":16555},{},[16556,16560],{"type":16,"tag":151,"props":16557,"children":16558},{},[16559],{"type":22,"value":14621},{"type":16,"tag":151,"props":16561,"children":16562},{},[16563],{"type":22,"value":16564},"20",{"type":16,"tag":157,"props":16566,"children":16567},{"encoding":159},[16568],{"type":22,"value":16569},"1.0 \\cdot 10^{20}",{"type":16,"tag":103,"props":16571,"children":16573},{"className":16572,"ariaHidden":167},[166],[16574,16600],{"type":16,"tag":103,"props":16575,"children":16577},{"className":16576},[172],[16578,16582,16587,16591,16596],{"type":16,"tag":103,"props":16579,"children":16581},{"className":16580,"style":1375},[177],[],{"type":16,"tag":103,"props":16583,"children":16585},{"className":16584},[183],[16586],{"type":22,"value":16548},{"type":16,"tag":103,"props":16588,"children":16590},{"className":16589,"style":1497},[632],[],{"type":16,"tag":103,"props":16592,"children":16594},{"className":16593},[243],[16595],{"type":22,"value":3753},{"type":16,"tag":103,"props":16597,"children":16599},{"className":16598,"style":1497},[632],[],{"type":16,"tag":103,"props":16601,"children":16603},{"className":16602},[172],[16604,16608,16613],{"type":16,"tag":103,"props":16605,"children":16607},{"className":16606,"style":11235},[177],[],{"type":16,"tag":103,"props":16609,"children":16611},{"className":16610},[183],[16612],{"type":22,"value":155},{"type":16,"tag":103,"props":16614,"children":16616},{"className":16615},[183],[16617,16622],{"type":16,"tag":103,"props":16618,"children":16620},{"className":16619},[183],[16621],{"type":22,"value":5746},{"type":16,"tag":103,"props":16623,"children":16625},{"className":16624},[194],[16626],{"type":16,"tag":103,"props":16627,"children":16629},{"className":16628},[199],[16630],{"type":16,"tag":103,"props":16631,"children":16633},{"className":16632},[205],[16634],{"type":16,"tag":103,"props":16635,"children":16637},{"className":16636,"style":11235},[210],[16638],{"type":16,"tag":103,"props":16639,"children":16640},{"style":3043},[16641,16645],{"type":16,"tag":103,"props":16642,"children":16644},{"className":16643,"style":221},[220],[],{"type":16,"tag":103,"props":16646,"children":16648},{"className":16647},[226,227,228,229],[16649],{"type":16,"tag":103,"props":16650,"children":16652},{"className":16651},[183,229],[16653],{"type":16,"tag":103,"props":16654,"children":16656},{"className":16655},[183,229],[16657],{"type":22,"value":16564},{"type":16,"tag":11127,"props":16659,"children":16660},{"align":16459},[16661],{"type":22,"value":16498},{"type":16,"tag":11095,"props":16663,"children":16664},{},[16665,16670,16675,16680,16814],{"type":16,"tag":11127,"props":16666,"children":16667},{},[16668],{"type":22,"value":16669},"GNMT + RL [31]",{"type":16,"tag":11127,"props":16671,"children":16672},{"align":16459},[16673],{"type":22,"value":16674},"24.6",{"type":16,"tag":11127,"props":16676,"children":16677},{"align":16459},[16678],{"type":22,"value":16679},"39.92",{"type":16,"tag":11127,"props":16681,"children":16682},{"align":16459},[16683],{"type":16,"tag":103,"props":16684,"children":16686},{"className":16685},[106],[16687,16726],{"type":16,"tag":103,"props":16688,"children":16690},{"className":16689},[111],[16691],{"type":16,"tag":114,"props":16692,"children":16693},{"xmlns":116},[16694],{"type":16,"tag":119,"props":16695,"children":16696},{},[16697,16721],{"type":16,"tag":123,"props":16698,"children":16699},{},[16700,16705,16709],{"type":16,"tag":151,"props":16701,"children":16702},{},[16703],{"type":22,"value":16704},"2.3",{"type":16,"tag":145,"props":16706,"children":16707},{},[16708],{"type":22,"value":3753},{"type":16,"tag":2662,"props":16710,"children":16711},{},[16712,16716],{"type":16,"tag":151,"props":16713,"children":16714},{},[16715],{"type":22,"value":14621},{"type":16,"tag":151,"props":16717,"children":16718},{},[16719],{"type":22,"value":16720},"19",{"type":16,"tag":157,"props":16722,"children":16723},{"encoding":159},[16724],{"type":22,"value":16725},"2.3 \\cdot 10^{19}",{"type":16,"tag":103,"props":16727,"children":16729},{"className":16728,"ariaHidden":167},[166],[16730,16756],{"type":16,"tag":103,"props":16731,"children":16733},{"className":16732},[172],[16734,16738,16743,16747,16752],{"type":16,"tag":103,"props":16735,"children":16737},{"className":16736,"style":1375},[177],[],{"type":16,"tag":103,"props":16739,"children":16741},{"className":16740},[183],[16742],{"type":22,"value":16704},{"type":16,"tag":103,"props":16744,"children":16746},{"className":16745,"style":1497},[632],[],{"type":16,"tag":103,"props":16748,"children":16750},{"className":16749},[243],[16751],{"type":22,"value":3753},{"type":16,"tag":103,"props":16753,"children":16755},{"className":16754,"style":1497},[632],[],{"type":16,"tag":103,"props":16757,"children":16759},{"className":16758},[172],[16760,16764,16769],{"type":16,"tag":103,"props":16761,"children":16763},{"className":16762,"style":11235},[177],[],{"type":16,"tag":103,"props":16765,"children":16767},{"className":16766},[183],[16768],{"type":22,"value":155},{"type":16,"tag":103,"props":16770,"children":16772},{"className":16771},[183],[16773,16778],{"type":16,"tag":103,"props":16774,"children":16776},{"className":16775},[183],[16777],{"type":22,"value":5746},{"type":16,"tag":103,"props":16779,"children":16781},{"className":16780},[194],[16782],{"type":16,"tag":103,"props":16783,"children":16785},{"className":16784},[199],[16786],{"type":16,"tag":103,"props":16787,"children":16789},{"className":16788},[205],[16790],{"type":16,"tag":103,"props":16791,"children":16793},{"className":16792,"style":11235},[210],[16794],{"type":16,"tag":103,"props":16795,"children":16796},{"style":3043},[16797,16801],{"type":16,"tag":103,"props":16798,"children":16800},{"className":16799,"style":221},[220],[],{"type":16,"tag":103,"props":16802,"children":16804},{"className":16803},[226,227,228,229],[16805],{"type":16,"tag":103,"props":16806,"children":16808},{"className":16807},[183,229],[16809],{"type":16,"tag":103,"props":16810,"children":16812},{"className":16811},[183,229],[16813],{"type":22,"value":16720},{"type":16,"tag":11127,"props":16815,"children":16816},{"align":16459},[16817],{"type":16,"tag":103,"props":16818,"children":16820},{"className":16819},[106],[16821,16859],{"type":16,"tag":103,"props":16822,"children":16824},{"className":16823},[111],[16825],{"type":16,"tag":114,"props":16826,"children":16827},{"xmlns":116},[16828],{"type":16,"tag":119,"props":16829,"children":16830},{},[16831,16854],{"type":16,"tag":123,"props":16832,"children":16833},{},[16834,16839,16843],{"type":16,"tag":151,"props":16835,"children":16836},{},[16837],{"type":22,"value":16838},"1.4",{"type":16,"tag":145,"props":16840,"children":16841},{},[16842],{"type":22,"value":3753},{"type":16,"tag":2662,"props":16844,"children":16845},{},[16846,16850],{"type":16,"tag":151,"props":16847,"children":16848},{},[16849],{"type":22,"value":14621},{"type":16,"tag":151,"props":16851,"children":16852},{},[16853],{"type":22,"value":16564},{"type":16,"tag":157,"props":16855,"children":16856},{"encoding":159},[16857],{"type":22,"value":16858},"1.4 \\cdot 10^{20}",{"type":16,"tag":103,"props":16860,"children":16862},{"className":16861,"ariaHidden":167},[166],[16863,16889],{"type":16,"tag":103,"props":16864,"children":16866},{"className":16865},[172],[16867,16871,16876,16880,16885],{"type":16,"tag":103,"props":16868,"children":16870},{"className":16869,"style":1375},[177],[],{"type":16,"tag":103,"props":16872,"children":16874},{"className":16873},[183],[16875],{"type":22,"value":16838},{"type":16,"tag":103,"props":16877,"children":16879},{"className":16878,"style":1497},[632],[],{"type":16,"tag":103,"props":16881,"children":16883},{"className":16882},[243],[16884],{"type":22,"value":3753},{"type":16,"tag":103,"props":16886,"children":16888},{"className":16887,"style":1497},[632],[],{"type":16,"tag":103,"props":16890,"children":16892},{"className":16891},[172],[16893,16897,16902],{"type":16,"tag":103,"props":16894,"children":16896},{"className":16895,"style":11235},[177],[],{"type":16,"tag":103,"props":16898,"children":16900},{"className":16899},[183],[16901],{"type":22,"value":155},{"type":16,"tag":103,"props":16903,"children":16905},{"className":16904},[183],[16906,16911],{"type":16,"tag":103,"props":16907,"children":16909},{"className":16908},[183],[16910],{"type":22,"value":5746},{"type":16,"tag":103,"props":16912,"children":16914},{"className":16913},[194],[16915],{"type":16,"tag":103,"props":16916,"children":16918},{"className":16917},[199],[16919],{"type":16,"tag":103,"props":16920,"children":16922},{"className":16921},[205],[16923],{"type":16,"tag":103,"props":16924,"children":16926},{"className":16925,"style":11235},[210],[16927],{"type":16,"tag":103,"props":16928,"children":16929},{"style":3043},[16930,16934],{"type":16,"tag":103,"props":16931,"children":16933},{"className":16932,"style":221},[220],[],{"type":16,"tag":103,"props":16935,"children":16937},{"className":16936},[226,227,228,229],[16938],{"type":16,"tag":103,"props":16939,"children":16941},{"className":16940},[183,229],[16942],{"type":16,"tag":103,"props":16943,"children":16945},{"className":16944},[183,229],[16946],{"type":22,"value":16564},{"type":16,"tag":11095,"props":16948,"children":16949},{},[16950,16955,16960,16965,17099],{"type":16,"tag":11127,"props":16951,"children":16952},{},[16953],{"type":22,"value":16954},"ConvS2S [8]",{"type":16,"tag":11127,"props":16956,"children":16957},{"align":16459},[16958],{"type":22,"value":16959},"25.16",{"type":16,"tag":11127,"props":16961,"children":16962},{"align":16459},[16963],{"type":22,"value":16964},"40.46",{"type":16,"tag":11127,"props":16966,"children":16967},{"align":16459},[16968],{"type":16,"tag":103,"props":16969,"children":16971},{"className":16970},[106],[16972,17011],{"type":16,"tag":103,"props":16973,"children":16975},{"className":16974},[111],[16976],{"type":16,"tag":114,"props":16977,"children":16978},{"xmlns":116},[16979],{"type":16,"tag":119,"props":16980,"children":16981},{},[16982,17006],{"type":16,"tag":123,"props":16983,"children":16984},{},[16985,16990,16994],{"type":16,"tag":151,"props":16986,"children":16987},{},[16988],{"type":22,"value":16989},"9.6",{"type":16,"tag":145,"props":16991,"children":16992},{},[16993],{"type":22,"value":3753},{"type":16,"tag":2662,"props":16995,"children":16996},{},[16997,17001],{"type":16,"tag":151,"props":16998,"children":16999},{},[17000],{"type":22,"value":14621},{"type":16,"tag":151,"props":17002,"children":17003},{},[17004],{"type":22,"value":17005},"18",{"type":16,"tag":157,"props":17007,"children":17008},{"encoding":159},[17009],{"type":22,"value":17010},"9.6 \\cdot 10^{18}",{"type":16,"tag":103,"props":17012,"children":17014},{"className":17013,"ariaHidden":167},[166],[17015,17041],{"type":16,"tag":103,"props":17016,"children":17018},{"className":17017},[172],[17019,17023,17028,17032,17037],{"type":16,"tag":103,"props":17020,"children":17022},{"className":17021,"style":1375},[177],[],{"type":16,"tag":103,"props":17024,"children":17026},{"className":17025},[183],[17027],{"type":22,"value":16989},{"type":16,"tag":103,"props":17029,"children":17031},{"className":17030,"style":1497},[632],[],{"type":16,"tag":103,"props":17033,"children":17035},{"className":17034},[243],[17036],{"type":22,"value":3753},{"type":16,"tag":103,"props":17038,"children":17040},{"className":17039,"style":1497},[632],[],{"type":16,"tag":103,"props":17042,"children":17044},{"className":17043},[172],[17045,17049,17054],{"type":16,"tag":103,"props":17046,"children":17048},{"className":17047,"style":11235},[177],[],{"type":16,"tag":103,"props":17050,"children":17052},{"className":17051},[183],[17053],{"type":22,"value":155},{"type":16,"tag":103,"props":17055,"children":17057},{"className":17056},[183],[17058,17063],{"type":16,"tag":103,"props":17059,"children":17061},{"className":17060},[183],[17062],{"type":22,"value":5746},{"type":16,"tag":103,"props":17064,"children":17066},{"className":17065},[194],[17067],{"type":16,"tag":103,"props":17068,"children":17070},{"className":17069},[199],[17071],{"type":16,"tag":103,"props":17072,"children":17074},{"className":17073},[205],[17075],{"type":16,"tag":103,"props":17076,"children":17078},{"className":17077,"style":11235},[210],[17079],{"type":16,"tag":103,"props":17080,"children":17081},{"style":3043},[17082,17086],{"type":16,"tag":103,"props":17083,"children":17085},{"className":17084,"style":221},[220],[],{"type":16,"tag":103,"props":17087,"children":17089},{"className":17088},[226,227,228,229],[17090],{"type":16,"tag":103,"props":17091,"children":17093},{"className":17092},[183,229],[17094],{"type":16,"tag":103,"props":17095,"children":17097},{"className":17096},[183,229],[17098],{"type":22,"value":17005},{"type":16,"tag":11127,"props":17100,"children":17101},{"align":16459},[17102],{"type":16,"tag":103,"props":17103,"children":17105},{"className":17104},[106],[17106,17143],{"type":16,"tag":103,"props":17107,"children":17109},{"className":17108},[111],[17110],{"type":16,"tag":114,"props":17111,"children":17112},{"xmlns":116},[17113],{"type":16,"tag":119,"props":17114,"children":17115},{},[17116,17138],{"type":16,"tag":123,"props":17117,"children":17118},{},[17119,17123,17127],{"type":16,"tag":151,"props":17120,"children":17121},{},[17122],{"type":22,"value":15071},{"type":16,"tag":145,"props":17124,"children":17125},{},[17126],{"type":22,"value":3753},{"type":16,"tag":2662,"props":17128,"children":17129},{},[17130,17134],{"type":16,"tag":151,"props":17131,"children":17132},{},[17133],{"type":22,"value":14621},{"type":16,"tag":151,"props":17135,"children":17136},{},[17137],{"type":22,"value":16564},{"type":16,"tag":157,"props":17139,"children":17140},{"encoding":159},[17141],{"type":22,"value":17142},"1.5 \\cdot 10^{20}",{"type":16,"tag":103,"props":17144,"children":17146},{"className":17145,"ariaHidden":167},[166],[17147,17173],{"type":16,"tag":103,"props":17148,"children":17150},{"className":17149},[172],[17151,17155,17160,17164,17169],{"type":16,"tag":103,"props":17152,"children":17154},{"className":17153,"style":1375},[177],[],{"type":16,"tag":103,"props":17156,"children":17158},{"className":17157},[183],[17159],{"type":22,"value":15071},{"type":16,"tag":103,"props":17161,"children":17163},{"className":17162,"style":1497},[632],[],{"type":16,"tag":103,"props":17165,"children":17167},{"className":17166},[243],[17168],{"type":22,"value":3753},{"type":16,"tag":103,"props":17170,"children":17172},{"className":17171,"style":1497},[632],[],{"type":16,"tag":103,"props":17174,"children":17176},{"className":17175},[172],[17177,17181,17186],{"type":16,"tag":103,"props":17178,"children":17180},{"className":17179,"style":11235},[177],[],{"type":16,"tag":103,"props":17182,"children":17184},{"className":17183},[183],[17185],{"type":22,"value":155},{"type":16,"tag":103,"props":17187,"children":17189},{"className":17188},[183],[17190,17195],{"type":16,"tag":103,"props":17191,"children":17193},{"className":17192},[183],[17194],{"type":22,"value":5746},{"type":16,"tag":103,"props":17196,"children":17198},{"className":17197},[194],[17199],{"type":16,"tag":103,"props":17200,"children":17202},{"className":17201},[199],[17203],{"type":16,"tag":103,"props":17204,"children":17206},{"className":17205},[205],[17207],{"type":16,"tag":103,"props":17208,"children":17210},{"className":17209,"style":11235},[210],[17211],{"type":16,"tag":103,"props":17212,"children":17213},{"style":3043},[17214,17218],{"type":16,"tag":103,"props":17215,"children":17217},{"className":17216,"style":221},[220],[],{"type":16,"tag":103,"props":17219,"children":17221},{"className":17220},[226,227,228,229],[17222],{"type":16,"tag":103,"props":17223,"children":17225},{"className":17224},[183,229],[17226],{"type":16,"tag":103,"props":17227,"children":17229},{"className":17228},[183,229],[17230],{"type":22,"value":16564},{"type":16,"tag":11095,"props":17232,"children":17233},{},[17234,17239,17244,17249,17382],{"type":16,"tag":11127,"props":17235,"children":17236},{},[17237],{"type":22,"value":17238},"MoE [26]",{"type":16,"tag":11127,"props":17240,"children":17241},{"align":16459},[17242],{"type":22,"value":17243},"26.03",{"type":16,"tag":11127,"props":17245,"children":17246},{"align":16459},[17247],{"type":22,"value":17248},"40.56",{"type":16,"tag":11127,"props":17250,"children":17251},{"align":16459},[17252],{"type":16,"tag":103,"props":17253,"children":17255},{"className":17254},[106],[17256,17294],{"type":16,"tag":103,"props":17257,"children":17259},{"className":17258},[111],[17260],{"type":16,"tag":114,"props":17261,"children":17262},{"xmlns":116},[17263],{"type":16,"tag":119,"props":17264,"children":17265},{},[17266,17289],{"type":16,"tag":123,"props":17267,"children":17268},{},[17269,17274,17278],{"type":16,"tag":151,"props":17270,"children":17271},{},[17272],{"type":22,"value":17273},"2.0",{"type":16,"tag":145,"props":17275,"children":17276},{},[17277],{"type":22,"value":3753},{"type":16,"tag":2662,"props":17279,"children":17280},{},[17281,17285],{"type":16,"tag":151,"props":17282,"children":17283},{},[17284],{"type":22,"value":14621},{"type":16,"tag":151,"props":17286,"children":17287},{},[17288],{"type":22,"value":16720},{"type":16,"tag":157,"props":17290,"children":17291},{"encoding":159},[17292],{"type":22,"value":17293},"2.0 \\cdot 10^{19}",{"type":16,"tag":103,"props":17295,"children":17297},{"className":17296,"ariaHidden":167},[166],[17298,17324],{"type":16,"tag":103,"props":17299,"children":17301},{"className":17300},[172],[17302,17306,17311,17315,17320],{"type":16,"tag":103,"props":17303,"children":17305},{"className":17304,"style":1375},[177],[],{"type":16,"tag":103,"props":17307,"children":17309},{"className":17308},[183],[17310],{"type":22,"value":17273},{"type":16,"tag":103,"props":17312,"children":17314},{"className":17313,"style":1497},[632],[],{"type":16,"tag":103,"props":17316,"children":17318},{"className":17317},[243],[17319],{"type":22,"value":3753},{"type":16,"tag":103,"props":17321,"children":17323},{"className":17322,"style":1497},[632],[],{"type":16,"tag":103,"props":17325,"children":17327},{"className":17326},[172],[17328,17332,17337],{"type":16,"tag":103,"props":17329,"children":17331},{"className":17330,"style":11235},[177],[],{"type":16,"tag":103,"props":17333,"children":17335},{"className":17334},[183],[17336],{"type":22,"value":155},{"type":16,"tag":103,"props":17338,"children":17340},{"className":17339},[183],[17341,17346],{"type":16,"tag":103,"props":17342,"children":17344},{"className":17343},[183],[17345],{"type":22,"value":5746},{"type":16,"tag":103,"props":17347,"children":17349},{"className":17348},[194],[17350],{"type":16,"tag":103,"props":17351,"children":17353},{"className":17352},[199],[17354],{"type":16,"tag":103,"props":17355,"children":17357},{"className":17356},[205],[17358],{"type":16,"tag":103,"props":17359,"children":17361},{"className":17360,"style":11235},[210],[17362],{"type":16,"tag":103,"props":17363,"children":17364},{"style":3043},[17365,17369],{"type":16,"tag":103,"props":17366,"children":17368},{"className":17367,"style":221},[220],[],{"type":16,"tag":103,"props":17370,"children":17372},{"className":17371},[226,227,228,229],[17373],{"type":16,"tag":103,"props":17374,"children":17376},{"className":17375},[183,229],[17377],{"type":16,"tag":103,"props":17378,"children":17380},{"className":17379},[183,229],[17381],{"type":22,"value":16720},{"type":16,"tag":11127,"props":17383,"children":17384},{"align":16459},[17385],{"type":16,"tag":103,"props":17386,"children":17388},{"className":17387},[106],[17389,17427],{"type":16,"tag":103,"props":17390,"children":17392},{"className":17391},[111],[17393],{"type":16,"tag":114,"props":17394,"children":17395},{"xmlns":116},[17396],{"type":16,"tag":119,"props":17397,"children":17398},{},[17399,17422],{"type":16,"tag":123,"props":17400,"children":17401},{},[17402,17407,17411],{"type":16,"tag":151,"props":17403,"children":17404},{},[17405],{"type":22,"value":17406},"1.2",{"type":16,"tag":145,"props":17408,"children":17409},{},[17410],{"type":22,"value":3753},{"type":16,"tag":2662,"props":17412,"children":17413},{},[17414,17418],{"type":16,"tag":151,"props":17415,"children":17416},{},[17417],{"type":22,"value":14621},{"type":16,"tag":151,"props":17419,"children":17420},{},[17421],{"type":22,"value":16564},{"type":16,"tag":157,"props":17423,"children":17424},{"encoding":159},[17425],{"type":22,"value":17426},"1.2 \\cdot 10^{20}",{"type":16,"tag":103,"props":17428,"children":17430},{"className":17429,"ariaHidden":167},[166],[17431,17457],{"type":16,"tag":103,"props":17432,"children":17434},{"className":17433},[172],[17435,17439,17444,17448,17453],{"type":16,"tag":103,"props":17436,"children":17438},{"className":17437,"style":1375},[177],[],{"type":16,"tag":103,"props":17440,"children":17442},{"className":17441},[183],[17443],{"type":22,"value":17406},{"type":16,"tag":103,"props":17445,"children":17447},{"className":17446,"style":1497},[632],[],{"type":16,"tag":103,"props":17449,"children":17451},{"className":17450},[243],[17452],{"type":22,"value":3753},{"type":16,"tag":103,"props":17454,"children":17456},{"className":17455,"style":1497},[632],[],{"type":16,"tag":103,"props":17458,"children":17460},{"className":17459},[172],[17461,17465,17470],{"type":16,"tag":103,"props":17462,"children":17464},{"className":17463,"style":11235},[177],[],{"type":16,"tag":103,"props":17466,"children":17468},{"className":17467},[183],[17469],{"type":22,"value":155},{"type":16,"tag":103,"props":17471,"children":17473},{"className":17472},[183],[17474,17479],{"type":16,"tag":103,"props":17475,"children":17477},{"className":17476},[183],[17478],{"type":22,"value":5746},{"type":16,"tag":103,"props":17480,"children":17482},{"className":17481},[194],[17483],{"type":16,"tag":103,"props":17484,"children":17486},{"className":17485},[199],[17487],{"type":16,"tag":103,"props":17488,"children":17490},{"className":17489},[205],[17491],{"type":16,"tag":103,"props":17492,"children":17494},{"className":17493,"style":11235},[210],[17495],{"type":16,"tag":103,"props":17496,"children":17497},{"style":3043},[17498,17502],{"type":16,"tag":103,"props":17499,"children":17501},{"className":17500,"style":221},[220],[],{"type":16,"tag":103,"props":17503,"children":17505},{"className":17504},[226,227,228,229],[17506],{"type":16,"tag":103,"props":17507,"children":17509},{"className":17508},[183,229],[17510],{"type":16,"tag":103,"props":17511,"children":17513},{"className":17512},[183,229],[17514],{"type":22,"value":16564},{"type":16,"tag":11095,"props":17516,"children":17517},{},[17518,17523,17527,17532,17536],{"type":16,"tag":11127,"props":17519,"children":17520},{},[17521],{"type":22,"value":17522},"Deep-Att + PosUnk Ensemble [32]",{"type":16,"tag":11127,"props":17524,"children":17525},{"align":16459},[17526],{"type":22,"value":16498},{"type":16,"tag":11127,"props":17528,"children":17529},{"align":16459},[17530],{"type":22,"value":17531},"40.4",{"type":16,"tag":11127,"props":17533,"children":17534},{"align":16459},[17535],{"type":22,"value":16498},{"type":16,"tag":11127,"props":17537,"children":17538},{"align":16459},[17539],{"type":16,"tag":103,"props":17540,"children":17542},{"className":17541},[106],[17543,17581],{"type":16,"tag":103,"props":17544,"children":17546},{"className":17545},[111],[17547],{"type":16,"tag":114,"props":17548,"children":17549},{"xmlns":116},[17550],{"type":16,"tag":119,"props":17551,"children":17552},{},[17553,17576],{"type":16,"tag":123,"props":17554,"children":17555},{},[17556,17561,17565],{"type":16,"tag":151,"props":17557,"children":17558},{},[17559],{"type":22,"value":17560},"8.0",{"type":16,"tag":145,"props":17562,"children":17563},{},[17564],{"type":22,"value":3753},{"type":16,"tag":2662,"props":17566,"children":17567},{},[17568,17572],{"type":16,"tag":151,"props":17569,"children":17570},{},[17571],{"type":22,"value":14621},{"type":16,"tag":151,"props":17573,"children":17574},{},[17575],{"type":22,"value":16564},{"type":16,"tag":157,"props":17577,"children":17578},{"encoding":159},[17579],{"type":22,"value":17580},"8.0 \\cdot 10^{20}",{"type":16,"tag":103,"props":17582,"children":17584},{"className":17583,"ariaHidden":167},[166],[17585,17611],{"type":16,"tag":103,"props":17586,"children":17588},{"className":17587},[172],[17589,17593,17598,17602,17607],{"type":16,"tag":103,"props":17590,"children":17592},{"className":17591,"style":1375},[177],[],{"type":16,"tag":103,"props":17594,"children":17596},{"className":17595},[183],[17597],{"type":22,"value":17560},{"type":16,"tag":103,"props":17599,"children":17601},{"className":17600,"style":1497},[632],[],{"type":16,"tag":103,"props":17603,"children":17605},{"className":17604},[243],[17606],{"type":22,"value":3753},{"type":16,"tag":103,"props":17608,"children":17610},{"className":17609,"style":1497},[632],[],{"type":16,"tag":103,"props":17612,"children":17614},{"className":17613},[172],[17615,17619,17624],{"type":16,"tag":103,"props":17616,"children":17618},{"className":17617,"style":11235},[177],[],{"type":16,"tag":103,"props":17620,"children":17622},{"className":17621},[183],[17623],{"type":22,"value":155},{"type":16,"tag":103,"props":17625,"children":17627},{"className":17626},[183],[17628,17633],{"type":16,"tag":103,"props":17629,"children":17631},{"className":17630},[183],[17632],{"type":22,"value":5746},{"type":16,"tag":103,"props":17634,"children":17636},{"className":17635},[194],[17637],{"type":16,"tag":103,"props":17638,"children":17640},{"className":17639},[199],[17641],{"type":16,"tag":103,"props":17642,"children":17644},{"className":17643},[205],[17645],{"type":16,"tag":103,"props":17646,"children":17648},{"className":17647,"style":11235},[210],[17649],{"type":16,"tag":103,"props":17650,"children":17651},{"style":3043},[17652,17656],{"type":16,"tag":103,"props":17653,"children":17655},{"className":17654,"style":221},[220],[],{"type":16,"tag":103,"props":17657,"children":17659},{"className":17658},[226,227,228,229],[17660],{"type":16,"tag":103,"props":17661,"children":17663},{"className":17662},[183,229],[17664],{"type":16,"tag":103,"props":17665,"children":17667},{"className":17666},[183,229],[17668],{"type":22,"value":16564},{"type":16,"tag":11095,"props":17670,"children":17671},{},[17672,17677,17682,17687,17820],{"type":16,"tag":11127,"props":17673,"children":17674},{},[17675],{"type":22,"value":17676},"GNMT + RL Ensemble [31]",{"type":16,"tag":11127,"props":17678,"children":17679},{"align":16459},[17680],{"type":22,"value":17681},"26.30",{"type":16,"tag":11127,"props":17683,"children":17684},{"align":16459},[17685],{"type":22,"value":17686},"41.16",{"type":16,"tag":11127,"props":17688,"children":17689},{"align":16459},[17690],{"type":16,"tag":103,"props":17691,"children":17693},{"className":17692},[106],[17694,17732],{"type":16,"tag":103,"props":17695,"children":17697},{"className":17696},[111],[17698],{"type":16,"tag":114,"props":17699,"children":17700},{"xmlns":116},[17701],{"type":16,"tag":119,"props":17702,"children":17703},{},[17704,17727],{"type":16,"tag":123,"props":17705,"children":17706},{},[17707,17712,17716],{"type":16,"tag":151,"props":17708,"children":17709},{},[17710],{"type":22,"value":17711},"1.8",{"type":16,"tag":145,"props":17713,"children":17714},{},[17715],{"type":22,"value":3753},{"type":16,"tag":2662,"props":17717,"children":17718},{},[17719,17723],{"type":16,"tag":151,"props":17720,"children":17721},{},[17722],{"type":22,"value":14621},{"type":16,"tag":151,"props":17724,"children":17725},{},[17726],{"type":22,"value":16564},{"type":16,"tag":157,"props":17728,"children":17729},{"encoding":159},[17730],{"type":22,"value":17731},"1.8 \\cdot 10^{20}",{"type":16,"tag":103,"props":17733,"children":17735},{"className":17734,"ariaHidden":167},[166],[17736,17762],{"type":16,"tag":103,"props":17737,"children":17739},{"className":17738},[172],[17740,17744,17749,17753,17758],{"type":16,"tag":103,"props":17741,"children":17743},{"className":17742,"style":1375},[177],[],{"type":16,"tag":103,"props":17745,"children":17747},{"className":17746},[183],[17748],{"type":22,"value":17711},{"type":16,"tag":103,"props":17750,"children":17752},{"className":17751,"style":1497},[632],[],{"type":16,"tag":103,"props":17754,"children":17756},{"className":17755},[243],[17757],{"type":22,"value":3753},{"type":16,"tag":103,"props":17759,"children":17761},{"className":17760,"style":1497},[632],[],{"type":16,"tag":103,"props":17763,"children":17765},{"className":17764},[172],[17766,17770,17775],{"type":16,"tag":103,"props":17767,"children":17769},{"className":17768,"style":11235},[177],[],{"type":16,"tag":103,"props":17771,"children":17773},{"className":17772},[183],[17774],{"type":22,"value":155},{"type":16,"tag":103,"props":17776,"children":17778},{"className":17777},[183],[17779,17784],{"type":16,"tag":103,"props":17780,"children":17782},{"className":17781},[183],[17783],{"type":22,"value":5746},{"type":16,"tag":103,"props":17785,"children":17787},{"className":17786},[194],[17788],{"type":16,"tag":103,"props":17789,"children":17791},{"className":17790},[199],[17792],{"type":16,"tag":103,"props":17793,"children":17795},{"className":17794},[205],[17796],{"type":16,"tag":103,"props":17797,"children":17799},{"className":17798,"style":11235},[210],[17800],{"type":16,"tag":103,"props":17801,"children":17802},{"style":3043},[17803,17807],{"type":16,"tag":103,"props":17804,"children":17806},{"className":17805,"style":221},[220],[],{"type":16,"tag":103,"props":17808,"children":17810},{"className":17809},[226,227,228,229],[17811],{"type":16,"tag":103,"props":17812,"children":17814},{"className":17813},[183,229],[17815],{"type":16,"tag":103,"props":17816,"children":17818},{"className":17817},[183,229],[17819],{"type":22,"value":16564},{"type":16,"tag":11127,"props":17821,"children":17822},{"align":16459},[17823],{"type":16,"tag":103,"props":17824,"children":17826},{"className":17825},[106],[17827,17866],{"type":16,"tag":103,"props":17828,"children":17830},{"className":17829},[111],[17831],{"type":16,"tag":114,"props":17832,"children":17833},{"xmlns":116},[17834],{"type":16,"tag":119,"props":17835,"children":17836},{},[17837,17861],{"type":16,"tag":123,"props":17838,"children":17839},{},[17840,17845,17849],{"type":16,"tag":151,"props":17841,"children":17842},{},[17843],{"type":22,"value":17844},"1.1",{"type":16,"tag":145,"props":17846,"children":17847},{},[17848],{"type":22,"value":3753},{"type":16,"tag":2662,"props":17850,"children":17851},{},[17852,17856],{"type":16,"tag":151,"props":17853,"children":17854},{},[17855],{"type":22,"value":14621},{"type":16,"tag":151,"props":17857,"children":17858},{},[17859],{"type":22,"value":17860},"21",{"type":16,"tag":157,"props":17862,"children":17863},{"encoding":159},[17864],{"type":22,"value":17865},"1.1 \\cdot 10^{21}",{"type":16,"tag":103,"props":17867,"children":17869},{"className":17868,"ariaHidden":167},[166],[17870,17896],{"type":16,"tag":103,"props":17871,"children":17873},{"className":17872},[172],[17874,17878,17883,17887,17892],{"type":16,"tag":103,"props":17875,"children":17877},{"className":17876,"style":1375},[177],[],{"type":16,"tag":103,"props":17879,"children":17881},{"className":17880},[183],[17882],{"type":22,"value":17844},{"type":16,"tag":103,"props":17884,"children":17886},{"className":17885,"style":1497},[632],[],{"type":16,"tag":103,"props":17888,"children":17890},{"className":17889},[243],[17891],{"type":22,"value":3753},{"type":16,"tag":103,"props":17893,"children":17895},{"className":17894,"style":1497},[632],[],{"type":16,"tag":103,"props":17897,"children":17899},{"className":17898},[172],[17900,17904,17909],{"type":16,"tag":103,"props":17901,"children":17903},{"className":17902,"style":11235},[177],[],{"type":16,"tag":103,"props":17905,"children":17907},{"className":17906},[183],[17908],{"type":22,"value":155},{"type":16,"tag":103,"props":17910,"children":17912},{"className":17911},[183],[17913,17918],{"type":16,"tag":103,"props":17914,"children":17916},{"className":17915},[183],[17917],{"type":22,"value":5746},{"type":16,"tag":103,"props":17919,"children":17921},{"className":17920},[194],[17922],{"type":16,"tag":103,"props":17923,"children":17925},{"className":17924},[199],[17926],{"type":16,"tag":103,"props":17927,"children":17929},{"className":17928},[205],[17930],{"type":16,"tag":103,"props":17931,"children":17933},{"className":17932,"style":11235},[210],[17934],{"type":16,"tag":103,"props":17935,"children":17936},{"style":3043},[17937,17941],{"type":16,"tag":103,"props":17938,"children":17940},{"className":17939,"style":221},[220],[],{"type":16,"tag":103,"props":17942,"children":17944},{"className":17943},[226,227,228,229],[17945],{"type":16,"tag":103,"props":17946,"children":17948},{"className":17947},[183,229],[17949],{"type":16,"tag":103,"props":17950,"children":17952},{"className":17951},[183,229],[17953],{"type":22,"value":17860},{"type":16,"tag":11095,"props":17955,"children":17956},{},[17957,17962,17967,17972,18105],{"type":16,"tag":11127,"props":17958,"children":17959},{},[17960],{"type":22,"value":17961},"ConvS2S Ensemble [8]",{"type":16,"tag":11127,"props":17963,"children":17964},{"align":16459},[17965],{"type":22,"value":17966},"26.36",{"type":16,"tag":11127,"props":17968,"children":17969},{"align":16459},[17970],{"type":22,"value":17971},"41.29",{"type":16,"tag":11127,"props":17973,"children":17974},{"align":16459},[17975],{"type":16,"tag":103,"props":17976,"children":17978},{"className":17977},[106],[17979,18017],{"type":16,"tag":103,"props":17980,"children":17982},{"className":17981},[111],[17983],{"type":16,"tag":114,"props":17984,"children":17985},{"xmlns":116},[17986],{"type":16,"tag":119,"props":17987,"children":17988},{},[17989,18012],{"type":16,"tag":123,"props":17990,"children":17991},{},[17992,17997,18001],{"type":16,"tag":151,"props":17993,"children":17994},{},[17995],{"type":22,"value":17996},"7.7",{"type":16,"tag":145,"props":17998,"children":17999},{},[18000],{"type":22,"value":3753},{"type":16,"tag":2662,"props":18002,"children":18003},{},[18004,18008],{"type":16,"tag":151,"props":18005,"children":18006},{},[18007],{"type":22,"value":14621},{"type":16,"tag":151,"props":18009,"children":18010},{},[18011],{"type":22,"value":16720},{"type":16,"tag":157,"props":18013,"children":18014},{"encoding":159},[18015],{"type":22,"value":18016},"7.7 \\cdot 10^{19}",{"type":16,"tag":103,"props":18018,"children":18020},{"className":18019,"ariaHidden":167},[166],[18021,18047],{"type":16,"tag":103,"props":18022,"children":18024},{"className":18023},[172],[18025,18029,18034,18038,18043],{"type":16,"tag":103,"props":18026,"children":18028},{"className":18027,"style":1375},[177],[],{"type":16,"tag":103,"props":18030,"children":18032},{"className":18031},[183],[18033],{"type":22,"value":17996},{"type":16,"tag":103,"props":18035,"children":18037},{"className":18036,"style":1497},[632],[],{"type":16,"tag":103,"props":18039,"children":18041},{"className":18040},[243],[18042],{"type":22,"value":3753},{"type":16,"tag":103,"props":18044,"children":18046},{"className":18045,"style":1497},[632],[],{"type":16,"tag":103,"props":18048,"children":18050},{"className":18049},[172],[18051,18055,18060],{"type":16,"tag":103,"props":18052,"children":18054},{"className":18053,"style":11235},[177],[],{"type":16,"tag":103,"props":18056,"children":18058},{"className":18057},[183],[18059],{"type":22,"value":155},{"type":16,"tag":103,"props":18061,"children":18063},{"className":18062},[183],[18064,18069],{"type":16,"tag":103,"props":18065,"children":18067},{"className":18066},[183],[18068],{"type":22,"value":5746},{"type":16,"tag":103,"props":18070,"children":18072},{"className":18071},[194],[18073],{"type":16,"tag":103,"props":18074,"children":18076},{"className":18075},[199],[18077],{"type":16,"tag":103,"props":18078,"children":18080},{"className":18079},[205],[18081],{"type":16,"tag":103,"props":18082,"children":18084},{"className":18083,"style":11235},[210],[18085],{"type":16,"tag":103,"props":18086,"children":18087},{"style":3043},[18088,18092],{"type":16,"tag":103,"props":18089,"children":18091},{"className":18090,"style":221},[220],[],{"type":16,"tag":103,"props":18093,"children":18095},{"className":18094},[226,227,228,229],[18096],{"type":16,"tag":103,"props":18097,"children":18099},{"className":18098},[183,229],[18100],{"type":16,"tag":103,"props":18101,"children":18103},{"className":18102},[183,229],[18104],{"type":22,"value":16720},{"type":16,"tag":11127,"props":18106,"children":18107},{"align":16459},[18108],{"type":16,"tag":103,"props":18109,"children":18111},{"className":18110},[106],[18112,18149],{"type":16,"tag":103,"props":18113,"children":18115},{"className":18114},[111],[18116],{"type":16,"tag":114,"props":18117,"children":18118},{"xmlns":116},[18119],{"type":16,"tag":119,"props":18120,"children":18121},{},[18122,18144],{"type":16,"tag":123,"props":18123,"children":18124},{},[18125,18129,18133],{"type":16,"tag":151,"props":18126,"children":18127},{},[18128],{"type":22,"value":17406},{"type":16,"tag":145,"props":18130,"children":18131},{},[18132],{"type":22,"value":3753},{"type":16,"tag":2662,"props":18134,"children":18135},{},[18136,18140],{"type":16,"tag":151,"props":18137,"children":18138},{},[18139],{"type":22,"value":14621},{"type":16,"tag":151,"props":18141,"children":18142},{},[18143],{"type":22,"value":17860},{"type":16,"tag":157,"props":18145,"children":18146},{"encoding":159},[18147],{"type":22,"value":18148},"1.2 \\cdot 10^{21}",{"type":16,"tag":103,"props":18150,"children":18152},{"className":18151,"ariaHidden":167},[166],[18153,18179],{"type":16,"tag":103,"props":18154,"children":18156},{"className":18155},[172],[18157,18161,18166,18170,18175],{"type":16,"tag":103,"props":18158,"children":18160},{"className":18159,"style":1375},[177],[],{"type":16,"tag":103,"props":18162,"children":18164},{"className":18163},[183],[18165],{"type":22,"value":17406},{"type":16,"tag":103,"props":18167,"children":18169},{"className":18168,"style":1497},[632],[],{"type":16,"tag":103,"props":18171,"children":18173},{"className":18172},[243],[18174],{"type":22,"value":3753},{"type":16,"tag":103,"props":18176,"children":18178},{"className":18177,"style":1497},[632],[],{"type":16,"tag":103,"props":18180,"children":18182},{"className":18181},[172],[18183,18187,18192],{"type":16,"tag":103,"props":18184,"children":18186},{"className":18185,"style":11235},[177],[],{"type":16,"tag":103,"props":18188,"children":18190},{"className":18189},[183],[18191],{"type":22,"value":155},{"type":16,"tag":103,"props":18193,"children":18195},{"className":18194},[183],[18196,18201],{"type":16,"tag":103,"props":18197,"children":18199},{"className":18198},[183],[18200],{"type":22,"value":5746},{"type":16,"tag":103,"props":18202,"children":18204},{"className":18203},[194],[18205],{"type":16,"tag":103,"props":18206,"children":18208},{"className":18207},[199],[18209],{"type":16,"tag":103,"props":18210,"children":18212},{"className":18211},[205],[18213],{"type":16,"tag":103,"props":18214,"children":18216},{"className":18215,"style":11235},[210],[18217],{"type":16,"tag":103,"props":18218,"children":18219},{"style":3043},[18220,18224],{"type":16,"tag":103,"props":18221,"children":18223},{"className":18222,"style":221},[220],[],{"type":16,"tag":103,"props":18225,"children":18227},{"className":18226},[226,227,228,229],[18228],{"type":16,"tag":103,"props":18229,"children":18231},{"className":18230},[183,229],[18232],{"type":16,"tag":103,"props":18233,"children":18235},{"className":18234},[183,229],[18236],{"type":22,"value":17860},{"type":16,"tag":11095,"props":18238,"children":18239},{},[18240,18245,18250,18255,18388],{"type":16,"tag":11127,"props":18241,"children":18242},{},[18243],{"type":22,"value":18244},"Transformer（base）",{"type":16,"tag":11127,"props":18246,"children":18247},{"align":16459},[18248],{"type":22,"value":18249},"27.3",{"type":16,"tag":11127,"props":18251,"children":18252},{"align":16459},[18253],{"type":22,"value":18254},"38.1",{"type":16,"tag":11127,"props":18256,"children":18257},{"align":16459},[18258],{"type":16,"tag":103,"props":18259,"children":18261},{"className":18260},[106],[18262,18300],{"type":16,"tag":103,"props":18263,"children":18265},{"className":18264},[111],[18266],{"type":16,"tag":114,"props":18267,"children":18268},{"xmlns":116},[18269],{"type":16,"tag":119,"props":18270,"children":18271},{},[18272,18295],{"type":16,"tag":123,"props":18273,"children":18274},{},[18275,18280,18284],{"type":16,"tag":151,"props":18276,"children":18277},{},[18278],{"type":22,"value":18279},"3.3",{"type":16,"tag":145,"props":18281,"children":18282},{},[18283],{"type":22,"value":3753},{"type":16,"tag":2662,"props":18285,"children":18286},{},[18287,18291],{"type":16,"tag":151,"props":18288,"children":18289},{},[18290],{"type":22,"value":14621},{"type":16,"tag":151,"props":18292,"children":18293},{},[18294],{"type":22,"value":17005},{"type":16,"tag":157,"props":18296,"children":18297},{"encoding":159},[18298],{"type":22,"value":18299},"3.3 \\cdot 10^{18}",{"type":16,"tag":103,"props":18301,"children":18303},{"className":18302,"ariaHidden":167},[166],[18304,18330],{"type":16,"tag":103,"props":18305,"children":18307},{"className":18306},[172],[18308,18312,18317,18321,18326],{"type":16,"tag":103,"props":18309,"children":18311},{"className":18310,"style":1375},[177],[],{"type":16,"tag":103,"props":18313,"children":18315},{"className":18314},[183],[18316],{"type":22,"value":18279},{"type":16,"tag":103,"props":18318,"children":18320},{"className":18319,"style":1497},[632],[],{"type":16,"tag":103,"props":18322,"children":18324},{"className":18323},[243],[18325],{"type":22,"value":3753},{"type":16,"tag":103,"props":18327,"children":18329},{"className":18328,"style":1497},[632],[],{"type":16,"tag":103,"props":18331,"children":18333},{"className":18332},[172],[18334,18338,18343],{"type":16,"tag":103,"props":18335,"children":18337},{"className":18336,"style":11235},[177],[],{"type":16,"tag":103,"props":18339,"children":18341},{"className":18340},[183],[18342],{"type":22,"value":155},{"type":16,"tag":103,"props":18344,"children":18346},{"className":18345},[183],[18347,18352],{"type":16,"tag":103,"props":18348,"children":18350},{"className":18349},[183],[18351],{"type":22,"value":5746},{"type":16,"tag":103,"props":18353,"children":18355},{"className":18354},[194],[18356],{"type":16,"tag":103,"props":18357,"children":18359},{"className":18358},[199],[18360],{"type":16,"tag":103,"props":18361,"children":18363},{"className":18362},[205],[18364],{"type":16,"tag":103,"props":18365,"children":18367},{"className":18366,"style":11235},[210],[18368],{"type":16,"tag":103,"props":18369,"children":18370},{"style":3043},[18371,18375],{"type":16,"tag":103,"props":18372,"children":18374},{"className":18373,"style":221},[220],[],{"type":16,"tag":103,"props":18376,"children":18378},{"className":18377},[226,227,228,229],[18379],{"type":16,"tag":103,"props":18380,"children":18382},{"className":18381},[183,229],[18383],{"type":16,"tag":103,"props":18384,"children":18386},{"className":18385},[183,229],[18387],{"type":22,"value":17005},{"type":16,"tag":11127,"props":18389,"children":18390},{"align":16459},[18391],{"type":22,"value":16498},{"type":16,"tag":11095,"props":18393,"children":18394},{},[18395,18400,18405,18410,18541],{"type":16,"tag":11127,"props":18396,"children":18397},{},[18398],{"type":22,"value":18399},"Transformer（big）",{"type":16,"tag":11127,"props":18401,"children":18402},{"align":16459},[18403],{"type":22,"value":18404},"28.4",{"type":16,"tag":11127,"props":18406,"children":18407},{"align":16459},[18408],{"type":22,"value":18409},"41.0",{"type":16,"tag":11127,"props":18411,"children":18412},{"align":16459},[18413],{"type":16,"tag":103,"props":18414,"children":18416},{"className":18415},[106],[18417,18453],{"type":16,"tag":103,"props":18418,"children":18420},{"className":18419},[111],[18421],{"type":16,"tag":114,"props":18422,"children":18423},{"xmlns":116},[18424],{"type":16,"tag":119,"props":18425,"children":18426},{},[18427,18449],{"type":16,"tag":123,"props":18428,"children":18429},{},[18430,18434,18438],{"type":16,"tag":151,"props":18431,"children":18432},{},[18433],{"type":22,"value":16704},{"type":16,"tag":145,"props":18435,"children":18436},{},[18437],{"type":22,"value":3753},{"type":16,"tag":2662,"props":18439,"children":18440},{},[18441,18445],{"type":16,"tag":151,"props":18442,"children":18443},{},[18444],{"type":22,"value":14621},{"type":16,"tag":151,"props":18446,"children":18447},{},[18448],{"type":22,"value":16720},{"type":16,"tag":157,"props":18450,"children":18451},{"encoding":159},[18452],{"type":22,"value":16725},{"type":16,"tag":103,"props":18454,"children":18456},{"className":18455,"ariaHidden":167},[166],[18457,18483],{"type":16,"tag":103,"props":18458,"children":18460},{"className":18459},[172],[18461,18465,18470,18474,18479],{"type":16,"tag":103,"props":18462,"children":18464},{"className":18463,"style":1375},[177],[],{"type":16,"tag":103,"props":18466,"children":18468},{"className":18467},[183],[18469],{"type":22,"value":16704},{"type":16,"tag":103,"props":18471,"children":18473},{"className":18472,"style":1497},[632],[],{"type":16,"tag":103,"props":18475,"children":18477},{"className":18476},[243],[18478],{"type":22,"value":3753},{"type":16,"tag":103,"props":18480,"children":18482},{"className":18481,"style":1497},[632],[],{"type":16,"tag":103,"props":18484,"children":18486},{"className":18485},[172],[18487,18491,18496],{"type":16,"tag":103,"props":18488,"children":18490},{"className":18489,"style":11235},[177],[],{"type":16,"tag":103,"props":18492,"children":18494},{"className":18493},[183],[18495],{"type":22,"value":155},{"type":16,"tag":103,"props":18497,"children":18499},{"className":18498},[183],[18500,18505],{"type":16,"tag":103,"props":18501,"children":18503},{"className":18502},[183],[18504],{"type":22,"value":5746},{"type":16,"tag":103,"props":18506,"children":18508},{"className":18507},[194],[18509],{"type":16,"tag":103,"props":18510,"children":18512},{"className":18511},[199],[18513],{"type":16,"tag":103,"props":18514,"children":18516},{"className":18515},[205],[18517],{"type":16,"tag":103,"props":18518,"children":18520},{"className":18519,"style":11235},[210],[18521],{"type":16,"tag":103,"props":18522,"children":18523},{"style":3043},[18524,18528],{"type":16,"tag":103,"props":18525,"children":18527},{"className":18526,"style":221},[220],[],{"type":16,"tag":103,"props":18529,"children":18531},{"className":18530},[226,227,228,229],[18532],{"type":16,"tag":103,"props":18533,"children":18535},{"className":18534},[183,229],[18536],{"type":16,"tag":103,"props":18537,"children":18539},{"className":18538},[183,229],[18540],{"type":22,"value":16720},{"type":16,"tag":11127,"props":18542,"children":18543},{"align":16459},[18544],{"type":22,"value":16498},{"type":16,"tag":1284,"props":18546,"children":18548},{"id":18547},"_62-模型变体",[18549],{"type":22,"value":18550},"6.2 模型变体",{"type":16,"tag":25,"props":18552,"children":18553},{},[18554,18556,18563],{"type":22,"value":18555},"为了评估 Transformer 不同组件的重要性，我们用不同方式改变 base 模型，并测量英德翻译开发集 ",{"type":16,"tag":18557,"props":18558,"children":18560},"code",{"className":18559},[],[18561],{"type":22,"value":18562},"newstest2013",{"type":22,"value":18564}," 上性能的变化。我们使用上一节所述的 beam search，但不进行 checkpoint 平均。结果见表 3。",{"type":16,"tag":25,"props":18566,"children":18567},{},[18568],{"type":22,"value":18569},"在表 3 的 A 行中，我们改变注意力头数量、注意力键和值的维度，同时保持计算量不变，具体方法见第 3.2.2 节。单头注意力比最佳设置低 0.9 BLEU；但头数过多时，质量也会下降。",{"type":16,"tag":25,"props":18571,"children":18572},{},[18573,18575,18676],{"type":22,"value":18574},"在 B 行中，降低注意力键的维度 ",{"type":16,"tag":103,"props":18576,"children":18578},{"className":18577},[106],[18579,18607],{"type":16,"tag":103,"props":18580,"children":18582},{"className":18581},[111],[18583],{"type":16,"tag":114,"props":18584,"children":18585},{"xmlns":116},[18586],{"type":16,"tag":119,"props":18587,"children":18588},{},[18589,18603],{"type":16,"tag":123,"props":18590,"children":18591},{},[18592],{"type":16,"tag":127,"props":18593,"children":18594},{},[18595,18599],{"type":16,"tag":131,"props":18596,"children":18597},{},[18598],{"type":22,"value":1650},{"type":16,"tag":131,"props":18600,"children":18601},{},[18602],{"type":22,"value":2032},{"type":16,"tag":157,"props":18604,"children":18605},{"encoding":159},[18606],{"type":22,"value":2037},{"type":16,"tag":103,"props":18608,"children":18610},{"className":18609,"ariaHidden":167},[166],[18611],{"type":16,"tag":103,"props":18612,"children":18614},{"className":18613},[172],[18615,18619],{"type":16,"tag":103,"props":18616,"children":18618},{"className":18617,"style":361},[177],[],{"type":16,"tag":103,"props":18620,"children":18622},{"className":18621},[183],[18623,18628],{"type":16,"tag":103,"props":18624,"children":18626},{"className":18625},[183,188],[18627],{"type":22,"value":1650},{"type":16,"tag":103,"props":18629,"children":18631},{"className":18630},[194],[18632],{"type":16,"tag":103,"props":18633,"children":18635},{"className":18634},[199,200],[18636,18665],{"type":16,"tag":103,"props":18637,"children":18639},{"className":18638},[205],[18640,18660],{"type":16,"tag":103,"props":18641,"children":18643},{"className":18642,"style":1727},[210],[18644],{"type":16,"tag":103,"props":18645,"children":18646},{"style":215},[18647,18651],{"type":16,"tag":103,"props":18648,"children":18650},{"className":18649,"style":221},[220],[],{"type":16,"tag":103,"props":18652,"children":18654},{"className":18653},[226,227,228,229],[18655],{"type":16,"tag":103,"props":18656,"children":18658},{"className":18657,"style":2089},[183,188,229],[18659],{"type":22,"value":2032},{"type":16,"tag":103,"props":18661,"children":18663},{"className":18662},[254],[18664],{"type":22,"value":257},{"type":16,"tag":103,"props":18666,"children":18668},{"className":18667},[205],[18669],{"type":16,"tag":103,"props":18670,"children":18672},{"className":18671,"style":417},[210],[18673],{"type":16,"tag":103,"props":18674,"children":18675},{},[],{"type":22,"value":18677}," 会损害模型质量。这说明确定兼容性并不容易，比点积更复杂的兼容性函数可能会有帮助。",{"type":16,"tag":25,"props":18679,"children":18680},{},[18681],{"type":22,"value":18682},"在 C 行和 D 行中可以看到，正如预期，更大的模型效果更好，而 dropout 对避免过拟合非常有帮助。在 E 行中，我们用可学习的位置嵌入 [8] 替换正弦位置编码，结果与 base 模型几乎相同。",{"type":16,"tag":1284,"props":18684,"children":18686},{"id":18685},"表-3transformer-架构变体",[18687],{"type":22,"value":18688},"表 3：Transformer 架构变体",{"type":16,"tag":25,"props":18690,"children":18691},{},[18692,18694,18699],{"type":22,"value":18693},"未列出的参数与 base 模型相同。所有指标都在英德翻译开发集 ",{"type":16,"tag":18557,"props":18695,"children":18697},{"className":18696},[],[18698],{"type":22,"value":18562},{"type":22,"value":18700}," 上测量。列出的 perplexity 是按照字节对编码计算的每个词片段 perplexity，不应与按词计算的 perplexity 直接比较。",{"type":16,"tag":11087,"props":18702,"children":18703},{},[18704,19532],{"type":16,"tag":11091,"props":18705,"children":18706},{},[18707],{"type":16,"tag":11095,"props":18708,"children":18709},{},[18710,18715,18760,18891,19011,19056,19160,19264,19392,19512,19517,19522,19527],{"type":16,"tag":11099,"props":18711,"children":18712},{},[18713],{"type":22,"value":18714},"配置",{"type":16,"tag":11099,"props":18716,"children":18717},{"align":16459},[18718],{"type":16,"tag":103,"props":18719,"children":18721},{"className":18720},[106],[18722,18743],{"type":16,"tag":103,"props":18723,"children":18725},{"className":18724},[111],[18726],{"type":16,"tag":114,"props":18727,"children":18728},{"xmlns":116},[18729],{"type":16,"tag":119,"props":18730,"children":18731},{},[18732,18739],{"type":16,"tag":123,"props":18733,"children":18734},{},[18735],{"type":16,"tag":131,"props":18736,"children":18737},{},[18738],{"type":22,"value":1321},{"type":16,"tag":157,"props":18740,"children":18741},{"encoding":159},[18742],{"type":22,"value":1321},{"type":16,"tag":103,"props":18744,"children":18746},{"className":18745,"ariaHidden":167},[166],[18747],{"type":16,"tag":103,"props":18748,"children":18750},{"className":18749},[172],[18751,18755],{"type":16,"tag":103,"props":18752,"children":18754},{"className":18753,"style":1347},[177],[],{"type":16,"tag":103,"props":18756,"children":18758},{"className":18757,"style":1352},[183,188],[18759],{"type":22,"value":1321},{"type":16,"tag":11099,"props":18761,"children":18762},{"align":16459},[18763],{"type":16,"tag":103,"props":18764,"children":18766},{"className":18765},[106],[18767,18814],{"type":16,"tag":103,"props":18768,"children":18770},{"className":18769},[111],[18771],{"type":16,"tag":114,"props":18772,"children":18773},{"xmlns":116},[18774],{"type":16,"tag":119,"props":18775,"children":18776},{},[18777,18810],{"type":16,"tag":123,"props":18778,"children":18779},{},[18780],{"type":16,"tag":127,"props":18781,"children":18782},{},[18783,18787],{"type":16,"tag":131,"props":18784,"children":18785},{},[18786],{"type":22,"value":1650},{"type":16,"tag":123,"props":18788,"children":18789},{},[18790,18794,18798,18802,18806],{"type":16,"tag":131,"props":18791,"children":18792},{"mathvariant":1409},[18793],{"type":22,"value":1090},{"type":16,"tag":131,"props":18795,"children":18796},{"mathvariant":1409},[18797],{"type":22,"value":1662},{"type":16,"tag":131,"props":18799,"children":18800},{"mathvariant":1409},[18801],{"type":22,"value":1650},{"type":16,"tag":131,"props":18803,"children":18804},{"mathvariant":1409},[18805],{"type":22,"value":1671},{"type":16,"tag":131,"props":18807,"children":18808},{"mathvariant":1409},[18809],{"type":22,"value":1676},{"type":16,"tag":157,"props":18811,"children":18812},{"encoding":159},[18813],{"type":22,"value":4216},{"type":16,"tag":103,"props":18815,"children":18817},{"className":18816,"ariaHidden":167},[166],[18818],{"type":16,"tag":103,"props":18819,"children":18821},{"className":18820},[172],[18822,18826],{"type":16,"tag":103,"props":18823,"children":18825},{"className":18824,"style":361},[177],[],{"type":16,"tag":103,"props":18827,"children":18829},{"className":18828},[183],[18830,18835],{"type":16,"tag":103,"props":18831,"children":18833},{"className":18832},[183,188],[18834],{"type":22,"value":1650},{"type":16,"tag":103,"props":18836,"children":18838},{"className":18837},[194],[18839],{"type":16,"tag":103,"props":18840,"children":18842},{"className":18841},[199,200],[18843,18880],{"type":16,"tag":103,"props":18844,"children":18846},{"className":18845},[205],[18847,18875],{"type":16,"tag":103,"props":18848,"children":18850},{"className":18849,"style":1727},[210],[18851],{"type":16,"tag":103,"props":18852,"children":18853},{"style":215},[18854,18858],{"type":16,"tag":103,"props":18855,"children":18857},{"className":18856,"style":221},[220],[],{"type":16,"tag":103,"props":18859,"children":18861},{"className":18860},[226,227,228,229],[18862],{"type":16,"tag":103,"props":18863,"children":18865},{"className":18864},[183,229],[18866],{"type":16,"tag":103,"props":18867,"children":18869},{"className":18868},[183,229],[18870],{"type":16,"tag":103,"props":18871,"children":18873},{"className":18872},[183,1481,229],[18874],{"type":22,"value":1753},{"type":16,"tag":103,"props":18876,"children":18878},{"className":18877},[254],[18879],{"type":22,"value":257},{"type":16,"tag":103,"props":18881,"children":18883},{"className":18882},[205],[18884],{"type":16,"tag":103,"props":18885,"children":18887},{"className":18886,"style":417},[210],[18888],{"type":16,"tag":103,"props":18889,"children":18890},{},[],{"type":16,"tag":11099,"props":18892,"children":18893},{"align":16459},[18894],{"type":16,"tag":103,"props":18895,"children":18897},{"className":18896},[106],[18898,18934],{"type":16,"tag":103,"props":18899,"children":18901},{"className":18900},[111],[18902],{"type":16,"tag":114,"props":18903,"children":18904},{"xmlns":116},[18905],{"type":16,"tag":119,"props":18906,"children":18907},{},[18908,18929],{"type":16,"tag":123,"props":18909,"children":18910},{},[18911],{"type":16,"tag":127,"props":18912,"children":18913},{},[18914,18918],{"type":16,"tag":131,"props":18915,"children":18916},{},[18917],{"type":22,"value":1650},{"type":16,"tag":123,"props":18919,"children":18920},{},[18921,18925],{"type":16,"tag":131,"props":18922,"children":18923},{"mathvariant":1409},[18924],{"type":22,"value":8680},{"type":16,"tag":131,"props":18926,"children":18927},{"mathvariant":1409},[18928],{"type":22,"value":8680},{"type":16,"tag":157,"props":18930,"children":18931},{"encoding":159},[18932],{"type":22,"value":18933},"d_{\\mathrm{ff}}",{"type":16,"tag":103,"props":18935,"children":18937},{"className":18936,"ariaHidden":167},[166],[18938],{"type":16,"tag":103,"props":18939,"children":18941},{"className":18940},[172],[18942,18946],{"type":16,"tag":103,"props":18943,"children":18945},{"className":18944,"style":361},[177],[],{"type":16,"tag":103,"props":18947,"children":18949},{"className":18948},[183],[18950,18955],{"type":16,"tag":103,"props":18951,"children":18953},{"className":18952},[183,188],[18954],{"type":22,"value":1650},{"type":16,"tag":103,"props":18956,"children":18958},{"className":18957},[194],[18959],{"type":16,"tag":103,"props":18960,"children":18962},{"className":18961},[199,200],[18963,19000],{"type":16,"tag":103,"props":18964,"children":18966},{"className":18965},[205],[18967,18995],{"type":16,"tag":103,"props":18968,"children":18970},{"className":18969,"style":1727},[210],[18971],{"type":16,"tag":103,"props":18972,"children":18973},{"style":215},[18974,18978],{"type":16,"tag":103,"props":18975,"children":18977},{"className":18976,"style":221},[220],[],{"type":16,"tag":103,"props":18979,"children":18981},{"className":18980},[226,227,228,229],[18982],{"type":16,"tag":103,"props":18983,"children":18985},{"className":18984},[183,229],[18986],{"type":16,"tag":103,"props":18987,"children":18989},{"className":18988},[183,229],[18990],{"type":16,"tag":103,"props":18991,"children":18993},{"className":18992,"style":8758},[183,1481,229],[18994],{"type":22,"value":8761},{"type":16,"tag":103,"props":18996,"children":18998},{"className":18997},[254],[18999],{"type":22,"value":257},{"type":16,"tag":103,"props":19001,"children":19003},{"className":19002},[205],[19004],{"type":16,"tag":103,"props":19005,"children":19007},{"className":19006,"style":417},[210],[19008],{"type":16,"tag":103,"props":19009,"children":19010},{},[],{"type":16,"tag":11099,"props":19012,"children":19013},{"align":16459},[19014],{"type":16,"tag":103,"props":19015,"children":19017},{"className":19016},[106],[19018,19039],{"type":16,"tag":103,"props":19019,"children":19021},{"className":19020},[111],[19022],{"type":16,"tag":114,"props":19023,"children":19024},{"xmlns":116},[19025],{"type":16,"tag":119,"props":19026,"children":19027},{},[19028,19035],{"type":16,"tag":123,"props":19029,"children":19030},{},[19031],{"type":16,"tag":131,"props":19032,"children":19033},{},[19034],{"type":22,"value":135},{"type":16,"tag":157,"props":19036,"children":19037},{"encoding":159},[19038],{"type":22,"value":135},{"type":16,"tag":103,"props":19040,"children":19042},{"className":19041,"ariaHidden":167},[166],[19043],{"type":16,"tag":103,"props":19044,"children":19046},{"className":19045},[172],[19047,19051],{"type":16,"tag":103,"props":19048,"children":19050},{"className":19049,"style":3719},[177],[],{"type":16,"tag":103,"props":19052,"children":19054},{"className":19053},[183,188],[19055],{"type":22,"value":135},{"type":16,"tag":11099,"props":19057,"children":19058},{"align":16459},[19059],{"type":16,"tag":103,"props":19060,"children":19062},{"className":19061},[106],[19063,19091],{"type":16,"tag":103,"props":19064,"children":19066},{"className":19065},[111],[19067],{"type":16,"tag":114,"props":19068,"children":19069},{"xmlns":116},[19070],{"type":16,"tag":119,"props":19071,"children":19072},{},[19073,19087],{"type":16,"tag":123,"props":19074,"children":19075},{},[19076],{"type":16,"tag":127,"props":19077,"children":19078},{},[19079,19083],{"type":16,"tag":131,"props":19080,"children":19081},{},[19082],{"type":22,"value":1650},{"type":16,"tag":131,"props":19084,"children":19085},{},[19086],{"type":22,"value":2032},{"type":16,"tag":157,"props":19088,"children":19089},{"encoding":159},[19090],{"type":22,"value":2037},{"type":16,"tag":103,"props":19092,"children":19094},{"className":19093,"ariaHidden":167},[166],[19095],{"type":16,"tag":103,"props":19096,"children":19098},{"className":19097},[172],[19099,19103],{"type":16,"tag":103,"props":19100,"children":19102},{"className":19101,"style":361},[177],[],{"type":16,"tag":103,"props":19104,"children":19106},{"className":19105},[183],[19107,19112],{"type":16,"tag":103,"props":19108,"children":19110},{"className":19109},[183,188],[19111],{"type":22,"value":1650},{"type":16,"tag":103,"props":19113,"children":19115},{"className":19114},[194],[19116],{"type":16,"tag":103,"props":19117,"children":19119},{"className":19118},[199,200],[19120,19149],{"type":16,"tag":103,"props":19121,"children":19123},{"className":19122},[205],[19124,19144],{"type":16,"tag":103,"props":19125,"children":19127},{"className":19126,"style":1727},[210],[19128],{"type":16,"tag":103,"props":19129,"children":19130},{"style":215},[19131,19135],{"type":16,"tag":103,"props":19132,"children":19134},{"className":19133,"style":221},[220],[],{"type":16,"tag":103,"props":19136,"children":19138},{"className":19137},[226,227,228,229],[19139],{"type":16,"tag":103,"props":19140,"children":19142},{"className":19141,"style":2089},[183,188,229],[19143],{"type":22,"value":2032},{"type":16,"tag":103,"props":19145,"children":19147},{"className":19146},[254],[19148],{"type":22,"value":257},{"type":16,"tag":103,"props":19150,"children":19152},{"className":19151},[205],[19153],{"type":16,"tag":103,"props":19154,"children":19156},{"className":19155,"style":417},[210],[19157],{"type":16,"tag":103,"props":19158,"children":19159},{},[],{"type":16,"tag":11099,"props":19161,"children":19162},{"align":16459},[19163],{"type":16,"tag":103,"props":19164,"children":19166},{"className":19165},[106],[19167,19195],{"type":16,"tag":103,"props":19168,"children":19170},{"className":19169},[111],[19171],{"type":16,"tag":114,"props":19172,"children":19173},{"xmlns":116},[19174],{"type":16,"tag":119,"props":19175,"children":19176},{},[19177,19191],{"type":16,"tag":123,"props":19178,"children":19179},{},[19180],{"type":16,"tag":127,"props":19181,"children":19182},{},[19183,19187],{"type":16,"tag":131,"props":19184,"children":19185},{},[19186],{"type":22,"value":1650},{"type":16,"tag":131,"props":19188,"children":19189},{},[19190],{"type":22,"value":2138},{"type":16,"tag":157,"props":19192,"children":19193},{"encoding":159},[19194],{"type":22,"value":2143},{"type":16,"tag":103,"props":19196,"children":19198},{"className":19197,"ariaHidden":167},[166],[19199],{"type":16,"tag":103,"props":19200,"children":19202},{"className":19201},[172],[19203,19207],{"type":16,"tag":103,"props":19204,"children":19206},{"className":19205,"style":361},[177],[],{"type":16,"tag":103,"props":19208,"children":19210},{"className":19209},[183],[19211,19216],{"type":16,"tag":103,"props":19212,"children":19214},{"className":19213},[183,188],[19215],{"type":22,"value":1650},{"type":16,"tag":103,"props":19217,"children":19219},{"className":19218},[194],[19220],{"type":16,"tag":103,"props":19221,"children":19223},{"className":19222},[199,200],[19224,19253],{"type":16,"tag":103,"props":19225,"children":19227},{"className":19226},[205],[19228,19248],{"type":16,"tag":103,"props":19229,"children":19231},{"className":19230,"style":678},[210],[19232],{"type":16,"tag":103,"props":19233,"children":19234},{"style":215},[19235,19239],{"type":16,"tag":103,"props":19236,"children":19238},{"className":19237,"style":221},[220],[],{"type":16,"tag":103,"props":19240,"children":19242},{"className":19241},[226,227,228,229],[19243],{"type":16,"tag":103,"props":19244,"children":19246},{"className":19245,"style":1124},[183,188,229],[19247],{"type":22,"value":2138},{"type":16,"tag":103,"props":19249,"children":19251},{"className":19250},[254],[19252],{"type":22,"value":257},{"type":16,"tag":103,"props":19254,"children":19256},{"className":19255},[205],[19257],{"type":16,"tag":103,"props":19258,"children":19260},{"className":19259,"style":417},[210],[19261],{"type":16,"tag":103,"props":19262,"children":19263},{},[],{"type":16,"tag":11099,"props":19265,"children":19266},{"align":16459},[19267],{"type":16,"tag":103,"props":19268,"children":19270},{"className":19269},[106],[19271,19315],{"type":16,"tag":103,"props":19272,"children":19274},{"className":19273},[111],[19275],{"type":16,"tag":114,"props":19276,"children":19277},{"xmlns":116},[19278],{"type":16,"tag":119,"props":19279,"children":19280},{},[19281,19310],{"type":16,"tag":123,"props":19282,"children":19283},{},[19284],{"type":16,"tag":127,"props":19285,"children":19286},{},[19287,19291],{"type":16,"tag":131,"props":19288,"children":19289},{},[19290],{"type":22,"value":15849},{"type":16,"tag":123,"props":19292,"children":19293},{},[19294,19298,19302,19306],{"type":16,"tag":131,"props":19295,"children":19296},{"mathvariant":1409},[19297],{"type":22,"value":1650},{"type":16,"tag":131,"props":19299,"children":19300},{"mathvariant":1409},[19301],{"type":22,"value":11062},{"type":16,"tag":131,"props":19303,"children":19304},{"mathvariant":1409},[19305],{"type":22,"value":1662},{"type":16,"tag":131,"props":19307,"children":19308},{"mathvariant":1409},[19309],{"type":22,"value":25},{"type":16,"tag":157,"props":19311,"children":19312},{"encoding":159},[19313],{"type":22,"value":19314},"P_{\\mathrm{drop}}",{"type":16,"tag":103,"props":19316,"children":19318},{"className":19317,"ariaHidden":167},[166],[19319],{"type":16,"tag":103,"props":19320,"children":19322},{"className":19321},[172],[19323,19327],{"type":16,"tag":103,"props":19324,"children":19326},{"className":19325,"style":10677},[177],[],{"type":16,"tag":103,"props":19328,"children":19330},{"className":19329},[183],[19331,19336],{"type":16,"tag":103,"props":19332,"children":19334},{"className":19333,"style":3056},[183,188],[19335],{"type":22,"value":15849},{"type":16,"tag":103,"props":19337,"children":19339},{"className":19338},[194],[19340],{"type":16,"tag":103,"props":19341,"children":19343},{"className":19342},[199,200],[19344,19381],{"type":16,"tag":103,"props":19345,"children":19347},{"className":19346},[205],[19348,19376],{"type":16,"tag":103,"props":19349,"children":19351},{"className":19350,"style":1727},[210],[19352],{"type":16,"tag":103,"props":19353,"children":19354},{"style":8195},[19355,19359],{"type":16,"tag":103,"props":19356,"children":19358},{"className":19357,"style":221},[220],[],{"type":16,"tag":103,"props":19360,"children":19362},{"className":19361},[226,227,228,229],[19363],{"type":16,"tag":103,"props":19364,"children":19366},{"className":19365},[183,229],[19367],{"type":16,"tag":103,"props":19368,"children":19370},{"className":19369},[183,229],[19371],{"type":16,"tag":103,"props":19372,"children":19374},{"className":19373},[183,1481,229],[19375],{"type":22,"value":15944},{"type":16,"tag":103,"props":19377,"children":19379},{"className":19378},[254],[19380],{"type":22,"value":257},{"type":16,"tag":103,"props":19382,"children":19384},{"className":19383},[205],[19385],{"type":16,"tag":103,"props":19386,"children":19388},{"className":19387,"style":10754},[210],[19389],{"type":16,"tag":103,"props":19390,"children":19391},{},[],{"type":16,"tag":11099,"props":19393,"children":19394},{"align":16459},[19395],{"type":16,"tag":103,"props":19396,"children":19398},{"className":19397},[106],[19399,19435],{"type":16,"tag":103,"props":19400,"children":19402},{"className":19401},[111],[19403],{"type":16,"tag":114,"props":19404,"children":19405},{"xmlns":116},[19406],{"type":16,"tag":119,"props":19407,"children":19408},{},[19409,19430],{"type":16,"tag":123,"props":19410,"children":19411},{},[19412],{"type":16,"tag":127,"props":19413,"children":19414},{},[19415,19419],{"type":16,"tag":131,"props":19416,"children":19417},{},[19418],{"type":22,"value":14609},{"type":16,"tag":123,"props":19420,"children":19421},{},[19422,19426],{"type":16,"tag":131,"props":19423,"children":19424},{"mathvariant":1409},[19425],{"type":22,"value":1676},{"type":16,"tag":131,"props":19427,"children":19428},{"mathvariant":1409},[19429],{"type":22,"value":9340},{"type":16,"tag":157,"props":19431,"children":19432},{"encoding":159},[19433],{"type":22,"value":19434},"\\epsilon_{\\mathrm{ls}}",{"type":16,"tag":103,"props":19436,"children":19438},{"className":19437,"ariaHidden":167},[166],[19439],{"type":16,"tag":103,"props":19440,"children":19442},{"className":19441},[172],[19443,19447],{"type":16,"tag":103,"props":19444,"children":19446},{"className":19445,"style":16057},[177],[],{"type":16,"tag":103,"props":19448,"children":19450},{"className":19449},[183],[19451,19456],{"type":16,"tag":103,"props":19452,"children":19454},{"className":19453},[183,188],[19455],{"type":22,"value":14609},{"type":16,"tag":103,"props":19457,"children":19459},{"className":19458},[194],[19460],{"type":16,"tag":103,"props":19461,"children":19463},{"className":19462},[199,200],[19464,19501],{"type":16,"tag":103,"props":19465,"children":19467},{"className":19466},[205],[19468,19496],{"type":16,"tag":103,"props":19469,"children":19471},{"className":19470,"style":1727},[210],[19472],{"type":16,"tag":103,"props":19473,"children":19474},{"style":215},[19475,19479],{"type":16,"tag":103,"props":19476,"children":19478},{"className":19477,"style":221},[220],[],{"type":16,"tag":103,"props":19480,"children":19482},{"className":19481},[226,227,228,229],[19483],{"type":16,"tag":103,"props":19484,"children":19486},{"className":19485},[183,229],[19487],{"type":16,"tag":103,"props":19488,"children":19490},{"className":19489},[183,229],[19491],{"type":16,"tag":103,"props":19492,"children":19494},{"className":19493},[183,1481,229],[19495],{"type":22,"value":16108},{"type":16,"tag":103,"props":19497,"children":19499},{"className":19498},[254],[19500],{"type":22,"value":257},{"type":16,"tag":103,"props":19502,"children":19504},{"className":19503},[205],[19505],{"type":16,"tag":103,"props":19506,"children":19508},{"className":19507,"style":417},[210],[19509],{"type":16,"tag":103,"props":19510,"children":19511},{},[],{"type":16,"tag":11099,"props":19513,"children":19514},{"align":16459},[19515],{"type":22,"value":19516},"训练步数",{"type":16,"tag":11099,"props":19518,"children":19519},{"align":16459},[19520],{"type":22,"value":19521},"PPL",{"type":16,"tag":11099,"props":19523,"children":19524},{"align":16459},[19525],{"type":22,"value":19526},"BLEU",{"type":16,"tag":11099,"props":19528,"children":19529},{"align":16459},[19530],{"type":22,"value":19531},"参数量（百万）",{"type":16,"tag":11120,"props":19533,"children":19534},{},[19535,19594,19652,19712,19771,19829,20026,20219,20356,20492,20627,20850,21073,21282,21493,21551,21609,21666,21724,21780],{"type":16,"tag":11095,"props":19536,"children":19537},{},[19538,19542,19546,19550,19554,19558,19562,19566,19570,19574,19579,19584,19589],{"type":16,"tag":11127,"props":19539,"children":19540},{},[19541],{"type":22,"value":172},{"type":16,"tag":11127,"props":19543,"children":19544},{"align":16459},[19545],{"type":22,"value":1330},{"type":16,"tag":11127,"props":19547,"children":19548},{"align":16459},[19549],{"type":22,"value":1685},{"type":16,"tag":11127,"props":19551,"children":19552},{"align":16459},[19553],{"type":22,"value":8693},{"type":16,"tag":11127,"props":19555,"children":19556},{"align":16459},[19557],{"type":22,"value":7392},{"type":16,"tag":11127,"props":19559,"children":19560},{"align":16459},[19561],{"type":22,"value":7536},{"type":16,"tag":11127,"props":19563,"children":19564},{"align":16459},[19565],{"type":22,"value":7536},{"type":16,"tag":11127,"props":19567,"children":19568},{"align":16459},[19569],{"type":22,"value":15877},{"type":16,"tag":11127,"props":19571,"children":19572},{"align":16459},[19573],{"type":22,"value":15877},{"type":16,"tag":11127,"props":19575,"children":19576},{"align":16459},[19577],{"type":22,"value":19578},"100K",{"type":16,"tag":11127,"props":19580,"children":19581},{"align":16459},[19582],{"type":22,"value":19583},"4.92",{"type":16,"tag":11127,"props":19585,"children":19586},{"align":16459},[19587],{"type":22,"value":19588},"25.8",{"type":16,"tag":11127,"props":19590,"children":19591},{"align":16459},[19592],{"type":22,"value":19593},"65",{"type":16,"tag":11095,"props":19595,"children":19596},{},[19597,19602,19606,19610,19614,19618,19622,19626,19630,19634,19638,19643,19648],{"type":16,"tag":11127,"props":19598,"children":19599},{},[19600],{"type":22,"value":19601},"A（1 头）",{"type":16,"tag":11127,"props":19603,"children":19604},{"align":16459},[19605],{"type":22,"value":1330},{"type":16,"tag":11127,"props":19607,"children":19608},{"align":16459},[19609],{"type":22,"value":1685},{"type":16,"tag":11127,"props":19611,"children":19612},{"align":16459},[19613],{"type":22,"value":8693},{"type":16,"tag":11127,"props":19615,"children":19616},{"align":16459},[19617],{"type":22,"value":155},{"type":16,"tag":11127,"props":19619,"children":19620},{"align":16459},[19621],{"type":22,"value":1685},{"type":16,"tag":11127,"props":19623,"children":19624},{"align":16459},[19625],{"type":22,"value":1685},{"type":16,"tag":11127,"props":19627,"children":19628},{"align":16459},[19629],{"type":22,"value":15877},{"type":16,"tag":11127,"props":19631,"children":19632},{"align":16459},[19633],{"type":22,"value":15877},{"type":16,"tag":11127,"props":19635,"children":19636},{"align":16459},[19637],{"type":22,"value":19578},{"type":16,"tag":11127,"props":19639,"children":19640},{"align":16459},[19641],{"type":22,"value":19642},"5.29",{"type":16,"tag":11127,"props":19644,"children":19645},{"align":16459},[19646],{"type":22,"value":19647},"24.9",{"type":16,"tag":11127,"props":19649,"children":19650},{"align":16459},[19651],{"type":22,"value":16498},{"type":16,"tag":11095,"props":19653,"children":19654},{},[19655,19660,19664,19668,19672,19677,19682,19686,19690,19694,19698,19703,19708],{"type":16,"tag":11127,"props":19656,"children":19657},{},[19658],{"type":22,"value":19659},"A（4 头）",{"type":16,"tag":11127,"props":19661,"children":19662},{"align":16459},[19663],{"type":22,"value":1330},{"type":16,"tag":11127,"props":19665,"children":19666},{"align":16459},[19667],{"type":22,"value":1685},{"type":16,"tag":11127,"props":19669,"children":19670},{"align":16459},[19671],{"type":22,"value":8693},{"type":16,"tag":11127,"props":19673,"children":19674},{"align":16459},[19675],{"type":22,"value":19676},"4",{"type":16,"tag":11127,"props":19678,"children":19679},{"align":16459},[19680],{"type":22,"value":19681},"128",{"type":16,"tag":11127,"props":19683,"children":19684},{"align":16459},[19685],{"type":22,"value":19681},{"type":16,"tag":11127,"props":19687,"children":19688},{"align":16459},[19689],{"type":22,"value":15877},{"type":16,"tag":11127,"props":19691,"children":19692},{"align":16459},[19693],{"type":22,"value":15877},{"type":16,"tag":11127,"props":19695,"children":19696},{"align":16459},[19697],{"type":22,"value":19578},{"type":16,"tag":11127,"props":19699,"children":19700},{"align":16459},[19701],{"type":22,"value":19702},"5.00",{"type":16,"tag":11127,"props":19704,"children":19705},{"align":16459},[19706],{"type":22,"value":19707},"25.5",{"type":16,"tag":11127,"props":19709,"children":19710},{"align":16459},[19711],{"type":22,"value":16498},{"type":16,"tag":11095,"props":19713,"children":19714},{},[19715,19720,19724,19728,19732,19737,19742,19746,19750,19754,19758,19763,19767],{"type":16,"tag":11127,"props":19716,"children":19717},{},[19718],{"type":22,"value":19719},"A（16 头）",{"type":16,"tag":11127,"props":19721,"children":19722},{"align":16459},[19723],{"type":22,"value":1330},{"type":16,"tag":11127,"props":19725,"children":19726},{"align":16459},[19727],{"type":22,"value":1685},{"type":16,"tag":11127,"props":19729,"children":19730},{"align":16459},[19731],{"type":22,"value":8693},{"type":16,"tag":11127,"props":19733,"children":19734},{"align":16459},[19735],{"type":22,"value":19736},"16",{"type":16,"tag":11127,"props":19738,"children":19739},{"align":16459},[19740],{"type":22,"value":19741},"32",{"type":16,"tag":11127,"props":19743,"children":19744},{"align":16459},[19745],{"type":22,"value":19741},{"type":16,"tag":11127,"props":19747,"children":19748},{"align":16459},[19749],{"type":22,"value":15877},{"type":16,"tag":11127,"props":19751,"children":19752},{"align":16459},[19753],{"type":22,"value":15877},{"type":16,"tag":11127,"props":19755,"children":19756},{"align":16459},[19757],{"type":22,"value":19578},{"type":16,"tag":11127,"props":19759,"children":19760},{"align":16459},[19761],{"type":22,"value":19762},"4.91",{"type":16,"tag":11127,"props":19764,"children":19765},{"align":16459},[19766],{"type":22,"value":19588},{"type":16,"tag":11127,"props":19768,"children":19769},{"align":16459},[19770],{"type":22,"value":16498},{"type":16,"tag":11095,"props":19772,"children":19773},{},[19774,19779,19783,19787,19791,19795,19799,19803,19807,19811,19815,19820,19825],{"type":16,"tag":11127,"props":19775,"children":19776},{},[19777],{"type":22,"value":19778},"A（32 头）",{"type":16,"tag":11127,"props":19780,"children":19781},{"align":16459},[19782],{"type":22,"value":1330},{"type":16,"tag":11127,"props":19784,"children":19785},{"align":16459},[19786],{"type":22,"value":1685},{"type":16,"tag":11127,"props":19788,"children":19789},{"align":16459},[19790],{"type":22,"value":8693},{"type":16,"tag":11127,"props":19792,"children":19793},{"align":16459},[19794],{"type":22,"value":19741},{"type":16,"tag":11127,"props":19796,"children":19797},{"align":16459},[19798],{"type":22,"value":19736},{"type":16,"tag":11127,"props":19800,"children":19801},{"align":16459},[19802],{"type":22,"value":19736},{"type":16,"tag":11127,"props":19804,"children":19805},{"align":16459},[19806],{"type":22,"value":15877},{"type":16,"tag":11127,"props":19808,"children":19809},{"align":16459},[19810],{"type":22,"value":15877},{"type":16,"tag":11127,"props":19812,"children":19813},{"align":16459},[19814],{"type":22,"value":19578},{"type":16,"tag":11127,"props":19816,"children":19817},{"align":16459},[19818],{"type":22,"value":19819},"5.01",{"type":16,"tag":11127,"props":19821,"children":19822},{"align":16459},[19823],{"type":22,"value":19824},"25.4",{"type":16,"tag":11127,"props":19826,"children":19827},{"align":16459},[19828],{"type":22,"value":16498},{"type":16,"tag":11095,"props":19830,"children":19831},{},[19832,19975,19979,19983,19987,19991,19995,19999,20003,20007,20011,20016,20021],{"type":16,"tag":11127,"props":19833,"children":19834},{},[19835,19837,19973],{"type":22,"value":19836},"B（",{"type":16,"tag":103,"props":19838,"children":19840},{"className":19839},[106],[19841,19878],{"type":16,"tag":103,"props":19842,"children":19844},{"className":19843},[111],[19845],{"type":16,"tag":114,"props":19846,"children":19847},{"xmlns":116},[19848],{"type":16,"tag":119,"props":19849,"children":19850},{},[19851,19873],{"type":16,"tag":123,"props":19852,"children":19853},{},[19854,19865,19869],{"type":16,"tag":127,"props":19855,"children":19856},{},[19857,19861],{"type":16,"tag":131,"props":19858,"children":19859},{},[19860],{"type":22,"value":1650},{"type":16,"tag":131,"props":19862,"children":19863},{},[19864],{"type":22,"value":2032},{"type":16,"tag":145,"props":19866,"children":19867},{},[19868],{"type":22,"value":746},{"type":16,"tag":151,"props":19870,"children":19871},{},[19872],{"type":22,"value":19736},{"type":16,"tag":157,"props":19874,"children":19875},{"encoding":159},[19876],{"type":22,"value":19877},"d_k = 16",{"type":16,"tag":103,"props":19879,"children":19881},{"className":19880,"ariaHidden":167},[166],[19882,19960],{"type":16,"tag":103,"props":19883,"children":19885},{"className":19884},[172],[19886,19890,19947,19951,19956],{"type":16,"tag":103,"props":19887,"children":19889},{"className":19888,"style":361},[177],[],{"type":16,"tag":103,"props":19891,"children":19893},{"className":19892},[183],[19894,19899],{"type":16,"tag":103,"props":19895,"children":19897},{"className":19896},[183,188],[19898],{"type":22,"value":1650},{"type":16,"tag":103,"props":19900,"children":19902},{"className":19901},[194],[19903],{"type":16,"tag":103,"props":19904,"children":19906},{"className":19905},[199,200],[19907,19936],{"type":16,"tag":103,"props":19908,"children":19910},{"className":19909},[205],[19911,19931],{"type":16,"tag":103,"props":19912,"children":19914},{"className":19913,"style":1727},[210],[19915],{"type":16,"tag":103,"props":19916,"children":19917},{"style":215},[19918,19922],{"type":16,"tag":103,"props":19919,"children":19921},{"className":19920,"style":221},[220],[],{"type":16,"tag":103,"props":19923,"children":19925},{"className":19924},[226,227,228,229],[19926],{"type":16,"tag":103,"props":19927,"children":19929},{"className":19928,"style":2089},[183,188,229],[19930],{"type":22,"value":2032},{"type":16,"tag":103,"props":19932,"children":19934},{"className":19933},[254],[19935],{"type":22,"value":257},{"type":16,"tag":103,"props":19937,"children":19939},{"className":19938},[205],[19940],{"type":16,"tag":103,"props":19941,"children":19943},{"className":19942,"style":417},[210],[19944],{"type":16,"tag":103,"props":19945,"children":19946},{},[],{"type":16,"tag":103,"props":19948,"children":19950},{"className":19949,"style":816},[632],[],{"type":16,"tag":103,"props":19952,"children":19954},{"className":19953},[821],[19955],{"type":22,"value":746},{"type":16,"tag":103,"props":19957,"children":19959},{"className":19958,"style":816},[632],[],{"type":16,"tag":103,"props":19961,"children":19963},{"className":19962},[172],[19964,19968],{"type":16,"tag":103,"props":19965,"children":19967},{"className":19966,"style":1375},[177],[],{"type":16,"tag":103,"props":19969,"children":19971},{"className":19970},[183],[19972],{"type":22,"value":19736},{"type":22,"value":19974},"）",{"type":16,"tag":11127,"props":19976,"children":19977},{"align":16459},[19978],{"type":22,"value":1330},{"type":16,"tag":11127,"props":19980,"children":19981},{"align":16459},[19982],{"type":22,"value":1685},{"type":16,"tag":11127,"props":19984,"children":19985},{"align":16459},[19986],{"type":22,"value":8693},{"type":16,"tag":11127,"props":19988,"children":19989},{"align":16459},[19990],{"type":22,"value":7392},{"type":16,"tag":11127,"props":19992,"children":19993},{"align":16459},[19994],{"type":22,"value":19736},{"type":16,"tag":11127,"props":19996,"children":19997},{"align":16459},[19998],{"type":22,"value":7536},{"type":16,"tag":11127,"props":20000,"children":20001},{"align":16459},[20002],{"type":22,"value":15877},{"type":16,"tag":11127,"props":20004,"children":20005},{"align":16459},[20006],{"type":22,"value":15877},{"type":16,"tag":11127,"props":20008,"children":20009},{"align":16459},[20010],{"type":22,"value":19578},{"type":16,"tag":11127,"props":20012,"children":20013},{"align":16459},[20014],{"type":22,"value":20015},"5.16",{"type":16,"tag":11127,"props":20017,"children":20018},{"align":16459},[20019],{"type":22,"value":20020},"25.1",{"type":16,"tag":11127,"props":20022,"children":20023},{"align":16459},[20024],{"type":22,"value":20025},"58",{"type":16,"tag":11095,"props":20027,"children":20028},{},[20029,20170,20174,20178,20182,20186,20190,20194,20198,20202,20206,20210,20214],{"type":16,"tag":11127,"props":20030,"children":20031},{},[20032,20033,20169],{"type":22,"value":19836},{"type":16,"tag":103,"props":20034,"children":20036},{"className":20035},[106],[20037,20074],{"type":16,"tag":103,"props":20038,"children":20040},{"className":20039},[111],[20041],{"type":16,"tag":114,"props":20042,"children":20043},{"xmlns":116},[20044],{"type":16,"tag":119,"props":20045,"children":20046},{},[20047,20069],{"type":16,"tag":123,"props":20048,"children":20049},{},[20050,20061,20065],{"type":16,"tag":127,"props":20051,"children":20052},{},[20053,20057],{"type":16,"tag":131,"props":20054,"children":20055},{},[20056],{"type":22,"value":1650},{"type":16,"tag":131,"props":20058,"children":20059},{},[20060],{"type":22,"value":2032},{"type":16,"tag":145,"props":20062,"children":20063},{},[20064],{"type":22,"value":746},{"type":16,"tag":151,"props":20066,"children":20067},{},[20068],{"type":22,"value":19741},{"type":16,"tag":157,"props":20070,"children":20071},{"encoding":159},[20072],{"type":22,"value":20073},"d_k = 32",{"type":16,"tag":103,"props":20075,"children":20077},{"className":20076,"ariaHidden":167},[166],[20078,20156],{"type":16,"tag":103,"props":20079,"children":20081},{"className":20080},[172],[20082,20086,20143,20147,20152],{"type":16,"tag":103,"props":20083,"children":20085},{"className":20084,"style":361},[177],[],{"type":16,"tag":103,"props":20087,"children":20089},{"className":20088},[183],[20090,20095],{"type":16,"tag":103,"props":20091,"children":20093},{"className":20092},[183,188],[20094],{"type":22,"value":1650},{"type":16,"tag":103,"props":20096,"children":20098},{"className":20097},[194],[20099],{"type":16,"tag":103,"props":20100,"children":20102},{"className":20101},[199,200],[20103,20132],{"type":16,"tag":103,"props":20104,"children":20106},{"className":20105},[205],[20107,20127],{"type":16,"tag":103,"props":20108,"children":20110},{"className":20109,"style":1727},[210],[20111],{"type":16,"tag":103,"props":20112,"children":20113},{"style":215},[20114,20118],{"type":16,"tag":103,"props":20115,"children":20117},{"className":20116,"style":221},[220],[],{"type":16,"tag":103,"props":20119,"children":20121},{"className":20120},[226,227,228,229],[20122],{"type":16,"tag":103,"props":20123,"children":20125},{"className":20124,"style":2089},[183,188,229],[20126],{"type":22,"value":2032},{"type":16,"tag":103,"props":20128,"children":20130},{"className":20129},[254],[20131],{"type":22,"value":257},{"type":16,"tag":103,"props":20133,"children":20135},{"className":20134},[205],[20136],{"type":16,"tag":103,"props":20137,"children":20139},{"className":20138,"style":417},[210],[20140],{"type":16,"tag":103,"props":20141,"children":20142},{},[],{"type":16,"tag":103,"props":20144,"children":20146},{"className":20145,"style":816},[632],[],{"type":16,"tag":103,"props":20148,"children":20150},{"className":20149},[821],[20151],{"type":22,"value":746},{"type":16,"tag":103,"props":20153,"children":20155},{"className":20154,"style":816},[632],[],{"type":16,"tag":103,"props":20157,"children":20159},{"className":20158},[172],[20160,20164],{"type":16,"tag":103,"props":20161,"children":20163},{"className":20162,"style":1375},[177],[],{"type":16,"tag":103,"props":20165,"children":20167},{"className":20166},[183],[20168],{"type":22,"value":19741},{"type":22,"value":19974},{"type":16,"tag":11127,"props":20171,"children":20172},{"align":16459},[20173],{"type":22,"value":1330},{"type":16,"tag":11127,"props":20175,"children":20176},{"align":16459},[20177],{"type":22,"value":1685},{"type":16,"tag":11127,"props":20179,"children":20180},{"align":16459},[20181],{"type":22,"value":8693},{"type":16,"tag":11127,"props":20183,"children":20184},{"align":16459},[20185],{"type":22,"value":7392},{"type":16,"tag":11127,"props":20187,"children":20188},{"align":16459},[20189],{"type":22,"value":19741},{"type":16,"tag":11127,"props":20191,"children":20192},{"align":16459},[20193],{"type":22,"value":7536},{"type":16,"tag":11127,"props":20195,"children":20196},{"align":16459},[20197],{"type":22,"value":15877},{"type":16,"tag":11127,"props":20199,"children":20200},{"align":16459},[20201],{"type":22,"value":15877},{"type":16,"tag":11127,"props":20203,"children":20204},{"align":16459},[20205],{"type":22,"value":19578},{"type":16,"tag":11127,"props":20207,"children":20208},{"align":16459},[20209],{"type":22,"value":19819},{"type":16,"tag":11127,"props":20211,"children":20212},{"align":16459},[20213],{"type":22,"value":19824},{"type":16,"tag":11127,"props":20215,"children":20216},{"align":16459},[20217],{"type":22,"value":20218},"60",{"type":16,"tag":11095,"props":20220,"children":20221},{},[20222,20305,20309,20313,20317,20321,20325,20329,20333,20337,20341,20346,20351],{"type":16,"tag":11127,"props":20223,"children":20224},{},[20225,20227,20304],{"type":22,"value":20226},"C（",{"type":16,"tag":103,"props":20228,"children":20230},{"className":20229},[106],[20231,20261],{"type":16,"tag":103,"props":20232,"children":20234},{"className":20233},[111],[20235],{"type":16,"tag":114,"props":20236,"children":20237},{"xmlns":116},[20238],{"type":16,"tag":119,"props":20239,"children":20240},{},[20241,20256],{"type":16,"tag":123,"props":20242,"children":20243},{},[20244,20248,20252],{"type":16,"tag":131,"props":20245,"children":20246},{},[20247],{"type":22,"value":1321},{"type":16,"tag":145,"props":20249,"children":20250},{},[20251],{"type":22,"value":746},{"type":16,"tag":151,"props":20253,"children":20254},{},[20255],{"type":22,"value":8050},{"type":16,"tag":157,"props":20257,"children":20258},{"encoding":159},[20259],{"type":22,"value":20260},"N = 2",{"type":16,"tag":103,"props":20262,"children":20264},{"className":20263,"ariaHidden":167},[166],[20265,20291],{"type":16,"tag":103,"props":20266,"children":20268},{"className":20267},[172],[20269,20273,20278,20282,20287],{"type":16,"tag":103,"props":20270,"children":20272},{"className":20271,"style":1347},[177],[],{"type":16,"tag":103,"props":20274,"children":20276},{"className":20275,"style":1352},[183,188],[20277],{"type":22,"value":1321},{"type":16,"tag":103,"props":20279,"children":20281},{"className":20280,"style":816},[632],[],{"type":16,"tag":103,"props":20283,"children":20285},{"className":20284},[821],[20286],{"type":22,"value":746},{"type":16,"tag":103,"props":20288,"children":20290},{"className":20289,"style":816},[632],[],{"type":16,"tag":103,"props":20292,"children":20294},{"className":20293},[172],[20295,20299],{"type":16,"tag":103,"props":20296,"children":20298},{"className":20297,"style":1375},[177],[],{"type":16,"tag":103,"props":20300,"children":20302},{"className":20301},[183],[20303],{"type":22,"value":8050},{"type":22,"value":19974},{"type":16,"tag":11127,"props":20306,"children":20307},{"align":16459},[20308],{"type":22,"value":8050},{"type":16,"tag":11127,"props":20310,"children":20311},{"align":16459},[20312],{"type":22,"value":1685},{"type":16,"tag":11127,"props":20314,"children":20315},{"align":16459},[20316],{"type":22,"value":8693},{"type":16,"tag":11127,"props":20318,"children":20319},{"align":16459},[20320],{"type":22,"value":7392},{"type":16,"tag":11127,"props":20322,"children":20323},{"align":16459},[20324],{"type":22,"value":7536},{"type":16,"tag":11127,"props":20326,"children":20327},{"align":16459},[20328],{"type":22,"value":7536},{"type":16,"tag":11127,"props":20330,"children":20331},{"align":16459},[20332],{"type":22,"value":15877},{"type":16,"tag":11127,"props":20334,"children":20335},{"align":16459},[20336],{"type":22,"value":15877},{"type":16,"tag":11127,"props":20338,"children":20339},{"align":16459},[20340],{"type":22,"value":19578},{"type":16,"tag":11127,"props":20342,"children":20343},{"align":16459},[20344],{"type":22,"value":20345},"6.11",{"type":16,"tag":11127,"props":20347,"children":20348},{"align":16459},[20349],{"type":22,"value":20350},"23.7",{"type":16,"tag":11127,"props":20352,"children":20353},{"align":16459},[20354],{"type":22,"value":20355},"36",{"type":16,"tag":11095,"props":20357,"children":20358},{},[20359,20441,20445,20449,20453,20457,20461,20465,20469,20473,20477,20482,20487],{"type":16,"tag":11127,"props":20360,"children":20361},{},[20362,20363,20440],{"type":22,"value":20226},{"type":16,"tag":103,"props":20364,"children":20366},{"className":20365},[106],[20367,20397],{"type":16,"tag":103,"props":20368,"children":20370},{"className":20369},[111],[20371],{"type":16,"tag":114,"props":20372,"children":20373},{"xmlns":116},[20374],{"type":16,"tag":119,"props":20375,"children":20376},{},[20377,20392],{"type":16,"tag":123,"props":20378,"children":20379},{},[20380,20384,20388],{"type":16,"tag":131,"props":20381,"children":20382},{},[20383],{"type":22,"value":1321},{"type":16,"tag":145,"props":20385,"children":20386},{},[20387],{"type":22,"value":746},{"type":16,"tag":151,"props":20389,"children":20390},{},[20391],{"type":22,"value":19676},{"type":16,"tag":157,"props":20393,"children":20394},{"encoding":159},[20395],{"type":22,"value":20396},"N = 4",{"type":16,"tag":103,"props":20398,"children":20400},{"className":20399,"ariaHidden":167},[166],[20401,20427],{"type":16,"tag":103,"props":20402,"children":20404},{"className":20403},[172],[20405,20409,20414,20418,20423],{"type":16,"tag":103,"props":20406,"children":20408},{"className":20407,"style":1347},[177],[],{"type":16,"tag":103,"props":20410,"children":20412},{"className":20411,"style":1352},[183,188],[20413],{"type":22,"value":1321},{"type":16,"tag":103,"props":20415,"children":20417},{"className":20416,"style":816},[632],[],{"type":16,"tag":103,"props":20419,"children":20421},{"className":20420},[821],[20422],{"type":22,"value":746},{"type":16,"tag":103,"props":20424,"children":20426},{"className":20425,"style":816},[632],[],{"type":16,"tag":103,"props":20428,"children":20430},{"className":20429},[172],[20431,20435],{"type":16,"tag":103,"props":20432,"children":20434},{"className":20433,"style":1375},[177],[],{"type":16,"tag":103,"props":20436,"children":20438},{"className":20437},[183],[20439],{"type":22,"value":19676},{"type":22,"value":19974},{"type":16,"tag":11127,"props":20442,"children":20443},{"align":16459},[20444],{"type":22,"value":19676},{"type":16,"tag":11127,"props":20446,"children":20447},{"align":16459},[20448],{"type":22,"value":1685},{"type":16,"tag":11127,"props":20450,"children":20451},{"align":16459},[20452],{"type":22,"value":8693},{"type":16,"tag":11127,"props":20454,"children":20455},{"align":16459},[20456],{"type":22,"value":7392},{"type":16,"tag":11127,"props":20458,"children":20459},{"align":16459},[20460],{"type":22,"value":7536},{"type":16,"tag":11127,"props":20462,"children":20463},{"align":16459},[20464],{"type":22,"value":7536},{"type":16,"tag":11127,"props":20466,"children":20467},{"align":16459},[20468],{"type":22,"value":15877},{"type":16,"tag":11127,"props":20470,"children":20471},{"align":16459},[20472],{"type":22,"value":15877},{"type":16,"tag":11127,"props":20474,"children":20475},{"align":16459},[20476],{"type":22,"value":19578},{"type":16,"tag":11127,"props":20478,"children":20479},{"align":16459},[20480],{"type":22,"value":20481},"5.19",{"type":16,"tag":11127,"props":20483,"children":20484},{"align":16459},[20485],{"type":22,"value":20486},"25.3",{"type":16,"tag":11127,"props":20488,"children":20489},{"align":16459},[20490],{"type":22,"value":20491},"50",{"type":16,"tag":11095,"props":20493,"children":20494},{},[20495,20577,20581,20585,20589,20593,20597,20601,20605,20609,20613,20618,20622],{"type":16,"tag":11127,"props":20496,"children":20497},{},[20498,20499,20576],{"type":22,"value":20226},{"type":16,"tag":103,"props":20500,"children":20502},{"className":20501},[106],[20503,20533],{"type":16,"tag":103,"props":20504,"children":20506},{"className":20505},[111],[20507],{"type":16,"tag":114,"props":20508,"children":20509},{"xmlns":116},[20510],{"type":16,"tag":119,"props":20511,"children":20512},{},[20513,20528],{"type":16,"tag":123,"props":20514,"children":20515},{},[20516,20520,20524],{"type":16,"tag":131,"props":20517,"children":20518},{},[20519],{"type":22,"value":1321},{"type":16,"tag":145,"props":20521,"children":20522},{},[20523],{"type":22,"value":746},{"type":16,"tag":151,"props":20525,"children":20526},{},[20527],{"type":22,"value":7392},{"type":16,"tag":157,"props":20529,"children":20530},{"encoding":159},[20531],{"type":22,"value":20532},"N = 8",{"type":16,"tag":103,"props":20534,"children":20536},{"className":20535,"ariaHidden":167},[166],[20537,20563],{"type":16,"tag":103,"props":20538,"children":20540},{"className":20539},[172],[20541,20545,20550,20554,20559],{"type":16,"tag":103,"props":20542,"children":20544},{"className":20543,"style":1347},[177],[],{"type":16,"tag":103,"props":20546,"children":20548},{"className":20547,"style":1352},[183,188],[20549],{"type":22,"value":1321},{"type":16,"tag":103,"props":20551,"children":20553},{"className":20552,"style":816},[632],[],{"type":16,"tag":103,"props":20555,"children":20557},{"className":20556},[821],[20558],{"type":22,"value":746},{"type":16,"tag":103,"props":20560,"children":20562},{"className":20561,"style":816},[632],[],{"type":16,"tag":103,"props":20564,"children":20566},{"className":20565},[172],[20567,20571],{"type":16,"tag":103,"props":20568,"children":20570},{"className":20569,"style":1375},[177],[],{"type":16,"tag":103,"props":20572,"children":20574},{"className":20573},[183],[20575],{"type":22,"value":7392},{"type":22,"value":19974},{"type":16,"tag":11127,"props":20578,"children":20579},{"align":16459},[20580],{"type":22,"value":7392},{"type":16,"tag":11127,"props":20582,"children":20583},{"align":16459},[20584],{"type":22,"value":1685},{"type":16,"tag":11127,"props":20586,"children":20587},{"align":16459},[20588],{"type":22,"value":8693},{"type":16,"tag":11127,"props":20590,"children":20591},{"align":16459},[20592],{"type":22,"value":7392},{"type":16,"tag":11127,"props":20594,"children":20595},{"align":16459},[20596],{"type":22,"value":7536},{"type":16,"tag":11127,"props":20598,"children":20599},{"align":16459},[20600],{"type":22,"value":7536},{"type":16,"tag":11127,"props":20602,"children":20603},{"align":16459},[20604],{"type":22,"value":15877},{"type":16,"tag":11127,"props":20606,"children":20607},{"align":16459},[20608],{"type":22,"value":15877},{"type":16,"tag":11127,"props":20610,"children":20611},{"align":16459},[20612],{"type":22,"value":19578},{"type":16,"tag":11127,"props":20614,"children":20615},{"align":16459},[20616],{"type":22,"value":20617},"4.88",{"type":16,"tag":11127,"props":20619,"children":20620},{"align":16459},[20621],{"type":22,"value":19707},{"type":16,"tag":11127,"props":20623,"children":20624},{"align":16459},[20625],{"type":22,"value":20626},"80",{"type":16,"tag":11095,"props":20628,"children":20629},{},[20630,20799,20803,20807,20811,20815,20819,20823,20827,20831,20835,20840,20845],{"type":16,"tag":11127,"props":20631,"children":20632},{},[20633,20634,20798],{"type":22,"value":20226},{"type":16,"tag":103,"props":20635,"children":20637},{"className":20636},[106],[20638,20695],{"type":16,"tag":103,"props":20639,"children":20641},{"className":20640},[111],[20642],{"type":16,"tag":114,"props":20643,"children":20644},{"xmlns":116},[20645],{"type":16,"tag":119,"props":20646,"children":20647},{},[20648,20690],{"type":16,"tag":123,"props":20649,"children":20650},{},[20651,20681,20685],{"type":16,"tag":127,"props":20652,"children":20653},{},[20654,20658],{"type":16,"tag":131,"props":20655,"children":20656},{},[20657],{"type":22,"value":1650},{"type":16,"tag":123,"props":20659,"children":20660},{},[20661,20665,20669,20673,20677],{"type":16,"tag":131,"props":20662,"children":20663},{"mathvariant":1409},[20664],{"type":22,"value":1090},{"type":16,"tag":131,"props":20666,"children":20667},{"mathvariant":1409},[20668],{"type":22,"value":1662},{"type":16,"tag":131,"props":20670,"children":20671},{"mathvariant":1409},[20672],{"type":22,"value":1650},{"type":16,"tag":131,"props":20674,"children":20675},{"mathvariant":1409},[20676],{"type":22,"value":1671},{"type":16,"tag":131,"props":20678,"children":20679},{"mathvariant":1409},[20680],{"type":22,"value":1676},{"type":16,"tag":145,"props":20682,"children":20683},{},[20684],{"type":22,"value":746},{"type":16,"tag":151,"props":20686,"children":20687},{},[20688],{"type":22,"value":20689},"256",{"type":16,"tag":157,"props":20691,"children":20692},{"encoding":159},[20693],{"type":22,"value":20694},"d_{\\mathrm{model}} = 256",{"type":16,"tag":103,"props":20696,"children":20698},{"className":20697,"ariaHidden":167},[166],[20699,20785],{"type":16,"tag":103,"props":20700,"children":20702},{"className":20701},[172],[20703,20707,20772,20776,20781],{"type":16,"tag":103,"props":20704,"children":20706},{"className":20705,"style":361},[177],[],{"type":16,"tag":103,"props":20708,"children":20710},{"className":20709},[183],[20711,20716],{"type":16,"tag":103,"props":20712,"children":20714},{"className":20713},[183,188],[20715],{"type":22,"value":1650},{"type":16,"tag":103,"props":20717,"children":20719},{"className":20718},[194],[20720],{"type":16,"tag":103,"props":20721,"children":20723},{"className":20722},[199,200],[20724,20761],{"type":16,"tag":103,"props":20725,"children":20727},{"className":20726},[205],[20728,20756],{"type":16,"tag":103,"props":20729,"children":20731},{"className":20730,"style":1727},[210],[20732],{"type":16,"tag":103,"props":20733,"children":20734},{"style":215},[20735,20739],{"type":16,"tag":103,"props":20736,"children":20738},{"className":20737,"style":221},[220],[],{"type":16,"tag":103,"props":20740,"children":20742},{"className":20741},[226,227,228,229],[20743],{"type":16,"tag":103,"props":20744,"children":20746},{"className":20745},[183,229],[20747],{"type":16,"tag":103,"props":20748,"children":20750},{"className":20749},[183,229],[20751],{"type":16,"tag":103,"props":20752,"children":20754},{"className":20753},[183,1481,229],[20755],{"type":22,"value":1753},{"type":16,"tag":103,"props":20757,"children":20759},{"className":20758},[254],[20760],{"type":22,"value":257},{"type":16,"tag":103,"props":20762,"children":20764},{"className":20763},[205],[20765],{"type":16,"tag":103,"props":20766,"children":20768},{"className":20767,"style":417},[210],[20769],{"type":16,"tag":103,"props":20770,"children":20771},{},[],{"type":16,"tag":103,"props":20773,"children":20775},{"className":20774,"style":816},[632],[],{"type":16,"tag":103,"props":20777,"children":20779},{"className":20778},[821],[20780],{"type":22,"value":746},{"type":16,"tag":103,"props":20782,"children":20784},{"className":20783,"style":816},[632],[],{"type":16,"tag":103,"props":20786,"children":20788},{"className":20787},[172],[20789,20793],{"type":16,"tag":103,"props":20790,"children":20792},{"className":20791,"style":1375},[177],[],{"type":16,"tag":103,"props":20794,"children":20796},{"className":20795},[183],[20797],{"type":22,"value":20689},{"type":22,"value":19974},{"type":16,"tag":11127,"props":20800,"children":20801},{"align":16459},[20802],{"type":22,"value":1330},{"type":16,"tag":11127,"props":20804,"children":20805},{"align":16459},[20806],{"type":22,"value":20689},{"type":16,"tag":11127,"props":20808,"children":20809},{"align":16459},[20810],{"type":22,"value":8693},{"type":16,"tag":11127,"props":20812,"children":20813},{"align":16459},[20814],{"type":22,"value":7392},{"type":16,"tag":11127,"props":20816,"children":20817},{"align":16459},[20818],{"type":22,"value":19741},{"type":16,"tag":11127,"props":20820,"children":20821},{"align":16459},[20822],{"type":22,"value":19741},{"type":16,"tag":11127,"props":20824,"children":20825},{"align":16459},[20826],{"type":22,"value":15877},{"type":16,"tag":11127,"props":20828,"children":20829},{"align":16459},[20830],{"type":22,"value":15877},{"type":16,"tag":11127,"props":20832,"children":20833},{"align":16459},[20834],{"type":22,"value":19578},{"type":16,"tag":11127,"props":20836,"children":20837},{"align":16459},[20838],{"type":22,"value":20839},"5.75",{"type":16,"tag":11127,"props":20841,"children":20842},{"align":16459},[20843],{"type":22,"value":20844},"24.5",{"type":16,"tag":11127,"props":20846,"children":20847},{"align":16459},[20848],{"type":22,"value":20849},"28",{"type":16,"tag":11095,"props":20851,"children":20852},{},[20853,21022,21026,21030,21034,21038,21042,21046,21050,21054,21058,21063,21068],{"type":16,"tag":11127,"props":20854,"children":20855},{},[20856,20857,21021],{"type":22,"value":20226},{"type":16,"tag":103,"props":20858,"children":20860},{"className":20859},[106],[20861,20918],{"type":16,"tag":103,"props":20862,"children":20864},{"className":20863},[111],[20865],{"type":16,"tag":114,"props":20866,"children":20867},{"xmlns":116},[20868],{"type":16,"tag":119,"props":20869,"children":20870},{},[20871,20913],{"type":16,"tag":123,"props":20872,"children":20873},{},[20874,20904,20908],{"type":16,"tag":127,"props":20875,"children":20876},{},[20877,20881],{"type":16,"tag":131,"props":20878,"children":20879},{},[20880],{"type":22,"value":1650},{"type":16,"tag":123,"props":20882,"children":20883},{},[20884,20888,20892,20896,20900],{"type":16,"tag":131,"props":20885,"children":20886},{"mathvariant":1409},[20887],{"type":22,"value":1090},{"type":16,"tag":131,"props":20889,"children":20890},{"mathvariant":1409},[20891],{"type":22,"value":1662},{"type":16,"tag":131,"props":20893,"children":20894},{"mathvariant":1409},[20895],{"type":22,"value":1650},{"type":16,"tag":131,"props":20897,"children":20898},{"mathvariant":1409},[20899],{"type":22,"value":1671},{"type":16,"tag":131,"props":20901,"children":20902},{"mathvariant":1409},[20903],{"type":22,"value":1676},{"type":16,"tag":145,"props":20905,"children":20906},{},[20907],{"type":22,"value":746},{"type":16,"tag":151,"props":20909,"children":20910},{},[20911],{"type":22,"value":20912},"1024",{"type":16,"tag":157,"props":20914,"children":20915},{"encoding":159},[20916],{"type":22,"value":20917},"d_{\\mathrm{model}} = 1024",{"type":16,"tag":103,"props":20919,"children":20921},{"className":20920,"ariaHidden":167},[166],[20922,21008],{"type":16,"tag":103,"props":20923,"children":20925},{"className":20924},[172],[20926,20930,20995,20999,21004],{"type":16,"tag":103,"props":20927,"children":20929},{"className":20928,"style":361},[177],[],{"type":16,"tag":103,"props":20931,"children":20933},{"className":20932},[183],[20934,20939],{"type":16,"tag":103,"props":20935,"children":20937},{"className":20936},[183,188],[20938],{"type":22,"value":1650},{"type":16,"tag":103,"props":20940,"children":20942},{"className":20941},[194],[20943],{"type":16,"tag":103,"props":20944,"children":20946},{"className":20945},[199,200],[20947,20984],{"type":16,"tag":103,"props":20948,"children":20950},{"className":20949},[205],[20951,20979],{"type":16,"tag":103,"props":20952,"children":20954},{"className":20953,"style":1727},[210],[20955],{"type":16,"tag":103,"props":20956,"children":20957},{"style":215},[20958,20962],{"type":16,"tag":103,"props":20959,"children":20961},{"className":20960,"style":221},[220],[],{"type":16,"tag":103,"props":20963,"children":20965},{"className":20964},[226,227,228,229],[20966],{"type":16,"tag":103,"props":20967,"children":20969},{"className":20968},[183,229],[20970],{"type":16,"tag":103,"props":20971,"children":20973},{"className":20972},[183,229],[20974],{"type":16,"tag":103,"props":20975,"children":20977},{"className":20976},[183,1481,229],[20978],{"type":22,"value":1753},{"type":16,"tag":103,"props":20980,"children":20982},{"className":20981},[254],[20983],{"type":22,"value":257},{"type":16,"tag":103,"props":20985,"children":20987},{"className":20986},[205],[20988],{"type":16,"tag":103,"props":20989,"children":20991},{"className":20990,"style":417},[210],[20992],{"type":16,"tag":103,"props":20993,"children":20994},{},[],{"type":16,"tag":103,"props":20996,"children":20998},{"className":20997,"style":816},[632],[],{"type":16,"tag":103,"props":21000,"children":21002},{"className":21001},[821],[21003],{"type":22,"value":746},{"type":16,"tag":103,"props":21005,"children":21007},{"className":21006,"style":816},[632],[],{"type":16,"tag":103,"props":21009,"children":21011},{"className":21010},[172],[21012,21016],{"type":16,"tag":103,"props":21013,"children":21015},{"className":21014,"style":1375},[177],[],{"type":16,"tag":103,"props":21017,"children":21019},{"className":21018},[183],[21020],{"type":22,"value":20912},{"type":22,"value":19974},{"type":16,"tag":11127,"props":21023,"children":21024},{"align":16459},[21025],{"type":22,"value":1330},{"type":16,"tag":11127,"props":21027,"children":21028},{"align":16459},[21029],{"type":22,"value":20912},{"type":16,"tag":11127,"props":21031,"children":21032},{"align":16459},[21033],{"type":22,"value":8693},{"type":16,"tag":11127,"props":21035,"children":21036},{"align":16459},[21037],{"type":22,"value":7392},{"type":16,"tag":11127,"props":21039,"children":21040},{"align":16459},[21041],{"type":22,"value":19681},{"type":16,"tag":11127,"props":21043,"children":21044},{"align":16459},[21045],{"type":22,"value":19681},{"type":16,"tag":11127,"props":21047,"children":21048},{"align":16459},[21049],{"type":22,"value":15877},{"type":16,"tag":11127,"props":21051,"children":21052},{"align":16459},[21053],{"type":22,"value":15877},{"type":16,"tag":11127,"props":21055,"children":21056},{"align":16459},[21057],{"type":22,"value":19578},{"type":16,"tag":11127,"props":21059,"children":21060},{"align":16459},[21061],{"type":22,"value":21062},"4.66",{"type":16,"tag":11127,"props":21064,"children":21065},{"align":16459},[21066],{"type":22,"value":21067},"26.0",{"type":16,"tag":11127,"props":21069,"children":21070},{"align":16459},[21071],{"type":22,"value":21072},"168",{"type":16,"tag":11095,"props":21074,"children":21075},{},[21076,21232,21236,21240,21244,21248,21252,21256,21260,21264,21268,21273,21277],{"type":16,"tag":11127,"props":21077,"children":21078},{},[21079,21080,21231],{"type":22,"value":20226},{"type":16,"tag":103,"props":21081,"children":21083},{"className":21082},[106],[21084,21128],{"type":16,"tag":103,"props":21085,"children":21087},{"className":21086},[111],[21088],{"type":16,"tag":114,"props":21089,"children":21090},{"xmlns":116},[21091],{"type":16,"tag":119,"props":21092,"children":21093},{},[21094,21123],{"type":16,"tag":123,"props":21095,"children":21096},{},[21097,21115,21119],{"type":16,"tag":127,"props":21098,"children":21099},{},[21100,21104],{"type":16,"tag":131,"props":21101,"children":21102},{},[21103],{"type":22,"value":1650},{"type":16,"tag":123,"props":21105,"children":21106},{},[21107,21111],{"type":16,"tag":131,"props":21108,"children":21109},{"mathvariant":1409},[21110],{"type":22,"value":8680},{"type":16,"tag":131,"props":21112,"children":21113},{"mathvariant":1409},[21114],{"type":22,"value":8680},{"type":16,"tag":145,"props":21116,"children":21117},{},[21118],{"type":22,"value":746},{"type":16,"tag":151,"props":21120,"children":21121},{},[21122],{"type":22,"value":20912},{"type":16,"tag":157,"props":21124,"children":21125},{"encoding":159},[21126],{"type":22,"value":21127},"d_{\\mathrm{ff}} = 1024",{"type":16,"tag":103,"props":21129,"children":21131},{"className":21130,"ariaHidden":167},[166],[21132,21218],{"type":16,"tag":103,"props":21133,"children":21135},{"className":21134},[172],[21136,21140,21205,21209,21214],{"type":16,"tag":103,"props":21137,"children":21139},{"className":21138,"style":361},[177],[],{"type":16,"tag":103,"props":21141,"children":21143},{"className":21142},[183],[21144,21149],{"type":16,"tag":103,"props":21145,"children":21147},{"className":21146},[183,188],[21148],{"type":22,"value":1650},{"type":16,"tag":103,"props":21150,"children":21152},{"className":21151},[194],[21153],{"type":16,"tag":103,"props":21154,"children":21156},{"className":21155},[199,200],[21157,21194],{"type":16,"tag":103,"props":21158,"children":21160},{"className":21159},[205],[21161,21189],{"type":16,"tag":103,"props":21162,"children":21164},{"className":21163,"style":1727},[210],[21165],{"type":16,"tag":103,"props":21166,"children":21167},{"style":215},[21168,21172],{"type":16,"tag":103,"props":21169,"children":21171},{"className":21170,"style":221},[220],[],{"type":16,"tag":103,"props":21173,"children":21175},{"className":21174},[226,227,228,229],[21176],{"type":16,"tag":103,"props":21177,"children":21179},{"className":21178},[183,229],[21180],{"type":16,"tag":103,"props":21181,"children":21183},{"className":21182},[183,229],[21184],{"type":16,"tag":103,"props":21185,"children":21187},{"className":21186,"style":8758},[183,1481,229],[21188],{"type":22,"value":8761},{"type":16,"tag":103,"props":21190,"children":21192},{"className":21191},[254],[21193],{"type":22,"value":257},{"type":16,"tag":103,"props":21195,"children":21197},{"className":21196},[205],[21198],{"type":16,"tag":103,"props":21199,"children":21201},{"className":21200,"style":417},[210],[21202],{"type":16,"tag":103,"props":21203,"children":21204},{},[],{"type":16,"tag":103,"props":21206,"children":21208},{"className":21207,"style":816},[632],[],{"type":16,"tag":103,"props":21210,"children":21212},{"className":21211},[821],[21213],{"type":22,"value":746},{"type":16,"tag":103,"props":21215,"children":21217},{"className":21216,"style":816},[632],[],{"type":16,"tag":103,"props":21219,"children":21221},{"className":21220},[172],[21222,21226],{"type":16,"tag":103,"props":21223,"children":21225},{"className":21224,"style":1375},[177],[],{"type":16,"tag":103,"props":21227,"children":21229},{"className":21228},[183],[21230],{"type":22,"value":20912},{"type":22,"value":19974},{"type":16,"tag":11127,"props":21233,"children":21234},{"align":16459},[21235],{"type":22,"value":1330},{"type":16,"tag":11127,"props":21237,"children":21238},{"align":16459},[21239],{"type":22,"value":1685},{"type":16,"tag":11127,"props":21241,"children":21242},{"align":16459},[21243],{"type":22,"value":20912},{"type":16,"tag":11127,"props":21245,"children":21246},{"align":16459},[21247],{"type":22,"value":7392},{"type":16,"tag":11127,"props":21249,"children":21250},{"align":16459},[21251],{"type":22,"value":7536},{"type":16,"tag":11127,"props":21253,"children":21254},{"align":16459},[21255],{"type":22,"value":7536},{"type":16,"tag":11127,"props":21257,"children":21258},{"align":16459},[21259],{"type":22,"value":15877},{"type":16,"tag":11127,"props":21261,"children":21262},{"align":16459},[21263],{"type":22,"value":15877},{"type":16,"tag":11127,"props":21265,"children":21266},{"align":16459},[21267],{"type":22,"value":19578},{"type":16,"tag":11127,"props":21269,"children":21270},{"align":16459},[21271],{"type":22,"value":21272},"5.12",{"type":16,"tag":11127,"props":21274,"children":21275},{"align":16459},[21276],{"type":22,"value":19824},{"type":16,"tag":11127,"props":21278,"children":21279},{"align":16459},[21280],{"type":22,"value":21281},"53",{"type":16,"tag":11095,"props":21283,"children":21284},{},[21285,21442,21446,21450,21454,21458,21462,21466,21470,21474,21478,21483,21488],{"type":16,"tag":11127,"props":21286,"children":21287},{},[21288,21289,21441],{"type":22,"value":20226},{"type":16,"tag":103,"props":21290,"children":21292},{"className":21291},[106],[21293,21338],{"type":16,"tag":103,"props":21294,"children":21296},{"className":21295},[111],[21297],{"type":16,"tag":114,"props":21298,"children":21299},{"xmlns":116},[21300],{"type":16,"tag":119,"props":21301,"children":21302},{},[21303,21333],{"type":16,"tag":123,"props":21304,"children":21305},{},[21306,21324,21328],{"type":16,"tag":127,"props":21307,"children":21308},{},[21309,21313],{"type":16,"tag":131,"props":21310,"children":21311},{},[21312],{"type":22,"value":1650},{"type":16,"tag":123,"props":21314,"children":21315},{},[21316,21320],{"type":16,"tag":131,"props":21317,"children":21318},{"mathvariant":1409},[21319],{"type":22,"value":8680},{"type":16,"tag":131,"props":21321,"children":21322},{"mathvariant":1409},[21323],{"type":22,"value":8680},{"type":16,"tag":145,"props":21325,"children":21326},{},[21327],{"type":22,"value":746},{"type":16,"tag":151,"props":21329,"children":21330},{},[21331],{"type":22,"value":21332},"4096",{"type":16,"tag":157,"props":21334,"children":21335},{"encoding":159},[21336],{"type":22,"value":21337},"d_{\\mathrm{ff}} = 4096",{"type":16,"tag":103,"props":21339,"children":21341},{"className":21340,"ariaHidden":167},[166],[21342,21428],{"type":16,"tag":103,"props":21343,"children":21345},{"className":21344},[172],[21346,21350,21415,21419,21424],{"type":16,"tag":103,"props":21347,"children":21349},{"className":21348,"style":361},[177],[],{"type":16,"tag":103,"props":21351,"children":21353},{"className":21352},[183],[21354,21359],{"type":16,"tag":103,"props":21355,"children":21357},{"className":21356},[183,188],[21358],{"type":22,"value":1650},{"type":16,"tag":103,"props":21360,"children":21362},{"className":21361},[194],[21363],{"type":16,"tag":103,"props":21364,"children":21366},{"className":21365},[199,200],[21367,21404],{"type":16,"tag":103,"props":21368,"children":21370},{"className":21369},[205],[21371,21399],{"type":16,"tag":103,"props":21372,"children":21374},{"className":21373,"style":1727},[210],[21375],{"type":16,"tag":103,"props":21376,"children":21377},{"style":215},[21378,21382],{"type":16,"tag":103,"props":21379,"children":21381},{"className":21380,"style":221},[220],[],{"type":16,"tag":103,"props":21383,"children":21385},{"className":21384},[226,227,228,229],[21386],{"type":16,"tag":103,"props":21387,"children":21389},{"className":21388},[183,229],[21390],{"type":16,"tag":103,"props":21391,"children":21393},{"className":21392},[183,229],[21394],{"type":16,"tag":103,"props":21395,"children":21397},{"className":21396,"style":8758},[183,1481,229],[21398],{"type":22,"value":8761},{"type":16,"tag":103,"props":21400,"children":21402},{"className":21401},[254],[21403],{"type":22,"value":257},{"type":16,"tag":103,"props":21405,"children":21407},{"className":21406},[205],[21408],{"type":16,"tag":103,"props":21409,"children":21411},{"className":21410,"style":417},[210],[21412],{"type":16,"tag":103,"props":21413,"children":21414},{},[],{"type":16,"tag":103,"props":21416,"children":21418},{"className":21417,"style":816},[632],[],{"type":16,"tag":103,"props":21420,"children":21422},{"className":21421},[821],[21423],{"type":22,"value":746},{"type":16,"tag":103,"props":21425,"children":21427},{"className":21426,"style":816},[632],[],{"type":16,"tag":103,"props":21429,"children":21431},{"className":21430},[172],[21432,21436],{"type":16,"tag":103,"props":21433,"children":21435},{"className":21434,"style":1375},[177],[],{"type":16,"tag":103,"props":21437,"children":21439},{"className":21438},[183],[21440],{"type":22,"value":21332},{"type":22,"value":19974},{"type":16,"tag":11127,"props":21443,"children":21444},{"align":16459},[21445],{"type":22,"value":1330},{"type":16,"tag":11127,"props":21447,"children":21448},{"align":16459},[21449],{"type":22,"value":1685},{"type":16,"tag":11127,"props":21451,"children":21452},{"align":16459},[21453],{"type":22,"value":21332},{"type":16,"tag":11127,"props":21455,"children":21456},{"align":16459},[21457],{"type":22,"value":7392},{"type":16,"tag":11127,"props":21459,"children":21460},{"align":16459},[21461],{"type":22,"value":7536},{"type":16,"tag":11127,"props":21463,"children":21464},{"align":16459},[21465],{"type":22,"value":7536},{"type":16,"tag":11127,"props":21467,"children":21468},{"align":16459},[21469],{"type":22,"value":15877},{"type":16,"tag":11127,"props":21471,"children":21472},{"align":16459},[21473],{"type":22,"value":15877},{"type":16,"tag":11127,"props":21475,"children":21476},{"align":16459},[21477],{"type":22,"value":19578},{"type":16,"tag":11127,"props":21479,"children":21480},{"align":16459},[21481],{"type":22,"value":21482},"4.75",{"type":16,"tag":11127,"props":21484,"children":21485},{"align":16459},[21486],{"type":22,"value":21487},"26.2",{"type":16,"tag":11127,"props":21489,"children":21490},{"align":16459},[21491],{"type":22,"value":21492},"90",{"type":16,"tag":11095,"props":21494,"children":21495},{},[21496,21501,21505,21509,21513,21517,21521,21525,21530,21534,21538,21543,21547],{"type":16,"tag":11127,"props":21497,"children":21498},{},[21499],{"type":22,"value":21500},"D（dropout=0.0）",{"type":16,"tag":11127,"props":21502,"children":21503},{"align":16459},[21504],{"type":22,"value":1330},{"type":16,"tag":11127,"props":21506,"children":21507},{"align":16459},[21508],{"type":22,"value":1685},{"type":16,"tag":11127,"props":21510,"children":21511},{"align":16459},[21512],{"type":22,"value":8693},{"type":16,"tag":11127,"props":21514,"children":21515},{"align":16459},[21516],{"type":22,"value":7392},{"type":16,"tag":11127,"props":21518,"children":21519},{"align":16459},[21520],{"type":22,"value":7536},{"type":16,"tag":11127,"props":21522,"children":21523},{"align":16459},[21524],{"type":22,"value":7536},{"type":16,"tag":11127,"props":21526,"children":21527},{"align":16459},[21528],{"type":22,"value":21529},"0.0",{"type":16,"tag":11127,"props":21531,"children":21532},{"align":16459},[21533],{"type":22,"value":15877},{"type":16,"tag":11127,"props":21535,"children":21536},{"align":16459},[21537],{"type":22,"value":19578},{"type":16,"tag":11127,"props":21539,"children":21540},{"align":16459},[21541],{"type":22,"value":21542},"5.77",{"type":16,"tag":11127,"props":21544,"children":21545},{"align":16459},[21546],{"type":22,"value":16674},{"type":16,"tag":11127,"props":21548,"children":21549},{"align":16459},[21550],{"type":22,"value":16498},{"type":16,"tag":11095,"props":21552,"children":21553},{},[21554,21559,21563,21567,21571,21575,21579,21583,21588,21592,21596,21601,21605],{"type":16,"tag":11127,"props":21555,"children":21556},{},[21557],{"type":22,"value":21558},"D（dropout=0.2）",{"type":16,"tag":11127,"props":21560,"children":21561},{"align":16459},[21562],{"type":22,"value":1330},{"type":16,"tag":11127,"props":21564,"children":21565},{"align":16459},[21566],{"type":22,"value":1685},{"type":16,"tag":11127,"props":21568,"children":21569},{"align":16459},[21570],{"type":22,"value":8693},{"type":16,"tag":11127,"props":21572,"children":21573},{"align":16459},[21574],{"type":22,"value":7392},{"type":16,"tag":11127,"props":21576,"children":21577},{"align":16459},[21578],{"type":22,"value":7536},{"type":16,"tag":11127,"props":21580,"children":21581},{"align":16459},[21582],{"type":22,"value":7536},{"type":16,"tag":11127,"props":21584,"children":21585},{"align":16459},[21586],{"type":22,"value":21587},"0.2",{"type":16,"tag":11127,"props":21589,"children":21590},{"align":16459},[21591],{"type":22,"value":15877},{"type":16,"tag":11127,"props":21593,"children":21594},{"align":16459},[21595],{"type":22,"value":19578},{"type":16,"tag":11127,"props":21597,"children":21598},{"align":16459},[21599],{"type":22,"value":21600},"4.95",{"type":16,"tag":11127,"props":21602,"children":21603},{"align":16459},[21604],{"type":22,"value":19707},{"type":16,"tag":11127,"props":21606,"children":21607},{"align":16459},[21608],{"type":22,"value":16498},{"type":16,"tag":11095,"props":21610,"children":21611},{},[21612,21617,21621,21625,21629,21633,21637,21641,21645,21649,21653,21658,21662],{"type":16,"tag":11127,"props":21613,"children":21614},{},[21615],{"type":22,"value":21616},"D（标签平滑=0.0）",{"type":16,"tag":11127,"props":21618,"children":21619},{"align":16459},[21620],{"type":22,"value":1330},{"type":16,"tag":11127,"props":21622,"children":21623},{"align":16459},[21624],{"type":22,"value":1685},{"type":16,"tag":11127,"props":21626,"children":21627},{"align":16459},[21628],{"type":22,"value":8693},{"type":16,"tag":11127,"props":21630,"children":21631},{"align":16459},[21632],{"type":22,"value":7392},{"type":16,"tag":11127,"props":21634,"children":21635},{"align":16459},[21636],{"type":22,"value":7536},{"type":16,"tag":11127,"props":21638,"children":21639},{"align":16459},[21640],{"type":22,"value":7536},{"type":16,"tag":11127,"props":21642,"children":21643},{"align":16459},[21644],{"type":22,"value":15877},{"type":16,"tag":11127,"props":21646,"children":21647},{"align":16459},[21648],{"type":22,"value":21529},{"type":16,"tag":11127,"props":21650,"children":21651},{"align":16459},[21652],{"type":22,"value":19578},{"type":16,"tag":11127,"props":21654,"children":21655},{"align":16459},[21656],{"type":22,"value":21657},"4.67",{"type":16,"tag":11127,"props":21659,"children":21660},{"align":16459},[21661],{"type":22,"value":20486},{"type":16,"tag":11127,"props":21663,"children":21664},{"align":16459},[21665],{"type":22,"value":16498},{"type":16,"tag":11095,"props":21667,"children":21668},{},[21669,21674,21678,21682,21686,21690,21694,21698,21702,21706,21710,21715,21720],{"type":16,"tag":11127,"props":21670,"children":21671},{},[21672],{"type":22,"value":21673},"D（标签平滑=0.2）",{"type":16,"tag":11127,"props":21675,"children":21676},{"align":16459},[21677],{"type":22,"value":1330},{"type":16,"tag":11127,"props":21679,"children":21680},{"align":16459},[21681],{"type":22,"value":1685},{"type":16,"tag":11127,"props":21683,"children":21684},{"align":16459},[21685],{"type":22,"value":8693},{"type":16,"tag":11127,"props":21687,"children":21688},{"align":16459},[21689],{"type":22,"value":7392},{"type":16,"tag":11127,"props":21691,"children":21692},{"align":16459},[21693],{"type":22,"value":7536},{"type":16,"tag":11127,"props":21695,"children":21696},{"align":16459},[21697],{"type":22,"value":7536},{"type":16,"tag":11127,"props":21699,"children":21700},{"align":16459},[21701],{"type":22,"value":15877},{"type":16,"tag":11127,"props":21703,"children":21704},{"align":16459},[21705],{"type":22,"value":21587},{"type":16,"tag":11127,"props":21707,"children":21708},{"align":16459},[21709],{"type":22,"value":19578},{"type":16,"tag":11127,"props":21711,"children":21712},{"align":16459},[21713],{"type":22,"value":21714},"5.47",{"type":16,"tag":11127,"props":21716,"children":21717},{"align":16459},[21718],{"type":22,"value":21719},"25.7",{"type":16,"tag":11127,"props":21721,"children":21722},{"align":16459},[21723],{"type":22,"value":16498},{"type":16,"tag":11095,"props":21725,"children":21726},{},[21727,21732,21736,21740,21744,21748,21752,21756,21760,21764,21768,21772,21776],{"type":16,"tag":11127,"props":21728,"children":21729},{},[21730],{"type":22,"value":21731},"E（可学习位置嵌入）",{"type":16,"tag":11127,"props":21733,"children":21734},{"align":16459},[21735],{"type":22,"value":1330},{"type":16,"tag":11127,"props":21737,"children":21738},{"align":16459},[21739],{"type":22,"value":1685},{"type":16,"tag":11127,"props":21741,"children":21742},{"align":16459},[21743],{"type":22,"value":8693},{"type":16,"tag":11127,"props":21745,"children":21746},{"align":16459},[21747],{"type":22,"value":7392},{"type":16,"tag":11127,"props":21749,"children":21750},{"align":16459},[21751],{"type":22,"value":7536},{"type":16,"tag":11127,"props":21753,"children":21754},{"align":16459},[21755],{"type":22,"value":7536},{"type":16,"tag":11127,"props":21757,"children":21758},{"align":16459},[21759],{"type":22,"value":15877},{"type":16,"tag":11127,"props":21761,"children":21762},{"align":16459},[21763],{"type":22,"value":15877},{"type":16,"tag":11127,"props":21765,"children":21766},{"align":16459},[21767],{"type":22,"value":19578},{"type":16,"tag":11127,"props":21769,"children":21770},{"align":16459},[21771],{"type":22,"value":19583},{"type":16,"tag":11127,"props":21773,"children":21774},{"align":16459},[21775],{"type":22,"value":21719},{"type":16,"tag":11127,"props":21777,"children":21778},{"align":16459},[21779],{"type":22,"value":16498},{"type":16,"tag":11095,"props":21781,"children":21782},{},[21783,21788,21792,21796,21800,21804,21808,21812,21817,21821,21826,21831,21836],{"type":16,"tag":11127,"props":21784,"children":21785},{},[21786],{"type":22,"value":21787},"big",{"type":16,"tag":11127,"props":21789,"children":21790},{"align":16459},[21791],{"type":22,"value":1330},{"type":16,"tag":11127,"props":21793,"children":21794},{"align":16459},[21795],{"type":22,"value":20912},{"type":16,"tag":11127,"props":21797,"children":21798},{"align":16459},[21799],{"type":22,"value":21332},{"type":16,"tag":11127,"props":21801,"children":21802},{"align":16459},[21803],{"type":22,"value":19736},{"type":16,"tag":11127,"props":21805,"children":21806},{"align":16459},[21807],{"type":22,"value":7536},{"type":16,"tag":11127,"props":21809,"children":21810},{"align":16459},[21811],{"type":22,"value":7536},{"type":16,"tag":11127,"props":21813,"children":21814},{"align":16459},[21815],{"type":22,"value":21816},"0.3",{"type":16,"tag":11127,"props":21818,"children":21819},{"align":16459},[21820],{"type":22,"value":15877},{"type":16,"tag":11127,"props":21822,"children":21823},{"align":16459},[21824],{"type":22,"value":21825},"300K",{"type":16,"tag":11127,"props":21827,"children":21828},{"align":16459},[21829],{"type":22,"value":21830},"4.33",{"type":16,"tag":11127,"props":21832,"children":21833},{"align":16459},[21834],{"type":22,"value":21835},"26.4",{"type":16,"tag":11127,"props":21837,"children":21838},{"align":16459},[21839],{"type":22,"value":21840},"213",{"type":16,"tag":66,"props":21842,"children":21844},{"id":21843},"_7-结论",[21845],{"type":22,"value":21846},"7 结论",{"type":16,"tag":25,"props":21848,"children":21849},{},[21850],{"type":22,"value":21851},"本文提出 Transformer，这是第一个完全建立在注意力机制上的序列转换模型，用多头自注意力替代了编码器-解码器架构中最常使用的循环层。",{"type":16,"tag":25,"props":21853,"children":21854},{},[21855],{"type":22,"value":21856},"在翻译任务上，Transformer 的训练速度显著快于基于循环层或卷积层的架构。在 WMT 2014 英德和英法翻译任务上，我们都取得了新的最佳结果。在英德任务上，我们的最佳模型甚至超过了此前报告的所有集成模型。",{"type":16,"tag":25,"props":21858,"children":21859},{},[21860],{"type":22,"value":21861},"我们对基于注意力的模型前景感到兴奋，并计划把它们应用到其他任务。我们计划将 Transformer 扩展到文本之外的输入和输出模态，并研究局部、受限的注意力机制，以高效处理图像、音频和视频等大规模输入输出。让生成过程变得更少依赖顺序，也是我们的另一个研究目标。",{"type":16,"tag":25,"props":21863,"children":21864},{},[21865,21867,21874],{"type":22,"value":21866},"训练和评估模型所用的代码地址为：",{"type":16,"tag":14791,"props":21868,"children":21872},{"href":21869,"rel":21870},"https:\u002F\u002Fgithub.com\u002Ftensorflow\u002Ftensor2tensor",[21871],"nofollow",[21873],{"type":22,"value":21869},{"type":22,"value":1266},{"type":16,"tag":66,"props":21876,"children":21878},{"id":21877},"致谢",[21879],{"type":22,"value":21877},{"type":16,"tag":25,"props":21881,"children":21882},{},[21883],{"type":22,"value":21884},"我们感谢 Nal Kalchbrenner 和 Stephan Gouws 提供富有帮助的评论、修正和启发。",{"type":16,"tag":66,"props":21886,"children":21888},{"id":21887},"参考文献",[21889],{"type":22,"value":21887},{"type":16,"tag":25,"props":21891,"children":21892},{},[21893],{"type":22,"value":21894},"以下条目保留原文书目信息，便于检索：",{"type":16,"tag":25,"props":21896,"children":21897},{},[21898],{"type":22,"value":21899},"[1] Jimmy Lei Ba, Jamie Ryan Kiros, and Geoffrey E. Hinton. Layer normalization. arXiv preprint arXiv:1607.06450, 2016.",{"type":16,"tag":25,"props":21901,"children":21902},{},[21903],{"type":22,"value":21904},"[2] Dzmitry Bahdanau, Kyunghyun Cho, and Yoshua Bengio. Neural machine translation by jointly learning to align and translate. CoRR, abs\u002F1409.0473, 2014.",{"type":16,"tag":25,"props":21906,"children":21907},{},[21908],{"type":22,"value":21909},"[3] Denny Britz, Anna Goldie, Minh-Thang Luong, and Quoc V. Le. Massive exploration of neural machine translation architectures. CoRR, abs\u002F1703.03906, 2017.",{"type":16,"tag":25,"props":21911,"children":21912},{},[21913],{"type":22,"value":21914},"[4] Jianpeng Cheng, Li Dong, and Mirella Lapata. Long short-term memory-networks for machine reading. arXiv preprint arXiv:1601.06733, 2016.",{"type":16,"tag":25,"props":21916,"children":21917},{},[21918],{"type":22,"value":21919},"[5] Kyunghyun Cho, Bart van Merrienboer, Caglar Gulcehre, Fethi Bougares, Holger Schwenk, and Yoshua Bengio. Learning phrase representations using RNN encoder-decoder for statistical machine translation. CoRR, abs\u002F1406.1078, 2014.",{"type":16,"tag":25,"props":21921,"children":21922},{},[21923],{"type":22,"value":21924},"[6] Francois Chollet. Xception: Deep learning with depthwise separable convolutions. arXiv preprint arXiv:1610.02357, 2016.",{"type":16,"tag":25,"props":21926,"children":21927},{},[21928],{"type":22,"value":21929},"[7] Junyoung Chung, Çaglar Gülçehre, Kyunghyun Cho, and Yoshua Bengio. Empirical evaluation of gated recurrent neural networks on sequence modeling. CoRR, abs\u002F1412.3555, 2014.",{"type":16,"tag":25,"props":21931,"children":21932},{},[21933],{"type":22,"value":21934},"[8] Jonas Gehring, Michael Auli, David Grangier, Denis Yarats, and Yann N. Dauphin. Convolutional sequence to sequence learning. arXiv preprint arXiv:1705.03122v2, 2017.",{"type":16,"tag":25,"props":21936,"children":21937},{},[21938],{"type":22,"value":21939},"[9] Alex Graves. Generating sequences with recurrent neural networks. arXiv preprint arXiv:1308.0850, 2013.",{"type":16,"tag":25,"props":21941,"children":21942},{},[21943],{"type":22,"value":21944},"[10] Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun. Deep residual learning for image recognition. In Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pages 770–778, 2016.",{"type":16,"tag":25,"props":21946,"children":21947},{},[21948],{"type":22,"value":21949},"[11] Sepp Hochreiter, Yoshua Bengio, Paolo Frasconi, and Jürgen Schmidhuber. Gradient flow in recurrent nets: the difficulty of learning long-term dependencies, 2001.",{"type":16,"tag":25,"props":21951,"children":21952},{},[21953],{"type":22,"value":21954},"[12] Sepp Hochreiter and Jürgen Schmidhuber. Long short-term memory. Neural Computation, 9(8):1735–1780, 1997.",{"type":16,"tag":25,"props":21956,"children":21957},{},[21958],{"type":22,"value":21959},"[13] Rafal Jozefowicz, Oriol Vinyals, Mike Schuster, Noam Shazeer, and Yonghui Wu. Exploring the limits of language modeling. arXiv preprint arXiv:1602.02410, 2016.",{"type":16,"tag":25,"props":21961,"children":21962},{},[21963],{"type":22,"value":21964},"[14] Łukasz Kaiser and Ilya Sutskever. Neural GPUs learn algorithms. In International Conference on Learning Representations (ICLR), 2016.",{"type":16,"tag":25,"props":21966,"children":21967},{},[21968],{"type":22,"value":21969},"[15] Nal Kalchbrenner, Lasse Espeholt, Karen Simonyan, Aaron van den Oord, Alex Graves, and Koray Kavukcuoglu. Neural machine translation in linear time. arXiv preprint arXiv:1610.10099v2, 2017.",{"type":16,"tag":25,"props":21971,"children":21972},{},[21973],{"type":22,"value":21974},"[16] Yoon Kim, Carl Denton, Luong Hoang, and Alexander M. Rush. Structured attention networks. In International Conference on Learning Representations, 2017.",{"type":16,"tag":25,"props":21976,"children":21977},{},[21978],{"type":22,"value":21979},"[17] Diederik Kingma and Jimmy Ba. Adam: A method for stochastic optimization. In ICLR, 2015.",{"type":16,"tag":25,"props":21981,"children":21982},{},[21983],{"type":22,"value":21984},"[18] Oleksii Kuchaiev and Boris Ginsburg. Factorization tricks for LSTM networks. arXiv preprint arXiv:1703.10722, 2017.",{"type":16,"tag":25,"props":21986,"children":21987},{},[21988],{"type":22,"value":21989},"[19] Zhouhan Lin, Minwei Feng, Cicero Nogueira dos Santos, Mo Yu, Bing Xiang, Bowen Zhou, and Yoshua Bengio. A structured self-attentive sentence embedding. arXiv preprint arXiv:1703.03130, 2017.",{"type":16,"tag":25,"props":21991,"children":21992},{},[21993],{"type":22,"value":21994},"[20] Samy Bengio and Łukasz Kaiser. Can active memory replace attention? In Advances in Neural Information Processing Systems (NIPS), 2016.",{"type":16,"tag":25,"props":21996,"children":21997},{},[21998],{"type":22,"value":21999},"[21] Minh-Thang Luong, Hieu Pham, and Christopher D. Manning. Effective approaches to attention-based neural machine translation. arXiv preprint arXiv:1508.04025, 2015.",{"type":16,"tag":25,"props":22001,"children":22002},{},[22003],{"type":22,"value":22004},"[22] Ankur Parikh, Oscar Täckström, Dipanjan Das, and Jakob Uszkoreit. A decomposable attention model. In Empirical Methods in Natural Language Processing, 2016.",{"type":16,"tag":25,"props":22006,"children":22007},{},[22008],{"type":22,"value":22009},"[23] Romain Paulus, Caiming Xiong, and Richard Socher. A deep reinforced model for abstractive summarization. arXiv preprint arXiv:1705.04304, 2017.",{"type":16,"tag":25,"props":22011,"children":22012},{},[22013],{"type":22,"value":22014},"[24] Ofir Press and Lior Wolf. Using the output embedding to improve language models. arXiv preprint arXiv:1608.05859, 2016.",{"type":16,"tag":25,"props":22016,"children":22017},{},[22018],{"type":22,"value":22019},"[25] Rico Sennrich, Barry Haddow, and Alexandra Birch. Neural machine translation of rare words with subword units. arXiv preprint arXiv:1508.07909, 2015.",{"type":16,"tag":25,"props":22021,"children":22022},{},[22023],{"type":22,"value":22024},"[26] Noam Shazeer, Azalia Mirhoseini, Krzysztof Maziarz, Andy Davis, Quoc Le, Geoffrey Hinton, and Jeff Dean. Outrageously large neural networks: The sparsely-gated mixture-of-experts layer. arXiv preprint arXiv:1701.06538, 2017.",{"type":16,"tag":25,"props":22026,"children":22027},{},[22028],{"type":22,"value":22029},"[27] Nitish Srivastava, Geoffrey E. Hinton, Alex Krizhevsky, Ilya Sutskever, and Ruslan Salakhutdinov. Dropout: a simple way to prevent neural networks from overfitting. Journal of Machine Learning Research, 15(1):1929–1958, 2014.",{"type":16,"tag":25,"props":22031,"children":22032},{},[22033],{"type":22,"value":22034},"[28] Sainbayar Sukhbaatar, Arthur Szlam, Jason Weston, and Rob Fergus. End-to-end memory networks. In Advances in Neural Information Processing Systems 28, pages 2440–2448, 2015.",{"type":16,"tag":25,"props":22036,"children":22037},{},[22038],{"type":22,"value":22039},"[29] Ilya Sutskever, Oriol Vinyals, and Quoc V. V. Le. Sequence to sequence learning with neural networks. In Advances in Neural Information Processing Systems, pages 3104–3112, 2014.",{"type":16,"tag":25,"props":22041,"children":22042},{},[22043],{"type":22,"value":22044},"[30] Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens, and Zbigniew Wojna. Rethinking the inception architecture for computer vision. CoRR, abs\u002F1512.00567, 2015.",{"type":16,"tag":25,"props":22046,"children":22047},{},[22048],{"type":22,"value":22049},"[31] Yonghui Wu, Mike Schuster, Zhifeng Chen, Quoc V. Le, Mohammad Norouzi, Wolfgang Macherey, Maxim Krikun, Yuan Cao, Qin Gao, Klaus Macherey, et al. Google’s neural machine translation system: Bridging the gap between human and machine translation. arXiv preprint arXiv:1609.08144, 2016.",{"type":16,"tag":25,"props":22051,"children":22052},{},[22053],{"type":22,"value":22054},"[32] Jie Zhou, Ying Cao, Xuguang Wang, Peng Li, and Wei Xu. Deep recurrent models with fast-forward connections for neural machine translation. CoRR, abs\u002F1606.04199, 2016.",{"title":22056,"searchDepth":22057,"depth":22058,"links":22059},"",3,2,[22060,22061,22062,22063,22071,22072,22078,22084,22085,22086],{"id":68,"depth":22058,"text":68},{"id":88,"depth":22058,"text":91},{"id":436,"depth":22058,"text":439},{"id":467,"depth":22058,"text":470,"children":22064},[22065,22066,22067,22068,22069,22070],{"id":1286,"depth":22057,"text":1289},{"id":1983,"depth":22057,"text":1986},{"id":7916,"depth":22057,"text":7919},{"id":8807,"depth":22057,"text":8810},{"id":9147,"depth":22057,"text":9150},{"id":10902,"depth":22057,"text":10905},{"id":12483,"depth":22058,"text":12486},{"id":14251,"depth":22058,"text":14254,"children":22073},[22074,22075,22076,22077],{"id":14262,"depth":22057,"text":14265},{"id":14283,"depth":22057,"text":14286},{"id":14299,"depth":22057,"text":14302},{"id":15806,"depth":22057,"text":15809},{"id":16155,"depth":22058,"text":16158,"children":22079},[22080,22081,22082,22083],{"id":16161,"depth":22057,"text":16164},{"id":16439,"depth":22057,"text":16442},{"id":18547,"depth":22057,"text":18550},{"id":18685,"depth":22057,"text":18688},{"id":21843,"depth":22058,"text":21846},{"id":21877,"depth":22058,"text":21877},{"id":21887,"depth":22058,"text":21887},30334,"\u002Fmaterials\u002Fcmu-15-779\u002Ftransformer-paper-zh.md",1795088941212]