From d7e10906113f703eadbc2f8d2889576f93050edf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:28:38 +0200 Subject: [PATCH] chore: sync content to repo (#10398) Co-authored-by: nilbuild <4921183+nilbuild@users.noreply.github.com> --- .../content/ai-native-applications@ZqFQmNizkU6ykTMCrmTWf.md | 3 +++ .../content/arithmetic-intensity@bPO6uwod30-ewwbo8i1eU.md | 3 +++ .../content/asynchronous-inference@59mMKSva9vkoMxHErNXOd.md | 3 +++ .../content/asynchronous-inference@LAD5cttwzdbMuWIv3XjG1.md | 3 +++ .../content/attention-mechanism@2KY3T5JVHQBR8pmAiuq0V.md | 3 +++ .../content/attention-optimization@KWM3V7_15_Aur0He9j2Va.md | 3 +++ .../content/attention-variants@6oTnNEsX_6Cu6lowEz9vg.md | 3 +++ .../content/auto-speech-recognition@6PRICRK6aRQCoThxnevB0.md | 3 +++ .../content/autoscaling@o_gotSHKIPtV5SgGN3Vja.md | 3 +++ .../content/benchmarking--profiling@TxK7B_FF38ms3eKPGug2w.md | 3 +++ .../content/bert-style-models@ef3EneMibOaDQkKcpipGY.md | 3 +++ .../content/blackwell@LihLABERuKBw30G_j48Fj.md | 3 +++ .../content/bottleneck-analysis@Gtd4g4MtB9zFfahRpbE2K.md | 3 +++ .../content/cache-aware-routing@3DR0Qvqu2qUTi9niU_63n.md | 3 +++ .../content/caching-approaches@2nGv7_8PTWYr7uqbudCcX.md | 3 +++ .../content/caching@PzEj5yq1lDQ324RWYpDdQ.md | 3 +++ .../content/capacity-management@xrr5Y1sbN2eyQyuA5LlAW.md | 3 +++ .../content/client-integration@3RkQpvRGWh1Hp_qHlTiXX.md | 3 +++ .../content/client-latency-overhead@gOWGIllIIxhic1NNRfugq.md | 3 +++ .../content/cold-starts@Q3zM9MN5lmHRAUWjdLdXR.md | 3 +++ .../content/compute@o_tknzA4tqcpbU6t5HO-T.md | 3 +++ .../content/consumer-vs-b2b@SFMvVdgTZv_POWFGWWM_s.md | 3 +++ .../content/containerization@--ohScNmF5LR04P8e95AP.md | 3 +++ .../content/continuous-batching@h_PDOMA-V-n7Pab3epuP3.md | 3 +++ .../content/cost-estimation@Gof4kW2LbkNCtH68c7Xvq.md | 3 +++ .../content/cuda@DwJJLFLPHGn9zdccTlLY2.md | 3 +++ .../content/dependency-management@wEaM8VyAo6MFsc3RXQ7Ar.md | 3 +++ .../content/desktop-inference@fQDl7_3nRGa8NFivOIgTP.md | 3 +++ .../content/diffusion-models@aagMwPPmi6ngfkIqZL-Mg.md | 3 +++ .../content/disaggregation@f6PWIqf4LubR7lQztaoB5.md | 3 +++ .../content/distillation@IdNkeL9pzy4lFv17ak1Vy.md | 3 +++ .../content/draft-target-decoding@m0EIHovHWGkWULN0a3C3v.md | 3 +++ .../content/eagle@0FZTg3_ZnTpMOQ078OOxE.md | 3 +++ .../content/embedding-model-inference@GmCkfSeVDJ8T8WLZ7DoOp.md | 3 +++ .../content/embedding-models@Tvt-vX1gWuJZJBHh5vfKu.md | 3 +++ .../content/encoder-decoder-arch@nORbxBOAtl9sNU4sIJgP3.md | 3 +++ .../content/end-to-end-metrics@tENGACXKSKVPmZy9a3D2z.md | 3 +++ .../content/expert-parallelism@62y78y-RXZT_QCUfd-YFE.md | 3 +++ .../content/few-step-image-generation@biVjpCA3bsKI7sa_aBIUE.md | 3 +++ .../content/fine-tuning@fWJplrAuUStEMNNLQc75G.md | 3 +++ .../content/flashattention@mQcj-H3R789o6nDh8tG9k.md | 3 +++ .../content/frameworks--libraries@D4uNOZgG0w-nFL3a89pDM.md | 3 +++ .../content/geo-aware-load-balancing@mzF2mHSj8cvr9mvr16R0N.md | 3 +++ .../content/gpu-architecture@00JLMvn8DUJBCw6-BxO0y.md | 3 +++ .../content/gpu-models@umz3jScWQViO9-lACUU4f.md | 3 +++ .../content/gpu-procurement@4mbGBgTu8AtrFHVYiffkB.md | 3 +++ .../content/grace--vera@kPT86kA0WwE_fWUAAKCTT.md | 3 +++ .../content/hopper@XQMbaW1tFdO8Os6mxcR2M.md | 3 +++ .../content/image--video-bottlenecks@0lRP9cmoS-_wxQSsIcRQg.md | 3 +++ .../content/image--video-generation@U0l_xCUiizbJfjfrlBaJ0.md | 3 +++ .../content/image--video-models@Mebx7HS-I-eNP5i3f6Clf.md | 3 +++ .../content/image-generation-arch@xsV4WevPzhfydyvxVzGcw.md | 3 +++ .../image-generation-inference@lnIHjq4nvCs3zvUln-jHR.md | 3 +++ .../content/indep-component-scaling@SKs0yuBEAyeTPQxLll3EM.md | 3 +++ .../content/inference-engines@01j1KkEB-nIXxP-rENcJ3.md | 3 +++ .../content/inference-vs-training@fVUfQYb8BP_hFMmd1S2-J.md | 3 +++ .../content/instances--interconnects@dAo4PMB_VnRFMSjyDrLj0.md | 3 +++ .../content/inter-token-latency@sre_s0dvpOP0DTM66a3hu.md | 3 +++ .../content/introduction@GL-9aJKTJT9s4maeT37Uf.md | 3 +++ .../content/kernel-selection--fusion@iKW1wqzd5EfRKvNIRshNN.md | 3 +++ .../content/kv-cache-storage@gkt2Gr34RJJf1NZgbmtXf.md | 3 +++ .../content/kv-cache@V678kX8FPChTZCe7Nqwhv.md | 3 +++ .../content/large-language-models@irgGvEnY1_WW1EPtuMKgC.md | 3 +++ .../content/latency-percentiles@8y280d5BPXrDZwtsHZCaX.md | 3 +++ .../content/latency-vs-throughput@kvNIi3aY5rn69_hYkDyUK.md | 3 +++ .../content/llm-based-models@M7K6d-Qw4BpM724LfM1u6.md | 3 +++ .../content/llm-bottlenecks@QVadKzBfLdegat-4wgOMX.md | 3 +++ .../content/local--edge-inference@FilQGz3geyyvHZGyKPW3G.md | 3 +++ .../content/locust@IQuJFfMP1dW0ELlip6LOd.md | 3 +++ .../content/long-context-handling@uL3Fn8XfS1Yv_PJd9R8wi.md | 3 +++ .../content/lora-aware-routing@KdceJEDBdy3TbzG5suE38.md | 3 +++ .../content/measuring-quality-impact@mK_4R8liXtl64iG7ONvgz.md | 3 +++ .../content/memory@5829d5dvXq9qlMSD1CG0N.md | 3 +++ .../content/mixture-of-experts-moe@7BddTfOkWmlL4HAkFBXtq.md | 3 +++ .../content/mobile-inference@kqzHHoiZ9y4tCS6MMlLHQ.md | 3 +++ .../content/model-evaluation@lVf5GoJF_PVTkTiQFTNVg.md | 3 +++ .../content/model-formats--runtimes@lTjWXB8YRwx11e8lxSgNm.md | 3 +++ .../content/model-parallelism@CiXsdHHL0e2DN4ICI5jeX.md | 3 +++ .../content/model-selection@jQAWZ1plw4_wPE1UvE_Ux.md | 3 +++ .../content/modelopt@zhiKpdZm8zOg4e3bm1OgN.md | 3 +++ .../content/multi-cloud@d78UiI_FZyLsU6O52btXg.md | 3 +++ .../content/multi-node-inference@FZx2xsqw_cmYLFS3GMAl-.md | 3 +++ .../content/n-gram@ZOclQMd7JASGXTRyctfW2.md | 3 +++ .../content/neural-networks@p2S0W3EDn768jbqzc7CE7.md | 3 +++ .../content/nims@KB7ExU9X49omOuk9D_Qal.md | 3 +++ .../content/nvidia-dynamo@KhTFecXEkHR4WX-_TYY3g.md | 3 +++ .../content/nvidia-dynamo@ctdMz6rHa4Cwl1x19V_5k.md | 3 +++ .../content/nvidia-genai-perf@zffK61pNUJDwkX7Ukql8y.md | 3 +++ .../content/nvidia@Uq3mE-0WIASHfcBDEnbLj.md | 3 +++ .../content/observability@w-dVv5dFNxD7Rqy16m4uL.md | 3 +++ .../content/omni-modal-models@plw2WiOoGxg64P-Kv-Xm7.md | 3 +++ .../content/online-vs-offline-inf@SkvPauxjmqok4TyNWSACL.md | 3 +++ .../content/onnx-runtime@Vmtbdre3SZzwq3FxOO-Bc.md | 3 +++ .../content/open-vs-closed-models@KAt5FzglOUtH7jLlzDn-I.md | 3 +++ .../content/opsbyte-ratio@uc2enkdtRqA5K7kbgrYJF.md | 3 +++ .../content/other-accelerators@am4Hh-14g6Awk0WcFmVN7.md | 3 +++ .../content/pagedattention@VcIRY4vSi7JkcV6g13vkN.md | 3 +++ .../content/performance-benchmarking@XXfjlqd0CYHwBR5-Kdmmi.md | 3 +++ .../content/pipeline-parallelism@9KLS3gWArBofLEULu3sQc.md | 3 +++ .../post-training-quantization@Zvvs8cB98IRee1okBILsS.md | 3 +++ .../content/prefill--decode-phases@hwKZmfYFiIu6plbiXtJRI.md | 3 +++ .../content/profiling-performance@S2GZw9etrrvEJzV9V-1C_.md | 3 +++ .../content/pytorch@AHVfxQRXmRoiTxC1eeMme.md | 3 +++ .../quantization-aware-training@QCXV4Utq0Ajs1krLM3fhg.md | 3 +++ .../content/quantization@DaSqcLHfnJ4_n9utVVTSk.md | 3 +++ .../content/real-time-factor@0-BRllrMiShLlytidGkST.md | 3 +++ .../content/reasoning-models@Y9YkcfblXHEvS-gdEWwaE.md | 3 +++ .../content/request-queueing@k-zXOGynI0B0yyT-n3raa.md | 3 +++ .../content/roofline-model@XF-wMQCqPZiWv1XmryPBP.md | 3 +++ .../content/routing--load-balancing@DSyTAtFC1TvC1P7yD10Nb.md | 3 +++ .../content/safetensors@jkbrf0_JDljWmVq_o9XE-.md | 3 +++ .../content/scale-to-zero@Pi87dWgu5NTIJRQUc169s.md | 3 +++ .../content/security-and-compliance@4x-4BFZH7bfNteD5ZAwTI.md | 3 +++ .../content/sglang-genai-bench@KHDnY7byKgmaPtC_0uDBM.md | 3 +++ .../content/sglang@YtlAf6qloKMmaSr9Bkkl_.md | 3 +++ .../content/shared-vs-dedicated-inf@2DsjfW9r4dOI3LDQU80qt.md | 3 +++ .../content/sizing--procurement@da0HGfSDr6_DV4ZJcrcIB.md | 3 +++ .../content/speculative-decoding@gMWgd10cxcQG_RG2oSAEH.md | 3 +++ .../content/speech-to-speech-models@moyS740HQoUUovilx1Yyt.md | 3 +++ .../content/speech@Lk8906Mpi3yxiSt8EC_BC.md | 3 +++ .../streaming--protocol-support@SGQCvuHAkg4TcJa_e2f-y.md | 3 +++ .../content/streaming-tts@3VbPNaSnzWZ-pC-lGFMrh.md | 3 +++ .../content/tensor-parallelism@csghcNLDywO8oTIZNXeyP.md | 3 +++ .../content/tensorrt-llm@n2_zIGkQgyYCMYPypz3vb.md | 3 +++ .../content/tensorrt@M4qjJSEAnsjrmP3A_O8HC.md | 3 +++ .../content/testing--deployment@ZlOtQtxJST10nuHjhbUx6.md | 3 +++ .../content/text-to-speech-tts@XnhhN1QVWzlp9DUwkii9-.md | 3 +++ .../content/the-inference-stack@uIhVVlMsykuxJCa02f-i-.md | 3 +++ .../content/time-to-first-token@xCgNhwbS_ELxEyRHD1enO.md | 3 +++ .../content/tokens-per-second@uxm_BIAyhfFmcE94k9i9J.md | 3 +++ .../content/transformer-blocks@xHM7qjigMmXEwxeMEeMWl.md | 3 +++ .../content/transformers--diffusers@H_e3KZ7p7enUCYE5M5Lr7.md | 3 +++ .../video-generation-inference@PqEH1ez5dg5tEsspgfAK1.md | 3 +++ .../content/video-generation-models@yTSrQfJsE-Mf4ZU_hJwYT.md | 3 +++ .../content/video-processing-for-vlms@7pRZu7xa2_43HVwKty9px.md | 3 +++ .../content/vision-language-models@JsjmQrN1V_dM1Kw6UnYLc.md | 3 +++ .../content/vllm@wcAtbab81IJdpFh6L-heP.md | 3 +++ .../content/voice-activity-detection@P0BA_p3xqpxkQsFHqM0AZ.md | 3 +++ .../content/writing-custom-kernels@1dWPVaSQ4TfVU3Obx3m19.md | 3 +++ .../content/zero-downtime-deplo@le4sPQ0h9l-vlmCB4G8ZY.md | 3 +++ 140 files changed, 420 insertions(+) create mode 100644 roadmaps/inference-engineering/content/ai-native-applications@ZqFQmNizkU6ykTMCrmTWf.md create mode 100644 roadmaps/inference-engineering/content/arithmetic-intensity@bPO6uwod30-ewwbo8i1eU.md create mode 100644 roadmaps/inference-engineering/content/asynchronous-inference@59mMKSva9vkoMxHErNXOd.md create mode 100644 roadmaps/inference-engineering/content/asynchronous-inference@LAD5cttwzdbMuWIv3XjG1.md create mode 100644 roadmaps/inference-engineering/content/attention-mechanism@2KY3T5JVHQBR8pmAiuq0V.md create mode 100644 roadmaps/inference-engineering/content/attention-optimization@KWM3V7_15_Aur0He9j2Va.md create mode 100644 roadmaps/inference-engineering/content/attention-variants@6oTnNEsX_6Cu6lowEz9vg.md create mode 100644 roadmaps/inference-engineering/content/auto-speech-recognition@6PRICRK6aRQCoThxnevB0.md create mode 100644 roadmaps/inference-engineering/content/autoscaling@o_gotSHKIPtV5SgGN3Vja.md create mode 100644 roadmaps/inference-engineering/content/benchmarking--profiling@TxK7B_FF38ms3eKPGug2w.md create mode 100644 roadmaps/inference-engineering/content/bert-style-models@ef3EneMibOaDQkKcpipGY.md create mode 100644 roadmaps/inference-engineering/content/blackwell@LihLABERuKBw30G_j48Fj.md create mode 100644 roadmaps/inference-engineering/content/bottleneck-analysis@Gtd4g4MtB9zFfahRpbE2K.md create mode 100644 roadmaps/inference-engineering/content/cache-aware-routing@3DR0Qvqu2qUTi9niU_63n.md create mode 100644 roadmaps/inference-engineering/content/caching-approaches@2nGv7_8PTWYr7uqbudCcX.md create mode 100644 roadmaps/inference-engineering/content/caching@PzEj5yq1lDQ324RWYpDdQ.md create mode 100644 roadmaps/inference-engineering/content/capacity-management@xrr5Y1sbN2eyQyuA5LlAW.md create mode 100644 roadmaps/inference-engineering/content/client-integration@3RkQpvRGWh1Hp_qHlTiXX.md create mode 100644 roadmaps/inference-engineering/content/client-latency-overhead@gOWGIllIIxhic1NNRfugq.md create mode 100644 roadmaps/inference-engineering/content/cold-starts@Q3zM9MN5lmHRAUWjdLdXR.md create mode 100644 roadmaps/inference-engineering/content/compute@o_tknzA4tqcpbU6t5HO-T.md create mode 100644 roadmaps/inference-engineering/content/consumer-vs-b2b@SFMvVdgTZv_POWFGWWM_s.md create mode 100644 roadmaps/inference-engineering/content/containerization@--ohScNmF5LR04P8e95AP.md create mode 100644 roadmaps/inference-engineering/content/continuous-batching@h_PDOMA-V-n7Pab3epuP3.md create mode 100644 roadmaps/inference-engineering/content/cost-estimation@Gof4kW2LbkNCtH68c7Xvq.md create mode 100644 roadmaps/inference-engineering/content/cuda@DwJJLFLPHGn9zdccTlLY2.md create mode 100644 roadmaps/inference-engineering/content/dependency-management@wEaM8VyAo6MFsc3RXQ7Ar.md create mode 100644 roadmaps/inference-engineering/content/desktop-inference@fQDl7_3nRGa8NFivOIgTP.md create mode 100644 roadmaps/inference-engineering/content/diffusion-models@aagMwPPmi6ngfkIqZL-Mg.md create mode 100644 roadmaps/inference-engineering/content/disaggregation@f6PWIqf4LubR7lQztaoB5.md create mode 100644 roadmaps/inference-engineering/content/distillation@IdNkeL9pzy4lFv17ak1Vy.md create mode 100644 roadmaps/inference-engineering/content/draft-target-decoding@m0EIHovHWGkWULN0a3C3v.md create mode 100644 roadmaps/inference-engineering/content/eagle@0FZTg3_ZnTpMOQ078OOxE.md create mode 100644 roadmaps/inference-engineering/content/embedding-model-inference@GmCkfSeVDJ8T8WLZ7DoOp.md create mode 100644 roadmaps/inference-engineering/content/embedding-models@Tvt-vX1gWuJZJBHh5vfKu.md create mode 100644 roadmaps/inference-engineering/content/encoder-decoder-arch@nORbxBOAtl9sNU4sIJgP3.md create mode 100644 roadmaps/inference-engineering/content/end-to-end-metrics@tENGACXKSKVPmZy9a3D2z.md create mode 100644 roadmaps/inference-engineering/content/expert-parallelism@62y78y-RXZT_QCUfd-YFE.md create mode 100644 roadmaps/inference-engineering/content/few-step-image-generation@biVjpCA3bsKI7sa_aBIUE.md create mode 100644 roadmaps/inference-engineering/content/fine-tuning@fWJplrAuUStEMNNLQc75G.md create mode 100644 roadmaps/inference-engineering/content/flashattention@mQcj-H3R789o6nDh8tG9k.md create mode 100644 roadmaps/inference-engineering/content/frameworks--libraries@D4uNOZgG0w-nFL3a89pDM.md create mode 100644 roadmaps/inference-engineering/content/geo-aware-load-balancing@mzF2mHSj8cvr9mvr16R0N.md create mode 100644 roadmaps/inference-engineering/content/gpu-architecture@00JLMvn8DUJBCw6-BxO0y.md create mode 100644 roadmaps/inference-engineering/content/gpu-models@umz3jScWQViO9-lACUU4f.md create mode 100644 roadmaps/inference-engineering/content/gpu-procurement@4mbGBgTu8AtrFHVYiffkB.md create mode 100644 roadmaps/inference-engineering/content/grace--vera@kPT86kA0WwE_fWUAAKCTT.md create mode 100644 roadmaps/inference-engineering/content/hopper@XQMbaW1tFdO8Os6mxcR2M.md create mode 100644 roadmaps/inference-engineering/content/image--video-bottlenecks@0lRP9cmoS-_wxQSsIcRQg.md create mode 100644 roadmaps/inference-engineering/content/image--video-generation@U0l_xCUiizbJfjfrlBaJ0.md create mode 100644 roadmaps/inference-engineering/content/image--video-models@Mebx7HS-I-eNP5i3f6Clf.md create mode 100644 roadmaps/inference-engineering/content/image-generation-arch@xsV4WevPzhfydyvxVzGcw.md create mode 100644 roadmaps/inference-engineering/content/image-generation-inference@lnIHjq4nvCs3zvUln-jHR.md create mode 100644 roadmaps/inference-engineering/content/indep-component-scaling@SKs0yuBEAyeTPQxLll3EM.md create mode 100644 roadmaps/inference-engineering/content/inference-engines@01j1KkEB-nIXxP-rENcJ3.md create mode 100644 roadmaps/inference-engineering/content/inference-vs-training@fVUfQYb8BP_hFMmd1S2-J.md create mode 100644 roadmaps/inference-engineering/content/instances--interconnects@dAo4PMB_VnRFMSjyDrLj0.md create mode 100644 roadmaps/inference-engineering/content/inter-token-latency@sre_s0dvpOP0DTM66a3hu.md create mode 100644 roadmaps/inference-engineering/content/introduction@GL-9aJKTJT9s4maeT37Uf.md create mode 100644 roadmaps/inference-engineering/content/kernel-selection--fusion@iKW1wqzd5EfRKvNIRshNN.md create mode 100644 roadmaps/inference-engineering/content/kv-cache-storage@gkt2Gr34RJJf1NZgbmtXf.md create mode 100644 roadmaps/inference-engineering/content/kv-cache@V678kX8FPChTZCe7Nqwhv.md create mode 100644 roadmaps/inference-engineering/content/large-language-models@irgGvEnY1_WW1EPtuMKgC.md create mode 100644 roadmaps/inference-engineering/content/latency-percentiles@8y280d5BPXrDZwtsHZCaX.md create mode 100644 roadmaps/inference-engineering/content/latency-vs-throughput@kvNIi3aY5rn69_hYkDyUK.md create mode 100644 roadmaps/inference-engineering/content/llm-based-models@M7K6d-Qw4BpM724LfM1u6.md create mode 100644 roadmaps/inference-engineering/content/llm-bottlenecks@QVadKzBfLdegat-4wgOMX.md create mode 100644 roadmaps/inference-engineering/content/local--edge-inference@FilQGz3geyyvHZGyKPW3G.md create mode 100644 roadmaps/inference-engineering/content/locust@IQuJFfMP1dW0ELlip6LOd.md create mode 100644 roadmaps/inference-engineering/content/long-context-handling@uL3Fn8XfS1Yv_PJd9R8wi.md create mode 100644 roadmaps/inference-engineering/content/lora-aware-routing@KdceJEDBdy3TbzG5suE38.md create mode 100644 roadmaps/inference-engineering/content/measuring-quality-impact@mK_4R8liXtl64iG7ONvgz.md create mode 100644 roadmaps/inference-engineering/content/memory@5829d5dvXq9qlMSD1CG0N.md create mode 100644 roadmaps/inference-engineering/content/mixture-of-experts-moe@7BddTfOkWmlL4HAkFBXtq.md create mode 100644 roadmaps/inference-engineering/content/mobile-inference@kqzHHoiZ9y4tCS6MMlLHQ.md create mode 100644 roadmaps/inference-engineering/content/model-evaluation@lVf5GoJF_PVTkTiQFTNVg.md create mode 100644 roadmaps/inference-engineering/content/model-formats--runtimes@lTjWXB8YRwx11e8lxSgNm.md create mode 100644 roadmaps/inference-engineering/content/model-parallelism@CiXsdHHL0e2DN4ICI5jeX.md create mode 100644 roadmaps/inference-engineering/content/model-selection@jQAWZ1plw4_wPE1UvE_Ux.md create mode 100644 roadmaps/inference-engineering/content/modelopt@zhiKpdZm8zOg4e3bm1OgN.md create mode 100644 roadmaps/inference-engineering/content/multi-cloud@d78UiI_FZyLsU6O52btXg.md create mode 100644 roadmaps/inference-engineering/content/multi-node-inference@FZx2xsqw_cmYLFS3GMAl-.md create mode 100644 roadmaps/inference-engineering/content/n-gram@ZOclQMd7JASGXTRyctfW2.md create mode 100644 roadmaps/inference-engineering/content/neural-networks@p2S0W3EDn768jbqzc7CE7.md create mode 100644 roadmaps/inference-engineering/content/nims@KB7ExU9X49omOuk9D_Qal.md create mode 100644 roadmaps/inference-engineering/content/nvidia-dynamo@KhTFecXEkHR4WX-_TYY3g.md create mode 100644 roadmaps/inference-engineering/content/nvidia-dynamo@ctdMz6rHa4Cwl1x19V_5k.md create mode 100644 roadmaps/inference-engineering/content/nvidia-genai-perf@zffK61pNUJDwkX7Ukql8y.md create mode 100644 roadmaps/inference-engineering/content/nvidia@Uq3mE-0WIASHfcBDEnbLj.md create mode 100644 roadmaps/inference-engineering/content/observability@w-dVv5dFNxD7Rqy16m4uL.md create mode 100644 roadmaps/inference-engineering/content/omni-modal-models@plw2WiOoGxg64P-Kv-Xm7.md create mode 100644 roadmaps/inference-engineering/content/online-vs-offline-inf@SkvPauxjmqok4TyNWSACL.md create mode 100644 roadmaps/inference-engineering/content/onnx-runtime@Vmtbdre3SZzwq3FxOO-Bc.md create mode 100644 roadmaps/inference-engineering/content/open-vs-closed-models@KAt5FzglOUtH7jLlzDn-I.md create mode 100644 roadmaps/inference-engineering/content/opsbyte-ratio@uc2enkdtRqA5K7kbgrYJF.md create mode 100644 roadmaps/inference-engineering/content/other-accelerators@am4Hh-14g6Awk0WcFmVN7.md create mode 100644 roadmaps/inference-engineering/content/pagedattention@VcIRY4vSi7JkcV6g13vkN.md create mode 100644 roadmaps/inference-engineering/content/performance-benchmarking@XXfjlqd0CYHwBR5-Kdmmi.md create mode 100644 roadmaps/inference-engineering/content/pipeline-parallelism@9KLS3gWArBofLEULu3sQc.md create mode 100644 roadmaps/inference-engineering/content/post-training-quantization@Zvvs8cB98IRee1okBILsS.md create mode 100644 roadmaps/inference-engineering/content/prefill--decode-phases@hwKZmfYFiIu6plbiXtJRI.md create mode 100644 roadmaps/inference-engineering/content/profiling-performance@S2GZw9etrrvEJzV9V-1C_.md create mode 100644 roadmaps/inference-engineering/content/pytorch@AHVfxQRXmRoiTxC1eeMme.md create mode 100644 roadmaps/inference-engineering/content/quantization-aware-training@QCXV4Utq0Ajs1krLM3fhg.md create mode 100644 roadmaps/inference-engineering/content/quantization@DaSqcLHfnJ4_n9utVVTSk.md create mode 100644 roadmaps/inference-engineering/content/real-time-factor@0-BRllrMiShLlytidGkST.md create mode 100644 roadmaps/inference-engineering/content/reasoning-models@Y9YkcfblXHEvS-gdEWwaE.md create mode 100644 roadmaps/inference-engineering/content/request-queueing@k-zXOGynI0B0yyT-n3raa.md create mode 100644 roadmaps/inference-engineering/content/roofline-model@XF-wMQCqPZiWv1XmryPBP.md create mode 100644 roadmaps/inference-engineering/content/routing--load-balancing@DSyTAtFC1TvC1P7yD10Nb.md create mode 100644 roadmaps/inference-engineering/content/safetensors@jkbrf0_JDljWmVq_o9XE-.md create mode 100644 roadmaps/inference-engineering/content/scale-to-zero@Pi87dWgu5NTIJRQUc169s.md create mode 100644 roadmaps/inference-engineering/content/security-and-compliance@4x-4BFZH7bfNteD5ZAwTI.md create mode 100644 roadmaps/inference-engineering/content/sglang-genai-bench@KHDnY7byKgmaPtC_0uDBM.md create mode 100644 roadmaps/inference-engineering/content/sglang@YtlAf6qloKMmaSr9Bkkl_.md create mode 100644 roadmaps/inference-engineering/content/shared-vs-dedicated-inf@2DsjfW9r4dOI3LDQU80qt.md create mode 100644 roadmaps/inference-engineering/content/sizing--procurement@da0HGfSDr6_DV4ZJcrcIB.md create mode 100644 roadmaps/inference-engineering/content/speculative-decoding@gMWgd10cxcQG_RG2oSAEH.md create mode 100644 roadmaps/inference-engineering/content/speech-to-speech-models@moyS740HQoUUovilx1Yyt.md create mode 100644 roadmaps/inference-engineering/content/speech@Lk8906Mpi3yxiSt8EC_BC.md create mode 100644 roadmaps/inference-engineering/content/streaming--protocol-support@SGQCvuHAkg4TcJa_e2f-y.md create mode 100644 roadmaps/inference-engineering/content/streaming-tts@3VbPNaSnzWZ-pC-lGFMrh.md create mode 100644 roadmaps/inference-engineering/content/tensor-parallelism@csghcNLDywO8oTIZNXeyP.md create mode 100644 roadmaps/inference-engineering/content/tensorrt-llm@n2_zIGkQgyYCMYPypz3vb.md create mode 100644 roadmaps/inference-engineering/content/tensorrt@M4qjJSEAnsjrmP3A_O8HC.md create mode 100644 roadmaps/inference-engineering/content/testing--deployment@ZlOtQtxJST10nuHjhbUx6.md create mode 100644 roadmaps/inference-engineering/content/text-to-speech-tts@XnhhN1QVWzlp9DUwkii9-.md create mode 100644 roadmaps/inference-engineering/content/the-inference-stack@uIhVVlMsykuxJCa02f-i-.md create mode 100644 roadmaps/inference-engineering/content/time-to-first-token@xCgNhwbS_ELxEyRHD1enO.md create mode 100644 roadmaps/inference-engineering/content/tokens-per-second@uxm_BIAyhfFmcE94k9i9J.md create mode 100644 roadmaps/inference-engineering/content/transformer-blocks@xHM7qjigMmXEwxeMEeMWl.md create mode 100644 roadmaps/inference-engineering/content/transformers--diffusers@H_e3KZ7p7enUCYE5M5Lr7.md create mode 100644 roadmaps/inference-engineering/content/video-generation-inference@PqEH1ez5dg5tEsspgfAK1.md create mode 100644 roadmaps/inference-engineering/content/video-generation-models@yTSrQfJsE-Mf4ZU_hJwYT.md create mode 100644 roadmaps/inference-engineering/content/video-processing-for-vlms@7pRZu7xa2_43HVwKty9px.md create mode 100644 roadmaps/inference-engineering/content/vision-language-models@JsjmQrN1V_dM1Kw6UnYLc.md create mode 100644 roadmaps/inference-engineering/content/vllm@wcAtbab81IJdpFh6L-heP.md create mode 100644 roadmaps/inference-engineering/content/voice-activity-detection@P0BA_p3xqpxkQsFHqM0AZ.md create mode 100644 roadmaps/inference-engineering/content/writing-custom-kernels@1dWPVaSQ4TfVU3Obx3m19.md create mode 100644 roadmaps/inference-engineering/content/zero-downtime-deplo@le4sPQ0h9l-vlmCB4G8ZY.md diff --git a/roadmaps/inference-engineering/content/ai-native-applications@ZqFQmNizkU6ykTMCrmTWf.md b/roadmaps/inference-engineering/content/ai-native-applications@ZqFQmNizkU6ykTMCrmTWf.md new file mode 100644 index 000000000..d1fc58ff3 --- /dev/null +++ b/roadmaps/inference-engineering/content/ai-native-applications@ZqFQmNizkU6ykTMCrmTWf.md @@ -0,0 +1,3 @@ +# AI-Native Applications + +AI-native applications are products where generative models power the core user experience, from code editors and voice agents to customer support and media generation. Each category has different inference requirements. Agents generate many inference calls per user action; chat applications prioritize TTFT; voice applications need low end-to-end latency for natural conversation. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/arithmetic-intensity@bPO6uwod30-ewwbo8i1eU.md b/roadmaps/inference-engineering/content/arithmetic-intensity@bPO6uwod30-ewwbo8i1eU.md new file mode 100644 index 000000000..f51a86c1b --- /dev/null +++ b/roadmaps/inference-engineering/content/arithmetic-intensity@bPO6uwod30-ewwbo8i1eU.md @@ -0,0 +1,3 @@ +# Arithmetic Intensity + +Arithmetic intensity is a property of an algorithm: the number of floating-point operations performed per byte of memory read or written. A large matrix multiplication has high arithmetic intensity because the same weights participate in many operations per load; a decode forward pass has low arithmetic intensity because each weight is loaded from VRAM and used once per token. Comparing a workload's arithmetic intensity against the GPU's ops:byte ratio is the fastest way to diagnose whether adding compute or adding bandwidth would help. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/asynchronous-inference@59mMKSva9vkoMxHErNXOd.md b/roadmaps/inference-engineering/content/asynchronous-inference@59mMKSva9vkoMxHErNXOd.md new file mode 100644 index 000000000..76d621229 --- /dev/null +++ b/roadmaps/inference-engineering/content/asynchronous-inference@59mMKSva9vkoMxHErNXOd.md @@ -0,0 +1,3 @@ +# Asynchronous Inference + +Asynchronous inference is a fire-and-forget request model where the client submits a job, receives an immediate acknowledgment, and retrieves the result later via a webhook. It is suited for throughput-sensitive, latency-insensitive workloads like bulk document embedding or corpus transcription. Async jobs have much longer time limits than synchronous requests and require robust server-side queuing. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/asynchronous-inference@LAD5cttwzdbMuWIv3XjG1.md b/roadmaps/inference-engineering/content/asynchronous-inference@LAD5cttwzdbMuWIv3XjG1.md new file mode 100644 index 000000000..76d621229 --- /dev/null +++ b/roadmaps/inference-engineering/content/asynchronous-inference@LAD5cttwzdbMuWIv3XjG1.md @@ -0,0 +1,3 @@ +# Asynchronous Inference + +Asynchronous inference is a fire-and-forget request model where the client submits a job, receives an immediate acknowledgment, and retrieves the result later via a webhook. It is suited for throughput-sensitive, latency-insensitive workloads like bulk document embedding or corpus transcription. Async jobs have much longer time limits than synchronous requests and require robust server-side queuing. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/attention-mechanism@2KY3T5JVHQBR8pmAiuq0V.md b/roadmaps/inference-engineering/content/attention-mechanism@2KY3T5JVHQBR8pmAiuq0V.md new file mode 100644 index 000000000..690ebb8d5 --- /dev/null +++ b/roadmaps/inference-engineering/content/attention-mechanism@2KY3T5JVHQBR8pmAiuq0V.md @@ -0,0 +1,3 @@ +# Attention Mechanism + +Attention is the mechanism transformers use to relate each token to all previous tokens via query, key, and value projections. During decode, each new token attends to every prior token, and the KV cache stores precomputed keys and values to avoid recomputation. Attention is the most expensive and most heavily optimized operation in LLM inference. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/attention-optimization@KWM3V7_15_Aur0He9j2Va.md b/roadmaps/inference-engineering/content/attention-optimization@KWM3V7_15_Aur0He9j2Va.md new file mode 100644 index 000000000..a3b6354aa --- /dev/null +++ b/roadmaps/inference-engineering/content/attention-optimization@KWM3V7_15_Aur0He9j2Va.md @@ -0,0 +1,3 @@ +# Attention Optimization + +Attention optimization refers to techniques designed to improve the computational efficiency and memory usage of the attention mechanism within transformer models. Because standard attention scales quadratically with input sequence length, these optimizations aim to reduce computation and memory overhead. By streamlining how tokens interact with one another, these methods allow models to process longer inputs faster and run on hardware with less available memory. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/attention-variants@6oTnNEsX_6Cu6lowEz9vg.md b/roadmaps/inference-engineering/content/attention-variants@6oTnNEsX_6Cu6lowEz9vg.md new file mode 100644 index 000000000..630466a14 --- /dev/null +++ b/roadmaps/inference-engineering/content/attention-variants@6oTnNEsX_6Cu6lowEz9vg.md @@ -0,0 +1,3 @@ +# Attention Variants + +Newer attention algorithms reduce the quadratic time complexity of standard attention. Sliding window attention limits each token to attending to the nearest w tokens, turning O(N²) into O(Nw). Linear attention approximates the softmax with a linear-time kernel. Compressed attention periodically compresses earlier context. These variants trade off some quality for better scaling on long sequences. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/auto-speech-recognition@6PRICRK6aRQCoThxnevB0.md b/roadmaps/inference-engineering/content/auto-speech-recognition@6PRICRK6aRQCoThxnevB0.md new file mode 100644 index 000000000..f6c51eaea --- /dev/null +++ b/roadmaps/inference-engineering/content/auto-speech-recognition@6PRICRK6aRQCoThxnevB0.md @@ -0,0 +1,3 @@ +# Automatic Speech Recognition (ASR) + +ASR models like Whisper take audio input and produce text transcripts. Whisper is an encoder-decoder model where the decoder, an autoregressive transformer, dominates inference time. TensorRT-LLM with in-flight batching for the decoder delivers the best performance. For real-time transcription, streaming via WebSockets with VAD-segmented chunks achieves round-trip targets of around 200 milliseconds per chunk on optimized deployments. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/autoscaling@o_gotSHKIPtV5SgGN3Vja.md b/roadmaps/inference-engineering/content/autoscaling@o_gotSHKIPtV5SgGN3Vja.md new file mode 100644 index 000000000..94757738d --- /dev/null +++ b/roadmaps/inference-engineering/content/autoscaling@o_gotSHKIPtV5SgGN3Vja.md @@ -0,0 +1,3 @@ +# Autoscaling + +Autoscaling dynamically adjusts the number of active model replicas to match incoming traffic, maintaining latency SLAs while minimizing idle GPU spend. Kubernetes is the standard orchestration layer, scaling replicas within a cluster based on traffic signals and utilization metrics. Key configuration parameters are min replicas, max replicas, the autoscaling window, the scale-down delay, and the concurrency target per replica. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/benchmarking--profiling@TxK7B_FF38ms3eKPGug2w.md b/roadmaps/inference-engineering/content/benchmarking--profiling@TxK7B_FF38ms3eKPGug2w.md new file mode 100644 index 000000000..7e1a02021 --- /dev/null +++ b/roadmaps/inference-engineering/content/benchmarking--profiling@TxK7B_FF38ms3eKPGug2w.md @@ -0,0 +1,3 @@ +# Benchmarking and Profiling + +Benchmarking and profiling are how inference engineers turn intuition into evidence. Benchmarking measures system-level throughput and latency under load; profiling zooms in to identify which specific operations are consuming time. Both are prerequisites for making optimization decisions with confidence. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/bert-style-models@ef3EneMibOaDQkKcpipGY.md b/roadmaps/inference-engineering/content/bert-style-models@ef3EneMibOaDQkKcpipGY.md new file mode 100644 index 000000000..adb912cc8 --- /dev/null +++ b/roadmaps/inference-engineering/content/bert-style-models@ef3EneMibOaDQkKcpipGY.md @@ -0,0 +1,3 @@ +# BERT-style Models + +BERT-style encoder-only models are the traditional architecture for text embeddings, producing a fixed-size vector by pooling the final hidden states of a bidirectional transformer. They are compact, fast, and well-suited for high-throughput indexing workloads where embedding quality is sufficient for the task. Models like BGE-small and all-MiniLM run efficiently on CPU or fractional GPU instances, making them the lowest-cost embedding option. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/blackwell@LihLABERuKBw30G_j48Fj.md b/roadmaps/inference-engineering/content/blackwell@LihLABERuKBw30G_j48Fj.md new file mode 100644 index 000000000..6f1509663 --- /dev/null +++ b/roadmaps/inference-engineering/content/blackwell@LihLABERuKBw30G_j48Fj.md @@ -0,0 +1,3 @@ +# Blackwell + +The Blackwell architecture, released in late 2024 with the B200 and B300, added FP4 Tensor Cores and new microscaling number formats including MXFP8, MXFP4, and NVFP4. B200 GPUs offer approximately 5 petaFLOPS of FP8 compute and up to 8 TB/s of memory bandwidth. They are the current gold standard for demanding inference workloads including large LLMs and video generation. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/bottleneck-analysis@Gtd4g4MtB9zFfahRpbE2K.md b/roadmaps/inference-engineering/content/bottleneck-analysis@Gtd4g4MtB9zFfahRpbE2K.md new file mode 100644 index 000000000..243426626 --- /dev/null +++ b/roadmaps/inference-engineering/content/bottleneck-analysis@Gtd4g4MtB9zFfahRpbE2K.md @@ -0,0 +1,3 @@ +# Bottleneck Analysis + +Bottleneck analysis identifies whether a given inference workload is limited by compute or by memory bandwidth, which determines which optimizations will actually improve performance. The core tool is the ops:byte ratio: comparing a GPU's peak FLOPS against its peak memory bandwidth reveals the hardware's balance point, and comparing a model operation's arithmetic intensity against that balance point reveals whether it is compute-bound or memory-bound. LLM prefill and image generation are compute-bound; LLM decode is memory-bound. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/cache-aware-routing@3DR0Qvqu2qUTi9niU_63n.md b/roadmaps/inference-engineering/content/cache-aware-routing@3DR0Qvqu2qUTi9niU_63n.md new file mode 100644 index 000000000..a472a923e --- /dev/null +++ b/roadmaps/inference-engineering/content/cache-aware-routing@3DR0Qvqu2qUTi9niU_63n.md @@ -0,0 +1,3 @@ +# Cache-Aware Routing + +In deployments with multiple replicas, cache-aware routing directs requests to the replica most likely to have a matching prefix already in its KV cache. Without this, a user in a long conversation may hit different replicas each turn, getting cache misses that increase latency. A global KV cache stored on networked SSD allows all replicas to eventually access any cached sequence. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/caching-approaches@2nGv7_8PTWYr7uqbudCcX.md b/roadmaps/inference-engineering/content/caching-approaches@2nGv7_8PTWYr7uqbudCcX.md new file mode 100644 index 000000000..7a1ffdee1 --- /dev/null +++ b/roadmaps/inference-engineering/content/caching-approaches@2nGv7_8PTWYr7uqbudCcX.md @@ -0,0 +1,3 @@ +# Caching Approaches + +KV cache reuse is the primary caching strategy in LLM inference, covering both prefix caching (reusing cached KV values across requests that share a common prefix) and KV cache offloading (moving less-used cache blocks to slower storage tiers to free GPU VRAM). A related concern is cache-aware routing, which directs requests to replicas most likely to already hold the relevant cached prefix. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/caching@PzEj5yq1lDQ324RWYpDdQ.md b/roadmaps/inference-engineering/content/caching@PzEj5yq1lDQ324RWYpDdQ.md new file mode 100644 index 000000000..e9ba595e9 --- /dev/null +++ b/roadmaps/inference-engineering/content/caching@PzEj5yq1lDQ324RWYpDdQ.md @@ -0,0 +1,3 @@ +# Caching + +Caching reuses previously computed KV values to avoid redundant prefill computation across requests that share common context. It is one of the most impactful techniques for reducing TTFT in workloads with long system prompts or multi-turn conversations. This group covers the approaches, storage tiers, routing implications, and long-context strategies that make caching effective at scale. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/capacity-management@xrr5Y1sbN2eyQyuA5LlAW.md b/roadmaps/inference-engineering/content/capacity-management@xrr5Y1sbN2eyQyuA5LlAW.md new file mode 100644 index 000000000..4c929709b --- /dev/null +++ b/roadmaps/inference-engineering/content/capacity-management@xrr5Y1sbN2eyQyuA5LlAW.md @@ -0,0 +1,3 @@ +# Capacity Management + +Multi-cloud inference pools GPU capacity from multiple cloud providers and regions into a single schedulable resource, treating distinct compute pools as fungible. This requires a global control plane for deployment and scaling decisions alongside per-cluster workload planes that serve traffic and report utilization. True multi-cloud unlocks greater total capacity, improved reliability through redundancy, lower latency via geographic proximity, and data sovereignty compliance. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/client-integration@3RkQpvRGWh1Hp_qHlTiXX.md b/roadmaps/inference-engineering/content/client-integration@3RkQpvRGWh1Hp_qHlTiXX.md new file mode 100644 index 000000000..baf437097 --- /dev/null +++ b/roadmaps/inference-engineering/content/client-integration@3RkQpvRGWh1Hp_qHlTiXX.md @@ -0,0 +1,3 @@ +# Client Integration + +The client layer connects application code to the inference service and contributes its own slice of end-to-end latency. This group covers the connection overhead, async request patterns, and protocol choices that determine how efficiently client and server communicate. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/client-latency-overhead@gOWGIllIIxhic1NNRfugq.md b/roadmaps/inference-engineering/content/client-latency-overhead@gOWGIllIIxhic1NNRfugq.md new file mode 100644 index 000000000..5d3d9b398 --- /dev/null +++ b/roadmaps/inference-engineering/content/client-latency-overhead@gOWGIllIIxhic1NNRfugq.md @@ -0,0 +1,3 @@ +# Client Latency Overhead + +Establishing a new connection between a client and inference server takes tens of milliseconds for TLS handshake, which can represent ten percent or more of a tight latency budget. Session reuse across requests eliminates this overhead for returning clients. Standard SDKs like the OpenAI SDK handle session reuse automatically; custom clients must implement it explicitly. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/cold-starts@Q3zM9MN5lmHRAUWjdLdXR.md b/roadmaps/inference-engineering/content/cold-starts@Q3zM9MN5lmHRAUWjdLdXR.md new file mode 100644 index 000000000..f0702c92a --- /dev/null +++ b/roadmaps/inference-engineering/content/cold-starts@Q3zM9MN5lmHRAUWjdLdXR.md @@ -0,0 +1,3 @@ +# Cold Starts + +A cold start is the time from zero replicas to serving the first request, including GPU provisioning, container image loading, model weight loading, and inference engine startup or compilation. Reducing cold start time is essential for autoscaling systems to scale down confidently and scale up responsively. Key techniques include quantized weights, local weight caching near the GPU instances, and caching compiled inference engines to disk. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/compute@o_tknzA4tqcpbU6t5HO-T.md b/roadmaps/inference-engineering/content/compute@o_tknzA4tqcpbU6t5HO-T.md new file mode 100644 index 000000000..59ee7e0af --- /dev/null +++ b/roadmaps/inference-engineering/content/compute@o_tknzA4tqcpbU6t5HO-T.md @@ -0,0 +1,3 @@ +# Compute + +A GPU's compute capability is determined by its Streaming Multiprocessors (SMs), each containing CUDA Cores for general arithmetic and Tensor Cores for matrix multiply-accumulate operations. Tensor Cores are the primary resource for inference, executing the matmuls that make up the majority of forward pass computation. GPU compute is measured in peak Tensor Core FLOPS at a given precision, and roughly doubles with each new NVIDIA architecture generation. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/consumer-vs-b2b@SFMvVdgTZv_POWFGWWM_s.md b/roadmaps/inference-engineering/content/consumer-vs-b2b@SFMvVdgTZv_POWFGWWM_s.md new file mode 100644 index 000000000..c0992a633 --- /dev/null +++ b/roadmaps/inference-engineering/content/consumer-vs-b2b@SFMvVdgTZv_POWFGWWM_s.md @@ -0,0 +1,3 @@ +# Consumer vs. B2B Workloads + +Consumer applications are cost-sensitive and have unpredictable traffic spikes, requiring flexible autoscaling and low marginal cost. Business-to-business applications prioritize uptime and consistent low latency, as they often sit in the critical path of revenue-generating workflows. Compliance requirements, including data residency and regulatory certifications, are more common in enterprise deployments. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/containerization@--ohScNmF5LR04P8e95AP.md b/roadmaps/inference-engineering/content/containerization@--ohScNmF5LR04P8e95AP.md new file mode 100644 index 000000000..af056fc81 --- /dev/null +++ b/roadmaps/inference-engineering/content/containerization@--ohScNmF5LR04P8e95AP.md @@ -0,0 +1,3 @@ +# Containerization + +Containerization packages an inference service and all its dependencies into a Docker image that runs consistently across environments. Inference containers include the CUDA toolkit, Python packages, the inference engine, and any system libraries like ffmpeg for audio processing. Using a pinned dependency tree prevents breaking changes from affecting running services and ensures reproducible builds. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/continuous-batching@h_PDOMA-V-n7Pab3epuP3.md b/roadmaps/inference-engineering/content/continuous-batching@h_PDOMA-V-n7Pab3epuP3.md new file mode 100644 index 000000000..f8251ba5f --- /dev/null +++ b/roadmaps/inference-engineering/content/continuous-batching@h_PDOMA-V-n7Pab3epuP3.md @@ -0,0 +1,3 @@ +# Continuous Batching + +Continuous batching operates at the token level, swapping requests in and out of the batch as they complete individual tokens rather than waiting for a full batch to finish. This keeps GPU utilization high and minimizes the latency penalty of batching compared to static or dynamic batching. All major inference engines implement continuous batching by default. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/cost-estimation@Gof4kW2LbkNCtH68c7Xvq.md b/roadmaps/inference-engineering/content/cost-estimation@Gof4kW2LbkNCtH68c7Xvq.md new file mode 100644 index 000000000..eb3eda9fd --- /dev/null +++ b/roadmaps/inference-engineering/content/cost-estimation@Gof4kW2LbkNCtH68c7Xvq.md @@ -0,0 +1,3 @@ +# Cost Estimation + +Dedicated inference cost depends on batch sizing, traffic patterns, and sequence lengths rather than simply on token count. Comparing dedicated versus API costs requires converting per-token API pricing into total cost over a representative time window and comparing it to GPU-hour costs at the same traffic volume. Total cost of ownership also includes engineering time for building and maintaining the inference system. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/cuda@DwJJLFLPHGn9zdccTlLY2.md b/roadmaps/inference-engineering/content/cuda@DwJJLFLPHGn9zdccTlLY2.md new file mode 100644 index 000000000..54cf7ecbf --- /dev/null +++ b/roadmaps/inference-engineering/content/cuda@DwJJLFLPHGn9zdccTlLY2.md @@ -0,0 +1,3 @@ +# CUDA + +CUDA is NVIDIA's proprietary platform for programming GPUs, consisting of a kernel language (C++), a driver for hardware management, and a runtime API for launching kernels. Every inference engine, framework, and library in the NVIDIA ecosystem depends on CUDA. Understanding CUDA is the foundation for diagnosing performance problems that tools and engines cannot solve automatically. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/dependency-management@wEaM8VyAo6MFsc3RXQ7Ar.md b/roadmaps/inference-engineering/content/dependency-management@wEaM8VyAo6MFsc3RXQ7Ar.md new file mode 100644 index 000000000..6d2d14816 --- /dev/null +++ b/roadmaps/inference-engineering/content/dependency-management@wEaM8VyAo6MFsc3RXQ7Ar.md @@ -0,0 +1,3 @@ +# Dependency Management + +Inference dependency chains are long and fragile due to the tight coupling between CUDA versions, framework versions, and hardware architectures. Pinning exact versions of every dependency prevents unexpected breakage when upstream packages release updates. Containers for newly released models often initially depend on pre-release software builds and need to be rebuilt on stable releases within days of the model launch. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/desktop-inference@fQDl7_3nRGa8NFivOIgTP.md b/roadmaps/inference-engineering/content/desktop-inference@fQDl7_3nRGa8NFivOIgTP.md new file mode 100644 index 000000000..d2846bde7 --- /dev/null +++ b/roadmaps/inference-engineering/content/desktop-inference@fQDl7_3nRGa8NFivOIgTP.md @@ -0,0 +1,3 @@ +# Desktop Inference + +Desktop inference runs on consumer hardware, primarily Apple M-series unified memory systems and high-end NVIDIA consumer GPUs. Apple hardware offers very large unified memory capacity at moderate bandwidth, enabling larger models than discrete consumer GPUs; NVIDIA consumer GPUs offer higher bandwidth with smaller capacity. Tools like Ollama and llama.cpp make it practical to run aggressively quantized models on consumer hardware with minimal setup. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/diffusion-models@aagMwPPmi6ngfkIqZL-Mg.md b/roadmaps/inference-engineering/content/diffusion-models@aagMwPPmi6ngfkIqZL-Mg.md new file mode 100644 index 000000000..e28f6c80d --- /dev/null +++ b/roadmaps/inference-engineering/content/diffusion-models@aagMwPPmi6ngfkIqZL-Mg.md @@ -0,0 +1,3 @@ +# Diffusion Models + +Diffusion models generate images by starting from random noise and iteratively denoising it over 30 to 50 steps, guided by a text prompt. Each step runs the denoiser twice, once with conditioning and once without, to apply classifier-free guidance. The entire process operates in a low-dimensional latent space rather than full pixel space to make attention computationally feasible. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/disaggregation@f6PWIqf4LubR7lQztaoB5.md b/roadmaps/inference-engineering/content/disaggregation@f6PWIqf4LubR7lQztaoB5.md new file mode 100644 index 000000000..cbfbeefd6 --- /dev/null +++ b/roadmaps/inference-engineering/content/disaggregation@f6PWIqf4LubR7lQztaoB5.md @@ -0,0 +1,3 @@ +# Disaggregation + +Disaggregation separates the prefill and decode phases onto independent GPU instances that can scale separately. The prefill engine processes the input sequence and sends the resulting KV cache to the decode engine via the hardware interconnect. This removes the resource contention between compute-heavy prefill and memory-heavy decode. Disaggregation is most effective at high traffic volumes, for large models, and with prefill-heavy workloads like code editors. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/distillation@IdNkeL9pzy4lFv17ak1Vy.md b/roadmaps/inference-engineering/content/distillation@IdNkeL9pzy4lFv17ak1Vy.md new file mode 100644 index 000000000..ffe8dae24 --- /dev/null +++ b/roadmaps/inference-engineering/content/distillation@IdNkeL9pzy4lFv17ak1Vy.md @@ -0,0 +1,3 @@ +# Distillation + +Distillation trains a smaller student model to emulate a larger teacher model by exposing it to the teacher's probability distributions, not just its final outputs. Unlike fine-tuning on synthetic data, distillation transfers the teacher's reasoning behavior. The resulting student model is smaller and faster while retaining much of the teacher's quality on the same tasks. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/draft-target-decoding@m0EIHovHWGkWULN0a3C3v.md b/roadmaps/inference-engineering/content/draft-target-decoding@m0EIHovHWGkWULN0a3C3v.md new file mode 100644 index 000000000..3b5bdd4fc --- /dev/null +++ b/roadmaps/inference-engineering/content/draft-target-decoding@m0EIHovHWGkWULN0a3C3v.md @@ -0,0 +1,3 @@ +# Draft-Target Speculative Decoding + +Draft-target speculation uses a small draft model, typically at least ten times smaller than the target, to generate draft tokens for the target model to verify. The draft model should share the same tokenizer and general behavior as the target, often being a smaller member of the same model family. It is the simplest speculation method to set up but introduces the most overhead due to running a separate model. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/eagle@0FZTg3_ZnTpMOQ078OOxE.md b/roadmaps/inference-engineering/content/eagle@0FZTg3_ZnTpMOQ078OOxE.md new file mode 100644 index 000000000..6524045c1 --- /dev/null +++ b/roadmaps/inference-engineering/content/eagle@0FZTg3_ZnTpMOQ078OOxE.md @@ -0,0 +1,3 @@ +# EAGLE + +EAGLE is a purpose-built speculation method where a small draft model is trained to accept hidden states from the target model as input. By seeing the target's internal representations, EAGLE achieves higher acceptance rates and longer draft sequences (up to eight tokens) than using a general-purpose small model. It integrates into the same forward pass as the target model, eliminating the orchestration overhead of separate draft and target model runs. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/embedding-model-inference@GmCkfSeVDJ8T8WLZ7DoOp.md b/roadmaps/inference-engineering/content/embedding-model-inference@GmCkfSeVDJ8T8WLZ7DoOp.md new file mode 100644 index 000000000..d3e4b1c27 --- /dev/null +++ b/roadmaps/inference-engineering/content/embedding-model-inference@GmCkfSeVDJ8T8WLZ7DoOp.md @@ -0,0 +1,3 @@ +# Embedding Model Inference + +TensorRT-LLM with its optimized XQA attention kernel offers the best performance for embedding models with LLM backbones. FP8 quantization of weights provides performance gains with minimal quality loss, verified by checking cosine similarity between quantized and original output vectors. Embedding models are small enough that horizontal scaling across individual GPU replicas is more effective than multi-GPU parallelism. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/embedding-models@Tvt-vX1gWuJZJBHh5vfKu.md b/roadmaps/inference-engineering/content/embedding-models@Tvt-vX1gWuJZJBHh5vfKu.md new file mode 100644 index 000000000..ffde49cfe --- /dev/null +++ b/roadmaps/inference-engineering/content/embedding-models@Tvt-vX1gWuJZJBHh5vfKu.md @@ -0,0 +1,3 @@ +# Embedding Models + +Embedding models convert variable-length text or image input into a fixed-length vector that encodes semantic meaning. These vectors are used in RAG, search, recommendation, and agent memory systems. Embedding inference has two distinct traffic profiles: high-throughput backfills for bulk indexing and low-latency lookups for real-time user queries, which are best served by separate deployments. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/encoder-decoder-arch@nORbxBOAtl9sNU4sIJgP3.md b/roadmaps/inference-engineering/content/encoder-decoder-arch@nORbxBOAtl9sNU4sIJgP3.md new file mode 100644 index 000000000..b08efba0c --- /dev/null +++ b/roadmaps/inference-engineering/content/encoder-decoder-arch@nORbxBOAtl9sNU4sIJgP3.md @@ -0,0 +1,3 @@ +# Encoder-Decoder Architectures + +Neural networks can be organized as encoder-only, decoder-only, or encoder-decoder models. Modern LLMs are decoder-only. Encoder-only models like BERT are used for text embeddings and classification. Encoder-decoder models like Whisper use an encoder to process input into an internal representation and a decoder to generate output tokens autoregressively. Each architecture has different inference characteristics and tooling requirements. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/end-to-end-metrics@tENGACXKSKVPmZy9a3D2z.md b/roadmaps/inference-engineering/content/end-to-end-metrics@tENGACXKSKVPmZy9a3D2z.md new file mode 100644 index 000000000..24403aa2d --- /dev/null +++ b/roadmaps/inference-engineering/content/end-to-end-metrics@tENGACXKSKVPmZy9a3D2z.md @@ -0,0 +1,3 @@ +# End-to-End Metrics + +End-to-end latency includes inference time, network round-trip, queue time, and client-side overhead. Inference-only metrics tell you how well the model server performs; end-to-end metrics tell you what users actually experience. When inference time is fast but end-to-end time is slow, the bottleneck is in infrastructure or client code, not the GPU. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/expert-parallelism@62y78y-RXZT_QCUfd-YFE.md b/roadmaps/inference-engineering/content/expert-parallelism@62y78y-RXZT_QCUfd-YFE.md new file mode 100644 index 000000000..97b5ecf00 --- /dev/null +++ b/roadmaps/inference-engineering/content/expert-parallelism@62y78y-RXZT_QCUfd-YFE.md @@ -0,0 +1,3 @@ +# Expert Parallelism + +Expert Parallelism distributes MoE experts across GPUs so each GPU hosts multiple complete experts. Tokens are routed to the GPU holding their activated experts, and each expert runs independently within a single GPU without splitting across multiple. EP has lower inter-GPU communication overhead than Tensor Parallelism and scales well to multi-node deployments via InfiniBand. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/few-step-image-generation@biVjpCA3bsKI7sa_aBIUE.md b/roadmaps/inference-engineering/content/few-step-image-generation@biVjpCA3bsKI7sa_aBIUE.md new file mode 100644 index 000000000..9517449ff --- /dev/null +++ b/roadmaps/inference-engineering/content/few-step-image-generation@biVjpCA3bsKI7sa_aBIUE.md @@ -0,0 +1,3 @@ +# Few-Step Image Generation + +Few-step image generation models produce usable images in eight or fewer denoising steps by training the model to collapse the standard 30 to 50 step process into a fraction of the iterations. Techniques include latent consistency distillation and adversarial training, both of which teach the model to reach high-quality outputs in fewer passes. They are a strong fit for latency-sensitive use cases like real-time filters, but produce noticeably lower quality than full-step models and are a model selection decision rather than an inference optimization technique. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/fine-tuning@fWJplrAuUStEMNNLQc75G.md b/roadmaps/inference-engineering/content/fine-tuning@fWJplrAuUStEMNNLQc75G.md new file mode 100644 index 000000000..86dc5d576 --- /dev/null +++ b/roadmaps/inference-engineering/content/fine-tuning@fWJplrAuUStEMNNLQc75G.md @@ -0,0 +1,3 @@ +# Fine-Tuning + +Fine-tuning takes a pretrained foundation model and adapts it to a specific domain by training on additional labeled data. A small fine-tuned model can match a much larger general model on constrained tasks like text-to-SQL. Smaller models are faster and cheaper to serve, so fine-tuning is one of the highest-leverage tools for hitting latency and cost targets. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/flashattention@mQcj-H3R789o6nDh8tG9k.md b/roadmaps/inference-engineering/content/flashattention@mQcj-H3R789o6nDh8tG9k.md new file mode 100644 index 000000000..a25edfa3b --- /dev/null +++ b/roadmaps/inference-engineering/content/flashattention@mQcj-H3R789o6nDh8tG9k.md @@ -0,0 +1,3 @@ +# FlashAttention + +FlashAttention is a series of hand-optimized CUDA kernels that compute attention with fewer reads and writes to GPU memory by fusing the attention algorithm into a single kernel. Standard attention stores intermediate matrices to memory between steps; FlashAttention eliminates those round-trips. FlashAttention 3 targets Hopper GPUs and FlashAttention 4 targets Blackwell GPUs. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/frameworks--libraries@D4uNOZgG0w-nFL3a89pDM.md b/roadmaps/inference-engineering/content/frameworks--libraries@D4uNOZgG0w-nFL3a89pDM.md new file mode 100644 index 000000000..185ba62e1 --- /dev/null +++ b/roadmaps/inference-engineering/content/frameworks--libraries@D4uNOZgG0w-nFL3a89pDM.md @@ -0,0 +1,3 @@ +# Frameworks and Libraries + +Frameworks and libraries sit above CUDA and below inference engines, providing the abstractions that inference engineers interact with most directly. This group covers PyTorch and the Hugging Face ecosystem that supply the model implementations and tooling the rest of the stack depends on. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/geo-aware-load-balancing@mzF2mHSj8cvr9mvr16R0N.md b/roadmaps/inference-engineering/content/geo-aware-load-balancing@mzF2mHSj8cvr9mvr16R0N.md new file mode 100644 index 000000000..0f443b46d --- /dev/null +++ b/roadmaps/inference-engineering/content/geo-aware-load-balancing@mzF2mHSj8cvr9mvr16R0N.md @@ -0,0 +1,3 @@ +# Geo-Aware Load Balancing + +A global load balancer routes requests to the GPU cluster geographically closest to the requesting user. Network latency accumulates at roughly 5 milliseconds per time zone traversed, so routing a request from Singapore to San Francisco adds 50 to 100 milliseconds of unnecessary overhead. Geo-aware routing ensures that spare capacity in a distant region is used only when nearby capacity is exhausted. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/gpu-architecture@00JLMvn8DUJBCw6-BxO0y.md b/roadmaps/inference-engineering/content/gpu-architecture@00JLMvn8DUJBCw6-BxO0y.md new file mode 100644 index 000000000..d2489002e --- /dev/null +++ b/roadmaps/inference-engineering/content/gpu-architecture@00JLMvn8DUJBCw6-BxO0y.md @@ -0,0 +1,3 @@ +# GPU Architecture + +GPU architecture determines the hard limits of what inference can achieve on a given device. This group covers compute, memory, interconnects, and sizing — the four dimensions an inference engineer needs to understand to make informed decisions about hardware selection, parallelism strategy, and optimization priorities. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/gpu-models@umz3jScWQViO9-lACUU4f.md b/roadmaps/inference-engineering/content/gpu-models@umz3jScWQViO9-lACUU4f.md new file mode 100644 index 000000000..2ca947b82 --- /dev/null +++ b/roadmaps/inference-engineering/content/gpu-models@umz3jScWQViO9-lACUU4f.md @@ -0,0 +1,3 @@ +# GPU Models + +Knowing the GPU landscape — which devices are available, what they offer, and how they compare — is practical knowledge for procurement decisions, capacity planning, and understanding which optimizations apply to which hardware. This group covers the NVIDIA architectures that dominate inference deployments and the alternatives worth knowing. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/gpu-procurement@4mbGBgTu8AtrFHVYiffkB.md b/roadmaps/inference-engineering/content/gpu-procurement@4mbGBgTu8AtrFHVYiffkB.md new file mode 100644 index 000000000..64705e8bc --- /dev/null +++ b/roadmaps/inference-engineering/content/gpu-procurement@4mbGBgTu8AtrFHVYiffkB.md @@ -0,0 +1,3 @@ +# GPU Procurement + +Cloud GPUs are available from hyperscalers like AWS and GCP, GPU-focused neoclouds like CoreWeave and Nebius, and secondary markets. Procurement mechanisms include reserved instances for predictable baseline load at discounted rates, on-demand instances for flexibility, and spot instances for cost-sensitive batch workloads that tolerate preemption. Large-scale deployments typically combine all three across multiple providers. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/grace--vera@kPT86kA0WwE_fWUAAKCTT.md b/roadmaps/inference-engineering/content/grace--vera@kPT86kA0WwE_fWUAAKCTT.md new file mode 100644 index 000000000..68e0c6f68 --- /dev/null +++ b/roadmaps/inference-engineering/content/grace--vera@kPT86kA0WwE_fWUAAKCTT.md @@ -0,0 +1,3 @@ +# Grace and Vera CPUs + +NVIDIA's ARM-based Grace and Vera CPUs connect to their paired GPUs via NVLink Chip-to-Chip at up to 900 GB/s, far higher than standard PCIe connections. This high-bandwidth CPU-GPU link makes offloading KV caches and LoRA weights to CPU memory practical, since data can be retrieved quickly enough to remain on the critical path. Grace pairs with Hopper and Blackwell; Vera pairs with the upcoming Rubin architecture. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/hopper@XQMbaW1tFdO8Os6mxcR2M.md b/roadmaps/inference-engineering/content/hopper@XQMbaW1tFdO8Os6mxcR2M.md new file mode 100644 index 000000000..fca29252e --- /dev/null +++ b/roadmaps/inference-engineering/content/hopper@XQMbaW1tFdO8Os6mxcR2M.md @@ -0,0 +1,3 @@ +# Hopper + +The Hopper architecture, released in 2022 with the H100 and later the H200, introduced FP8 Tensor Cores offering twice the compute throughput of FP16 and asynchronous data transfer features used by FlashAttention 3. H100 and H200 GPUs remain among the most widely deployed inference accelerators due to their maturity, broad software support, and relative availability compared to newer generations. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/image--video-bottlenecks@0lRP9cmoS-_wxQSsIcRQg.md b/roadmaps/inference-engineering/content/image--video-bottlenecks@0lRP9cmoS-_wxQSsIcRQg.md new file mode 100644 index 000000000..68ef64a65 --- /dev/null +++ b/roadmaps/inference-engineering/content/image--video-bottlenecks@0lRP9cmoS-_wxQSsIcRQg.md @@ -0,0 +1,3 @@ +# Image and Video Generation Bottlenecks + +Image and video generation models are compute-bound because each denoising step applies attention over the entire latent space in parallel, similar to LLM prefill. The models are also relatively small, so memory bandwidth is not the limiting factor. Optimizations focus on FLOPS efficiency rather than memory traffic reduction. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/image--video-generation@U0l_xCUiizbJfjfrlBaJ0.md b/roadmaps/inference-engineering/content/image--video-generation@U0l_xCUiizbJfjfrlBaJ0.md new file mode 100644 index 000000000..ebd8c2f21 --- /dev/null +++ b/roadmaps/inference-engineering/content/image--video-generation@U0l_xCUiizbJfjfrlBaJ0.md @@ -0,0 +1,3 @@ +# Image and Video Generation + +Image and video generation inference is compute-bound rather than memory-bound, making it a different optimization problem from LLM serving. This group covers the production patterns, kernel choices, and parallelism techniques specific to diffusion model deployment. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/image--video-models@Mebx7HS-I-eNP5i3f6Clf.md b/roadmaps/inference-engineering/content/image--video-models@Mebx7HS-I-eNP5i3f6Clf.md new file mode 100644 index 000000000..39bb76c16 --- /dev/null +++ b/roadmaps/inference-engineering/content/image--video-models@Mebx7HS-I-eNP5i3f6Clf.md @@ -0,0 +1,3 @@ +# Image & Video Models + +Image and video models are specialized artificial intelligence architectures designed to process, interpret, and generate visual data. These models utilize techniques like convolutional neural networks (CNNs), transformers, and diffusion processes to analyze pixel patterns for tasks such as object detection, image classification, and video frame synthesis. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/image-generation-arch@xsV4WevPzhfydyvxVzGcw.md b/roadmaps/inference-engineering/content/image-generation-arch@xsV4WevPzhfydyvxVzGcw.md new file mode 100644 index 000000000..f5ebf793e --- /dev/null +++ b/roadmaps/inference-engineering/content/image-generation-arch@xsV4WevPzhfydyvxVzGcw.md @@ -0,0 +1,3 @@ +# Image Generation Architecture + +Image generation models are pipelines of multiple components: a text encoder that converts the prompt into conditioning, a denoising model that iteratively refines a latent representation from noise into an image, and a variational autoencoder that converts the final latent to pixels. The denoising model is the largest and most compute-intensive component and runs up to 100 forward passes per image generation. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/image-generation-inference@lnIHjq4nvCs3zvUln-jHR.md b/roadmaps/inference-engineering/content/image-generation-inference@lnIHjq4nvCs3zvUln-jHR.md new file mode 100644 index 000000000..df9ed063d --- /dev/null +++ b/roadmaps/inference-engineering/content/image-generation-inference@lnIHjq4nvCs3zvUln-jHR.md @@ -0,0 +1,3 @@ +# Image Generation Inference + +Production image generation inference uses SGLang Diffusion, TensorRT, or hand-optimized PyTorch. The most important optimization target is the attention kernel, where FlashAttention 3 or 4 outperforms the default on Hopper and Blackwell respectively. GEMM kernels for linear layers can be quantized to FP8 safely. Torch compilation with manual kernel plugins caches the compiled engine to avoid multi-minute recompilation on every cold start. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/indep-component-scaling@SKs0yuBEAyeTPQxLll3EM.md b/roadmaps/inference-engineering/content/indep-component-scaling@SKs0yuBEAyeTPQxLll3EM.md new file mode 100644 index 000000000..aa63f1baf --- /dev/null +++ b/roadmaps/inference-engineering/content/indep-component-scaling@SKs0yuBEAyeTPQxLll3EM.md @@ -0,0 +1,3 @@ +# Independent Component Scaling + +Multi-model pipelines, like a voice agent that runs VAD, ASR, an LLM, and TTS in sequence, have components with different hardware requirements and traffic patterns. Each component should autoscale independently based on its own load. Running all components within the same Kubernetes cluster minimizes inter-component network latency, which can otherwise consume a significant fraction of a tight latency budget. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/inference-engines@01j1KkEB-nIXxP-rENcJ3.md b/roadmaps/inference-engineering/content/inference-engines@01j1KkEB-nIXxP-rENcJ3.md new file mode 100644 index 000000000..d336f7c04 --- /dev/null +++ b/roadmaps/inference-engineering/content/inference-engines@01j1KkEB-nIXxP-rENcJ3.md @@ -0,0 +1,3 @@ +# Inference Engines + +Inference engines are the core serving layer that takes model weights and handles requests at production scale. They implement continuous batching, memory management, parallelism, and optimization techniques so engineers do not have to implement these from scratch. This group covers the three dominant LLM engines and the orchestration platform that sits above them. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/inference-vs-training@fVUfQYb8BP_hFMmd1S2-J.md b/roadmaps/inference-engineering/content/inference-vs-training@fVUfQYb8BP_hFMmd1S2-J.md new file mode 100644 index 000000000..67d74abfe --- /dev/null +++ b/roadmaps/inference-engineering/content/inference-vs-training@fVUfQYb8BP_hFMmd1S2-J.md @@ -0,0 +1,3 @@ +# Inference vs. Training + +Training is the process of learning model weights from data. Inference is serving those weights in production to generate outputs. Training runs are compute-intensive batch jobs; inference is a continuous, latency-sensitive service. Where training success is measured in loss curves, inference success is measured in time to first token, tokens per second, and uptime. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/instances--interconnects@dAo4PMB_VnRFMSjyDrLj0.md b/roadmaps/inference-engineering/content/instances--interconnects@dAo4PMB_VnRFMSjyDrLj0.md new file mode 100644 index 000000000..6ae437c1a --- /dev/null +++ b/roadmaps/inference-engineering/content/instances--interconnects@dAo4PMB_VnRFMSjyDrLj0.md @@ -0,0 +1,3 @@ +# Instances and Interconnects + +Within a node, NVLink connects GPUs one-to-one at up to 1800 GB/s on Blackwell, and NVSwitch creates an all-to-all fabric across all eight GPUs. Between nodes, InfiniBand provides inter-node connectivity at substantially lower bandwidth than NVLink. Multi-Instance GPU (MIG) partitions a single GPU into up to seven isolated slices, each with dedicated compute and memory, allowing small models to run on fractions of a high-end GPU without resource contention. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/inter-token-latency@sre_s0dvpOP0DTM66a3hu.md b/roadmaps/inference-engineering/content/inter-token-latency@sre_s0dvpOP0DTM66a3hu.md new file mode 100644 index 000000000..bfdb690be --- /dev/null +++ b/roadmaps/inference-engineering/content/inter-token-latency@sre_s0dvpOP0DTM66a3hu.md @@ -0,0 +1,3 @@ +# Inter-Token Latency (ITL) + +ITL is the time between successive generated tokens during decode. An ITL of 10 milliseconds corresponds to 100 tokens per second. It is the per-step measure underlying TPS and is directly tied to how quickly the GPU can complete a single decode forward pass. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/introduction@GL-9aJKTJT9s4maeT37Uf.md b/roadmaps/inference-engineering/content/introduction@GL-9aJKTJT9s4maeT37Uf.md new file mode 100644 index 000000000..5b864a4b6 --- /dev/null +++ b/roadmaps/inference-engineering/content/introduction@GL-9aJKTJT9s4maeT37Uf.md @@ -0,0 +1,3 @@ +# Inference Engineer + +Inference engineering is the practice of making generative AI models faster, less expensive, and more reliable to serve in production. It spans the full stack from CUDA kernels and GPU architecture to Kubernetes autoscaling and multi-cloud capacity management. The field emerged alongside the explosion of open models and the shift from pay-per-token APIs to dedicated deployments, where engineers own latency, throughput, cost, and uptime directly. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/kernel-selection--fusion@iKW1wqzd5EfRKvNIRshNN.md b/roadmaps/inference-engineering/content/kernel-selection--fusion@iKW1wqzd5EfRKvNIRshNN.md new file mode 100644 index 000000000..238d77628 --- /dev/null +++ b/roadmaps/inference-engineering/content/kernel-selection--fusion@iKW1wqzd5EfRKvNIRshNN.md @@ -0,0 +1,3 @@ +# Kernel Selection and Fusion + +A CUDA kernel is a function that runs in parallel across thousands of GPU threads simultaneously. Kernel selection is choosing the best existing kernel for a specific operation; kernel fusion combines sequential kernels into one to eliminate unnecessary reads and writes to VRAM between operations. Libraries like cuBLAS, CUTLASS, CuTe, and FlashInfer provide pre-built kernels for common inference operations, and most inference engines perform kernel selection automatically. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/kv-cache-storage@gkt2Gr34RJJf1NZgbmtXf.md b/roadmaps/inference-engineering/content/kv-cache-storage@gkt2Gr34RJJf1NZgbmtXf.md new file mode 100644 index 000000000..3472c860c --- /dev/null +++ b/roadmaps/inference-engineering/content/kv-cache-storage@gkt2Gr34RJJf1NZgbmtXf.md @@ -0,0 +1,3 @@ +# KV Cache Storage + +KV caches can be stored in GPU VRAM (fastest, tens to hundreds of gigabytes), CPU RAM (slower, hundreds of gigabytes to terabytes), local SSD, or networked SSD. Moving less frequently accessed KV blocks to slower storage frees VRAM for active requests. NVIDIA Dynamo's KV Block Manager provides APIs for managing KV blocks across these storage tiers transparently. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/kv-cache@V678kX8FPChTZCe7Nqwhv.md b/roadmaps/inference-engineering/content/kv-cache@V678kX8FPChTZCe7Nqwhv.md new file mode 100644 index 000000000..b921c7003 --- /dev/null +++ b/roadmaps/inference-engineering/content/kv-cache@V678kX8FPChTZCe7Nqwhv.md @@ -0,0 +1,3 @@ +# KV Cache + +The KV cache stores the key and value tensors computed during attention for each previously processed token. Without it, attention would require recomputing every prior token on each decode step, making inference prohibitively slow. The KV cache lives in GPU memory and its size grows with sequence length and batch size, making it a major target for memory management techniques. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/large-language-models@irgGvEnY1_WW1EPtuMKgC.md b/roadmaps/inference-engineering/content/large-language-models@irgGvEnY1_WW1EPtuMKgC.md new file mode 100644 index 000000000..4133a9a98 --- /dev/null +++ b/roadmaps/inference-engineering/content/large-language-models@irgGvEnY1_WW1EPtuMKgC.md @@ -0,0 +1,3 @@ +# Large Language Models + +An LLM is a decoder-only transformer with three main components: an embedding layer that converts input tokens to vectors, a stack of transformer blocks that process those vectors, and an output language modeling head that converts the final hidden states into a probability distribution over the vocabulary. The configuration file for an open model specifies all architectural details including layer count, hidden dimensions, and attention heads. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/latency-percentiles@8y280d5BPXrDZwtsHZCaX.md b/roadmaps/inference-engineering/content/latency-percentiles@8y280d5BPXrDZwtsHZCaX.md new file mode 100644 index 000000000..28895cbf9 --- /dev/null +++ b/roadmaps/inference-engineering/content/latency-percentiles@8y280d5BPXrDZwtsHZCaX.md @@ -0,0 +1,3 @@ +# Latency Percentiles + +Mean latency hides outliers. LLM response times follow a right-skewed distribution, so engineers measure P50, P90, P95, and P99. A P99 of several seconds means one in a hundred users has a poor experience even if the median is fast. Reducing P90/P99 latency is as important as reducing average latency. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/latency-vs-throughput@kvNIi3aY5rn69_hYkDyUK.md b/roadmaps/inference-engineering/content/latency-vs-throughput@kvNIi3aY5rn69_hYkDyUK.md new file mode 100644 index 000000000..a8856c0e2 --- /dev/null +++ b/roadmaps/inference-engineering/content/latency-vs-throughput@kvNIi3aY5rn69_hYkDyUK.md @@ -0,0 +1,3 @@ +# Latency vs. Throughput + +Lower latency means faster responses per user; higher throughput means more total requests served per unit time. These goals are often in tension. Increasing batch size improves throughput but worsens per-user latency. Choosing the right balance is one of the primary decisions in configuring an inference service. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/llm-based-models@M7K6d-Qw4BpM724LfM1u6.md b/roadmaps/inference-engineering/content/llm-based-models@M7K6d-Qw4BpM724LfM1u6.md new file mode 100644 index 000000000..2b10806cb --- /dev/null +++ b/roadmaps/inference-engineering/content/llm-based-models@M7K6d-Qw4BpM724LfM1u6.md @@ -0,0 +1,3 @@ +# LLM-based Models + +LLM-based embedding models use a decoder-only transformer backbone, typically between one and eight billion parameters, to produce higher-quality semantic representations than BERT-style models. They apply a pooling operation to the final layer hidden states and are trained with contrastive objectives on large-scale retrieval datasets. The quality advantage is most pronounced on complex or cross-lingual retrieval tasks; the cost is higher compute per embedding and the need for GPU inference rather than CPU. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/llm-bottlenecks@QVadKzBfLdegat-4wgOMX.md b/roadmaps/inference-engineering/content/llm-bottlenecks@QVadKzBfLdegat-4wgOMX.md new file mode 100644 index 000000000..3eb73dde1 --- /dev/null +++ b/roadmaps/inference-engineering/content/llm-bottlenecks@QVadKzBfLdegat-4wgOMX.md @@ -0,0 +1,3 @@ +# LLM Inference Bottlenecks + +LLM prefill is compute-bound because it processes the entire input sequence as one large matrix multiplication, performing many operations per byte loaded. LLM decode is memory-bound because it generates tokens one at a time via vector-matrix multiplications that require loading all model weights from memory for each token while performing relatively few arithmetic operations. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/local--edge-inference@FilQGz3geyyvHZGyKPW3G.md b/roadmaps/inference-engineering/content/local--edge-inference@FilQGz3geyyvHZGyKPW3G.md new file mode 100644 index 000000000..c56b0e51a --- /dev/null +++ b/roadmaps/inference-engineering/content/local--edge-inference@FilQGz3geyyvHZGyKPW3G.md @@ -0,0 +1,3 @@ +# Local and Edge Inference + +Not all inference runs in a datacenter. Desktop and mobile deployments have different hardware constraints, tooling ecosystems, and use cases from cloud inference. This group covers the patterns and tools for running models on consumer hardware and mobile devices. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/locust@IQuJFfMP1dW0ELlip6LOd.md b/roadmaps/inference-engineering/content/locust@IQuJFfMP1dW0ELlip6LOd.md new file mode 100644 index 000000000..4b1cd915a --- /dev/null +++ b/roadmaps/inference-engineering/content/locust@IQuJFfMP1dW0ELlip6LOd.md @@ -0,0 +1,3 @@ +# Locust + +Locust is a general-purpose open-source load testing framework written in Python. In inference benchmarking it is used to simulate realistic user traffic patterns — including request arrival rates, concurrency ramps, and burst scenarios — that synthetic tools do not capture. It is particularly useful for end-to-end load testing that includes network overhead, queuing, and autoscaling behavior rather than raw model server performance alone. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/long-context-handling@uL3Fn8XfS1Yv_PJd9R8wi.md b/roadmaps/inference-engineering/content/long-context-handling@uL3Fn8XfS1Yv_PJd9R8wi.md new file mode 100644 index 000000000..4d827c5f6 --- /dev/null +++ b/roadmaps/inference-engineering/content/long-context-handling@uL3Fn8XfS1Yv_PJd9R8wi.md @@ -0,0 +1,3 @@ +# Long Context Handling + +Long sequences produce KV caches large enough to strain GPU memory, slow down decode, and increase prefill time. Techniques for handling long context include FlashAttention for memory-efficient attention computation, PagedAttention for non-contiguous KV storage, chunked prefill to interleave long prefill with decode, and model-level approaches like sliding window attention that limit how many prior tokens each token attends to. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/lora-aware-routing@KdceJEDBdy3TbzG5suE38.md b/roadmaps/inference-engineering/content/lora-aware-routing@KdceJEDBdy3TbzG5suE38.md new file mode 100644 index 000000000..aea6224bb --- /dev/null +++ b/roadmaps/inference-engineering/content/lora-aware-routing@KdceJEDBdy3TbzG5suE38.md @@ -0,0 +1,3 @@ +# LoRA-Aware Routing + +LoRA fine-tune weights are small adapter layers loaded on top of a base model to specialize its behavior for a specific task or customer. A deployment serving many LoRA variants for the same base model must route each request to a replica that already has the required LoRA loaded in GPU memory. Routing without this awareness forces constant LoRA swapping, adding latency and reducing throughput on each affected replica. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/measuring-quality-impact@mK_4R8liXtl64iG7ONvgz.md b/roadmaps/inference-engineering/content/measuring-quality-impact@mK_4R8liXtl64iG7ONvgz.md new file mode 100644 index 000000000..e16ed66a6 --- /dev/null +++ b/roadmaps/inference-engineering/content/measuring-quality-impact@mK_4R8liXtl64iG7ONvgz.md @@ -0,0 +1,3 @@ +# Measuring Quality Impact + +After quantizing a model, quality must be validated against the original. Three methods are used: perplexity scoring (lower is better, with minimal increase after quantization), standard intelligence benchmarks like MMLU or SWE-bench, and product-specific evals tailored to the actual use case. All three should show differences indistinguishable from noise before a quantized model is deployed. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/memory@5829d5dvXq9qlMSD1CG0N.md b/roadmaps/inference-engineering/content/memory@5829d5dvXq9qlMSD1CG0N.md new file mode 100644 index 000000000..e3b0e8d59 --- /dev/null +++ b/roadmaps/inference-engineering/content/memory@5829d5dvXq9qlMSD1CG0N.md @@ -0,0 +1,3 @@ +# Memory + +GPU memory determines what fits on a single device and how fast data can move through the compute units. VRAM capacity sets the upper limit on model size and KV cache; VRAM bandwidth determines how quickly the decode phase can load weights per token. Modern datacenter GPUs use high-bandwidth memory (HBM) in generations HBM3, HBM3e, and HBM4, stacked directly on the GPU die to maximize bandwidth. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/mixture-of-experts-moe@7BddTfOkWmlL4HAkFBXtq.md b/roadmaps/inference-engineering/content/mixture-of-experts-moe@7BddTfOkWmlL4HAkFBXtq.md new file mode 100644 index 000000000..5062fa767 --- /dev/null +++ b/roadmaps/inference-engineering/content/mixture-of-experts-moe@7BddTfOkWmlL4HAkFBXtq.md @@ -0,0 +1,3 @@ +# Mixture of Experts (MoE) + +Mixture of Experts models replace dense linear layers with a set of smaller expert matrices, activating only a subset of experts per token via a learned router. A model like Qwen3-235B activates only 22 billion of its 235 billion parameters per forward pass, making single-user inference efficient. In batched production serving, different tokens activate different experts, so nearly all parameters are active across the batch. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/mobile-inference@kqzHHoiZ9y4tCS6MMlLHQ.md b/roadmaps/inference-engineering/content/mobile-inference@kqzHHoiZ9y4tCS6MMlLHQ.md new file mode 100644 index 000000000..0c7e82e61 --- /dev/null +++ b/roadmaps/inference-engineering/content/mobile-inference@kqzHHoiZ9y4tCS6MMlLHQ.md @@ -0,0 +1,3 @@ +# Mobile Inference + +Mobile devices run inference using vendor frameworks: Google's AI Edge SDK for Android and Apple's Foundation Models and Core ML for iOS. Hardware limitations restrict practical model sizes to one to two billion parameters on current phones. Small fine-tuned models for tasks like transcription, translation, and on-device classification are the strongest fit for mobile edge inference. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/model-evaluation@lVf5GoJF_PVTkTiQFTNVg.md b/roadmaps/inference-engineering/content/model-evaluation@lVf5GoJF_PVTkTiQFTNVg.md new file mode 100644 index 000000000..37ab15cca --- /dev/null +++ b/roadmaps/inference-engineering/content/model-evaluation@lVf5GoJF_PVTkTiQFTNVg.md @@ -0,0 +1,3 @@ +# Model Evaluation + +Model evaluation, or evals, is the practice of systematically measuring model quality for a specific task or domain. Standard intelligence benchmarks like MMLU measure general capabilities but can be gamed; product-specific evals that test real use cases provide more reliable signal. High-confidence evals are a prerequisite before investing in inference optimization, ensuring the model is actually useful before making it fast. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/model-formats--runtimes@lTjWXB8YRwx11e8lxSgNm.md b/roadmaps/inference-engineering/content/model-formats--runtimes@lTjWXB8YRwx11e8lxSgNm.md new file mode 100644 index 000000000..4bd2ea829 --- /dev/null +++ b/roadmaps/inference-engineering/content/model-formats--runtimes@lTjWXB8YRwx11e8lxSgNm.md @@ -0,0 +1,3 @@ +### Model Formats and Runtimes + +Before a model can serve requests, its weights must be stored, loaded, and compiled into an executable form. This group covers the file formats used to distribute weights and the runtimes that compile them into optimized inference engines for specific hardware targets. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/model-parallelism@CiXsdHHL0e2DN4ICI5jeX.md b/roadmaps/inference-engineering/content/model-parallelism@CiXsdHHL0e2DN4ICI5jeX.md new file mode 100644 index 000000000..25f9a2701 --- /dev/null +++ b/roadmaps/inference-engineering/content/model-parallelism@CiXsdHHL0e2DN4ICI5jeX.md @@ -0,0 +1,3 @@ +# Model Parallelism + +When a model is too large to fit on a single GPU, or when serving latency requires more compute than one device can provide, model parallelism distributes the workload across multiple GPUs or nodes. This group covers the three parallelism strategies and their trade-offs in terms of communication overhead, latency, and hardware requirements. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/model-selection@jQAWZ1plw4_wPE1UvE_Ux.md b/roadmaps/inference-engineering/content/model-selection@jQAWZ1plw4_wPE1UvE_Ux.md new file mode 100644 index 000000000..ec0140381 --- /dev/null +++ b/roadmaps/inference-engineering/content/model-selection@jQAWZ1plw4_wPE1UvE_Ux.md @@ -0,0 +1,3 @@ +# Model Selection + +The most important decision in model performance optimization is which model to use. Smaller models with fewer parameters are faster and cheaper to serve than larger ones. The goal is to find the smallest model that meets quality requirements for the task. Model architecture also determines which inference engines and optimization techniques are available, so popular architectures with broad tooling support are preferable. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/modelopt@zhiKpdZm8zOg4e3bm1OgN.md b/roadmaps/inference-engineering/content/modelopt@zhiKpdZm8zOg4e3bm1OgN.md new file mode 100644 index 000000000..e6bc6e2e6 --- /dev/null +++ b/roadmaps/inference-engineering/content/modelopt@zhiKpdZm8zOg4e3bm1OgN.md @@ -0,0 +1,3 @@ +# ModelOpt + +ModelOpt is NVIDIA's toolkit for post-training quantization, supporting FP8, INT8, FP4, MXFP8, MXFP4, and NVFP4 output formats. It integrates with PyTorch and exports quantized models directly to TensorRT-LLM, vLLM, and SGLang. Calibration runs a small set of representative inputs through the model to compute per-layer scale factors that minimize the gap between quantized and original outputs. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/multi-cloud@d78UiI_FZyLsU6O52btXg.md b/roadmaps/inference-engineering/content/multi-cloud@d78UiI_FZyLsU6O52btXg.md new file mode 100644 index 000000000..497240a0c --- /dev/null +++ b/roadmaps/inference-engineering/content/multi-cloud@d78UiI_FZyLsU6O52btXg.md @@ -0,0 +1,3 @@ +# Multi-Cloud + +Multi-cloud inference distributes GPU capacity across providers and regions to maximize availability, minimize latency, and meet data residency requirements. This group covers procurement strategy, capacity management, geographic routing, and the reliability patterns that make large-scale inference resilient to hardware and provider failures. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/multi-node-inference@FZx2xsqw_cmYLFS3GMAl-.md b/roadmaps/inference-engineering/content/multi-node-inference@FZx2xsqw_cmYLFS3GMAl-.md new file mode 100644 index 000000000..df9b6df5c --- /dev/null +++ b/roadmaps/inference-engineering/content/multi-node-inference@FZx2xsqw_cmYLFS3GMAl-.md @@ -0,0 +1,3 @@ +# Multi-Node Inference + +Multi-node inference spreads a model across more than eight GPUs using InfiniBand for inter-node communication. It is necessary for the largest models where a single node lacks sufficient VRAM for weights and KV cache. InfiniBand bandwidth is an order of magnitude lower than NVLink, so parallelism strategies must minimize cross-node communication. For MoE models, Expert Parallelism is preferred over Tensor Parallelism between nodes. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/n-gram@ZOclQMd7JASGXTRyctfW2.md b/roadmaps/inference-engineering/content/n-gram@ZOclQMd7JASGXTRyctfW2.md new file mode 100644 index 000000000..c2661df62 --- /dev/null +++ b/roadmaps/inference-engineering/content/n-gram@ZOclQMd7JASGXTRyctfW2.md @@ -0,0 +1,3 @@ +# N-gram Speculation + +N-gram speculation builds a dictionary during prefill that maps token prefixes to observed n-gram continuations from the input. During decode, generated tokens are looked up in this dictionary and matching suffixes become draft tokens with no separate model required. It is most effective for code completion, where output frequently mirrors input syntax, and can produce very long draft sequences with high acceptance rates in that domain. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/neural-networks@p2S0W3EDn768jbqzc7CE7.md b/roadmaps/inference-engineering/content/neural-networks@p2S0W3EDn768jbqzc7CE7.md new file mode 100644 index 000000000..bcf41deed --- /dev/null +++ b/roadmaps/inference-engineering/content/neural-networks@p2S0W3EDn768jbqzc7CE7.md @@ -0,0 +1,3 @@ +# Neural Networks + +A neural network is a layered structure of nodes where each node applies a learned weight matrix and bias to its input. Multiple layers connected in sequence form the foundation for all generative AI models. Inference engineers need a working model of how neural networks compute outputs to understand where performance bottlenecks arise. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/nims@KB7ExU9X49omOuk9D_Qal.md b/roadmaps/inference-engineering/content/nims@KB7ExU9X49omOuk9D_Qal.md new file mode 100644 index 000000000..2fa8dff0a --- /dev/null +++ b/roadmaps/inference-engineering/content/nims@KB7ExU9X49omOuk9D_Qal.md @@ -0,0 +1,3 @@ +# NIMs + +NIMs are pre-built Docker containers provided by NVIDIA for popular open models, available as flexible multi-LLM containers or as GPU- and model-specific optimized containers. They provide a ready-made starting point for serving common architectures without building a custom container from scratch. For maximum control and custom configurations, starting from a less opinionated base image and building up is generally preferable. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/nvidia-dynamo@KhTFecXEkHR4WX-_TYY3g.md b/roadmaps/inference-engineering/content/nvidia-dynamo@KhTFecXEkHR4WX-_TYY3g.md new file mode 100644 index 000000000..75dcbc6e8 --- /dev/null +++ b/roadmaps/inference-engineering/content/nvidia-dynamo@KhTFecXEkHR4WX-_TYY3g.md @@ -0,0 +1,3 @@ +# NVIDIA Dynamo + +Dynamo supports conditional disaggregation, where requests are first sent to the decode engine and only routed to a prefill engine when the input is long or not already cached. The ratio of prefill to decode engines is configurable at runtime and can shift as traffic patterns change. This dynamic allocation prevents the prefill queue from becoming a bottleneck and avoids wasting decode engine resources on prefill-heavy traffic. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/nvidia-dynamo@ctdMz6rHa4Cwl1x19V_5k.md b/roadmaps/inference-engineering/content/nvidia-dynamo@ctdMz6rHa4Cwl1x19V_5k.md new file mode 100644 index 000000000..ec30a5544 --- /dev/null +++ b/roadmaps/inference-engineering/content/nvidia-dynamo@ctdMz6rHa4Cwl1x19V_5k.md @@ -0,0 +1,3 @@ +# NVIDIA Dynamo + +Dynamo is NVIDIA's open-source distributed serving platform that sits above inference engines and provides orchestration for large-scale deployments. It adds KV-aware routing, disaggregated prefill/decode serving, multi-node parallelism, and an SLA-based planner that dynamically adjusts the number of prefill and decode workers based on TTFT and TPS targets. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/nvidia-genai-perf@zffK61pNUJDwkX7Ukql8y.md b/roadmaps/inference-engineering/content/nvidia-genai-perf@zffK61pNUJDwkX7Ukql8y.md new file mode 100644 index 000000000..f36570e6d --- /dev/null +++ b/roadmaps/inference-engineering/content/nvidia-genai-perf@zffK61pNUJDwkX7Ukql8y.md @@ -0,0 +1,3 @@ +# NVIDIA GenAI-Perf + +GenAI-Perf is NVIDIA's inference benchmarking tool, designed to work across TensorRT-LLM, Triton Inference Server, and OpenAI-compatible endpoints. It generates synthetic workloads with configurable sequence length distributions and concurrency levels and outputs standard latency and throughput metrics including TTFT, ITL, and TPS percentiles. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/nvidia@Uq3mE-0WIASHfcBDEnbLj.md b/roadmaps/inference-engineering/content/nvidia@Uq3mE-0WIASHfcBDEnbLj.md new file mode 100644 index 000000000..11d902bb4 --- /dev/null +++ b/roadmaps/inference-engineering/content/nvidia@Uq3mE-0WIASHfcBDEnbLj.md @@ -0,0 +1,3 @@ +# NVIDIA + +NVIDIA GPUs are the dominant hardware for inference due to their combination of Tensor Core compute, HBM bandwidth, NVLink interconnects, and the CUDA software ecosystem. The three most relevant architectures for current inference deployments are Hopper, Blackwell, and the Grace and Vera CPU platforms that pair with them. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/observability@w-dVv5dFNxD7Rqy16m4uL.md b/roadmaps/inference-engineering/content/observability@w-dVv5dFNxD7Rqy16m4uL.md new file mode 100644 index 000000000..f836629ba --- /dev/null +++ b/roadmaps/inference-engineering/content/observability@w-dVv5dFNxD7Rqy16m4uL.md @@ -0,0 +1,3 @@ +# Observability + +Inference observability tracks total request volume, input and output sequence lengths, response codes, latency percentiles (P50, P90, P99 of TTFT, TPS, and end-to-end latency), replica count, GPU utilization, and queue depth. These metrics are interdependent; a latency spike could come from long input sequences, high concurrency, or a hardware issue. Observability must integrate with existing tooling like Grafana, Datadog, and PagerDuty to put inference metrics in context. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/omni-modal-models@plw2WiOoGxg64P-Kv-Xm7.md b/roadmaps/inference-engineering/content/omni-modal-models@plw2WiOoGxg64P-Kv-Xm7.md new file mode 100644 index 000000000..f1cf34fe6 --- /dev/null +++ b/roadmaps/inference-engineering/content/omni-modal-models@plw2WiOoGxg64P-Kv-Xm7.md @@ -0,0 +1,3 @@ +# Omni-Modal Models + +Omni models accept multiple input modalities and produce multiple output types in a single model. They avoid the latency of running separate specialized models in sequence but often lag behind dedicated models in quality within each modality. Production pipelines using VLMs typically coordinate preprocessing steps like OCR, audio transcription, and PDF extraction as separate independently scaling components. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/online-vs-offline-inf@SkvPauxjmqok4TyNWSACL.md b/roadmaps/inference-engineering/content/online-vs-offline-inf@SkvPauxjmqok4TyNWSACL.md new file mode 100644 index 000000000..9d120a7d3 --- /dev/null +++ b/roadmaps/inference-engineering/content/online-vs-offline-inf@SkvPauxjmqok4TyNWSACL.md @@ -0,0 +1,3 @@ +# Online vs. Offline Inference + +Online inference serves real-time user-facing requests where latency matters. Offline inference processes large batches of data asynchronously where throughput and cost matter more than per-request speed. The same model, like Whisper for transcription, may need separate deployments for each use case because the optimal configurations differ significantly. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/onnx-runtime@Vmtbdre3SZzwq3FxOO-Bc.md b/roadmaps/inference-engineering/content/onnx-runtime@Vmtbdre3SZzwq3FxOO-Bc.md new file mode 100644 index 000000000..931ee23f2 --- /dev/null +++ b/roadmaps/inference-engineering/content/onnx-runtime@Vmtbdre3SZzwq3FxOO-Bc.md @@ -0,0 +1,3 @@ +# ONNX Runtime + +ONNX Runtime is an open-source inference runtime that executes models exported to the ONNX format, which stores both weights and a computation graph for cross-platform portability. It supports multiple hardware backends including CUDA, DirectML, and CPU. It is widely used for embedding models, vision models, and pipelines that need to run across hardware environments without engine-specific configuration. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/open-vs-closed-models@KAt5FzglOUtH7jLlzDn-I.md b/roadmaps/inference-engineering/content/open-vs-closed-models@KAt5FzglOUtH7jLlzDn-I.md new file mode 100644 index 000000000..3fc3fea0d --- /dev/null +++ b/roadmaps/inference-engineering/content/open-vs-closed-models@KAt5FzglOUtH7jLlzDn-I.md @@ -0,0 +1,3 @@ +# Open vs. Closed Models + +Closed models are accessed exclusively through a vendor API; the weights are never released. Open models publish their weights for download and self-hosting. For inference engineers, this distinction determines the entire problem space: closed models require no infrastructure but offer no control over latency, cost, or configuration, while open models require building and operating a full inference stack but give complete control over every layer of it. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/opsbyte-ratio@uc2enkdtRqA5K7kbgrYJF.md b/roadmaps/inference-engineering/content/opsbyte-ratio@uc2enkdtRqA5K7kbgrYJF.md new file mode 100644 index 000000000..03af22b7e --- /dev/null +++ b/roadmaps/inference-engineering/content/opsbyte-ratio@uc2enkdtRqA5K7kbgrYJF.md @@ -0,0 +1,3 @@ +# Ops:Byte Ratio + +The ops:byte ratio is a property of GPU hardware that expresses the balance between compute throughput and memory bandwidth. It is calculated by dividing peak FLOPS by peak memory bandwidth, and sets the threshold that separates compute-bound from memory-bound workloads on that device. Different GPU generations have very different ops:byte ratios, which is why an optimization that works well on one generation may have no effect on another. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/other-accelerators@am4Hh-14g6Awk0WcFmVN7.md b/roadmaps/inference-engineering/content/other-accelerators@am4Hh-14g6Awk0WcFmVN7.md new file mode 100644 index 000000000..1ddef2370 --- /dev/null +++ b/roadmaps/inference-engineering/content/other-accelerators@am4Hh-14g6Awk0WcFmVN7.md @@ -0,0 +1,3 @@ +# Other Accelerators + +Alternatives to NVIDIA GPUs include AMD MI350, Google TPUs, AWS Inferentia and Trainium, Cerebras WSE-3, and Groq LPUs. Each offers a specific advantage: AMD provides CUDA-compatible tooling via ROCm; TPUs integrate deeply with Google Cloud and JAX; Inferentia and Trainium offer cost advantages within AWS. All face the shared challenge of building an ecosystem to match the breadth of NVIDIA's CUDA libraries and inference engine support. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/pagedattention@VcIRY4vSi7JkcV6g13vkN.md b/roadmaps/inference-engineering/content/pagedattention@VcIRY4vSi7JkcV6g13vkN.md new file mode 100644 index 000000000..cb06cce3e --- /dev/null +++ b/roadmaps/inference-engineering/content/pagedattention@VcIRY4vSi7JkcV6g13vkN.md @@ -0,0 +1,3 @@ +# PagedAttention + +PagedAttention stores the KV cache in fixed-size pages rather than a single contiguous memory block, allowing it to be distributed across GPU memory without fragmentation. It uses a lookup table to access pages on demand, reducing memory waste from over-allocation and enabling more requests to share GPU memory. It is the memory management technique that made vLLM's high throughput possible. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/performance-benchmarking@XXfjlqd0CYHwBR5-Kdmmi.md b/roadmaps/inference-engineering/content/performance-benchmarking@XXfjlqd0CYHwBR5-Kdmmi.md new file mode 100644 index 000000000..fa8360259 --- /dev/null +++ b/roadmaps/inference-engineering/content/performance-benchmarking@XXfjlqd0CYHwBR5-Kdmmi.md @@ -0,0 +1,3 @@ +# Performance Benchmarking + +Benchmarking measures inference service performance under realistic workloads. Good benchmarks simulate realistic input and output sequence lengths, concurrency patterns, and request contents rather than synthetic uniform inputs. Standard tools include SGLang Genai-bench, NVIDIA GenAI-Perf, and Locust. Always establish a baseline before applying optimizations and change one variable at a time. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/pipeline-parallelism@9KLS3gWArBofLEULu3sQc.md b/roadmaps/inference-engineering/content/pipeline-parallelism@9KLS3gWArBofLEULu3sQc.md new file mode 100644 index 000000000..d2534a0d2 --- /dev/null +++ b/roadmaps/inference-engineering/content/pipeline-parallelism@9KLS3gWArBofLEULu3sQc.md @@ -0,0 +1,3 @@ +# Pipeline Parallelism + +Pipeline Parallelism splits an LLM's layers into sequential stages and assigns each stage to a different GPU or node. Unlike Tensor Parallelism, each GPU holds complete layers rather than layer fragments. It introduces pipeline bubbles where GPUs wait for preceding stages, reducing efficiency. It is used for multi-node inference with dense models, typically combined with Tensor Parallelism within each node. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/post-training-quantization@Zvvs8cB98IRee1okBILsS.md b/roadmaps/inference-engineering/content/post-training-quantization@Zvvs8cB98IRee1okBILsS.md new file mode 100644 index 000000000..639431c98 --- /dev/null +++ b/roadmaps/inference-engineering/content/post-training-quantization@Zvvs8cB98IRee1okBILsS.md @@ -0,0 +1,3 @@ +Post-Training Quantization + +Post-training quantization converts finished model weights to lower precision after training is complete, using a small calibration dataset to compute per-layer scale factors that minimize the gap between quantized and original outputs. It is the standard approach for most deployments since it requires no retraining and can be applied to any existing model. The leading tool is NVIDIA ModelOpt, which exports quantized models in formats compatible with all major inference engines. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/prefill--decode-phases@hwKZmfYFiIu6plbiXtJRI.md b/roadmaps/inference-engineering/content/prefill--decode-phases@hwKZmfYFiIu6plbiXtJRI.md new file mode 100644 index 000000000..b81f01f3a --- /dev/null +++ b/roadmaps/inference-engineering/content/prefill--decode-phases@hwKZmfYFiIu6plbiXtJRI.md @@ -0,0 +1,3 @@ +# Prefill and Decode Phases + +LLM inference has two distinct phases. Prefill processes the entire input sequence in parallel to build the KV cache and is compute-bound. Decode generates output tokens one at a time by reading model weights from memory and is memory-bound. Each phase has different bottlenecks and responds to different optimization techniques. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/profiling-performance@S2GZw9etrrvEJzV9V-1C_.md b/roadmaps/inference-engineering/content/profiling-performance@S2GZw9etrrvEJzV9V-1C_.md new file mode 100644 index 000000000..120c93c12 --- /dev/null +++ b/roadmaps/inference-engineering/content/profiling-performance@S2GZw9etrrvEJzV9V-1C_.md @@ -0,0 +1,3 @@ +# Profiling Performance + +Profiling reveals where time is spent within a benchmark, identifying which operations consume the most compute or memory. Tools include PyTorch Profiler, NVIDIA Nsight Systems for system-wide GPU and CPU analysis, and NVIDIA Nsight Compute for per-kernel analysis. Profiling is most useful when writing custom inference code or contributing to inference frameworks, rather than configuring existing engines. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/pytorch@AHVfxQRXmRoiTxC1eeMme.md b/roadmaps/inference-engineering/content/pytorch@AHVfxQRXmRoiTxC1eeMme.md new file mode 100644 index 000000000..753411aeb --- /dev/null +++ b/roadmaps/inference-engineering/content/pytorch@AHVfxQRXmRoiTxC1eeMme.md @@ -0,0 +1,3 @@ +# PyTorch + +PyTorch is the standard Python framework for building and running neural networks. It abstracts CUDA into a tensor operation API and provides automatic differentiation for training. For inference, PyTorch supports torch.compile for automatic kernel optimization and serves as the foundation that all major inference engines build on top of. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/quantization-aware-training@QCXV4Utq0Ajs1krLM3fhg.md b/roadmaps/inference-engineering/content/quantization-aware-training@QCXV4Utq0Ajs1krLM3fhg.md new file mode 100644 index 000000000..077005c6a --- /dev/null +++ b/roadmaps/inference-engineering/content/quantization-aware-training@QCXV4Utq0Ajs1krLM3fhg.md @@ -0,0 +1,3 @@ +# Quantization-Aware Training + +Quantization-aware training computes scale factors and optimizes model weights jointly during training so the final model is already calibrated for the target low-precision format. This produces better quality at low precision than post-training quantization but requires retraining the model from scratch or from a checkpoint. Some labs release models trained this way, such as GPT-OSS in MXFP4. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/quantization@DaSqcLHfnJ4_n9utVVTSk.md b/roadmaps/inference-engineering/content/quantization@DaSqcLHfnJ4_n9utVVTSk.md new file mode 100644 index 000000000..62e4bf0e9 --- /dev/null +++ b/roadmaps/inference-engineering/content/quantization@DaSqcLHfnJ4_n9utVVTSk.md @@ -0,0 +1,3 @@ +# Quantization + +Quantization converts model weights and other values from their native precision to a lower-precision format. Cutting precision in half doubles memory bandwidth for decode and enables twice the FLOPS for prefill on compatible Tensor Cores. The tradeoff is potential quality loss from reduced numerical precision, which can compound across the many calculations in a forward pass. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/real-time-factor@0-BRllrMiShLlytidGkST.md b/roadmaps/inference-engineering/content/real-time-factor@0-BRllrMiShLlytidGkST.md new file mode 100644 index 000000000..300980eeb --- /dev/null +++ b/roadmaps/inference-engineering/content/real-time-factor@0-BRllrMiShLlytidGkST.md @@ -0,0 +1,3 @@ +# Real-Time Factor (RTF) + +RTF measures how quickly an ASR system can process audio relative to its duration. A system that transcribes one hour of audio in six seconds has an RTF of 600x. It is the standard throughput metric for long-file transcription workloads and improves roughly linearly with the number of GPUs used for parallel chunk processing. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/reasoning-models@Y9YkcfblXHEvS-gdEWwaE.md b/roadmaps/inference-engineering/content/reasoning-models@Y9YkcfblXHEvS-gdEWwaE.md new file mode 100644 index 000000000..5b7293769 --- /dev/null +++ b/roadmaps/inference-engineering/content/reasoning-models@Y9YkcfblXHEvS-gdEWwaE.md @@ -0,0 +1,3 @@ +# Reasoning Models + +Reasoning models generate an intermediate thinking sequence before producing their final output, adding a third token sequence alongside the input and output. This reasoning trace increases total token count significantly and changes inference economics: TTFT is higher, total latency is longer, and output token costs dominate. Inference engineers must account for the variable and sometimes very long reasoning sequences when sizing infrastructure and setting latency budgets. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/request-queueing@k-zXOGynI0B0yyT-n3raa.md b/roadmaps/inference-engineering/content/request-queueing@k-zXOGynI0B0yyT-n3raa.md new file mode 100644 index 000000000..07498de5e --- /dev/null +++ b/roadmaps/inference-engineering/content/request-queueing@k-zXOGynI0B0yyT-n3raa.md @@ -0,0 +1,3 @@ +# Request Queueing + +A request queue holds incoming traffic when all active replicas are at capacity while autoscaling brings new replicas online. A first-in, first-out queue is the standard implementation; priority queues can give paid users better position. Queues must be visible to newly started replicas so that requests are not held waiting for existing replicas when new capacity becomes available. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/roofline-model@XF-wMQCqPZiWv1XmryPBP.md b/roadmaps/inference-engineering/content/roofline-model@XF-wMQCqPZiWv1XmryPBP.md new file mode 100644 index 000000000..0cebec277 --- /dev/null +++ b/roadmaps/inference-engineering/content/roofline-model@XF-wMQCqPZiWv1XmryPBP.md @@ -0,0 +1,3 @@ +# Roofline Model + +The roofline model is a visual framework for reasoning about hardware performance limits. It plots arithmetic intensity on the x-axis against achievable performance on the y-axis, with two ceilings: a diagonal memory bandwidth ceiling for low-intensity workloads and a horizontal compute ceiling for high-intensity ones. A workload's position relative to these ceilings shows not just whether it is compute-bound or memory-bound, but how far it sits from the hardware's theoretical maximum — which tells you how much headroom remains before an optimization hits a hard limit. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/routing--load-balancing@DSyTAtFC1TvC1P7yD10Nb.md b/roadmaps/inference-engineering/content/routing--load-balancing@DSyTAtFC1TvC1P7yD10Nb.md new file mode 100644 index 000000000..36bb07050 --- /dev/null +++ b/roadmaps/inference-engineering/content/routing--load-balancing@DSyTAtFC1TvC1P7yD10Nb.md @@ -0,0 +1,3 @@ +# Routing and Load Balancing + +Routers make request-level decisions about which replica to send a request to, considering factors like KV cache prefix match and LoRA availability. Load balancers make system-level decisions to distribute traffic evenly across replicas. In systems with prefix caching, naive round-robin load balancing misses cache opportunities; intelligent routing increases cache hit rates and reduces TTFT. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/safetensors@jkbrf0_JDljWmVq_o9XE-.md b/roadmaps/inference-engineering/content/safetensors@jkbrf0_JDljWmVq_o9XE-.md new file mode 100644 index 000000000..e2fcd129f --- /dev/null +++ b/roadmaps/inference-engineering/content/safetensors@jkbrf0_JDljWmVq_o9XE-.md @@ -0,0 +1,3 @@ +# Safetensors + +Safetensors is the standard format for storing and distributing open model weights. It uses memory mapping for fast, zero-copy loading and prevents execution of arbitrary code during deserialization, making it safer than earlier pickle-based formats. All major inference engines load safetensors natively, and most open models on Hugging Face publish weights in this format. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/scale-to-zero@Pi87dWgu5NTIJRQUc169s.md b/roadmaps/inference-engineering/content/scale-to-zero@Pi87dWgu5NTIJRQUc169s.md new file mode 100644 index 000000000..39d3d1637 --- /dev/null +++ b/roadmaps/inference-engineering/content/scale-to-zero@Pi87dWgu5NTIJRQUc169s.md @@ -0,0 +1,3 @@ +# Scale to Zero + +Scale to zero allows the system to run zero replicas during periods of no traffic and spin up on demand. It requires fast cold starts and robust queuing to hold incoming requests while a replica starts. Scale to zero is well-suited for development environments and periodically accessed workloads, but not for latency-sensitive production applications with unpredictable light traffic. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/security-and-compliance@4x-4BFZH7bfNteD5ZAwTI.md b/roadmaps/inference-engineering/content/security-and-compliance@4x-4BFZH7bfNteD5ZAwTI.md new file mode 100644 index 000000000..94cc9df57 --- /dev/null +++ b/roadmaps/inference-engineering/content/security-and-compliance@4x-4BFZH7bfNteD5ZAwTI.md @@ -0,0 +1,3 @@ +# Security and Compliance + +Inference infrastructure must protect user inputs, model outputs, and model weights. Standard practices include data encryption, container security, network isolation, and access controls validated by third-party penetration testing. Regulated industries require compliance with frameworks like SOC 2 and HIPAA, which also apply to underlying cloud providers. Multi-cloud deployments support data residency requirements by routing each country's data to clusters in that country. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/sglang-genai-bench@KHDnY7byKgmaPtC_0uDBM.md b/roadmaps/inference-engineering/content/sglang-genai-bench@KHDnY7byKgmaPtC_0uDBM.md new file mode 100644 index 000000000..1d9274405 --- /dev/null +++ b/roadmaps/inference-engineering/content/sglang-genai-bench@KHDnY7byKgmaPtC_0uDBM.md @@ -0,0 +1,3 @@ +# SGLang Genai-bench + +SGLang Genai-bench is a benchmarking tool built into the SGLang project for measuring LLM inference throughput and latency under realistic request distributions. It supports variable input and output sequence lengths and concurrent request patterns, making it well-suited for validating SGLang and vLLM deployments before production. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/sglang@YtlAf6qloKMmaSr9Bkkl_.md b/roadmaps/inference-engineering/content/sglang@YtlAf6qloKMmaSr9Bkkl_.md new file mode 100644 index 000000000..9b16777c7 --- /dev/null +++ b/roadmaps/inference-engineering/content/sglang@YtlAf6qloKMmaSr9Bkkl_.md @@ -0,0 +1,3 @@ +# SGLang + +SGLang is a high-throughput inference engine with a flexible frontend language for customizing server behavior. It has strong support for MoE models like DeepSeek and Qwen and works closely with model labs to optimize new architectural features. SGLang Diffusion extends it to image and video generation models with pipeline abstractions and diffusion-specific parallelism. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/shared-vs-dedicated-inf@2DsjfW9r4dOI3LDQU80qt.md b/roadmaps/inference-engineering/content/shared-vs-dedicated-inf@2DsjfW9r4dOI3LDQU80qt.md new file mode 100644 index 000000000..0feb4dce3 --- /dev/null +++ b/roadmaps/inference-engineering/content/shared-vs-dedicated-inf@2DsjfW9r4dOI3LDQU80qt.md @@ -0,0 +1,3 @@ +# Shared vs. Dedicated Inference + +Shared inference means sending requests to a public API and paying per token. Dedicated inference means renting GPUs and running your own model server, paying per GPU-hour. Shared inference is simpler and has no cold start, but dedicated inference offers control over latency, cost at scale, model customization, and uptime. Most products start shared and move to dedicated as traffic grows. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/sizing--procurement@da0HGfSDr6_DV4ZJcrcIB.md b/roadmaps/inference-engineering/content/sizing--procurement@da0HGfSDr6_DV4ZJcrcIB.md new file mode 100644 index 000000000..96efcfae6 --- /dev/null +++ b/roadmaps/inference-engineering/content/sizing--procurement@da0HGfSDr6_DV4ZJcrcIB.md @@ -0,0 +1,3 @@ +# Sizing and Procurement + +Estimating GPU requirements starts with VRAM: roughly 1 GB per billion parameters at FP8 for weights, plus at least 50 percent headroom for the KV cache. This total is rounded up to the nearest available instance size. Cloud GPUs are available from hyperscalers like AWS and GCP, neoclouds like CoreWeave and Nebius, and spot markets; large deployments combine reserved instances for baseline load with on-demand and spot capacity for bursts. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/speculative-decoding@gMWgd10cxcQG_RG2oSAEH.md b/roadmaps/inference-engineering/content/speculative-decoding@gMWgd10cxcQG_RG2oSAEH.md new file mode 100644 index 000000000..dd768cce6 --- /dev/null +++ b/roadmaps/inference-engineering/content/speculative-decoding@gMWgd10cxcQG_RG2oSAEH.md @@ -0,0 +1,3 @@ +# Speculative Decoding + +Speculative decoding takes advantage of spare compute during memory-bound decode by generating multiple draft tokens and validating them in a single forward pass. If the target model accepts N draft tokens, the forward pass produces N+1 tokens instead of one. The performance gain depends on draft token cost, draft sequence length, and acceptance rate, which decreases at higher temperatures and larger batch sizes. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/speech-to-speech-models@moyS740HQoUUovilx1Yyt.md b/roadmaps/inference-engineering/content/speech-to-speech-models@moyS740HQoUUovilx1Yyt.md new file mode 100644 index 000000000..4273ef2e7 --- /dev/null +++ b/roadmaps/inference-engineering/content/speech-to-speech-models@moyS740HQoUUovilx1Yyt.md @@ -0,0 +1,3 @@ +# Speech-to-Speech Models + +Speech-to-speech models unify ASR, reasoning, and TTS into a single model by expanding the LLM vocabulary with audio input and output tokens, eliminating the latency of cascading three separate models in sequence. Current closed models like OpenAI's gpt-4o-realtime exist but remain more expensive and less capable than well-tuned cascading pipelines of dedicated ASR, LLM, and TTS models. Open speech-to-speech models are an active research area without commercially viable options at the time of writing. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/speech@Lk8906Mpi3yxiSt8EC_BC.md b/roadmaps/inference-engineering/content/speech@Lk8906Mpi3yxiSt8EC_BC.md new file mode 100644 index 000000000..a90d79f52 --- /dev/null +++ b/roadmaps/inference-engineering/content/speech@Lk8906Mpi3yxiSt8EC_BC.md @@ -0,0 +1,3 @@ +# Speech + +Speech inference covers the pipeline from raw audio to text and back, including the detection, transcription, diarization, and synthesis steps that make up voice applications. Each component has distinct inference characteristics and often runs on separate hardware to avoid resource contention. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/streaming--protocol-support@SGQCvuHAkg4TcJa_e2f-y.md b/roadmaps/inference-engineering/content/streaming--protocol-support@SGQCvuHAkg4TcJa_e2f-y.md new file mode 100644 index 000000000..b4bdafea8 --- /dev/null +++ b/roadmaps/inference-engineering/content/streaming--protocol-support@SGQCvuHAkg4TcJa_e2f-y.md @@ -0,0 +1,3 @@ +# Streaming and Protocol Support + +Text streaming over HTTP is sufficient for LLM chat applications. For real-time voice and video applications, WebSockets provide a persistent bidirectional connection for continuous unstructured data like audio chunks. gRPC provides bidirectional streaming for structured service-to-service communication with schema enforcement. Each WebSocket connection holds a slot on the server, and the concurrency limit must be set to match the GPU's sustainable load. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/streaming-tts@3VbPNaSnzWZ-pC-lGFMrh.md b/roadmaps/inference-engineering/content/streaming-tts@3VbPNaSnzWZ-pC-lGFMrh.md new file mode 100644 index 000000000..503a436d2 --- /dev/null +++ b/roadmaps/inference-engineering/content/streaming-tts@3VbPNaSnzWZ-pC-lGFMrh.md @@ -0,0 +1,3 @@ +# Streaming TTS + +Real-time TTS streams audio continuously over WebSockets rather than returning a complete file after generation finishes. The audio decoder converts token streams to waveforms as they are generated, implemented in compiled PyTorch with dynamic batching. After tuning the inference engine and decoder, the primary performance lever is how many concurrent real-time streams a single GPU replica can sustain. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/tensor-parallelism@csghcNLDywO8oTIZNXeyP.md b/roadmaps/inference-engineering/content/tensor-parallelism@csghcNLDywO8oTIZNXeyP.md new file mode 100644 index 000000000..07d8e654e --- /dev/null +++ b/roadmaps/inference-engineering/content/tensor-parallelism@csghcNLDywO8oTIZNXeyP.md @@ -0,0 +1,3 @@ +# Tensor Parallelism + +Tensor Parallelism splits each layer's weight matrices across multiple GPUs and distributes the matmul computation. Each GPU handles a shard of each layer and the results are combined via an all-reduce operation before the next layer. It reduces per-user latency by sharing both VRAM and compute across GPUs, but requires fast NVLink interconnects and is not well-suited for multi-node deployments. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/tensorrt-llm@n2_zIGkQgyYCMYPypz3vb.md b/roadmaps/inference-engineering/content/tensorrt-llm@n2_zIGkQgyYCMYPypz3vb.md new file mode 100644 index 000000000..9d418284c --- /dev/null +++ b/roadmaps/inference-engineering/content/tensorrt-llm@n2_zIGkQgyYCMYPypz3vb.md @@ -0,0 +1,3 @@ +# TensorRT-LLM + +TensorRT-LLM is NVIDIA's inference engine combining PyTorch-based model execution with proprietary NVIDIA kernels unavailable in other engines. It achieves the highest performance among the three major engines, with excellent support for Hopper and Blackwell architectures and NVIDIA-specific formats like NVFP4. The tradeoff is a steeper learning curve and narrower model support compared to vLLM or SGLang. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/tensorrt@M4qjJSEAnsjrmP3A_O8HC.md b/roadmaps/inference-engineering/content/tensorrt@M4qjJSEAnsjrmP3A_O8HC.md new file mode 100644 index 000000000..9d0e866ea --- /dev/null +++ b/roadmaps/inference-engineering/content/tensorrt@M4qjJSEAnsjrmP3A_O8HC.md @@ -0,0 +1,3 @@ +# TensorRT + +TensorRT is NVIDIA's high-performance inference runtime that compiles PyTorch or ONNX models into optimized engines using fused kernels, quantization, and hardware-specific tuning. It is the standard choice for image and video generation models and for any workload where maximum throughput on NVIDIA hardware is the primary goal. Compiled engines are cached to disk, eliminating recompilation overhead on subsequent cold starts. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/testing--deployment@ZlOtQtxJST10nuHjhbUx6.md b/roadmaps/inference-engineering/content/testing--deployment@ZlOtQtxJST10nuHjhbUx6.md new file mode 100644 index 000000000..f1d03943c --- /dev/null +++ b/roadmaps/inference-engineering/content/testing--deployment@ZlOtQtxJST10nuHjhbUx6.md @@ -0,0 +1,3 @@ +# Testing and Deployment + +Deploying inference services safely requires tools for rolling out changes without downtime, estimating costs before committing to a configuration, and maintaining visibility into system health once live. This group covers the deployment, cost, observability, and compliance concerns that apply once a service is ready for production traffic. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/text-to-speech-tts@XnhhN1QVWzlp9DUwkii9-.md b/roadmaps/inference-engineering/content/text-to-speech-tts@XnhhN1QVWzlp9DUwkii9-.md new file mode 100644 index 000000000..ba63358ab --- /dev/null +++ b/roadmaps/inference-engineering/content/text-to-speech-tts@XnhhN1QVWzlp9DUwkii9-.md @@ -0,0 +1,3 @@ +# Text-to-Speech (TTS) + +Modern TTS models like Orpheus TTS are fine-tuned LLMs with an expanded vocabulary that includes audio tokens. They generate audio token sequences decoded to waveforms by a small audio decoder. The key performance metrics are time to first byte of audio (TTFB) and the number of concurrent real-time streams a single GPU replica can sustain, rather than raw token throughput. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/the-inference-stack@uIhVVlMsykuxJCa02f-i-.md b/roadmaps/inference-engineering/content/the-inference-stack@uIhVVlMsykuxJCa02f-i-.md new file mode 100644 index 000000000..1f8bfb209 --- /dev/null +++ b/roadmaps/inference-engineering/content/the-inference-stack@uIhVVlMsykuxJCa02f-i-.md @@ -0,0 +1,3 @@ +# The Inference Stack + +A complete inference system has three layers: runtime, infrastructure, and tooling. The runtime layer optimizes a single model on a single instance using techniques like batching, caching, quantization, and speculation. The infrastructure layer scales across clusters, regions, and cloud providers while maintaining uptime. The tooling layer provides the right level of abstraction for engineers to control the system productively. All three must work together for mission-critical inference at scale. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/time-to-first-token@xCgNhwbS_ELxEyRHD1enO.md b/roadmaps/inference-engineering/content/time-to-first-token@xCgNhwbS_ELxEyRHD1enO.md new file mode 100644 index 000000000..261aa28b3 --- /dev/null +++ b/roadmaps/inference-engineering/content/time-to-first-token@xCgNhwbS_ELxEyRHD1enO.md @@ -0,0 +1,3 @@ +# Time to First Token (TTFT) + +TTFT measures how long it takes for a user to receive the first output token after submitting a request. It is determined primarily by the prefill phase and by queue time. For streaming applications like chatbots, TTFT is the most visible latency signal and directly affects perceived responsiveness. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/tokens-per-second@uxm_BIAyhfFmcE94k9i9J.md b/roadmaps/inference-engineering/content/tokens-per-second@uxm_BIAyhfFmcE94k9i9J.md new file mode 100644 index 000000000..97ddc562d --- /dev/null +++ b/roadmaps/inference-engineering/content/tokens-per-second@uxm_BIAyhfFmcE94k9i9J.md @@ -0,0 +1,3 @@ +# Tokens Per Second (TPS) + +TPS measures how many tokens are delivered to the user per second after the first token. It is determined by the decode phase and reflects memory bandwidth efficiency. Per-user TPS (also called perceived TPS) is a latency metric; total TPS across the inference service is a throughput metric. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/transformer-blocks@xHM7qjigMmXEwxeMEeMWl.md b/roadmaps/inference-engineering/content/transformer-blocks@xHM7qjigMmXEwxeMEeMWl.md new file mode 100644 index 000000000..98dc09529 --- /dev/null +++ b/roadmaps/inference-engineering/content/transformer-blocks@xHM7qjigMmXEwxeMEeMWl.md @@ -0,0 +1,3 @@ +# Transformer Blocks + +Transformer blocks are the repeated hidden layers of an LLM. Each block contains an attention sublayer, a feed-forward neural network sublayer, and normalization layers. The feed-forward network holds the majority of model parameters, while attention is the most computationally complex operation per token. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/transformers--diffusers@H_e3KZ7p7enUCYE5M5Lr7.md b/roadmaps/inference-engineering/content/transformers--diffusers@H_e3KZ7p7enUCYE5M5Lr7.md new file mode 100644 index 000000000..cde7c03ab --- /dev/null +++ b/roadmaps/inference-engineering/content/transformers--diffusers@H_e3KZ7p7enUCYE5M5Lr7.md @@ -0,0 +1,3 @@ +# Hugging Face Transformers and Diffusers + +The transformers and diffusers libraries provide reference implementations of popular LLM and image/video generation architectures. They are not designed for production serving but are essential for understanding model configurations, downloading weights, and prototyping inference. Model cards on Hugging Face use transformers or diffusers sample code as the canonical usage examples. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/video-generation-inference@PqEH1ez5dg5tEsspgfAK1.md b/roadmaps/inference-engineering/content/video-generation-inference@PqEH1ez5dg5tEsspgfAK1.md new file mode 100644 index 000000000..d0427fbd3 --- /dev/null +++ b/roadmaps/inference-engineering/content/video-generation-inference@PqEH1ez5dg5tEsspgfAK1.md @@ -0,0 +1,3 @@ +# Video Generation Inference + +Video generation models run on full nodes of eight GPUs with a batch size of one because the latent space is too large to batch effectively. Attention over the three-dimensional video latent space consumes 70 to 80 percent of compute time and is the primary optimization target. Attention caching techniques that reuse outputs from previous timesteps or transformer layers can improve speed by 30 to 40 percent. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/video-generation-models@yTSrQfJsE-Mf4ZU_hJwYT.md b/roadmaps/inference-engineering/content/video-generation-models@yTSrQfJsE-Mf4ZU_hJwYT.md new file mode 100644 index 000000000..6698c248d --- /dev/null +++ b/roadmaps/inference-engineering/content/video-generation-models@yTSrQfJsE-Mf4ZU_hJwYT.md @@ -0,0 +1,3 @@ +# Video Generation Models + +Video generation models extend image generation to three-dimensional latent space, representing width, height, and time simultaneously. They run the same number of denoising steps as image models but process far more data per step. Attention over the full video latent space dominates compute time, making video generation the most resource-intensive common inference workload. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/video-processing-for-vlms@7pRZu7xa2_43HVwKty9px.md b/roadmaps/inference-engineering/content/video-processing-for-vlms@7pRZu7xa2_43HVwKty9px.md new file mode 100644 index 000000000..ec7d9f600 --- /dev/null +++ b/roadmaps/inference-engineering/content/video-processing-for-vlms@7pRZu7xa2_43HVwKty9px.md @@ -0,0 +1,3 @@ +# Video Processing for VLMs + +One second of video contains 24 frames, each requiring roughly 1,000 tokens to represent at full resolution. This makes video inputs extremely long-context workloads. Frame rate reduction and downsampling are practically mandatory for running video through a VLM in a single request. Prefix caching and chunked prefill become critical for managing the resulting multi-thousand-token sequences. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/vision-language-models@JsjmQrN1V_dM1Kw6UnYLc.md b/roadmaps/inference-engineering/content/vision-language-models@JsjmQrN1V_dM1Kw6UnYLc.md new file mode 100644 index 000000000..704121da9 --- /dev/null +++ b/roadmaps/inference-engineering/content/vision-language-models@JsjmQrN1V_dM1Kw6UnYLc.md @@ -0,0 +1,3 @@ +# Vision Language Models (VLMs) + +VLMs combine a large language model with a small vision encoder that converts input images or video into a sequence of visual tokens. A high-resolution image typically adds about 1,000 tokens to the input sequence. All LLM inference optimization techniques apply to VLMs, but longer sequences make quantization, prefix caching, and disaggregation especially important. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/vllm@wcAtbab81IJdpFh6L-heP.md b/roadmaps/inference-engineering/content/vllm@wcAtbab81IJdpFh6L-heP.md new file mode 100644 index 000000000..dd2d4680c --- /dev/null +++ b/roadmaps/inference-engineering/content/vllm@wcAtbab81IJdpFh6L-heP.md @@ -0,0 +1,3 @@ +# vLLM + +vLLM is the most widely adopted LLM inference engine, offering broad support for model architectures, hardware platforms (NVIDIA, AMD, Google TPUs), and optimization techniques including prefix caching, quantization, speculative decoding, and parallelism. Its primary advantage is ease of adoption and the fastest day-zero support for newly released open models. It also supports multimodal inference via vLLM Omni. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/voice-activity-detection@P0BA_p3xqpxkQsFHqM0AZ.md b/roadmaps/inference-engineering/content/voice-activity-detection@P0BA_p3xqpxkQsFHqM0AZ.md new file mode 100644 index 000000000..1e8542f94 --- /dev/null +++ b/roadmaps/inference-engineering/content/voice-activity-detection@P0BA_p3xqpxkQsFHqM0AZ.md @@ -0,0 +1,3 @@ +# Voice Activity Detection (VAD) + +VAD is a lightweight model that monitors an audio stream or file and identifies segments containing speech, filtering out silence. It is a required preprocessing component in both real-time ASR pipelines and long-file transcription pipelines, splitting audio into discrete chunks at natural speech boundaries rather than fixed time intervals. VAD runs on dedicated hardware to avoid competing with the ASR model for GPU resources. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/writing-custom-kernels@1dWPVaSQ4TfVU3Obx3m19.md b/roadmaps/inference-engineering/content/writing-custom-kernels@1dWPVaSQ4TfVU3Obx3m19.md new file mode 100644 index 000000000..2901724f5 --- /dev/null +++ b/roadmaps/inference-engineering/content/writing-custom-kernels@1dWPVaSQ4TfVU3Obx3m19.md @@ -0,0 +1,3 @@ +# Writing Custom Kernels + +Custom kernels are written when existing libraries do not fully utilize a GPU's capabilities for a specific operation or architecture. Most production kernels are written in C++ using CUTLASS and CuTe for GEMM-style operations and FlashInfer for attention variants; Triton offers a Python-based alternative with a lower barrier to entry. Writing an effective kernel requires understanding the target GPU's memory hierarchy, SM occupancy, and Tensor Core instruction set — skills that sit at the intersection of software and hardware. \ No newline at end of file diff --git a/roadmaps/inference-engineering/content/zero-downtime-deplo@le4sPQ0h9l-vlmCB4G8ZY.md b/roadmaps/inference-engineering/content/zero-downtime-deplo@le4sPQ0h9l-vlmCB4G8ZY.md new file mode 100644 index 000000000..9a9aef8e0 --- /dev/null +++ b/roadmaps/inference-engineering/content/zero-downtime-deplo@le4sPQ0h9l-vlmCB4G8ZY.md @@ -0,0 +1,3 @@ +# Zero-Downtime Deployment + +Canary deployments update inference services without interrupting live traffic by routing a small percentage of requests to the new deployment, monitoring for issues, and gradually shifting traffic over. Blue-green deployments, which cut all traffic at once to a parallel environment, are impractical for large inference workloads because they require doubling GPU capacity during the transition. With autoscaling, canary deployments are low cost since the old service scales down as the new one takes traffic. \ No newline at end of file