• bitcoinBitcoin(BTC)$60,461.002.96%
  • ethereumEthereum(ETH)$1,699.272.14%
  • tetherTether(USDT)$1.00-0.01%
  • binancecoinBNB(BNB)$552.283.30%
  • usd-coinUSDC(USDC)$1.00-0.01%
  • rippleXRP(XRP)$0.921.68%
  • solanaSolana(SOL)$70.011.08%
  • tronTRON(TRX)$0.3049110.72%
  • Figure HelocFigure Heloc(FIGR_HELOC)$1.073.37%
  • HyperliquidHyperliquid(HYPE)$62.174.06%
  • zcashZcash(ZEC)$786.712.17%
  • moneroMonero(XMR)$481.142.59%
  • dogecoinDogecoin(DOGE)$0.0548961.40%
  • leo-tokenLEO Token(LEO)$9.160.33%
  • whitebitWhiteBIT Coin(WBT)$58.042.42%
  • chainlinkChainlink(LINK)$8.762.83%
  • USDSUSDS(USDS)$1.00-0.03%
  • cardanoCardano(ADA)$0.1447542.39%
  • Ethena USDeEthena USDe(USDE)$1.000.01%
  • stellarStellar(XLM)$0.1281611.35%
  • daiDai(DAI)$1.00-0.01%
  • bitcoin-cashBitcoin Cash(BCH)$197.241.77%
  • nearNEAR Protocol(NEAR)$2.986.33%
  • litecoinLitecoin(LTC)$39.570.47%
  • avalanche-2Avalanche(AVAX)$6.652.32%
  • CantonCanton(CC)$0.073562-1.49%
  • Ripple USDRipple USD(RLUSD)$1.00-0.01%
  • tether-goldTether Gold(XAUT)$3,544.640.59%
  • USD1USD1(USD1)$1.000.02%
  • uniswapUniswap(UNI)$4.683.96%
  • the-open-networkGram (prev. Toncoin)(GRAM)$0.982.99%
  • suiSui(SUI)$0.653.10%
  • Global DollarGlobal Dollar(USDG)$1.00-0.01%
  • Ondo US Dollar YieldOndo US Dollar Yield(USDY)$1.150.13%
  • RainRain(RAIN)$0.003286-6.87%
  • paypal-usdPayPal USD(PYUSD)$1.00-0.03%
  • hedera-hashgraphHedera(HBAR)$0.0513861.15%
  • BittensorBittensor(TAO)$190.263.98%
  • shiba-inuShiba Inu(SHIB)$0.0000041.65%
  • crypto-com-chainCronos(CRO)$0.0407640.09%
  • Spiko Amundi Overnight Swap Fund (EUR)Spiko Amundi Overnight Swap Fund (EUR)(EURSAFO)$1.120.00%
  • quant-networkQuant(QNT)$140.934.88%
  • Circle USYCCircle USYC(USYC)$1.140.00%
  • MemeCoreMemeCore(M)$0.76-1.98%
  • Falcon USDFalcon USD(USDF)$1.00-0.01%
  • okbOKB(OKB)$80.402.34%
  • United StablesUnited Stables(U)$1.000.04%
  • pax-goldPAX Gold(PAXG)$3,547.380.54%
  • BlackRock USD Institutional Digital Liquidity FundBlackRock USD Institutional Digital Liquidity Fund(BUIDL)$1.000.00%
  • aaveAave(AAVE)$98.14-2.24%
TradePoint.io
  • Main
  • AI & Technology
  • Stock Charts
  • Market & News
  • Business
  • Finance Tips
  • Trade Tube
  • Blog
  • Shop
No Result
View All Result
TradePoint.io
No Result
View All Result

Inside NVIDIA’s cuDNN Graph API: Fusion, Autotuning, and Plan Reuse with cuDNN Frontend

September 15, 2026
in AI & Technology
Reading Time: 3 mins read
A A
Inside NVIDIA’s cuDNN Graph API: Fusion, Autotuning, and Plan Reuse with cuDNN Frontend
ShareShareShareShareShare
@section("5. SDPA (Flash Attention) with causal masking")
def sdpa_demo():
   if not HAS_SDPA:
       raise RuntimeError(f"fused SDPA needs SM80+ (Ampere), this GPU is sm_{SM}")
   b, h, s, d = 4, 16, 1024, 64
   scale = 1.0 / math.sqrt(d)
   SDPA_FLOPS = 4 * b * h * s * s * d * 0.5
   q = torch.randn(b, h, s, d, device=DEV, dtype=DTYPE)
   k = torch.randn(b, h, s, d, device=DEV, dtype=DTYPE)
   v = torch.randn(b, h, s, d, device=DEV, dtype=DTYPE)
   o = torch.empty(b, h, s, d, device=DEV, dtype=DTYPE)
   g = cudnn.pygraph(
       handle=HANDLE, name="sdpa",
       io_data_type=TORCH2CUDNN[DTYPE],
       intermediate_data_type=cudnn.data_type.FLOAT,
       compute_data_type=cudnn.data_type.FLOAT,
   )
   Q, Kt, V = tensor_of(g, q, "Q"), tensor_of(g, k, "K"), tensor_of(g, v, "V")
   causal = True
   try:
       O, _stats = g.sdpa(name="sdpa", q=Q, k=Kt, v=V,
                          is_inference=True, attn_scale=scale, use_causal_mask=True)
   except TypeError:
       try:
           O, _stats = g.sdpa(name="sdpa", q=Q, k=Kt, v=V,
                              is_inference=True, attn_scale=scale,
                              diagonal_alignment=cudnn.diagonal_alignment.TOP_LEFT,
                              right_bound=0)
       except Exception:
           causal = False
           O, _stats = g.sdpa(name="sdpa", q=Q, k=Kt, v=V,
                              is_inference=True, attn_scale=scale)
   print(f"    causal masking: {causal}")
   O.set_output(True).set_data_type(TORCH2CUDNN[DTYPE])
   O.set_dim(list(o.size())).set_stride(list(o.stride()))
   build(g)
   ws = workspace_for(g)
   pack = {Q: q, Kt: k, V: v, O: o}
   g.execute(pack, ws)
   torch.cuda.synchronize()
   ref = torch.nn.functional.scaled_dot_product_attention(q, k, v, is_causal=causal, scale=scale)
   rel = ((o.float() - ref.float()).abs().max() / ref.float().abs().max()).item()
   print(f"    shape   : b{b} h{h} s{s} d{d}   workspace {ws.numel()/1024:.1f} KiB")
   print(f"    rel err : {rel:.2e}")
   ms = bench(lambda: g.execute(pack, ws))
   ms_t = bench(lambda: torch.nn.functional.scaled_dot_product_attention(
       q, k, v, is_causal=causal, scale=scale))
   print()
   report("cuDNN FE SDPA", ms, SDPA_FLOPS)
   report("torch SDPA (backend's choice)", ms_t, SDPA_FLOPS)
   print("    Note: torch may already be dispatching to cuDNN or FlashAttention,")
   print("    so parity here is the expected, healthy outcome.")
   return f"{ms:.3f} ms, {tflops(SDPA_FLOPS, ms):.1f} TFLOP/s"
sdpa_demo()
@section("6. Serialize a built graph, reload it, execute by UID")
def serialization():
   Bsz, M, Kd, Nd = 8, 256, 512, 256
   a = torch.randn(Bsz, M, Kd, device=DEV, dtype=DTYPE)
   bm = torch.randn(Bsz, Kd, Nd, device=DEV, dtype=DTYPE)
   out = torch.empty(Bsz, M, Nd, device=DEV, dtype=DTYPE)
   UID_A, UID_B, UID_C = 1, 2, 3
   g = cudnn.pygraph(
       handle=HANDLE, name="serializable_mm",
       io_data_type=TORCH2CUDNN[DTYPE],
       intermediate_data_type=cudnn.data_type.FLOAT,
       compute_data_type=cudnn.data_type.FLOAT,
   )
   A = tensor_of(g, a, "A").set_uid(UID_A)
   Bt = tensor_of(g, bm, "B").set_uid(UID_B)
   C = g.matmul(A=A, B=Bt, compute_data_type=cudnn.data_type.FLOAT)
   C.set_output(True).set_data_type(TORCH2CUDNN[DTYPE]).set_uid(UID_C)
   t0 = time.perf_counter()
   build(g)
   cold_ms = (time.perf_counter() - t0) * 1e3
   blob = g.serialize()
   print(f"    cold build      : {cold_ms:.1f} ms")
   print(f"    serialized plan : {len(blob)} bytes (cache this to disk / ship it)")
   t0 = time.perf_counter()
   g2 = cudnn.pygraph()
   try:
       g2.deserialize(HANDLE, blob)
   except TypeError:
       g2.deserialize(blob)
   warm_ms = (time.perf_counter() - t0) * 1e3
   print(f"    deserialize     : {warm_ms:.1f} ms  -> {cold_ms/max(warm_ms,1e-6):.1f}x faster startup")
   ws = torch.empty(max(g2.get_workspace_size(), 1), device=DEV, dtype=torch.uint8)
   g2.execute({UID_A: a, UID_B: bm, UID_C: out}, ws, handle=HANDLE)
   torch.cuda.synchronize()
   ref = torch.bmm(a.float(), bm.float())
   rel = ((out.float() - ref).abs().max() / ref.abs().max()).item()
   print(f"    rel err after reload: {rel:.2e}")
   return f"{len(blob)} B blob, reload {cold_ms/max(warm_ms,1e-6):.1f}x faster than rebuild"
serialization()

Credit: Source link

YOU MAY ALSO LIKE

Google AI Previews Tool Retrieval in the Gemini API for Large Tool Libraries

Tesla Rolls Out FSD V15 with Reality View and Vision Upgrades – Unite.AI

ShareTweetSendSharePin

Related Posts

Google AI Previews Tool Retrieval in the Gemini API for Large Tool Libraries
AI & Technology

Google AI Previews Tool Retrieval in the Gemini API for Large Tool Libraries

November 22, 2026
Tesla Rolls Out FSD V15 with Reality View and Vision Upgrades – Unite.AI
AI & Technology

Tesla Rolls Out FSD V15 with Reality View and Vision Upgrades – Unite.AI

November 22, 2026
OpenRouter Cuts Three Providers After AI Prompt Exposure Findings – Unite.AI
AI & Technology

OpenRouter Cuts Three Providers After AI Prompt Exposure Findings – Unite.AI

November 22, 2026
Is It Actually Worth Upgrading To The Pixel 11?
AI & Technology

Is It Actually Worth Upgrading To The Pixel 11?

November 22, 2026
Next Post
Israeli strike in Lebanon injures reporter and cameraman

Israeli strike in Lebanon injures reporter and cameraman

Leave a Reply Cancel reply

Your email address will not be published. Required fields are marked *

Search

No Result
View All Result
Take-Two shatters sales record as ‘GTA 6’ launches to massive fanfare

Take-Two shatters sales record as ‘GTA 6’ launches to massive fanfare

November 20, 2026
Kansas City wins 1st NWSL Championship as Rodman, Spirit lose 3rd straight final – ESPN

Kansas City wins 1st NWSL Championship as Rodman, Spirit lose 3rd straight final – ESPN

November 22, 2026
Genesis AI Hits .5B Valuation in Robotics Bet

Genesis AI Hits $3.5B Valuation in Robotics Bet

November 19, 2026

About

Learn more

Our Services

Legal

Privacy Policy

Terms of Use

Bloggers

Learn more

Article Links

Contact

Advertise

Ask us anything

©2020- TradePoint.io - All rights reserved!

Tradepoint.io, being just a publishing and technology platform, is not a registered broker-dealer or investment adviser. So we do not provide investment advice. Rather, brokerage services are provided to clients of Tradepoint.io by independent SEC-registered broker-dealers and members of FINRA/SIPC. Every form of investing carries some risk and past performance is not a guarantee of future results. “Tradepoint.io“, “Instant Investing” and “My Trading Tools” are registered trademarks of Apperbuild, LLC.

This website is operated by Apperbuild, LLC. We have no link to any brokerage firm and we do not provide investment advice. Every information and resource we provide is solely for the education of our readers. © 2020 Apperbuild, LLC. All rights reserved.

No Result
View All Result
  • Main
  • AI & Technology
  • Stock Charts
  • Market & News
  • Business
  • Finance Tips
  • Trade Tube
  • Blog
  • Shop

© 2023 - TradePoint.io - All Rights Reserved!