data/flash_bench.csv

dtype,d,seq_len,impl,fwd_ms,bwd_ms,e2e_ms
float32,16,128,pytorch,0.12465907633304596,0.5204430669546127,0.6451021432876587
float32,16,128,triton_flash,0.01187089178711176,0.6185732772573829,0.6304441690444946
float32,16,256,pytorch,0.11905621737241745,0.5208603218197823,0.6399165391921997
float32,16,256,triton_flash,0.01621900126338005,0.6140681095421314,0.6302871108055115
float32,16,512,pytorch,0.1308063417673111,0.5299631208181381,0.6607694625854492
float32,16,512,triton_flash,0.02488689310848713,0.6082778703421354,0.6331647634506226
float32,16,1024,pytorch,0.13858072459697723,0.5407632142305374,0.6793439388275146
float32,16,1024,triton_flash,0.0437011681497097,0.6018016971647739,0.6455028653144836
float32,16,2048,pytorch,0.17344558238983154,0.4973258972167969,0.6707714796066284
float32,16,2048,triton_flash,0.08229740709066391,0.5784110799431801,0.660708487033844
float32,16,4096,pytorch,0.5376604795455933,0.8414074182510376,1.3790678977966309
float32,16,4096,triton_flash,0.1530105173587799,0.5653564035892487,0.7183669209480286
float32,16,8192,pytorch,1.924333095550537,3.206986427307129,5.131319522857666
float32,16,8192,triton_flash,0.31498947739601135,2.016775220632553,2.3317646980285645
float32,16,16384,pytorch,7.583169937133789,12.579959869384766,20.163129806518555
float32,16,16384,triton_flash,0.6879677176475525,8.019616663455963,8.707584381103516
float32,16,32768,pytorch,35.15679168701172,50.200782775878906,85.35757446289062
float32,16,32768,triton_flash,1.959969162940979,31.643615126609802,33.60358428955078
float32,16,65536,pytorch,,,
float32,16,65536,triton_flash,7.498465061187744,,
float32,32,128,pytorch,0.12362661212682724,0.5246296301484108,0.648256242275238
float32,32,128,triton_flash,0.012916319072246552,0.6020736172795296,0.6149899363517761
float32,32,256,pytorch,0.12605075538158417,0.5191686302423477,0.6452193856239319
float32,32,256,triton_flash,0.01763702742755413,0.6070405710488558,0.6246775984764099
float32,32,512,pytorch,0.1342126876115799,0.5378362387418747,0.6720489263534546
float32,32,512,triton_flash,0.028019318357110023,0.611276438459754,0.639295756816864
float32,32,1024,pytorch,0.13301046192646027,0.534347340464592,0.6673578023910522
float32,32,1024,triton_flash,0.047974180430173874,0.5937763340771198,0.6417505145072937
float32,32,2048,pytorch,0.17799162864685059,0.49950695037841797,0.6774985790252686
float32,32,2048,triton_flash,0.08926910161972046,0.5637074112892151,0.6529765129089355
float32,32,4096,pytorch,0.5586839318275452,0.8496330380439758,1.408316969871521
float32,32,4096,triton_flash,0.17790304124355316,0.6057605296373367,0.7836635708808899
float32,32,8192,pytorch,2.025404453277588,3.181220054626465,5.206624507904053
float32,32,8192,triton_flash,0.34378400444984436,2.1998461186885834,2.5436301231384277
float32,32,16384,pytorch,7.915563106536865,13.130709171295166,21.04627227783203
float32,32,16384,triton_flash,0.8448686003684998,8.676181852817535,9.521050453186035
float32,32,32768,pytorch,36.462181091308594,51.96021270751953,88.42239379882812
float32,32,32768,triton_flash,2.5638539791107178,34.236248254776,36.80010223388672
float32,32,65536,pytorch,,,
float32,32,65536,triton_flash,10.114856719970703,,
float32,64,128,pytorch,0.13064712285995483,0.5412216186523438,0.6718687415122986
float32,64,128,triton_flash,0.014301666058599949,0.6242158925160766,0.6385175585746765
float32,64,256,pytorch,0.13519208133220673,0.5409094244241714,0.6761015057563782
float32,64,256,triton_flash,0.020251400768756866,0.6249312087893486,0.6451826095581055
float32,64,512,pytorch,0.13928042352199554,0.5488250404596329,0.6881054639816284
float32,64,512,triton_flash,0.03301728516817093,0.6177325770258904,0.6507498621940613
float32,64,1024,pytorch,0.13825109601020813,0.5642800033092499,0.702531099319458
float32,64,1024,triton_flash,0.059274349361658096,0.6094330213963985,0.6687073707580566
float32,64,2048,pytorch,0.20861725509166718,0.4898863583803177,0.6985036134719849
float32,64,2048,triton_flash,0.11078418791294098,0.5565537661314011,0.667337954044342
float32,64,4096,pytorch,0.6637576222419739,1.0878928303718567,1.7516504526138306
float32,64,4096,triton_flash,0.21500347554683685,0.9016343504190445,1.1166378259658813
float32,64,8192,pytorch,2.456576108932495,4.298021078109741,6.754597187042236
float32,64,8192,triton_flash,0.44004735350608826,3.473032623529434,3.9130799770355225
float32,64,16384,pytorch,9.893942832946777,16.94670581817627,26.840648651123047
float32,64,16384,triton_flash,1.2568162679672241,13.379923462867737,14.636739730834961
float32,64,32768,pytorch,43.36127853393555,66.06643295288086,109.4277114868164
float32,64,32768,triton_flash,4.112161159515381,51.91156244277954,56.02372360229492
float32,64,65536,pytorch,,,
float32,64,65536,triton_flash,17.249839782714844,,
float32,128,128,pytorch,0.11557603627443314,0.5213436409831047,0.6369196772575378
float32,128,128,triton_flash,0.020308636128902435,0.548356868326664,0.5686655044555664
float32,128,256,pytorch,0.1258781999349594,0.517955556511879,0.6438337564468384
float32,128,256,triton_flash,0.03286709636449814,0.5871860161423683,0.6200531125068665
float32,128,512,pytorch,0.12279844284057617,0.5262462496757507,0.6490446925163269
float32,128,512,triton_flash,0.05700226500630379,0.5593465976417065,0.6163488626480103
float32,128,1024,pytorch,0.13184888660907745,0.5270644873380661,0.6589133739471436
float32,128,1024,triton_flash,0.10707969963550568,0.5175818651914597,0.6246615648269653
float32,128,2048,pytorch,0.26466867327690125,0.4545592963695526,0.7192279696464539
float32,128,2048,triton_flash,0.2079632580280304,0.4668257534503937,0.6747890114784241
float32,128,4096,pytorch,0.8985901474952698,1.6054357886314392,2.504025936126709
float32,128,4096,triton_flash,0.5308263897895813,1.5403229594230652,2.0711493492126465
float32,128,8192,pytorch,3.398730516433716,6.066050291061401,9.464780807495117
float32,128,8192,triton_flash,1.7835369110107422,5.751260280609131,7.534797191619873
float32,128,16384,pytorch,13.55659008026123,24.33305072784424,37.88964080810547
float32,128,16384,triton_flash,5.626699447631836,22.632482528686523,28.25918197631836
float32,128,32768,pytorch,57.47541427612305,94.37968826293945,151.8551025390625
float32,128,32768,triton_flash,20.629276275634766,87.6577033996582,108.28697967529297
float32,128,65536,pytorch,,,
float32,128,65536,triton_flash,82.4273910522461,,
bfloat16,16,128,pytorch,0.12711696326732635,0.5149972587823868,0.6421142220497131
bfloat16,16,128,triton_flash,0.008603598922491074,0.6982087232172489,0.70681232213974
bfloat16,16,256,pytorch,0.12618540227413177,0.5115032941102982,0.6376886963844299
bfloat16,16,256,triton_flash,0.00967857614159584,0.7260054610669613,0.7356840372085571
bfloat16,16,512,pytorch,0.13286401331424713,0.5010183304548264,0.6338823437690735
bfloat16,16,512,triton_flash,0.012342417612671852,0.7070387359708548,0.7193811535835266
bfloat16,16,1024,pytorch,0.13355091214179993,0.5078878104686737,0.6414387226104736
bfloat16,16,1024,triton_flash,0.01767667382955551,0.7104356065392494,0.7281122803688049
bfloat16,16,2048,pytorch,0.1293325573205948,0.4994489401578903,0.6287814974784851
bfloat16,16,2048,triton_flash,0.029038693755865097,0.706197950989008,0.735236644744873
bfloat16,16,4096,pytorch,0.38011330366134644,0.4506337642669678,0.8307470679283142
bfloat16,16,4096,triton_flash,0.050581689924001694,0.6927209533751011,0.7433026432991028
bfloat16,16,8192,pytorch,1.2742047309875488,1.594050645828247,2.868255376815796
bfloat16,16,8192,triton_flash,0.1428881287574768,1.8423064351081848,1.9851945638656616
bfloat16,16,16384,pytorch,4.730341911315918,6.144253730773926,10.874595642089844
bfloat16,16,16384,triton_flash,0.38894155621528625,7.240185469388962,7.629127025604248
bfloat16,16,32768,pytorch,18.75957679748535,25.83331871032715,44.5928955078125
bfloat16,16,32768,triton_flash,1.1940759420394897,28.488611817359924,29.682687759399414
bfloat16,16,65536,pytorch,81.2759017944336,,
bfloat16,16,65536,triton_flash,4.586021423339844,,
bfloat16,32,128,pytorch,0.12433042377233505,0.494148887693882,0.618479311466217
bfloat16,32,128,triton_flash,0.009724904783070087,0.6880407435819507,0.6977656483650208
bfloat16,32,256,pytorch,0.12406639009714127,0.5154918059706688,0.6395581960678101
bfloat16,32,256,triton_flash,0.011119954288005829,0.726523645222187,0.7376435995101929
bfloat16,32,512,pytorch,0.136015385389328,0.4990067183971405,0.6350221037864685
bfloat16,32,512,triton_flash,0.014299283735454082,0.7058156030252576,0.7201148867607117
bfloat16,32,1024,pytorch,0.1360682100057602,0.519484356045723,0.6555525660514832
bfloat16,32,1024,triton_flash,0.02039397694170475,0.7169098164886236,0.7373037934303284
bfloat16,32,2048,pytorch,0.1369975209236145,0.4935304522514343,0.6305279731750488
bfloat16,32,2048,triton_flash,0.034046389162540436,0.6962083801627159,0.7302547693252563
bfloat16,32,4096,pytorch,0.37971213459968567,0.424212783575058,0.8039249181747437
bfloat16,32,4096,triton_flash,0.06118820980191231,0.6868196837604046,0.7480078935623169
bfloat16,32,8192,pytorch,1.27370023727417,1.5661463737487793,2.839846611022949
bfloat16,32,8192,triton_flash,0.16339819133281708,1.8648579865694046,2.0282561779022217
bfloat16,32,16384,pytorch,4.763161659240723,6.127760887145996,10.890922546386719
bfloat16,32,16384,triton_flash,0.457584023475647,7.265423655509949,7.723007678985596
bfloat16,32,32768,pytorch,18.752717971801758,25.137460708618164,43.89017868041992
bfloat16,32,32768,triton_flash,1.4546339511871338,28.55010199546814,30.004735946655273
bfloat16,32,65536,pytorch,81.31942749023438,,
bfloat16,32,65536,triton_flash,5.500489234924316,,
bfloat16,64,128,pytorch,0.12202516943216324,0.5286194309592247,0.6506446003913879
bfloat16,64,128,triton_flash,0.01123123150318861,0.6899659158661962,0.7011971473693848
bfloat16,64,256,pytorch,0.12677983939647675,0.5092562586069107,0.6360360980033875
bfloat16,64,256,triton_flash,0.013002940453588963,0.7110446011647582,0.7240475416183472
bfloat16,64,512,pytorch,0.12693709135055542,0.5119546055793762,0.6388916969299316
bfloat16,64,512,triton_flash,0.018755920231342316,0.7090413495898247,0.727797269821167
bfloat16,64,1024,pytorch,0.12441147863864899,0.5012811869382858,0.6256926655769348
bfloat16,64,1024,triton_flash,0.029328707605600357,0.7002786956727505,0.7296074032783508
bfloat16,64,2048,pytorch,0.1312282681465149,0.49995213747024536,0.6311804056167603
bfloat16,64,2048,triton_flash,0.05081166326999664,0.6975466459989548,0.7483583092689514
bfloat16,64,4096,pytorch,0.3881649076938629,0.43819281458854675,0.8263577222824097
bfloat16,64,4096,triton_flash,0.09388783574104309,0.7349584400653839,0.828846275806427
bfloat16,64,8192,pytorch,1.3175185918807983,1.611681580543518,2.9292001724243164
bfloat16,64,8192,triton_flash,0.25424426794052124,2.7295268177986145,2.9837710857391357
bfloat16,64,16384,pytorch,4.852447032928467,6.232293605804443,11.08474063873291
bfloat16,64,16384,triton_flash,0.6809776425361633,10.348224818706512,11.029202461242676
bfloat16,64,32768,pytorch,19.179828643798828,25.648845672607422,44.82867431640625
bfloat16,64,32768,triton_flash,2.2446329593658447,39.73476004600525,41.979393005371094
bfloat16,64,65536,pytorch,82.95680236816406,,
bfloat16,64,65536,triton_flash,8.716426849365234,,
bfloat16,128,128,pytorch,0.12723277509212494,0.5115050226449966,0.6387377977371216
bfloat16,128,128,triton_flash,0.01540232915431261,0.6653519151732326,0.6807542443275452
bfloat16,128,256,pytorch,0.13112647831439972,0.5077687948942184,0.6388952732086182
bfloat16,128,256,triton_flash,0.020781781524419785,0.7840208746492863,0.804802656173706
bfloat16,128,512,pytorch,0.12178614735603333,0.605666309595108,0.7274524569511414
bfloat16,128,512,triton_flash,0.032612621784210205,0.7706170082092285,0.8032296299934387
bfloat16,128,1024,pytorch,0.11929361522197723,0.6057620793581009,0.7250556945800781
bfloat16,128,1024,triton_flash,0.054898958653211594,0.7685160227119923,0.8234149813652039
bfloat16,128,2048,pytorch,0.136956125497818,0.6126578152179718,0.7496139407157898
bfloat16,128,2048,triton_flash,0.10154028981924057,0.7202378883957863,0.8217781782150269
bfloat16,128,4096,pytorch,0.4057762622833252,0.472406268119812,0.8781825304031372
bfloat16,128,4096,triton_flash,0.19298826158046722,1.169649377465248,1.3626376390457153
bfloat16,128,8192,pytorch,1.3657245635986328,1.679750680923462,3.0454752445220947
bfloat16,128,8192,triton_flash,0.740023136138916,4.2113847732543945,4.9514079093933105
bfloat16,128,16384,pytorch,5.030723571777344,6.44927978515625,11.480003356933594
bfloat16,128,16384,triton_flash,2.1923351287841797,16.3962345123291,18.58856964111328
bfloat16,128,32768,pytorch,19.894840240478516,26.423755645751953,46.31859588623047
bfloat16,128,32768,triton_flash,7.266067028045654,62.56561326980591,69.83168029785156
bfloat16,128,65536,pytorch,88.90726470947266,,
bfloat16,128,65536,triton_flash,29.05224609375,,