^j1ddlZddlZddlmZmZddlZddlmZddlmZm Z m Z m Z m Z ddl mZej ddej dedej"d ej d ej f d Zej ddej dedej"d ej&d ej d eed ej fdZej ddej dedej"d eej&d eed ej d ej fdZej ddej,ej zdej.dej0edeej"d eej&d eed ej deed ej fdZej ddej,dej0edeej"d eej&deed eeded e e eej,gfdZej d dej"dededed ej"f dZy)!N)CallableOptional)Int32Int64Float32Boolean const_expr)utilsvalopreduction_bufferinit_valreturnctjjtjj}}tj|j d}||z||z}}|dk(r||||f<tjj |} ||kr|||f} tjj| |S)zDreduction_buffer has shape (num_warps / warp_per_row, warps_per_row)r r)cutearchlane_idxwarp_idxsizeshapebarrierwarp_reduction) r r rrrr warps_per_rowrow_idxcol_idxblock_reduce_vals e/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_vendor/quack/reduce.py block_reducers ++-tyy/A/A/ChHII.44Q78M=0(]2JWG1}-0')*II-+GX,=> 99 # #$4b 99mbar_ptrphasec tjj}tjjtjj }}|j \} \} } || z|| z} } |dk(ritjj 5| | z}tjj||| z|jjzdzddd|| kr1tj|tj|| | |ff||tjj|||nd|}tj| | ztjj}t!j"|D]L}||tjjzz}|tj$|dgks?|||| |f}Ntjj'||S#1swYxYw)zRreduction_buffer has shape (num_warps / warps_per_row, (warps_per_row, cluster_n))rNpeer_cta_rank_in_clusterr"r mode)rrblock_idx_in_clusterrrr elect_onembarrier_arrive_and_expect_tx element_typewidthr store_shared_remote elem_pointer mbarrier_waitceil_div WARP_SIZEcutlassrange_constexprrr)r r rr!rr"cta_rank_in_clusterrrrows_per_blockr cluster_nrr num_warpsrnum_iteriidxs rcluster_reducer=s))88:++-tyy/A/A/ChH1A1G1G.N.]I=0(]2JWG1} YY " &6I II 3 3I%(8(E(E(K(KKqP   ) !!    /'GEX;Y1Z [ %-   IIHU5FEAN}}]Y6 8K8KLH  $ $X .TTYY0000 +1#6 6!"24DWc\4RS T 99 # #$4b 99)  s AG77Hc\t|durt||||St||||||S)zPPerform either block or cluster reduction based on whether mbar_ptr is provided.N)rr"r)r rr=)r r rr!r"rs rblock_or_cluster_reducer@Es:(d"#C%5IIc2'7Yabbr xthreads_per_rowhook_fnc^tt|tjr|j ||d}n|}tj j tjtj jt|jtk(rtjjnttj jt tj j"tj$i|} tjj'|| t!|tjj(}t|dur|t|durF|j*d\} } | dk(s |Jdt| dkDxs| dkDrt-|| ||||}|S)zXreduction_buffer must have shape (num_warps / warps_per_row, (warps_per_row, cluster_n))rrreduction_profilethreads_in_groupNr /mbar_ptr must be provided for cluster reductionr?)r isinstancer TensorSSAreduce ReductionOpADDoperatoraddMAXdtyperrfmaxmaxMINminMULmulrr3rr@) rAr rBrr!r"rrCr warp_oprr8s r row_reducerZUsh*Q/0hhrHhB hll  177g;M0NdiinnTW c hll    G )) " " _dii.A.AB # C '%& "$./#3#9#9!#< yA~!5 = 5 ma'89q= 9)W.PXC Jr return_exp_xc  |jtk(sJd tjj |j tj jtj dtjjt|tjj}tjtj}tjj||z||zz d} tjj | j tj j ddt"j$t|tjj} t'|dur|t'|dur|j(\} \} } | dk(s |Jd t'| dkDxs| dkDr|j*t,k(sJd tjj/tjj1}}|| z|| z}}t'|dur6|dk(rt3j4|| |||f<tjj7tj }d} || krt3j8|||f\}} tjj |tjj}| tjj;||z dz} tjj | t"j$} t'|r'| tjj;||z dz} |}n9tjj=}|dk(ritjj?5| | z}tjjA||| z|j*jBzd zddd|| krEt3jDt3j4|| t3jF||||ff|| tjjI|||nd tjJ| | ztjj}tjL|t}|jOtj tjL|t}|jOdtQjR|D]a}||tjjzz}|tjT|dgks?t3j8|||f\||<||<c|jWj tj jtj d}tjj |tjj}d} tQjR|D]2}| ||tjj;|||z dzz } 4tjj | t"j$} t'|r'| tjj;||z dz} |}|| t'|r| fSdfS#1swYxYw)Nzx must be of type Float32rrErGT)fastmathr rIz+reduction_buffer must be of type cute.Int64r$r%r'r(),rRrrrrrLrMrQinfrSrVr3mathlog2eexp2rNrOrPr rr-rrrr f32x2_to_i64r i64_to_f32x2expr*r+r,r.r/r0r1r2make_rmem_tensorfillr4r5rload)rArBrr!rCr"r[max_xlog2_eexp_x sum_exp_xr7rr8rrrrmax_x_single_warp max_x_finalr6r9r:sum_exp_x_single_warpr;r<s ronline_softmax_reducerq~s 77g ::: e II $ $ !!%% PQR _dii.A.AB % E YYtvv F IINN1v:84N HE (( T%%))C1 M _dii.A.AB)I '%& "$./5E5K5K22 A~!5 = 5 ma'89q= 9#00E9 = 9"&!3!3!5tyy7I7I7KhH'=8(]:RWG(d*+q=9>9K9KES\9]$Wg%56 !!#%,[[L! m+383E3E(():;40%y#ii667H$))..Y TYY]]+<{+JUY]ZZ  II44Y M l+TYY]]5;+>]NNE#&*ii&D&D&F#q=,,.$2]$B  ??$% 14D4Q4Q4W4WW[\\ i'--**5)<**,wBU8V.W!19  ''ARXY'Z==)BDIIDWDWX$($9$9(G$L!!&& |4(,(=(=h(P%%**3/ 00:A"Q)<)<%< #ii66{DIINNS   00:A!6q!9DIIMM)!,{:T=J="I!II44Y M l+TYY]]5;+>]NNE# )z,'?e JJT JJQs 7A[[ X elem_per_lane subwarp_size warp_sizec @|dk\r)|dkr$|dttj|zk(sJ|dkr,||zdk(r$|dttj|zk(sJtjj |z}tj |tj|}tj|dg}|d|zksJtjttj|ttj||zdD]f}tjtj|dD]7}tjj||d|z|z}|||z||<9htjttjtj|dgdz ddD]} d| z} tj| dD]} |d | f} |d | | zf} t|| z }tjtj| dD]P}| || |}}|r|n|| |<|r|n|| |<tjj| || |z}| ||z| |<R|d S) aX For warp reduction, we use Swap Shuffle The normal way to reduction among threads: use shuffle to let *** the first half of threads *** have *** whole data *** from the second half of threads. After each step of reduction, a half of threads won't work in the following steps. That is, as the reduction progresses, the efficiency of shuffle & reduction instructions gradually change from 1/2, 1/4 to 1/32 (the worst case). To overcome this shortcoming, for a NxN matrix to be reduced among N threads as a 1XN vectors, we use swap & shuffle aiming to let *** each half of threads *** have *** a half of data *** from the other half of threads. After reduction, each half of threads should deal with a (N/2)x(N/2) sub-matrix independently in the following step. We can recursively do this until the problem size is 1. r rr(T) unroll_full)offsetN)Nr)intr`rarrrlogical_divide make_layoutrr4rangeshuffle_sync_bflyr5r)rrrsrtrurnumvecr;vshfl_vallogmmrfrg_Afrg_B should_swaploweruppers rsum_swap_shufflers  B  ATYY|% R  $ ) c$))I"677 7 8yy!!#|3H At// >?A YYqs #F R<' '' ']] DIIf DIIi<.G$H IW[#tyy|> #Ayy221Q4a<@W2XHQ4(?AaD ## ''DIIdii6L,M(NQR(RTVXZ[/ Iqd3 /AdAgJEdAEkNE%hl33K]]499U#3F /$Qxqu$/5Ua$/5Ua 9966uQxLHX6Y 8h.a /  //" W:r )r^)r^N)Nr^)NNNr^N)NNNNF)r r rw)r`rOtypingrrr4 cutlass.cuterrrrrr r jitNumericTensorrPointerr=r@rKrM Constexprr{rZboolrqrr rrsE %>>]` :  :# :7;{{ :NRll : \\ :  :  !! ": ":":kk":ll ": ll ": E? ": \\": ":J " c  c ckk ct||$ c E? c ll c \\ c  c /3'+! "&% ~~ $% %&&s+%t{{+ % t||$ % E? %ll%h % \\% %P/3'+"&!_K ~~_K&&s+_Kt{{+_Kt||$ _K h  _K E? _K_Kw01_K _KDTV5 {{5#&5:=5NQ5 [[5 5r