^ju ddlZddlZddlmcmZddlmZddlm Z ddl m Z m Z ddl mZejeZde j$de j&d efd Zde j$de j&d d d dfd Zde j$d e j.fdZe eeefZde j.d dfdZde j.d efdZde j$ded d fdZde j$fdZ d e j.fdZ!dZ"dZ#y)N) is_symbolic) OrderedSet)configir)Vxcomm_buffer_typereturnct|}t|tjryt|tjr|j sy|j }t|tjryt|tjryt|tjrt|jsyy)ze Check if an input can be realized as a comm buffer of the specified `comm_buffer_type`. TF) _get_data isinstancerLoopsBuffershould_allocateget_output_specCommBufferLayout FixedLayoutFlexibleLayoutr get_numel)r r datalayouts h/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_inductor/comm_lowering.pycan_realize_as_comm_bufferr7s Qfd}|j@fd}|jBfd}y#t$rt j dYywxYw)z4 Register lowerings for the comm subsystem. zRInductor support for distributed collectives depends on building torch.distributedNr)add_layout_constraintcloneconstrain_to_fx_stridescopy_register_loweringc&||Sr3)fnrYr[r]s rregister_comm_loweringz7register_comm_lowerings..register_comm_loweringsb"9: $$rr5r6rrr ct|||r t|||S|}tjrG|j t j jj|jtjj|}tjjjj ||||Sr3)rCrOr reorder_for_compute_comm_overlapr rr.no_fuse_buffer_namesr,r/r ExternKernelrequire_contiguous_AllReduce_Kernelcreate_inplace all_reduce_rN)r5r6rc10drZs r _all_reducez,register_comm_lowerings.._all_reduces 0Y K'Y C CCj  2 2 KKM GG ( ( , ,S\\^ <oo005 ++    $ $      rct|||r!|t|||}t||Stjj |}tj jjj||||Sr3) rCrOr1rrerfrgrhrirN)r5r6rretrjr\s r _all_reduce_z-register_comm_lowerings.._all_reduce_s 0Y K$S)Z@C c "Joo005 ++    $ $      rc|Dcgc] }| }}tjjjj||||Scc}wr3rrQrhall_reduce_coalesced_rN)rUr6rr5rjrZs r_all_reduce_coalescedz6register_comm_lowerings.._all_reduce_coalescedsR(./%*// ++  & & . .      0sActtjjjj||||Sr3rp)rUr6rrjs r_all_reduce_coalesced_z7register_comm_lowerings.._all_reduce_coalesced_s6 ++  & & . .      rcHtjj|||Sr3)rWall_gather_into_tensorrN)r5 group_sizerrjs r_all_gather_into_tensorz8register_comm_lowerings.._all_gather_into_tensor(s(#  ' ' / /      rc tjtjjtj j jj|||Sr3) rErFrrGrHrQrR all_gather_into_tensor_coalescedrN)rUrwrrjs r!_all_gather_into_tensor_coalescedzBregister_comm_lowerings.._all_gather_into_tensor_coalesced1sM LL    4 455==    rcxtjjjj|||||SN)out)rrQrhall_gather_into_tensor_outrN)r5rwrr~rjs r_all_gather_into_tensor_outz._all_gather_into_tensor_out=s> ++  + + 3 3    ,  rcJtjj||||Sr3)rWreduce_scatter_tensorrN)r5r6rwrrjs r_reduce_scatter_tensorz7register_comm_lowerings.._reduce_scatter_tensorHs+#  & & . .       rcztjjjj||||||Sr})rrQrhreduce_scatter_tensor_outrN)r5r6rwrr~rjs r_reduce_scatter_tensor_outz;register_comm_lowerings.._reduce_scatter_tensor_outRsA ++  * * 2 2     ,  rc tjtjjtj j jj||||Sr3) rErFrrGrHrQrRreduce_scatter_tensor_coalescedrN)rUr6rwrrjs r _reduce_scatter_tensor_coalescedzAregister_comm_lowerings.._reduce_scatter_tensor_coalesced^sP LL    4 444<<    rcJtjj||||Sr3)rWall_to_all_singlerN)r5output_split_sizesinput_split_sizesrrjs r_all_to_all_singlez3register_comm_lowerings.._all_to_all_singleks+#  " " * *       rc|}tjjjj||||Sr3rrQrh broadcast_rN)r5srcrrjrZs r _broadcastz+register_comm_lowerings.._broadcastus:Cj ++ OO # #S#z  rcttjjjj||||Sr3r)r5rrrjs r _broadcast_z,register_comm_lowerings.._broadcast_}s1 ++ OO # #S#z  rcxttjjjj ||||Sr3)rWrJrK_dtensorshard_dim_alltoallrN)r5 gather_dim shard_dimrs r_shard_dim_alltoallz4register_comm_lowerings.._shard_dim_alltoalls4# II   1 1 9 9       rct|r|Stjjjj ||Sr3)r4r _WaitKernel create_wait wait_tensorrN)r5rjs r _wait_tensorz-register_comm_lowerings.._wait_tensors5 C J ""4#3#3#;#;SA rctjj|}tjj ||||Sr3)rrerfrWisendrN)r5dsttagrrjs r_isendz'register_comm_lowerings.._isends4oo005#DJJ$6$6S#zRRrctjj|}tjj j j |||||Sr3)rrerfrQrhirecvrN)r5rrrrjs r_irecvz'register_comm_lowerings.._irecvsGoo005 ++ JJ  S#z  rc |Dcgc]!}tjj|#}}jj}t j j5tjj||||||\}}} } } ddd r J|d| t|D]H\} } | j| dk(st j j| jJ|dj}tjtj|||  }g}t!t||D]\}\} }}| dk(r|j"j%tj&tj(||||j*j%|j|j%|tj,tjj/||t0|fg}|j%tj2j5||Scc}w#1swYxYw)N rr)device)rrerf batch_p2p_opsrNrr. fake_moderQprocess_kernelzipr mark_buffer_mutatedr/ get_deviceMultiOutputLayout enumeratemutation_outputsappendMutationOutput NoneLayout alias_names MultiOutputtensor_to_layoutlistrGrH)op_list peer_listtag_listtensorsrtrTexample_output tensor_argsnon_tensor_argsunflatten_argsunbacked_bindingsop tensor_argrpackedresultsiex_out placeholderrjs r_batch_p2p_opsz/register_comm_lowerings.._batch_p2p_opss#BIJQ2??55a8JJ##++ WW   $$33   !  %E2C1D&EE$!';7 CNB    W}++J,?,?,AB C Q**,%%   /      "+C.,Q"R A AAvW}''..%%bmm6&BAvN""))!**,7q!!nn((99&AAYK r||22;?@ AaK  s&I +II)"rJrK_c10d_functional all_reduceAttributeErrorloginfoloweringrYrZr[r\r]rrGstrriall_reduce_coalescedrqrvrzrrrrr broadcastrrrrrrr)rarkrnrrrtrxr{rrrrrrrrrrrrrYrjrZr[r\r]s @@@@@@rregister_comm_loweringsrs ""--% 99 % %DDOO, \\C  -<D,,- \\C  .0D5567D6678D778 9 DAAB  C  D;;<=D667 8 D::; < D@@A  B  D223 4 DNN+,DOO,- EII..AAB C D,,-.DJJ'S(SDJJ'(D../101}      s$JJ=<J=c   tjjjddlm}|jjjj|jjjjddl m}dtj dtj"d d d tj fd dtj d t$d tj ffd |jdtj dt$d t$ffd }|jdtj dt$d t$dtj ffd }|jdtj dtj dt$d t$ffd }|jdtj dtj dt$d t$dtj f fd }|j&dtj dt$d t$ffd }|j(dtj dt$d t$dtj ffd }|j*dtj dt$d t$ffd }|j,dtj dt$d t$ffd } |j.dtj dt$d t$dtj ffd } |j0dtj dt$dt2d t$dtj f fd } |j4dtj d t$dtj ffd } |j6dtj d t$dt8dtj ffd } |j:dtj dtj d!tj d"tj d t$f fd# }|j< d-dtj dtj d!tj d"tj d t$f fd$ }|j>dtj dtj d%tj d"tj d t$f fd& }|j@ d.d'tj d(tj dt2d t$dt$f fd) }|jB d.d(tj d t$dt$ffd* }|jDdtj d t$ffd+ }|jFdtj dt$d t$ffd, }y#t$rt j dYywxYw)/zH Register lowerings for symmetric memory (symm_mem) operations. z7symm_mem ops not available, skipping symm_mem loweringsNr)register_out_variantr)r]r5r rrr c|jtjj|j |j |j |j}t||||S)z Fallback: insert a Pointwise identity copy allocated in P2P via CommBufferLayout. Used when we don't control the input's allocation. )rdtypeinner_fnranges) r r PointwiserHrr; make_loaderget_sizer#)r5r rcopys r_copy_input_to_comm_bufferz?register_symm_mem_lowerings.._copy_input_to_comm_buffers` ||"">>#--/__&<<> #  t%5zB rct|tjjr't |tjj||S|tjj|S)a Ensure inp is in P2P memory for a symm_mem collective. If inductor controls the buffer's allocation (ComputedBuffer, or any buffer with FlexibleLayout/FixedLayout), switch its layout to CommBufferLayout in-place, zero-copy. If inductor does not control allocation (e.g. InputBuffer), insert a Pointwise identity copy into a new CommBufferLayout buffer. This adds an extra Triton kernel. Returns the possibly new TensorBox. TODO(tianrengao): eliminate the extra kernel for static-shape InputBuffers by pre-allocating P2P memory in the wrapper and DMA .copy_() )rrr?r@r#)r5rrs r_maybe_realize_symm_memz._maybe_realize_symm_memsZ$ &c2+<+<+E+E F "3(9(9(B(BJ OJ-!!** rr6c ||}tjtjjtj j j j|||Sr3)rErFrrGrHrIrMrNr5r6rrrLs r_symm_mem_one_shot_all_reducezBregister_symm_mem_lowerings.._symm_mem_one_shot_all_reduce$s[ &c:6 LL      $ $,,44    rr~c ||}tjtjjtj j j j||||Sr3)rErFrrGrHrIone_shot_all_reduce_outrNr5r6rr~rrLs r!_symm_mem_one_shot_all_reduce_outzFregister_symm_mem_lowerings.._symm_mem_one_shot_all_reduce_out5s^&c:6 LL      $ $0088    r symm_buffer local_inputc ||}tjtjjtj j j j||||Sr3)rErFrrGrHrIone_shot_all_reduce_copyrN)rrr6rrrLs r"_symm_mem_one_shot_all_reduce_copyzGregister_symm_mem_lowerings.._symm_mem_one_shot_all_reduce_copyHs^.k:F  LL      $ $1199    rc ||}tjtjjtj j j j|||||Sr3)rErFrrGrHrIone_shot_all_reduce_copy_outrN)rrr6rr~rrLs r&_symm_mem_one_shot_all_reduce_copy_outzKregister_symm_mem_lowerings.._symm_mem_one_shot_all_reduce_copy_out[sa.k:F  LL      $ $55==    rc||}tjjjj||||Sr3)rrIrHtwo_shot_all_reduce_rNrs r_symm_mem_two_shot_all_reduce_zCregister_symm_mem_lowerings.._symm_mem_two_shot_all_reduce_pD &c:6    ) ) 1 1      routputc ||}tjtjjtj j j j||||Sr3)rErFrrGrHrItwo_shot_all_reduce_outrN)r5r6rrrrLs r!_symm_mem_two_shot_all_reduce_outzFregister_symm_mem_lowerings.._symm_mem_two_shot_all_reduce_outs^&c:6 LL      $ $0088    rc||}tjjjj||||Sr3)rrIrHmultimem_all_reduce_rNrs r_symm_mem_multimem_all_reduce_zCregister_symm_mem_lowerings.._symm_mem_multimem_all_reduce_rrc ||}tjtjjtj j j j|||Sr3)rErFrrGrHrImultimem_one_shot_all_reducerNrs r&_symm_mem_multimem_one_shot_all_reducezKregister_symm_mem_lowerings.._symm_mem_multimem_one_shot_all_reduces[ &c:6 LL      $ $55==    rc ||}tjtjjtj j j j||||Sr3)rErFrrGrHrI multimem_one_shot_all_reduce_outrNrs r*_symm_mem_multimem_one_shot_all_reduce_outzOregister_symm_mem_lowerings.._symm_mem_multimem_one_shot_all_reduce_outs^&c:6 LL      $ $99AA    rrootc ||}tjtjjtj j j j|||||Sr3)rErFrrGrHrImultimem_one_shot_reduce_outrN)r5r6rrr~rrLs r&_symm_mem_multimem_one_shot_reduce_outzKregister_symm_mem_lowerings.._symm_mem_multimem_one_shot_reduce_outsa&c:6 LL      $ $55==    rc ||}tjtjjtj j j j|||Sr3)rErFrrGrHrImultimem_all_gather_outrN)r5rr~rrLs r!_symm_mem_multimem_all_gather_outzFregister_symm_mem_lowerings.._symm_mem_multimem_all_gather_outs[ &c:6 LL      $ $0088    rsplit_last_dimc ||}tjtjjtj j j j||||Sr3)rErFrrGrHrIreduce_scatter_outrN)r5rr rrrLs r_symm_mem_reduce_scatter_outzAregister_symm_mem_lowerings.._symm_mem_reduce_scatter_outs^&c:6 LL      $ $++33    r in_splitsout_splits_offsetsc||}||}tjjjj|||||yr3)rrIrHall_to_all_vdevrN)r5r~rrrrrLs r_symm_mem_all_to_all_vdevz>register_symm_mem_lowerings.._symm_mem_all_to_all_vdevsT&c:6%c:6    $ $ , ,       rc ||}||}tjjjj||||||yr3)rrIrHall_to_all_vdev_2drN)r5r~rrr major_alignrrLs r_symm_mem_all_to_all_vdev_2dzAregister_symm_mem_lowerings.._symm_mem_all_to_all_vdev_2dsW&c:6%c:6    ' ' / /       rin_splits_offsetsc||}||}tjjjj|||||yr3)rrIrHall_to_all_vdev_2d_offsetrN)r5r~rrrrrLs r#_symm_mem_all_to_all_vdev_2d_offsetzHregister_symm_mem_lowerings.._symm_mem_all_to_all_vdev_2d_offset(sT&c:6%c:6    . . 6 6       rin_tileout_tilec||}||}tjjjj|||||yr3)rrIrH tile_reducerN)rrrrr6rrLs r_symm_mem_tile_reducez:register_symm_mem_lowerings.._symm_mem_tile_reduce<sT*':>*8Z@    ( (       rct|D]\}}||||<||}tjjjj |||||yr3)rrrIrHmulti_root_tile_reducerN) in_tilesrrootsrr6rrrrLs r _symm_mem_multi_root_tile_reducezEregister_symm_mem_lowerings.._symm_mem_multi_root_tile_reducePsr$H- GJAw1':FHQK G*8Z@    + + 3 3       rcFtjj||Sr3)rW_low_contention_all_gatherrN)r5rrLs r#_symm_mem_low_contention_all_gatherzHregister_symm_mem_lowerings.._symm_mem_low_contention_all_gatheres'$  / / 7 7    rcHtjj|||Sr3)rW_low_contention_reduce_scatterrN)r5r6rrLs r'_symm_mem_low_contention_reduce_scatterzLregister_symm_mem_lowerings.._symm_mem_low_contention_reduce_scatterts*$  3 3 ; ;      rr3)r9)$rJrKrLrMrrrtorch._library._out_variantrrNrrrrr]rrGr?rrrrrrrintr rboolrrrr r#r(r+)rr]rrrrrrrrrrr rrrrr!r&r)r,rrrLs @@@rregister_symm_mem_loweringsr0s 99%% $$ A$$,,((00))11--55 , \\++C  & \\ 8x334 \\   5 x778 \\   \\  9 $x889 \\ \\    : $x<<= \\ \\    \\  > (x445  \\   6 x778 \\     9 $x445  \\   6 x<<= \\   > x@@A \\   \\  B $x<<= \\     \\  > (x778 \\  \\ 9 x223 \\     4 $x//0 \\ \\<<LL   1&x223  \\ \\<<LL   4*x99: \\ \\<<LL   ;&x++,  ,,   -&x667  ,,   8(x::;  \\    <  x>>? \\   @ e  JKs&YY'&Y')$loggingrJtorch.utils._pytreeutils_pytreerEtorch._inductor.utilsrtorch.utils._ordered_setrrr virtualizedr getLogger__name__rrGr?r/rr#rSr tupler.rr+r1r4rCrOrWrr0r_rrr<s@ $$-/g!T ||')'8'8 <% ||%''%?% %P    "))  1Zc3h0288t8= =d= ?" bll %299% Upk r