Ë
    ÄÓ^jC'  ã                   ój  — U d dl mZ d dlZd dlmZmZmZmZ d dlm	Z	 ej                  j                  Zej                  j                  Zej                  j                  Zej                  j                  Zej                  j                   ej"                  j                   ej$                  j                   ej&                  j                   ej(                  j                   ej*                  j                   ej,                  j                   ej.                  j                   ej0                  j                   ej2                  j                   ej4                  j                   ej6                  j                   ej8                  j                   ej:                  j                   ej<                  j                   ej>                  j                   hZ e!ejD                  jF                     e$d<   ejJ                  j                   ejL                  j                   ejN                  j                   ejP                  j                   ejR                  j                   ejT                  j                   ejT                  j                   ejV                  j                   ejX                  j                   ejZ                  j                   ej\                  j                   ej^                  j                   ej`                  j                   ejN                  j                   ejb                  j                   ejP                  j                   ej                  j                   ejd                  j                   ejf                  j                   ejh                  j                   ejj                  j                   hZ6e!ejD                  jF                     e$d<   ej<                  j                   ej>                  j                   ejV                  j                   hZ7e!ejD                  jF                     e$d<   e!jq                  e6e «      Z9 G d„ d	«      Z:y)
é    )ÚAnyN)Ú_resolve_process_groupÚFakeWorkÚProcessGroupÚWork)Útree_map_onlyÚnon_functional_collectivesÚfunctional_collectivesÚsync_opsc            
       ó>  — e Zd Zej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  hZej                  j
                  ej                  j
                  ej                  j
                  ej                   j
                  ej"                  j
                  ej$                  j
                  ej&                  j
                  ej(                  j
                  hZej.                  j
                  ej                  j
                  ej0                  j
                  ej2                  j
                  ej4                  j
                  ej6                  j
                  ej8                  j
                  ej:                  j
                  ej8                  j
                  ej>                  j
                  h
Z ejB                  j
                  ejD                  j
                  ejB                  j
                  ejF                  j
                  ejF                  j
                  ejH                  j
                  ejJ                  j
                  e&jN                  j
                  hZ(ejR                  j
                  hZ*ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej                   j
                  ej                  j
                  ej$                  j
                  ej&                  j
                  ejH                  j
                  ejJ                  j
                  h
Z+ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej"                  j
                  ej(                  j
                  ej                  j
                  hZ,ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej"                  j
                  ej                  j
                  ej                  j
                  ej4                  j
                  ej6                  j
                  h
Z-ej                  j
                  ej$                  j
                  hZ.ej8                  j
                  ej8                  j
                  hZ/ej                  j
                  ej.                  j
                  ej                  j
                  ej0                  j
                  ej2                  j
                  ejB                  j
                  ejB                  j
                  hZ0ejF                  j
                  ejF                  j
                  e&jN                  j
                  hZ1ej>                  j
                  ejD                  j
                  hZ2e3de4de5fd„«       Z6e3de7fd„«       Z8e3de5fd„«       Z9e3de:fd„«       Z;y)ÚCollectiveOpÚargÚreturnc                 ór   ‡— dŠdt         j                  ddfˆfd„}t        t         j                  || «       ‰S )z?Calculate total memory consumed by the tensors in the argument.r   Útr   Nc                 óH   •— ‰| j                  «       j                  «       z  Šy )N)Úuntyped_storageÚnbytes)r   Útotal_memorys    €út/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/distributed/_tools/fake_collectives.pyÚ	sum_bytesz+CollectiveOp.sum_tensors.<locals>.sum_bytesÐ   s   ø€ à˜A×-Ñ-Ó/×6Ñ6Ó8Ñ8‰Ló    )ÚtorchÚTensorr   )r   r   r   s     @r   Úsum_tensorszCollectiveOp.sum_tensorsË   s9   ø€ ð ˆð	9œŸ™ð 	9¨$õ 	9ô 	”e—l‘l I¨sÔ3ØÐr   c                 ó”  — | t         j                  v rt        j                  |d   «      S | t         j                  v rt        j                  |d   «      S | t         j
                  v rt        |d   «      S | t         j                  v rt        |d   «      S | t         j                  v rt        |d   «      S t        d| › dt        › �«      ‚)zKRetrieve the process group for collective operations, except `wait_tensor`.é   é   é   é   úFunc ú not found in )r   ÚPG_ARG_1r   ÚunboxÚPG_ARG_2ÚPG_ARG_3r   ÚPG_ARG_4ÚPG_ARG_5Ú	TypeErrorÚcollective_ops)ÚfuncÚargss     r   Úget_process_groupzCollectiveOp.get_process_group×   sº   € ð ”<×(Ñ(Ñ(Ü×%Ñ% d¨1¡gÓ.Ð.Ø”<×(Ñ(Ñ(Ü×%Ñ% d¨1¡gÓ.Ð.Ø”<×(Ñ(Ñ(Ü)¨$¨q©'Ó2Ð2Ø”<×(Ñ(Ñ(Ü)¨$¨q©'Ó2Ð2Ø”<×(Ñ(Ñ(Ü)¨$¨q©'Ó2Ð2Ü˜% ˜v ^´NÐ3CÐDÓEÐEr   c                 ó^  — | t         j                  v rt         j                  |d   «      S | t         j                  v rt         j                  |d   «      S | t         j                  v r|j                  «       j                  «       S | t         j                  v r!|d   j                  «       j                  «       S | t        j                  j                  u r!|d   j                  «       j                  «       S | t        j                  j                  u r8t        t         j                  |d   «      t         j                  |d   «      «      S | t        j                  j                  u rJt        |d   j                  «       j                  «       |d   j                  «       j                  «       «      S | t        j                  j                  k(  r!|d   j                  «       j                  «       S | t         j                   v rt         j                  |«      S | t         j"                  v r@|d   j                  «       j                  «       |j                  «       j                  «       z   S | t        j$                  j                  u rt         j                  |d   «      S t'        d| › dt(        › �«      ‚)zdCompute the communication tensor size, except for `wait_tensor`, `barrier`, and `monitored_barrier`.r   r   éÿÿÿÿr   zUnknown function: z in )r   ÚCOMM_TENSOR_ARG_0r   ÚCOMM_TENSOR_ARG_1ÚCOMM_TENSOR_ARG_RESr   r   Ú"COMM_TENSOR_SINGLE_UNTYPED_STORAGEÚc10dÚ_reduce_scatter_base_ÚdefaultÚ	alltoall_ÚmaxÚalltoall_base_Ú_c10d_functionalÚall_gather_into_tensor_outÚCOMM_TENSOR_RES_SUMÚCOMM_TENSOR_ARG_0_AND_RESÚbatch_p2p_opsr)   r*   )r+   Úresr,   Úkwargss       r   Úget_comm_tensor_sizez!CollectiveOp.get_comm_tensor_sizeæ   s8  € ð ”<×1Ñ1Ñ1Ü×+Ñ+¨D°©GÓ4Ð4Ø”<×1Ñ1Ñ1Ü×+Ñ+¨D°©GÓ4Ð4Ø”<×3Ñ3Ñ3Ø×&Ñ&Ó(×/Ñ/Ó1Ð1Ø”<×BÑBÑBØ˜‘7×*Ñ*Ó,×3Ñ3Ó5Ð5Ø”4×-Ñ-×5Ñ5Ñ5Ø˜‘7×*Ñ*Ó,×3Ñ3Ó5Ð5Ø”4—>‘>×)Ñ)Ñ)äÜ×(Ñ(¨¨a©Ó1´<×3KÑ3KÈDÐQRÉGÓ3Tóð ð ”4×&Ñ&×.Ñ.Ñ.äØ�Q‘×'Ñ'Ó)×0Ñ0Ó2°D¸±G×4KÑ4KÓ4M×4TÑ4TÓ4Vóð ð Ô#×>Ñ>×FÑFÒFØ˜‘8×+Ñ+Ó-×4Ñ4Ó6Ð6Ø”<×3Ñ3Ñ3Ü×+Ñ+¨CÓ0Ð0Ø”<×9Ñ9Ñ9à˜‘7×*Ñ*Ó,×3Ñ3Ó5¸×8KÑ8KÓ8M×8TÑ8TÓ8VÑVÐVØÔ#×1Ñ1×9Ñ9Ñ9Ü×+Ñ+¨D°©GÓ4Ð4ÜÐ,¨T¨F°$´~Ð6FÐGÓHÐHr   c                 óÎ   — | t         j                  v rt        j                  |«      S | t         j                  v rt        j                  |d   «      S t        d| › dt        › �«      ‚)Nr   r!   r"   )r   ÚWKr   r$   ÚWK_ARG_1r)   r*   )r+   r?   s     r   Úget_workzCollectiveOp.get_work  sW   € à”<—?‘?Ñ"Ü—>‘> #Ó&Ð&Ø”\×*Ñ*Ñ*Ü—>‘> # a¡&Ó)Ð)Ü˜% ˜v ^´NÐ3CÐDÓEÐEr   N)<Ú__name__Ú
__module__Ú__qualname__r4   Ú
broadcast_r6   Ú
allreduce_Úreduce_ÚsendÚrecv_Úrecv_any_source_Úbarrierr#   Ú
allgather_Ú_allgather_base_Úreduce_scatter_r5   Úgather_Úscatter_r7   r9   r%   r:   Ú	broadcastÚ
all_reduceÚall_reduce_Úall_reduce_coalescedÚall_reduce_coalesced_Úall_gather_into_tensorr;   Ú_c10d_functional_autogradÚ all_gather_into_tensor_coalescedr&   Úreduce_scatter_tensorÚreduce_scatter_tensor_coalescedÚall_to_all_singleÚisendÚirecvÚ_dtensorÚshard_dim_alltoallr'   r>   r(   rD   rC   r0   r1   r2   r3   r=   r<   Ústaticmethodr   Úintr   r   r-   rA   r   rE   © r   r   r   r   M   sE  „ ð 	�‰×ÑØ�‰×ÑØ�‰×ÑØ�	‰	×ÑØ�
‰
×ÑØ×Ñ×%Ñ%Ø�‰×Ñð	€Hð 	�‰×ÑØ×Ñ×%Ñ%Ø×Ñ×$Ñ$Ø×"Ñ"×*Ñ*Ø�‰×ÑØ�‰×ÑØ�‰×ÑØ×Ñ×#Ñ#ð€Hð 	×"Ñ"×*Ñ*Ø×#Ñ#×+Ñ+Ø×#Ñ#×+Ñ+Ø×$Ñ$×,Ñ,Ø×-Ñ-×5Ñ5Ø×.Ñ.×6Ñ6Ø×/Ñ/×7Ñ7Ø×3Ñ3×;Ñ;Ø!×8Ñ8×@Ñ@Ø×9Ñ9×AÑAð€Hð 	×.Ñ.×6Ñ6Ø×8Ñ8×@Ñ@Ø!×7Ñ7×?Ñ?Ø×*Ñ*×2Ñ2Ø!×3Ñ3×;Ñ;Ø×Ñ×&Ñ&Ø×Ñ×&Ñ&Ø×#Ñ#×+Ñ+ð	€Hð 	×&Ñ&×.Ñ.ð€Hð
 	�‰×ÑØ�‰×ÑØ�‰×ÑØ×Ñ×$Ñ$Ø×"Ñ"×*Ñ*Ø×Ñ×%Ñ%Ø�‰×ÑØ�‰×ÑØ×Ñ×&Ñ&Ø×Ñ×&Ñ&ð€Hð 	�	‰	×ÑØ�
‰
×ÑØ×Ñ×%Ñ%Ø�‰×ÑØ�‰×ÑØ×Ñ×#Ñ#Ø�‰×Ñð
€Bð 	�‰×ÑØ�	‰	×ÑØ�
‰
×ÑØ×Ñ×%Ñ%Ø�‰×ÑØ�‰×ÑØ�‰×ÑØ�‰×ÑØ×-Ñ-×5Ñ5Ø×.Ñ.×6Ñ6ðÐð" 	×Ñ×$Ñ$Ø�‰×ÑðÐð 	×/Ñ/×7Ñ7Ø!×8Ñ8×@Ñ@ðÐð 	×Ñ×%Ñ%Ø×"Ñ"×*Ñ*Ø×#Ñ#×+Ñ+Ø×#Ñ#×+Ñ+Ø×$Ñ$×,Ñ,Ø×.Ñ.×6Ñ6Ø!×7Ñ7×?Ñ?ð*Ð&ð 	×*Ñ*×2Ñ2Ø!×3Ñ3×;Ñ;Ø×#Ñ#×+Ñ+ð!Ðð 	×9Ñ9×AÑAØ×8Ñ8×@Ñ@ðÐð
 ð	˜ð 	 ò 	ó ð	ð ðF¨ò Fó ðFð ðI¸ò Ió ðIðB ðF˜tò Fó ñFr   r   );Útypingr   r   Útorch._C._distributed_c10dr   r   r   r   Útorch.utils._pytreer   Úopsr4   r:   r[   rb   rI   r6   rJ   rK   rL   rM   rN   rP   rR   r5   rQ   rS   rT   r7   r9   rO   Úmonitored_barrier_r	   ÚsetÚ_opsÚ
OpOverloadÚ__annotations__rU   rV   rZ   r]   Úreduce_scatter_tensor_outr_   Úwait_tensorrW   rX   rY   r;   r\   r^   r`   ra   r>   rc   r
   r   Úunionr*   r   rf   r   r   ú<module>rs      s  ðÞ ã ÷ó õ .ð ‡y�y‡~�~€Ø—9‘9×-Ñ-Ð Ø!ŸI™I×?Ñ?Ð Ø�9‰9×Ñ€ð 	‡O�O×ÑØ‡O�O×ÑØ‡L�L×ÑØ‡I�I×ÑØ‡J�J×ÑØ×Ñ×!Ñ!Ø‡O�O×ÑØ×Ñ× Ñ Ø×Ñ×&Ñ&Ø×Ñ×!Ñ!Ø‡L�L×ÑØ‡M�M×ÑØ‡N�N×ÑØ×Ñ×ÑØ‡L�L×ÑØ×Ñ×#Ñ#ð!:Ð ˜C §
¡
× 5Ñ 5Ñ6ó ð& ×Ñ×&Ñ&Ø×Ñ×'Ñ'Ø×+Ñ+×3Ñ3Ø×*Ñ*×2Ñ2Ø×.Ñ.×6Ñ6Ø×&Ñ&×.Ñ.Ø×/Ñ/×7Ñ7Ø× Ñ ×(Ñ(Ø× Ñ ×(Ñ(Ø×)Ñ)×1Ñ1Ø×*Ñ*×2Ñ2Ø×/Ñ/×7Ñ7Ø×5Ñ5×=Ñ=Ø×4Ñ4×<Ñ<Ø×4Ñ4×<Ñ<Ø×3Ñ3×;Ñ;Ø×Ñ×'Ñ'Ø×Ñ×"Ñ"Ø×Ñ×"Ñ"Ø×"Ñ"×*Ñ*Ø×Ñ×'Ñ'ð+6Ð ˜˜EŸJ™J×1Ñ1Ñ2ó ð2 	‡L�L×ÑØ×Ñ×#Ñ#Ø× Ñ ×(Ñ(ð(€ˆ#ˆe�j‰j×#Ñ#Ñ
$ó ð —‘Ð1Ð3MÓN€÷AFò AFr   