
    ^j                        U d dl Z d dlZd dlZd dlZd dlmZ d dlZeej                  z  Z	 e j                  e      ZdefdZdefdZ e       r"ej                   j#                         s ed      ej                   j&                  Zej                   j*                  Zej                   j.                  Zej                   j2                  Zej                   j6                  Z e       rd dlmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z8 d Z9i Z:e;eejx                  f   e=d	<   dd
edefdZ>ej~                  dk7  rd dlm@Z@ ddlAmBZBmCZC ddlD ddlDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZM ddlNmOZO ddlPmQZQmRZRmPZP  e4        y G d d      ZSeSej                  d   _U        eSej                  d   _1        y)    N)	timedeltareturnc                  6    t        t        j                  d      S )a  
    Return ``True`` if the distributed package is available.

    Otherwise,
    ``torch.distributed`` does not expose any other APIs. Currently,
    ``torch.distributed`` is available on Linux, MacOS and Windows. Set
    ``USE_DISTRIBUTED=1`` to enable it when building PyTorch from source.
    Currently, the default value is ``USE_DISTRIBUTED=1`` for Linux and Windows,
    ``USE_DISTRIBUTED=0`` for MacOS.
    
_c10d_init)hasattrtorch_C     e/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/distributed/__init__.pyis_availabler      s     588\**r   c                  D    ddl } | j                  j                  d      duS )z-Check if the spmd_types package is installed.r   N
spmd_types)importlib.utilutil	find_spec)	importlibs    r   _is_spmd_types_availabler      s    >>##L1==r   z&Failed to initialize torch.distributed)_broadcast_coalesced"_compute_bucket_assignment_by_size_ControlCollectives_DEFAULT_FIRST_BUCKET_BYTES_make_nccl_premul_sum_register_builtin_comm_hook_register_comm_hook_StoreCollectives_test_python_store_verify_params_across_processesBackendBuiltinCommHookType
DebugLevel	FileStoreget_debug_level
GradBucketLoggerPrefixStoreProcessGroupReducerset_debug_levelset_debug_level_from_envStoreTCPStoreWorkc                  H    ddl  G fddj                        }  |        S )z
        Supports using PDB from inside a multiprocessing child process.

        Usage:
        _make_distributed_pdb().set_trace()
        r   Nc                       e Zd Z fdZy)._make_distributed_pdb.<locals>._DistributedPdbc                     t         j                  }	 t        d      5 t         _         j                  j                  | g|i | d d d        |t         _        y # 1 sw Y   xY w# |t         _        w xY w)Nz
/dev/stdin)sysstdinopenPdbinteraction)selfargskwargs_stdinpdbs       r   r6   z:_make_distributed_pdb.<locals>._DistributedPdb.interactionY   sg    'l+ Csy+++DB4B6BC !'CIC C !'CIs"   A( *AA( A%!A( (A5N)__name__
__module____qualname__r6   )r;   s   r   _DistributedPdbr0   X   s    'r   r?   )r;   r5   )r?   r;   s    @r   _make_distributed_pdbr@   M   s#     		'cgg 	'   r   _breakpoint_cacherankskipc                 b   |dkD  rdt        t        t        j                                     }t        j                  |d      dz   }|t        |<   ||k  rt        j                  d|       y|]t        j                  j                  j                  D ]6  }t        j                  j                  j                  t        |      |       8 t               | k(  r/t               }|j!                  d|  d       |j#                          t        j$                  j'                         }t        j$                  j)                         }t        j$                  j+                  d       	 t-                t        j$                  j+                  |       ~y# t        j$                  j+                  |       ~w xY w)	aD  
        Set a breakpoint, but only on a single rank.  All other ranks will wait for you to be
        done with the breakpoint before continuing.

        Args:
            rank (int): Which rank to break on.  Default: ``0``
            skip (int): Skip the first ``skip`` calls to this breakpoint. Default: ``0``.
        r      zSkip the breakpoint, counter=%dN)secondszS
!!! ATTENTION !!!

Type 'up' to get to the frame that called dist.breakpoint(rank=z)
F)hashstr	traceback
format_excrA   getlogwarningr   distributeddistributed_c10d_pg_map_set_pg_timeoutr   get_rankr@   message	set_tracer	   _meta_in_tls_dispatch_include_DisableTorchDispatch!_set_meta_in_tls_dispatch_includebarrier)	rB   rC   	timeout_skeycountergroupr;   meta_in_tlsguards	            r   
breakpointr_   e   s[    !8s9//123C'++C3a7G%,c"$=wG  **;;CC !!22BBi0%
 :')CKKRRVQWWZ\ MMO hh<<>..02259	IHH66{C HH66{Cs   !
F "F.win32)	HashStorerE   )
DeviceMeshinit_device_mesh)*)	_all_gather_base_coalescing_manager_CoalescingManager_create_process_group_wrapper_get_process_group_name_rank_not_in_group_reduce_scatter_base_time_estimatorget_node_local_rank)_remote_device)_create_store_from_optionsregister_rendezvous_handler
rendezvousc                       e Zd Zy)_StubN)r<   r=   r>   r
   r   r   rs   rs      s    r   rs   ztorch.distributed)r   r   i  )Vloggingr2   rI   typingdatetimer   r   intSymIntRankType	getLoggerr<   rL   boolr   r   r	   r   RuntimeError
_DistError	DistError_DistBackendErrorDistBackendError_DistNetworkErrorDistNetworkError_DistStoreErrorDistStoreError_DistQueueEmptyErrorQueueEmptyErrortorch._C._distributed_c10dr   r   r   r   r   r   r   r   r   r   r   _Backendr    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   _Workr@   rA   dictAny__annotations__r_   platformra   device_meshrb   rc   rO   re   rf   rg   rh   ri   rj   rk   rl   rm   remote_devicern   rq   ro   rp   rs   modules	GroupNamer
   r   r   <module>r      s    
      g!+d +>$ > >%((--/
?
@@ HH	88-- 88-- ))((//>      8!, 02tCO,1( ( (T ||w89
 $
 
 
 .    27CKK#$.49CKK#$1r   