
    ^j                     d    d dl Z d dlZd dlmZmZ ddlmZ  e       rd dlZe G d d             Zy)    N)asdict	dataclass   )is_torch_availablec                       e Zd ZU dZdZedz  ed<   dZee	e	f   dz  ed<   dZ
eed<   dZeed<   dZedz  ed<   dZeed	<   dZeed
<   d Zededd fd       ZdefdZde	fdZde	ej,                  z  fdZd Zy)DistributedConfigaK  
    Configuration for native distributed training (FSDP2 + TP).

    Args:
        tp_size (`int`, *optional*):
            Number of devices for tensor parallelism. If `None` and `fsdp_size` is set, defaults to 1.
        tp_plan (`dict`, *optional*):
            Tensor parallel sharding plan. Leave as `None` to use the model's `base_model_tp_plan`.
            Set explicitly to override.
        enable_sequence_parallel (`bool`, *optional*, defaults to `False`):
            Reserved for sequence parallelism. Not wired up yet.
        enable_expert_parallel (`bool`, *optional*, defaults to `False`):
            Route MoE models through the expert-parallel path (``base_model_ep_plan``).
        fsdp_size (`int`, *optional*):
            Number of devices for FSDP (data parallelism). If `None` and `tp_size` is set, defaults to 1.
        fsdp_cpu_offload (`bool`, *optional*, defaults to `False`):
            Whether to enable CPU offloading for FSDP2.
        fsdp_mixed_precision (`bool`, *optional*, defaults to `False`):
            Whether to enable mixed precision for FSDP2.
    Ntp_sizetp_planFenable_sequence_parallelenable_expert_parallel	fsdp_sizefsdp_cpu_offloadfsdp_mixed_precisionc           	         | j                   | j                  y | j                   d| _         | j                  d| _        t        j                  j	                         rt        j                  j                         rdt        j                  j                         }| j                   | j                  z  |k7  r)t        d| j                    d| j                   d| d      y y y )N   z	tp_size (z) * fsdp_size (z) is not equal to world_size ())r	   r   torchdistributedis_availableis_initializedget_world_sizeRuntimeError)self
world_sizes     w/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/distributed/configuration_utils.py__post_init__zDistributedConfig.__post_init__9   s    <<DNN$:<<DL>>!DN))+0A0A0P0P0R**99;J||dnn,
:"~_T^^<LLjkujvvwx  ; 1S+    config_dictreturnc           
          i ||}| j                   j                         D ch c]  }|j                   }} | di |j                         D ci c]  \  }}||v s|| c}}S c c}w c c}}w )N )__dataclass_fields__valuesnameitems)clsr   kwargsmergedf
valid_keyskvs           r   	from_dictzDistributedConfig.from_dictI   sm    *K*6*&)&>&>&E&E&GHaffH
HJv||~Itq!jadIJJ IIs   A'A,A,c                     t        |       S )N)r   r   s    r   to_dictzDistributedConfig.to_dictO   s    d|r   c                 R    t        j                  | j                         d      dz   S )Nr   )indent
)jsondumpsr0   r/   s    r   to_json_stringz DistributedConfig.to_json_stringR   s    zz$,,.3d::r   json_file_pathc                     t        |dd      5 }|j                  | j                                d d d        y # 1 sw Y   y xY w)Nwzutf-8)encoding)openwriter6   )r   r7   r)   s      r   to_json_filezDistributedConfig.to_json_fileU   s:    .#8 	+AGGD'')*	+ 	+ 	+s	    8Ac                 T    | j                   j                   d| j                          S )N )	__class____name__r6   r/   s    r   __repr__zDistributedConfig.__repr__Y   s(    ..))*!D,?,?,A+BCCr   )rA   
__module____qualname____doc__r	   int__annotations__r
   dictstrr   boolr   r   r   r   r   classmethodr-   r0   r6   osPathLiker=   rB   r!   r   r   r   r      s    * GS4Z%)GT#s(^d")%*d*#(D( IsTz "d"!&$&  KD K7J K K
 ; ;+3+< +Dr   r   )	r4   rL   dataclassesr   r   utilsr   r   r   r!   r   r   <module>rP      s:     	 ) &  ?D ?D ?Dr   