Ë
    ÄÓ^jÚ  ã                   óN  — d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZmZmZmZ d dlmZ d dlmZmZmZmZ d„ Z	 ddej.                  d	ed
ededej2                  dej4                  dz  defd„Zdej.                  d	ededefd„Zdededz  dej.                  fd„Zy)é    N)Ú_get_device_module)Údistributed_c10d)ÚShardÚShardedTensorÚShardedTensorMetadataÚTensorProperties)ÚShardMetadata)Ú
DeviceMeshÚDTensorÚ	Replicater   c                 ó¾   — |j                  «       dk(  rd| › d|› �S |j                  «       dk(  r"d| › d|› dt        |«      j                  «       › �S d| › d|› d| |z  › �S )NÚcpuzrank:ú/Úhpuú:)Úlowerr   Úcurrent_device)ÚrankÚdevice_typeÚnum_devices_per_nodes      ún/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/distributed/fsdp/_shard_utils.pyÚ_get_remote_device_strr      s€   € Ø×ÑÓ˜eÒ#Ø�t�f˜A˜k˜]Ð+Ð+Ø	×	Ñ	Ó	 Ò	%Ø�t�f˜A˜k˜]¨!Ô,>¸{Ó,K×,ZÑ,ZÓ,\Ð+]Ð^Ð^à�t�f˜A˜k˜]¨!¨DÐ3GÑ,GÐ+HÐIÐIó    Útensorr   Ú
world_sizer   ÚpgÚdeviceÚreturnc                 óR  — | j                  |d¬«      }t        |«      |kD  rx||   j                  «       }| j                  «       D �cg c]  }d‘Œ }	}t	        j
                  | j                  «       d   |z  «      |z  |	d<   t        j                  ||	|«      g}
ng }
|D �cg c]  }t        |j                  «       «      ‘Œ }}dgt        t        j                  |D �cg c]  }|d   ‘Œ	 c}«      «      dd z   }dgt        |d   «      dz
  z  }	|D �cg c]  }|g|	z   ‘Œ
 }}|€t        j                  |«      j                  n|j                  }t        t        |«      «      D �cg c]#  }t        t!        j"                  ||«      ||«      ‘Œ% }}t        |«      t        |«      k7  st        |«      t        |«      k7  r/t%        dt        |«      › dt        |«      › dt        |«      › �«      ‚t'        |||«      D ���cg c]  \  }}}t)        |||«      ‘Œ }}}}t+        || j                  «       t-        | j.                  | j0                  dt2        j4                  | j7                  «       ¬	«      ¬
«      }t9        j:                  |
||¬«      S c c}w c c}w c c}w c c}w c c}w c c}}}w )zŸ
    Shard a tensor to chunks along the first dimension. The local rank will get its
    corresponding chunk as the local shard to create a ShardedTensor.
    r   )ÚdimNéÿÿÿÿé   zQExpected chunk_sizes, chunk_offsets, and placements to have the same length, got z, F)ÚdtypeÚlayoutÚrequires_gradÚmemory_formatÚ
pin_memory)Úshards_metadataÚsizeÚtensor_properties)Úsharded_tensor_metadataÚprocess_group)ÚchunkÚlenÚcloner)   ÚmathÚceilr   Úfrom_tensor_and_offsetsÚlistÚ	itertoolsÚ
accumulater   Ú_get_pg_default_deviceÚtypeÚranger   ÚdistÚget_global_rankÚAssertionErrorÚzipr	   r   r   r#   r$   ÚtorchÚcontiguous_formatÚ	is_pinnedr   Ú+_init_from_local_shards_and_global_metadata)r   r   r   r   r   r   ÚchunksÚlocal_shardÚ_ÚoffsetsÚlocal_shardsr-   Úchunk_sizesÚ
chunk_sizeÚdim0_offsetsÚd0Úchunk_offsetsr   ÚrÚ
placementsÚoffsetr)   Ú	placementÚshard_metadatar+   s                            r   Ú_create_chunk_sharded_tensorrP      sª  € ð �\‰\˜*¨!ˆ\Ó,€FÜ
ˆ6ƒ{�TÒØ˜T‘l×(Ñ(Ó*ˆØ$Ÿk™k›mÖ,˜’1Ð,ˆÐ,Ü—Y‘Y˜vŸ{™{›}¨QÑ/°*Ñ<Ó=ÀÑDˆ�‰
Ü×5Ñ5°kÀ7ÈDÓQÐR‰àˆð 4:Ö:¨%”4˜Ÿ
™
›Õ%Ð:€KÐ:Ø�3œÜ×Ñ¸kÖJ°
˜j¨›mÒJÓKóà	€rðñ €Lð ˆc”S˜ Q™Ó(¨1Ñ,Ñ-€GØ.:Ö;¨�b�T˜G“^Ð;€MÐ;ð ˆ>ô 	×/Ñ/°Ó3×8Ò8à�[‰[ð ô ”s˜;Ó'Ó(öð ô 	Ü× Ñ   QÓ'ØØ õ	
ð€Jð ô ˆ;Óœ3˜}Ó-Ò-´°[Ó1AÄSÈÃ_Ò1TÜðÜ�{Ó#Ð$ B¤s¨=Ó'9Ð&:¸"¼SÀ»_Ð<MðOó
ð 	
ô (+¨=¸+ÀzÓ'R÷ð á#ˆF�D˜)ô 	�f˜d IÕ.ð€Nò ô 4Ø&Ø�[‰[‹]Ü*Ø—,‘,Ø—=‘=ØÜ×1Ñ1Ø×'Ñ'Ó)ô
ô
Ðô ×DÑDØÐ.EÐUWôð ùò] -ùò ;ùâJùò <ùòùôs$   Á	J	Â  JÃJÄJÅ#(JÇ;J"Údevice_meshc                 óh  — | j                  «       j                  «       } t        |j                  «      D �cg c]  }t	        «       ‘Œ }}t        |j                  «      D �cg c]  }t	        «       ‘Œ }}t        d«      |d<   t        j                  | ||d¬«      j                  |¬«      S c c}w c c}w )zš
    Shard a tensor to chunks along the first dimension. The local rank will get its
    corresponding chunk as the local tensor to create a DTensor.
    r   r!   F)Ú	run_check)rL   )	Údetachr/   r8   Úndimr   ÚDShardr   Ú
from_localÚredistribute)r   r   rQ   rC   Úreplicate_placementsÚshard_placementss         r   Ú_create_chunk_dtensorr[   _   s£   € ð �]‰]‹_×"Ñ"Ó$€Fô 27°{×7GÑ7GÓ1HÖI¨AœI�KÐIÐÐIÜ-2°;×3CÑ3CÓ-DÖE¨œ	�ÐEÐÐEÜ! !›9Ð�RÑä×ÑØ�Ð1¸Uôç�lØ#ð ó ðùò	 JùÚEs   ¶B*Á B/Ú	root_meshc                 óú   — || j                   k7  rt        d«      ‚t        t        j                  | j
                  «      «      }t        «       |d<   | j                  | j                   |¬«      } | j                  «       S )zT
    All gather a DTensor in its sharded dimension and return the local tensor.
    z2The device mesh of a tensor should be a root mesh.r!   )rQ   rL   )	rQ   r;   r3   ÚcopyÚdeepcopyrL   r   rX   Úto_local)r   r\   rL   s      r   Ú_all_gather_dtensorra   x   st   € ð �F×&Ñ&Ò&ÜÐQÓRÐRä”d—m‘m F×$5Ñ$5Ó6Ó7€Jô “[€Jˆr�NØ× Ñ Ø×&Ñ&Øð !ó €Fð
 �?‰?ÓÐr   )N)r^   r4   r0   r=   Útorch.distributedÚdistributedr9   Útorch._utilsr   r   Ú'torch.distributed._shard.sharded_tensorr   r   r   r   Ú&torch.distributed._shard.sharding_specr	   Útorch.distributed.tensorr
   r   r   rV   r   ÚTensorÚintÚProcessGroupr   rP   r[   ra   © r   r   ú<module>rl      së   ðã Û Û ã Ý  Ý +Ý .÷ó õ Aß TÓ TòJð #'ñ?Ø�L‰Lð?à
ð?ð ð?ð ð	?ð
 	×Ñð?ð �L‰L˜4Ñð?ð ó?ðDØ�L‰Lðà
ðð ðð ó	ð2Øðà˜DÑ ðð ‡\�\ôr   