
    ^j7B                        d dl mZ d dlZd dlmZ ddlmZ er
d dlZd dlm	Z	  e       r9d dlZd dlm	Z	 d dl
mZ d dlmZ  eed	      s eed
      rej                  e_         G d d      ZddZy)    )annotationsN)TYPE_CHECKING   )is_torch_available)DTensor)%compute_local_shape_and_global_offset)Shardlocal_shard_size_and_offset_local_shard_size_and_offsetc                      e Zd ZdZd
dZ	 d	 	 	 	 	 ddZ	 	 	 	 	 	 	 	 	 	 ddZ	 	 	 	 	 	 	 	 ddZ	 	 	 	 	 	 	 	 	 	 ddZddZ	dd	Z
y)DtensorShardOperationu
  Shard-on-read: slice a full disk tensor down to this rank's local
    DTensor shard, for any combination of placements on a 1-D or n-D mesh.  It's on
    read because instructions are made so the cpu only fetches on disk the parts we want.

    Placements primer
    -----------------
    Each mesh dim carries one placement describing how it slices the tensor:

    | Placement                | Local data on each rank of the mesh dim         |
    |--------------------------|-------------------------------------------------|
    | Replicate                | full tensor (no slicing)                        |
    | Shard(d)                 | contiguous chunk of dim d (rows r*c .. (r+1)*c) |
    | _StridedShard(d, sf=N)   | one chunk from each of N groups along dim d,   |
    |                          | concatenated together (interleaved layout)      |

    Different scenarios of different placements
    ------------------------------------------
    Placement tuples are ordered outermost-first; for a 2-D (fsdp, tp) mesh
    the tuple is (fsdp_placement, tp_placement).

    | Scenario                                          | Placements                                  |
    |---------------------------------------------------|---------------------------------------------|
    | TP-only, non-fused (e.g. q_proj/k_proj/v_proj)    | [Shard(d)]                                   |
    | TP-only, fused gate/up                            | [_StridedShard(d, sf=2)]                     |
    | TP + FSDP, same tensor dim (contiguous TP case)   | [Shard(d), Shard(d)]                         |
    | TP + FSDP, same tensor dim (fused/interleaved TP) | [_StridedShard(d, sf=tp_size), Shard(d)]     |
    | TP + FSDP, different dims                         | [Shard(d1), Shard(d2)]                       |

    Loading (this class)
    --------------------
    During from_pretrained, each rank looks up every tensor key, but does not load the weight bytes yet (safetensors get_slice).
    When a weight is needed, shard_tensor indexes that slice to read only this rank's local shard through Dtensor logic which provides
    placement logics

    Depending on how the checkpoint was saved, the weight loader either gives us one big tensor or many small ones:

    1. One stacked tensor: the checkpoint has one key with all experts together, shaped [num_experts, in, out].
       The weight loader passes it straight through; we slice out this rank's piece.

    2. One tensor per expert:  the checkpoint has a separate key for each expert (expert 0, expert 1, …), each shaped [in, out].
       The weight loader feeds them in one at a time. If this rank doesn't own a given expert,
       we skip it. Later, `MergeModulelist` will stack the owned expert we kept to create the rank's local shard
    c                    |j                   | _         t        |j                        | _        |j                  | _        t        |j                  | j                   | j                        \  }}|d   | _        |d   | _        y Nr   )	device_meshtuple
placementsndim
param_ndimr   shape_axis0_offset_axis0_local_size)selfparamlocal_shapeoffsetss       r/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/distributed/sharding_utils.py__init__zDtensorShardOperation.__init__R   sl     ,, 0 01**DU[[RVRbRbdhdsdstW %QZ!,Q    Nc           	     V   t        |t        j                        rt        |j                        n|j                         }t        | j                        D cg c]  \  }}t        |d      s||f }}}||s|d   j                  ||      S |D 	cg c]  }	g  }
}	|D ]h  \  }}| j                  |      }|j                         |j                         }}| j                  |j                        }|
|   j                  |||f       j |D cg c]  }d|fg	 }}t        |
      D ]\  \  }}||   }|D ]H  \  }}}|j!                         r| j#                  |||      }+| j%                  ||||j&                        }J |||<   ^ t)        d |D              }|r| j+                  ||||      S g }|D ]5  }t-        |      dkD  r|d   nd\  }}|j                  t/        ||             7 |t1        |         j                  ||      S |D cg c]#  \  }}||| j                  |j                        f% }}}t)        d |D              }| j2                  |cxk  xr | j2                  | j4                  z   k  nc }|r|sy|D 	cg c]  }	g  }}	|D ]X  \  }}	}|dkD  s|d	z
  }| j                  |      }|j                         |j                         }}||   j                  ||f       Z |D cg c]  }d|fg	 }}t        |      D ],  \  }}||   }|D ]  \  }}| j#                  |||      } |||<   . g }|D ])  }|r|d   nd\  }}|j                  t/        ||             + |t1        |         j                  ||      S c c}}w c c}	w c c}w c c}}w c c}	w c c}w )
a  Return this rank's local shard of a checkpoint tensor.

        Two layouts (example param shape [N, in, out]):

        - tensor_idx is None: one stacked [N, in, out] tensor;
          slice every sharded dim (including axis 0).
        - tensor_idx given: one [in, out] tensor per expert;
          return None if this rank does not own that expert, else slice
          inner dims only. Surviving pieces are stacked by MergeModulelist
          into this rank's local [n_local, in, out] shard.
        dimN.devicedtyper   c              3  D   K   | ]  \  }}|j                            y wN)is_shard).0_	placements      r   	<genexpr>z5DtensorShardOperation.shard_tensor.<locals>.<genexpr>   s      #`I	(:(:(<$<#`s    )r   r   c              3  .   K   | ]  \  }}}|d k(    yw)r   N )r'   r(   	param_dims      r   r*   z5DtensorShardOperation.shard_tensor.<locals>.<genexpr>   s     ^Ayi1n^s      )
isinstancetorchTensorlistr   	get_shape	enumerater   hasattrto_get_sub_meshget_local_ranksize_normalize_param_dimr    appendr&   _compute_contiguous_slice_compute_strided_slicesplit_factorany_slice_and_catlenslicer   r   r   )r   source
tensor_idxr"   r#   source_shapemesh_dimr)   dim_placementsr(   planned_ops_by_dimsub_meshrank
world_sizedim_idxr9   intervals_by_dimplanned_ops	intervalshas_strided_shardslice_partsstartendnormalized_dim_placementshas_axis0_shardowns_tensor_idxplanned_ops_by_source_dimr-   
source_dimintervals_by_source_dims                                 r   shard_tensorz"DtensorShardOperation.shard_tensor\   s2    .8-MtFLL)SYScScSe=Ft=W
&9h	[bclns[tXy!
 

 !c{~~V5~AA
 /;!;"!;!;'5 R#)--h7#+#:#:#<hmmoj33IMMB"7+22ItZ3PQ	R 9EE!TEE(12D(E 6$,W5	3> u/ItZ ))+$($B$B9dT^$_	$($?$?	4Q[]f]s]s$t		u
 -6 )6 !$#`Q_#` ` !**63CVUSS !1 :I14Y!1C1JE3&&uUC'89: eK0144F%4PP
 hv%
PcPXZcXy$";";IMM"JK%
! %

 ^D]^^,,
hT=O=ORVRhRh=hh? 2>$>AR$>!$>&? 	Q"Ha1}&]
--h7#+#:#:#<hmmoj)*5<<dJ=OP	Q <H"H4QI;"H"H'01J'K 	<#J/
;I$/ X j ::9dJW	X2;#J/		< 0 	2I)21JE3uUC01	2 eK(),,F%,HHe
 "<  F4%
  %? #Is*   N-N	NN(N	N!4N&c                .   g }|D ]  \  }}t        j                  ||z
  |z        }t        |      D ]_  }	||	|z  z   }
t        |
|z   |      }||
z
  }|dkD  s%t	        j
                  |||      \  }}|dkD  sE|
|z   }|j                  |||z   f       a  |S r   )mathceilrangeminr	   r
   r;   )r   rO   rJ   rK   r>   local_intervalsinterval_startinterval_endgroup_width	group_idxgroup_start	group_end	group_lenlocal_shard_sizelocal_shard_offsetshard_starts                   r   r=   z,DtensorShardOperation._compute_strided_slice   s     ,5 	^(NL))\N%Bl$RSK"<0 ^	,y;/FFk 9<H	%3	q=;@;\;\!:t<8$&8 (!+&14F&F'..[K[=[/\]^		^( r   c                   t        d |D              }t        j                  |||      \  }}||z   }|dk(  rg S t        |      dk(  r|d   \  }}	||z   ||z   fgS g }
d}|D ],  \  }}||z
  }|dkD  s|
j	                  |||z   |f       ||z  }. g }|
D ]G  \  }}}t        ||      }t        ||      }||k  s%|||z
  z   }|||z
  z   }|j	                  ||f       I |S )Nc              3  ,   K   | ]  \  }}||z
    y wr%   r,   )r'   rR   rS   s      r   r*   zBDtensorShardOperation._compute_contiguous_slice.<locals>.<genexpr>   s     EZUCS5[Es   r   r.   )sumr	   r
   rA   r;   maxr_   )r   rO   rJ   rK   flat_total_lenlocal_flat_lenlocal_flat_startlocal_flat_endsource_startr(   flat_segmentsidx
source_endinterval_lenr`   interval_flat_startinterval_flat_endoverlap_flat_startoverlap_flat_endsource_overlap_startsource_overlap_ends                        r   r<   z/DtensorShardOperation._compute_contiguous_slice   sY    E9EE+0+L+L^]gim+n(()N:QI y>Q'lOL!!$44l^6STUU (1 	$$L*%4La$$c3+=|%LM|#		$ DQ 	S@!2L!$%8:J!K"#4nE!$44'37IL_7_'`$%15EH[5[%\"&&(<>P'QR	S r   c                   t        |      D cg c]  \  }}t        |      dkD  s| }}}t        |      dkD  rt        d      |r|d   nd }g }	t        |      D ]H  \  }}||k(  r|	j                  t	        d              &|d   \  }
}|	j                  t	        |
|             J ||t        |	         j                  ||      S t        |	      }g }||   D ]5  \  }}g |d | t	        ||      ||dz   d  }|j                  ||          7 t        j                  ||      j                  ||      S c c}}w )Nr.   zUCurrent shard-on-read only supports disjoint ranges on a single checkpoint dimension.r   r!   )r    )	r4   rA   
ValueErrorr;   rB   r   r6   r0   cat)r   rC   rO   r"   r#   rL   dim_intervalsmulti_interval_dims
concat_dimbase_slicesrR   rS   base_slices_tupleinterval_tensorsra   rb   interval_slicess                    r   r@   z$DtensorShardOperation._slice_and_cat  s    FOyEYt+A7M]`an]ors]swtt"#a' tuu/B(+
&/	&: 	6"G]*$""5;/ +1-
s""5#45	6 %,-00e0LL "+.,5j,A 	=(NL";J/nl3 #:>#34O
 ##F?$;<	= yy)z:==VSX=YYA us
   D:D:c                    | j                   j                  dk(  r| j                   S | j                   | j                   j                  |      S )Nr.   )r   r   mesh_dim_names)r   rF   s     r   r7   z#DtensorShardOperation._get_sub_mesh5  sD      A%### 0 0 ? ? IJJr   c                .    |dk\  r|S | j                   |z   S r   )r   )r   r    s     r   r:   z*DtensorShardOperation._normalize_param_dim:  s    Qhs9DOOc$99r   )r   r   )NNN)rC   torch.TensorrD   z
int | Nonereturnztorch.Tensor | None)
rO   list[tuple[int, int]]rJ   intrK   r   r>   r   r   r   )rO   r   rJ   r   rK   r   r   r   )
rC   r   rO   zlist[list[tuple[int, int]]]r"   ztorch.device | str | int | Noner#   ztorch.dtype | Noner   r   )rF   r   )r    r   r   r   )__name__
__module____qualname____doc__r   rZ   r=   r<   r@   r7   r:   r,   r   r   r   r   %   s    *X0 W[aI"aI0:aI	aIF.69GJZ]	60.0690GJ0	0d'Z'Z /'Z 0	'Z
 "'Z 
'ZRK
:r   r   c                    t        j                  | j                         |j                  |j                  d|j
                  t        |j                                     S )zeWrap `local_tensor` as a DTensor that mirrors `ref`'s mesh, placements,
    global shape, and stride.F)	run_checkr   stride)r   
from_local
contiguousr   r   r   r   r   )local_tensorrefs     r   _dtensor_from_local_liker   ?  sI     !iiSZZ\" r   )r   r   r   r   r   r   )
__future__r   r\   typingr   utilsr   r0   torch.distributed.tensorr   torch.distributed.tensor._utilsr   (torch.distributed.tensor.placement_typesr	   r5   r   r
   r   r   r,   r   r   <module>r      sg    #    & 00U> 578WULj=k,1,N,N)W: W:t
r   