^j6 zddlZddlmZmZddlZddlmZddlmcm cm Z ddl mcm ZddlmZmZmZmZddlmZddlmZddlmZddlmZddlmZdd lmZdd lm Z m!Z!m"Z"mZ#dd l$m%Z%m&Z&d gZ'd e!de(ejRejRffdZ*d e!de+de(ejRejRffdZ,d e!de(ejRejRffdZ-de!de+defdZ.de!dej^defdZ0de!dej^fdZ1de jdd ejfde+de jdfdZ4d ejfde+de+de+dej^dejff dZ5d ejfde+d e de!fd!Z6d ejfde(ejfe7effd"Z8d e!d#e dzdejffd$Z9Gd%d eZ:y)&N)Anycast)Shard ShardedTensorShardedTensorMetadataTensorProperties) ShardMetadata)ChunkShardingSpec)_set_fsdp_flattened)FSDPExtensions)_create_chunk_sharded_tensor)_remote_device) DeviceMeshDTensor Replicater)_flatten_tensor_unflatten_tensorDTensorExtensionstensorreturnc|j}|jdk7r td|jd}dgt |j z}|j d}|jdj r3tt|j}|j ||z}|||<tj||jj fS)N&Only 1D DeviceMeshes currently handledr)mesh_dim) device_meshndimAssertionError placementslensizeis_shardrDSharddimtorchSize _local_tensor)rr placementoffsets num_chunks shard_dim chunk_sizes q/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/distributed/tensor/parallel/fsdp.py_get_boxr-s$$K1EFF!!!$IcC &&G!!1!-J$$&+// [[+z9 '  JJw !5!5!:!:!< ==idxcxt|\}}tj|Dcgc]}||z c}|fScc}wN)r-r$r%)rr/r(r vals r, _get_box_forr31s6V$MGT JJW5cc 5 6 ==5s 7ch|j}|j}|tt||dS)Nr)rget_coordinaterr3)rrcoords r,_get_local_boxr76s7$$K  & & (E } a ))r.dt current_rankc|j}|jdk7r tdt|\}}t t |t |d|d|j jS)Nrrrank:/ shard_offsets shard_sizesr')rrrr7r listr&device)r8r9meshr(sizess r,_create_shard_md_from_dtrD>sh >>D yyA~EFF#B'NGU 7mK,q)9)9)@)@(AB r.dt_pgc  g}tj|}|dkDrdnd}|jdjr|j }nd}t |D]a}t ||\}}|jtt|t|d|dkDr|n|d|jjct||j t|j|j|j S)Nrrr;r<r=)dtypelayout requires_grad)shards_metadatar tensor_properties)distget_rankrr!r ranger3appendr r@r&rArrrGrHrI) r8rE shards_mdmy_rankscapegoat_rank shard_countir(rCs r,!_create_sharded_tensor_md_from_dtrUKs ImmE"G!A+Q1N }}Q  "jjl  ;   %b!, "7m Ka!eNA2CSCSCZCZB[\      !! WWY*((99**   r.cn|j}|jdk7r td|jS)Nrr)rrr get_group)r8rBs r, _get_dt_pgrXss/ >>D yyA~EFF >> r.specrankct|ts|Sd}|jD]G}tt|}|j |k(s'|j |j k7sEd}n|rtj|}t|jD]o\}}tt|}|j |k(s*|j |j k7sHt d|d|j |j|<q|S)z Rewrite ``spec`` to match the device of ``tensor``. FSDP.sharded_optim_state_dict sneakly ships optimizer state to CPU so if the original ShardingSpec produces CUDA metadata, ST construction bombs. FTr;r<) isinstancer rrrrZrAcopydeepcopy enumerate)rYrrZrewriteprTr's r,_rewrite_spec_if_neededrbzs d- . G __  # 668t  fmm ;G   }}T"%doo6 TLAy^Y7I~~4'I,<,<,>&--,O%3eD66==/4R%S"  T Kr. world_sizenum_devices_per_nodepgc t|turt|jdk7rt|j }t |||||}|jd}t|tj|jg}tj|j} d| j_ tj|| |jd} | St|tur|j } | j"dk7r t d|j$}t |||t&j(j+|}t-|} t|t/|t1j2| g}t5|| } d| j_ tj|| | d} | St |||||S)NrrF)sharded_tensor_metadata process_group init_rrefsr)typerr local_shardsr local_tensorr rr]r^metadatarKrI+_init_from_local_shards_and_global_metadata_process_grouprrrr&r$ accelerator device_countrXrDrLrMrU) rrZrcrdre inner_paraminner_stouter_local_shardshardsst_metast_outerrrEs r, _chunk_tensorrxs F|}$ v""$ % * ))+ /      #//1!4 (DMM*;*D*DE F -- 1227!!/ LL $+ //    f ((   q  !IJ J** /       * * ,   6" (4VT]]5=QR S 4FEB27!!/ LL $+  +       r.rc||jnd}| td|jdkrtd|jdd|jj }t |t jrt |tst|jDcgc] }t}}t|jDcgc] }t}}td|d<tj|||d j|| S|j}|d}|j}t|jDcgc] }t}}||d <t|jD cgc] } t}} td|d <||d <tj|||d j|| Scc}wcc}wcc}wcc} w) z Shard a tensor to chunks along the first dimension. The local rank will gets its corresponding chunk as the local tensor to create a DTensor. Nz4No parent device_mesh is found for FSDP device_mesh.z!Found parent device_mesh of ndim=,zbut meshes must be at least 2D.rF) run_checkrr)_get_root_mesh RuntimeErrorrdetachcloner\r$TensorrrNrr" from_local redistributerto_local) rrZr root_mesh_replicate_placementsshard_placements tp_placements tp_placementrTs r,_chunk_dtensorrs1<0G **,TIQRR~~/ /?q A -  ]]_ " " $F &%,,' 670K6;9>>5JK KK16y~~1FGAIKGG$Qi!! I3u ,!'   )) $Q' "6;9>>5JK KK#/R 16y~~1FGAIKGG%ay+!! I3u ,!'   9 LG* LGs*GGG G%ctt|j}t|dk(r?t |dj tur!|dj }|j}|}|t|dkDr|fSgfS)Nrr)rrrkrrjr)rru inner_tensors r,_pre_load_state_dictr&sz- ( 5 5 7F 6{aD!1!12mCay'' **, c&kAoF 662 66r. parent_meshc(||jk7rtttj|j }t t|dz D]}t||<|j|j|}|jS)zGAll gather a DTensor in its FSDP dimension and return the local tensor.rr}) rrr@r]r^rrNrrrr)rrrrTs r,_all_gather_dtensorr2s f(((dmmF$5$567J3z?Q& '$! 1 $  &&!F ?? r.ceZdZdZdfd Zdej deej edzffdZ dej dedej fdZ ddej d e d e d e d e jd ejdzdej fdZdej d e dedej fdZdej deej eeffdZdededzdej fdZxZS)rz DTensorExtension is the TensorFlattener extension needed for 2D FSDP + TP. This is the implementation for FSDPExtensions defined in https://github.com/pytorch/pytorch/blob/main/torch/distributed/fsdp/_fsdp_extensions.py rNct|d|_||_tj j |j|_yr1)super__init__compute_stream device_handler$_dynamodisablepost_unflatten_transform)selfr __class__s r,rzDTensorExtensions.__init__Os@ "*). (=(=  ) )) %r.rct|Sr1)rrrs r,pre_flatten_transformz'DTensorExtensions.pre_flatten_transformYsv&&r.param_extensionc|jxs|jj}|jj|5t |||j|j}t ||cdddS#1swYyxYw)N)rr)rrcurrent_streamstreamrr )rrrrresults r,rz*DTensorExtensions.post_unflatten_transform_s}$$K(:(:(I(I(K    & &v .  '"00#22 F  '   s 0A>>BrZrcrdrerAc t|||||Sr1)rx)rrrZrcrdrerAs r, chunk_tensorzDTensorExtensions.chunk_tensorrsVT:7KRPPr.rct|||Sr1)r)rrrZrs r, chunk_dtensorzDTensorExtensions.chunk_dtensor}s fdK88r.ct|Sr1)rrs r,pre_load_state_dict_transformz/DTensorExtensions.pre_load_state_dict_transforms$F++r.rct||Sr1)r)rrrs r,all_gather_dtensorz$DTensorExtensions.all_gather_dtensors #6;77r.)rNr1)__name__ __module__ __qualname____doc__rr$rtuplerrrintrL ProcessGrouprArrrr@rrrr __classcell__)rs@r,rrGsV ' ' u||S4Z' (' ll58 4'+ Q  Q Q Q " Q    Q t# Q  Q9 99 9  9, , u||T%[( ), 88 $&8  8r.);r]typingrrr$torch.distributed distributedrL&torch.distributed._shard.sharding_spec_shard sharding_spec shard_spec"torch.distributed.distributed_c10ddistributed_c10dc10d'torch.distributed._shard.sharded_tensorrrrrr :torch.distributed._shard.sharding_spec.chunk_sharding_specr $torch.distributed.fsdp._common_utilsr 'torch.distributed.fsdp._fsdp_extensionsr #torch.distributed.fsdp._shard_utilsr torch.distributed.remote_devicertorch.distributed.tensorrrrr"6torch.distributed.tensor.parallel._data_parallel_utilsrr__all__rr%r-rr3r7rDrrUrX ShardingSpecrrbrxrr@rrrr.r,rs5  ;;11 AXDBL:TT  >W>uzz5::'=!>>$>>s>uUZZ5K/L> *7*uUZZ-C'D*    %%))%%P7t00  ! !+0<<?B>I LLI I I  I  I  \\ I X> LL> > >  > B 7 LL 7 5<<e $% 7 d" \\*I8I8r.