^j ddlZddlZddlZddlZddlZddlZddlZddlmZm Z ddl m Z ddlm Z ddl mZddlmZmZddlmZddlmZmZmZdd lmZddlZddlmZddlmZddlmcm Z!dd l"m#Z#dd l$m%Z%dd l&m'Z'm(Z(m)Z*dd l+m,Z,ddl-m.Z.m/Z/ddl0m1Z1ddl2m3Z3m4Z4m5Z5ddl6m7Z7ddl8m9Z9m:Z:m;Z;ddlZ>m?Z?ddl@mAZAmBZBmCZCmDZDddlmEZEmFZFddlGmHZIddlJmKZKmLZLmMZMmNZNmOZOddlPmQZQmRZRmSZSmTZTmUZUmVZVmWZWddlXmYZYeVr)eZde[dejjZ^ndZ^eTrdZ_dZ`ejjZ^n.eUrdZ_dZ`n'eWrd Z_d!Z`ejjZ^nd"Z_d#Z`d$Z^Gd%d&eZbGd'd(eZcGd)d*ejeZed+ejd,ejd-e fd.Zgd/Zh dd+ejd0eifd1Zjdd2Zkd3Zld4Zmdd+ejd5eifd6Znd+ejd7eifd8Zod+ejd9eifd:ZpGd;d<ZqGd=d>eeZrGd?d@eeZsGdAdBesZtGdCdDesZuGdEdFeeZvGdGdHevZwGdIdJejZxGdKdLesZyGdMdNejZzGdOdPejZ|GdQdRejZ}ejdSe fdTZejdUe fdVZejdWe fdXZejdYe fdZZejd[e fd\Zeejd]e fd^Zeejd_e fd`Zeejdae fdbZeejdce fddZdee d-e dfedgefdhZ ddiejdjejdkeedlffdmZejeWdnGdodpeMZGdqdrZGdsdteeLZGdudveeKZddwedzfdxZGdydzejZGd{d|ejZGd}d~ejZy)N)ABCabstractmethod)Callable) nullcontext)deepcopy)autoEnumwraps)Anycast no_type_check)mock) checkpoint) DeviceMesh) CPUOffload fully_shardFullyShardedDataParallel) TrainingState)FSDPParamGroupRegisterPostBackwardFunction)#NO_RESHARD_AFTER_FORWARD_STRATEGIES)BackwardPrefetchMixedPrecisionShardingStrategy)ShardedGradScaler)always_wrap_policyModuleWrapPolicywrap)distribute_tensorDTensorShard)ColwiseParallelparallelize_moduleRowwiseParallelSequenceParallel)TransformerDecoderLayerTransformerEncoderLayer)DistributedDataParallel)MultiProcContinuousTestMultiProcessTestCaseMultiThreadedTestCase run_subtests TEST_SKIPS) FILE_SCHEMAget_cycles_per_ms set_rng_seed TEST_CUDATEST_HPUTEST_WITH_ROCMTEST_XPU) has_tritoncudancclzhpu:0hcclxpuxcclcpuglooc(eZdZeZeZy) FSDPInitModeN)__name__ __module__ __qualname__rNO_FSDP RECURSIVEn/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/testing/_internal/common_fsdp.pyrBrB^sfGIrIrBc6eZdZeZeZeZy)DEVICEInitModeN)rCrDrEr DEVICE_BEFORE DEVICE_AFTER DEVICE_NEVERrHrIrJrLrLgsFM6L6LrIrLceZdZdZedeejdffdZedejfdZ ed dZ e ede d e de jfd Zy) FSDPTestModelzZThis defines the interface expected from all models used commonly for FSDP unit tests.return.cy)z*Returns an input for the model as a tuple.NrHselfdevices rJ get_inputzFSDPTestModel.get_inputt rIcy)z,Returns the loss given the input and output.NrH)rUinputoutputs rJget_losszFSDPTestModel.get_lossyrXrINcy)z named_buffersdistget_world_sizerangeall_gather_objectAssertionErrorzip)rmrnro param_nameparamnamed_module_states buffer_namebuffer world_size_olist rank0_statesstatep1p2s rJ_assert_module_statesrsC"'!7!7!9 J U\\^'')*#(#6#6#8 K fmmo))+,$$]3J , -aT -E -5"5]K8LCDDqr = !@A A #L% E  GQWa b"   .s*D*D' D%c4tjtSN)rhrV DEVICE_TYPErHrIrJ get_devtypers << $$rI zero_buffersc|rtj|n t}|5|jD]/}t j 5|j ddd1|rB|jD]/}t j 5|j ddd1dddy#1swYxYw#1swYPxYw#1swYyxYw)zBZeros the parameters and optionally buffers of ``model`` in place.N)FSDPsummon_full_paramsr parametersrhno_gradzero_buffers)rmr summon_fullctxrrs rJ _zero_modelrs -8$ ! !% ([]C #%%' E      --/ #]]_#LLN## # ##  ## ##s;(C  B43C C! C 4B= 9C C C  Cct|s|jt}|r|j|jSr)torhalf state_dict)rm cpu_offloadrs rJ_get_state_dictrs. %     rIc jdj|Dcgc]}||t|ndc}Scc}w)Nrnone)joinstr)test_name_mappingrass rJ subtest_namers7 88IMNAam 3q6 " ?N Ns0cx|jD];\}}|jtjdk7s)|j||<=|dk(r|ndg}t j |t tttjf|d}|D]}||jt||<|S)Nr>r) itemsrVrhr>rxbroadcast_object_listr dictrrirr)rankrr~rrs rJ_broadcast_state_dictrs(--/1 E <<5<<. .%*YY[Jz "1 19Z$ /Eu%d3 ,-uQx8J H !+J!7!:!:;!G :H rIrecursectj||5tt|j cdddS#1swYyxYw)a[ Returns the full unsharded parameters of ``model``. Any FSDP-managed parameters offloaded to CPU are moved to GPU in the returned list. Args: recurse (bool): If ``False``, only unshards the parameters immediate to ``model``; if ``True``, recurses through the module hierarchy rooted at ``model``. )rN)rrrlistr)rmrs rJget_full_paramsrs?   82U--/01222s "AA move_to_devicec4|r|jtS|Sr)rr)rmrs rJ_move_to_devicers$2588K ==rI wrap_fsdpc(|s|St|g|i|Srr)rmrrarbs rJ_maybe_wrap_fsdprs!5CtE'CD'CF'CCrIc:eZdZdedefdZdefdZdefdZdZy) DummyProcessGrouprsizec ||_||_yr)_rank_size)rUrrs rJ__init__zDummyProcessGroup.__init__s  rIrRc|jSr)rrUs rJrzDummyProcessGroup.rank zzrIc|jSr)rrs rJrzDummyProcessGroup.sizerrIcBtj}d}||_|S)Ncdtjj}|jd|S)Nr@)rhfuturesFuture set_result)futures rJ get_futurez/DummyProcessGroup.allreduce..get_futures'+0==+?+?+AF   a MrI)rMockr)rUrarb dist_waitrs rJ allreducezDummyProcessGroup.allreduces"IIK   * rIN)rCrDrEintrrrrrHrIrJrrs2Scc rIrceZdZdejdededeffd ZdZdZ dZ d Z e ddejd e ded eeefd zdeded ej$ezfdZdZxZS)TransformerWithSharedParamsrrdevice_init_modeadd_bn deterministicct||j|_|j|_|rt j dd}d}tj|||_ tj|dddd|_ tj|||_ |jj|j_|jd|jjj!|f|jd t j"|j$t j& d|_|r)t jj+|j(nt jj-|_|t0j2k(r|j5t6}|r|j9yy) Nrr8g?)d_modelnum_encoder_layersnum_decoder_layersdim_feedforwarddropout vocab_bias long_buffer)dtype)superrrrrrh manual_seedrk Embedding embed_tokens Transformer transformerLinear output_projweightregister_buffernew_ones zeros_likerlongbs BatchNorm1dIdentitybnrLrMrreval)rUrrrrrd_vocabr __class__s rJrz$TransformerWithSharedParams.__init__sa JJL **,    a LL':>>    99Wg6#'"3"3":":  $++22;;WJG      T__EJJ ? 39%((&&tww/uxx?P?P?R ~;; ;77;'D  IIK rIc,tjd|jztjd|j d|j }tj|j dz|j d|j }||fS)Nr@ rVr7)rhrrarangeviewr)rUrVsrctgts rJrWz%TransformerWithSharedParams.get_input8sl !dii-(ll2f-221dgg>ll477Q;v6;;AtwwGSzrIc|j|}||jz|jj|z}|j|}|j |}|j ||}|j |Sr)rrrtype_asrrr)rUsrc_idstgt_idsrrxs rJforwardz#TransformerWithSharedParams.forward>sv(DOO#d&6&6&>&>s&CC(ggcl   S# &""rIc|\}}tjj|jd|j d|jddS)Nsum) reduction)rk functional cross_entropyrr)rUrZr[rrs rJr\z$TransformerWithSharedParams.get_lossFsI3}}** KKFKKO ,chhrle+  rIc$|jyrbackwardr^s rJr`z(TransformerWithSharedParams.run_backwardL  rINfsdp_init_mode fsdp_kwargsrRcN|i}|tjk(r&t|tr|d}n|}t ||||S|tj k(rd|vrt tth}n|jd}d|vr8|dtjtjhvrt|tsd}n|}t|tr|d} n|} t | |||} t| |fd|i|} |tjk(r| j!t"} | St%d|)au Initializes a :class:`TransformerWithSharedParams` instance. Args: fsdp_init_mode (FSDPInitMode): If ``NO_FSDP``, then does not wrap any modules with FSDP. If ``RECURSIVE``, then wraps with top-level FSDP. By default, the top-level FSDP uses the ``ModuleWrapPolicy`` for encoder and decoder layers, but a different auto wrap policy may be specified via ``fsdp_kwargs``. device_init_mode (DEVICEInitMode): Determines model movement to DEVICE. fsdp_kwargs (Optional[Dict[str, Any]]): Optional keyword arguments forwarded to the FSDP constructor. deterministic (bool): Whether to make the model deterministic across constructions. add_bn (bool): Whether to include batch norm in the model. Nrauto_wrap_policysharding_strategyUnsupported FSDP init mode: )rBrF isinstancergrrGrr(r'popr HYBRID_SHARD_HYBRID_SHARD_ZERO2rrLrNrr ValueError) rrr rr rrpgrfsdp_pg tformer_pgm fsdp_models rJrdz TransformerWithSharedParams.initOsX6  K \11 1%'1X.$fm |55 5!4#3//$ $/??3E#F ${2 34$113C3W3WXY"5%0%'"1X " +,fmA"2 J  >#>#>>']];7  77GHIIrIc|jgSr)rrs rJget_ignored_modulesz/TransformerWithSharedParams.get_ignored_moduless  !!rI)NFT)rCrDrErx ProcessGrouprLboolrrWrr\r`rjrBrrr rkrlrrdr __classcell__rs@rJrr s(  ()( (  (T #  .2# KJ  KJ$KJ)KJ#s(^d* KJ  KJ  KJ T KJKJZ"rIrceZdZdejdededeffd ZdZdZ dZ d Z e ddejd e ded eeefd zded ej$f dZxZS)NestedWrappedModulerrrrrc t|j|_j|_|t j k(}fd}|rtjdtjttjdd||tj|ttjdd|ttjdd||ttjdd|ttjdd||_ y)Nc&r t|fiS|Srrlayerr rrrs rJ _maybe_wrapz1NestedWrappedModule.__init__.._maybe_wrapE58K88LrIrrr7r)rrrrrrLrMrhrrk Sequentialrrmodule rUrrrrrr rr'rs `` ` rJrzNestedWrappedModule.__init__s JJL **,)^-I-II     a mm BIIaO^ <   !R0@. QR#BIIb"$5~F   "a(8.I J BIIaO^ <   rIcxtjd|jztjdd|fS)Nr@r7rr)rhrrrandrTs rJrWzNestedWrappedModule.get_inputs. !dii-( 1a/11rIc$|j|Srr*rUrs rJrzNestedWrappedModule.forward{{1~rIc&|j}|Sr)rrUrZr[r_s rJr\zNestedWrappedModule.get_lossszz| rIc$|jyrrr^s rJr`z NestedWrappedModule.run_backwardr rINr r rRc|i}|tjk(rt|d||S|tjk(r:t|fd||d|}|tj k(r|j t}|Std|)a Initializes a :class:`NestedWrappedModule` instance. Args: fsdp_init_mode (FSDPInitMode): If ``NO_FSDP``, then does not wrap any modules with FSDP. If ``RECURSIVE``, then wraps some nested modules with FSDP but not the top-level module. The model may later be wrapped with a top-level FSDP external to this method if desired. device_init_mode (DEVICEInitMode): Determines model movement to DEVICE. fsdp_kwargs (Optional[Dict[str, Any]]): Optional keyword arguments forwarded to the FSDP constructor. deterministic (bool): Whether to make the model deterministic across constructions. FrrrTr) rBrFr"rGrLrNrrr)rrr rr rrs rJrdzNestedWrappedModule.inits.  K \11 1&!1+   |55 5,!1+   J >#>#>>']];7  77GHIIrINF)rCrDrErxrrrLrrWrr\r`rjrBrrr rkrlrdrr s@rJr"r"s     )    @2 .2# +J  +J$+J)+J#s(^d* +J  +J  +J+JrIr"c heZdZe ddej dededee e fdzde f fd Z xZ S) AlwaysWrapNestedWrappedModuleNrrr rr rc0ttt |tj|||}|tjk(r|S|tj k(rB|xsi}t |fdti|}|tjk(r|jt}|Sy)z Initializes a :class:`NestedWrappedModule` instance, but unlike :meth:`NestedWrappedModule.init`, for the ``RECURSIVE`` init mode, this wraps with top-level FSDP and the ``always_wrap_policy()`` auto wrap policy. )rrr rr rrN) rr9rdrBrFrGrrrLrNrr)rrr rr rrmrrs rJrdz"AlwaysWrapNestedWrappedModule.inits )+H '//-#'    \11 1L |55 5%+KeX6HXKXJ>#>#>>']];7   6rIr7)rCrDrErjrxrrBrLrrr rrdrr s@rJr9r9s^ .2#   $)#s(^d*   rIr9c eZdZdejdededeffd Zed dZ e d dejde ded e e e fdzdef d ZxZS) NonUniformReqGradNWMrrrrrc tt| j|_j |_|t jk(}fd}|rtjdtjttjdd||tj|ttjdd|ttjdd||tjttjdd|ttjdd||_y)Nc&r t|fiS|Srrr%s rJr'z2NonUniformReqGradNWM.__init__.._maybe_wrap5r(rIrrr7r)rr"rrrrrLrMrhrrkr)rrr*r+s `` ` rJrzNonUniformReqGradNWM.__init__"s !413JJL **,)^-I-II     a mm BIIaO^ <   !R0@. QR#BIIb"$5~F   #BIIb!$4nE#BIIaO^D   rINc|jD]-\}}tj||r|jd/yr7)rurematchrequires_grad_)rm req_grad_masknps rJ_set_nonuniform_req_gradz-NonUniformReqGradNWM._set_nonuniform_req_gradLs:**, (DAq88M1-  ' (rIr r ctjd}|tjk(r't |d||}tj |||S|tj k(rT|i}t |fd||d|}|tjk(r|jt}tj |||Std|)a Initializes a :class:`NestedWrappedModule` instance, but unlike :meth:`NestedWrappedModule.init`, it wraps a second :class:`torch.nn.Sequential` container to enable the desired non-uniform ``requires_grad`` ``use_orig_params=True`` tests. For both ``RECURSIVE`` and ``NO_FSDP`` init modes, freezes all parameters except the last two to validate ``ShardedGradScaler`` support for ranks with no (non-zero sized) local shards in FSDP ``use_orig_params=True`` mode. zmodule\.2.*\.1.*Fr6Tr) r@compilerBrFr<rFrGrLrNrrr)rrr rr rreq_grad_pattern ddp_modelrs rJrdzNonUniformReqGradNWM.initRs(::&9: \11 1,!1+ I ! 9 9)EU V  |55 5" -!1+   J >#>#>>']];7 9 9*FV W 77GHIIrIrer7)rCrDrErxrrrLrrjrFrBrrr rdrr s@rJr<r<!s(   ( ( ) (  ( T((  .2# +J  +J$+J)+J#s(^d* +J  +J+JrIr<c eZdZdZdej dedeffd ZdZdZ dZ d Z e d e ed ededed ef d ZxZS)ModuleWithDelayzThis class wraps a :class:`FSDPTestModel` to optionally add a delay after computing the loss and/or before the gradient reduction.r*delay_after_loss_msdelay_before_reduction_mscLt|||_||_||_yr)rrrMrNr*)rUr*rMrNrs rJrzModuleWithDelay.__init__s' #6 )B& rIc8|jj|Sr)r*rWrTs rJrWzModuleWithDelay.get_inputs{{$$V,,rIc$|j|Srr/r0s rJrzModuleWithDelay.forwardr1rIcB|jj||}|jdkDrststr$t j |jdz |Str=tjjt|jtz|SNr) r*r\rMr3r5timesleepr2rhr9_sleeprr0r3s rJr\zModuleWithDelay.get_losssy{{##E62  # #a '8 433d:;  !!#d&>&>ARAT&T"UV rIctjjfd}tjd|5j j |dddy#1swYyxYw)NcjdkDrrtr>tjj t jt zn.tstr"tjjdz |i|SrS) rNr2rhr9rWrr0r3r5rUrV)rarborig_reduce_scatterrUs rJ_delayed_reduce_scatterz=ModuleWithDelay.run_backward.._delayed_reduce_scattersk--1JJ%%D::=N=PPQJJt==DE&77 7rIz'torch.distributed.reduce_scatter_single)rh distributedreduce_scatter_singlerpatchr*r`)rUr_r[rZs` @rJr`zModuleWithDelay.run_backwardsW#//EE 8ZZ 57N  + KK $ $T * + + +s AA' module_class model_args model_kwargsc<t|j|i|||S)aA Args: module_class (Type[FSDPTestModel]): Wrapped module class to which to add delays. model_args: Positional arguments forwarded to the ``module_class`` ``init()``. delay_after_loss_ms (int): Delay after computing the loss/before the optimizer step (in ms). delay_before_reduction_ms (int): Delay before reduce-scattering gradients (in ms). model_kwargs: Keyword arguments forwarded to the ``module_class`` ``init()``. )rLrd)r_rMrNr`ras rJrdzModuleWithDelay.inits,* L  z :\ :  %  rI)rCrDrErfrkrlrrrWrr\r`rjtyperQr rdrr s@rJrLrLsF   ! $' -+$ =)  ! $'     rIrLc~eZdZeej ddddfdejdedede e e fdzde d e d e fd Zy) NestedWrappedModuleWithDelayNFrrrr rr rrMrNc Dtjt|||||||S)Nrrr rr rrMrN)rLrdr"rgs rJrdz!NestedWrappedModuleWithDelay.inits4## )-#' 3&?$  rI)rCrDrErjrLrNrxrrBrrr rrrdrHrIrJreres,:+F+F-1##$)*    $ ) #s(^d*    !  $'  rIrec$eZdZfdZdZxZS)DummyDDPc0t|||_yr)rrr*)rUr*rs rJrzDummyDDP.__init__s  rIc&|j|i|Srr/rUrarbs rJrzDummyDDP.forwardst{{D+F++rIrCrDrErrrr s@rJriris ,rIriceZdZdejdedededef fd ZdZ dZ e d dejd e ded e eefd zdedef d ZxZS)MixtureOfExpertsrrrrdelay_before_free_msrc t|||||||_||_||_|t j k(|_|r"tjd|jzd}d}d} ttj|||j} td| jD|_| jD] } d| _ |rtjdttj|||j} |rHtj$j'|jg} t)| | fi|} t)| |fi|} tj*ttj| ||j| | ttj|| |j|_y) N)rrrrr*rrrc3<K|]}|jywr)numel).0rEs rJ z,MixtureOfExperts.__init__.. s$L1QWWY$LTr)rrrrrprrLrMrrhrrrrkrrrnum_expert_paramsexpertr\ new_grouprr)r*)rUrrrrrprr d_expertd_sharedd_inputryrEshared expert_grouprs rJrzMixtureOfExperts.__init__s -'   $8!"..2N2NN    b499n - 8X!>@S@ST!$$L8I8I8K$L!L""$ AAH     a  8X!>@S@ST  ,,66L&,>+>F&%7;7Fmm BIIgx8$:M:M N   BIIh8$:M:M N   rIcfjdkDrjd}t|tretj j jjfd}tjd|5j|cdddSj|S#1swYxYw)Nrr8ctr>tjjt j t zn.tstr"tjj dz |i|S)NrT) r2rhr9rWrrpr0r3r5rUrV)rarb orig_reshardrUs rJ_delayed_reshardz2MixtureOfExperts.forward.._delayed_reshard)s]  )) 9 9** {{1~ **s ;B''B0c|j|jstj5|j D]v}t |dr|j |j j|jtjj|j |jx dddyy#1swYyxYw)Nryrq) r rrhrrhasattrgraddiv_rr\ all_reducerr)rUr_rEs rJr`zMixtureOfExperts.run_backward;s ~~ O*OAq(+ vv) DOO4))44QVV4::4N O O O O Os-CACCNr r c|i}|tjk(rt|d|||S|tjk(r;t|fd|||d|}|tj k(r|j t}|Std|)a Initializes a :class:`MixtureOfExperts` instance. Args: fsdp_init_mode (FSDPInitMode): If ``NO_FSDP``, then does not wrap any modules with FSDP. If ``RECURSIVE``, then wraps some nested modules with FSDP, including the expert and shared layers, but not the top-level module. The model may later be wrapped with a top-level FSDP external to this method if desired. device_init_mode (DEVICEInitMode): Determines model movement to DEVICE. fsdp_kwargs (Optional[Dict[str, Any]]): Optional keyword arguments forwarded to the FSDP constructor. deterministic (bool): Whether to make the model deterministic across constructions. delay_before_free_ms (int): Delay before resharding expert parameters in the forward pass (in ms). F)rrrprTr) rBrFrorGrLrNrrr)rrr rr rrprs rJrdzMixtureOfExperts.initGs4  K \11 1#!1%9+  |55 5)!1%9+   J >#>#>>']];7  77GHIIrI)NFr)rCrDrErxrrrLrrrr`rjrBrrr rdrr s@rJroros2   2 2 ) 2 " 2  2 h0 O .2#$% 0J  0J$0J)0J#s(^d* 0J  0J " 0J0JrIroc eZdZ ddddddedej dzded ed ef fd Zd ejd ejfdZ dZ xZ S)MLPNTFr7)bias with_bufferdim_multiplierdimrVrrrc t|tj|||z|||_tj||z||||_|r)|j dtj|f|yd|_ y)N)rVrrr) rrrkrin_projout_projrrhrandnr)rUrrVrrrrs rJrz MLP.__init__|so yyns&:6PTU  .3"6FQUV   5;;vf+M NDKrIrrRc|j|}tj|}|j|}tj|}|j||jz}|Sr)rFrelurr)rUrzs rJrz MLP.forwardsS LLO FF1I MM!  FF1I ;; "DKKArIc|j4tjjj |jyyr)rrhrkrdnormal_rs rJreset_parameterszMLP.reset_parameterss+ ;; " HHMM ! !$++ . #rIr) rCrDrErrhrVrrrirrrr s@rJrr{sv'+ !  t#    "%,,/rIrcFeZdZdddedeffdZdededed dfd ZxZS) MLPStackF)with_seq_parallelmlp_dimrct|dt|t|dg}|r&|jtj|dt ||||_y)N)rFr)rappendrk LayerNormrrr)rUrrmodulesrs rJrzMLPStack.__init__sX  * L  * $   NN2<<e< = '"!2rItp_meshdp_meshuse_activation_checkpointingrRc tdtdtdtdtd|jrttdn td}|jrt d|d<t ||||D]8}t |tjr|r t|t|fd |i|:t|fd |i||S) NF)use_local_outputr@)output_layouts)z 0.in_projz 0.out_projz 1.in_projz 1.out_projz 2.in_projz 2.out_proj) sequence_dim3) device_meshparallelize_planmesh) r#r%rr"r&r$rrkrrr)rUrrrr rr*s rJ parallelizezMLPStack.parallelizes)%@)5A(%@)5A(%@%%*qB "    ! !$4!$D S !4WGWX =F&",,/+6"    0=$- 0=$-rrZrarbct|dkDr|d}nd|vr|d}ntd|d|||||i|S)Nrr[z,Cannot get reduce-scatter output from args: z kwargs: )lenr|)clsrZrorarbr[s rJreduce_scatter_with_assertr_sa 4y1}a V !;D6F8 T  f  / //rIreplicated_modulesharded_moduleprefixes_to_ignore.ct|j|jdD]\\}}\}}|}|D]} |j| d}|j|||j |t t |t s td|j|j} } t| tdtdfk(r tdt|| | } |j|j| j|j|j|j|j!|jt|j| | } |j |jt t |jt s td|j|jj| jy)NTrsz&Expected sharded_param to be a DTensorrzmFSDP's (Shard(0), Shard(0)) layout differs from distribute_tensor(), so we cannot check for equality using itz+Expected sharded_param.grad to be a DTensor)r}rureplace assertEqualassertIsInstancer!rr|r placementsrgr"r to_localr assertIsNoneassertIsNotNone)rrrrreplicated_namereplicated_param sharded_name sharded_paramclean_sharded_nameprefixrrsharded_ref_paramsharded_ref_grads rJcheck_sharded_parityrrs OR**,'')OTJ+*-JlM *( HF!3!;!;FB!G  H );< ]G4-1 !IJ J(44m6N6Nj  q58 4 4 ; ..>jQ  ..02C2L2L2NO  (   ]// 0  M../,-=-B-BD*U ]//9-,,g6 !NO O  **3357G7P7P7RS9TrIznot-support-multithreadc@eZdZedZfdZdZdZdZxZ S)FSDPTestMultiThreadctSr DEVICE_COUNTrs rJrzFSDPTestMultiThread.world_sizerIcBt||jyr)rsetUp_spawn_threadsrUrs rJr zFSDPTestMultiThread.setUps   rIc t|g|i|Srr-rls rJr-z FSDPTestMultiThread.run_subtestsD242622rIc@tjjyrrh_dynamoresetrs rJperThreadSetUpz"FSDPTestMultiThread.perThreadSetUp rIc@tjjyrrrs rJperThreadTearDownz%FSDPTestMultiThread.perThreadTearDownrrI) rCrDrEpropertyrr r-rrrr s@rJrrs) 3rIrc#TeZdZdZdZdZdZdZedZ d"d e jd e d e d ed edzde dedzde de deeefdzfdZdddedddddddddf deededededzde de dededzdedzdedzde de de de d eeefdzdeeefdzf d!Zy)# FSDPTestMixinz Mixin class containing shared test utilities for FSDP tests. Provides common helper methods for both FSDPTest and FSDPTestContinuous. c<|j||jyr)rr)rUrrs rJ_check_cpu_offloadz FSDPTestMixin._check_cpu_offloads j&<&<=rIc<|j||jyr)rbackward_prefetch)rUrrs rJ_check_backward_prefetchz&FSDPTestMixin._check_backward_prefetchs *J,H,HIrIc<|j||jyr)rforward_prefetch)rUrr"s rJ_check_forward_prefetchz%FSDPTestMixin._check_forward_prefetchs ):+F+FGrIc t|g|i|Srrrls rJr-zFSDPTestMixin.run_subtestsrrIc ||}||_||_|jdd}td|jd|jt dk7r^t jj|jkr3tjtd|jj |r`t jjjj j#}t%j&d|j||nDt%j&|j(t*t-|j|j d} |jt2z} t4st6rt jj9| | g} t%j:| t j<j?tA|jC||t j<j?t%j:| t%jDy#t.$r=} d | j0d vr&tjtd jd} ~ wwxYw)Nfake_pgF dist init r=, world=r> multi-gpu-fakebackendrrstore init_methodr,rr recompilerbackend_unavailable device_ids)#r file_namegetprintrrrh accelerator device_countsysexitr. exit_codetesting _internalr\r& FakeStorerxinit_process_groupr/DISTRIBUTED_BACKENDr RuntimeErrorrarr2r5set_device_indexrrrr1run_testdestroy_process_group rr test_namer4piperbrUr&r-er3 device_ids rJ_runzFSDPTestMixin._runs9~ "**Y. TYYKx/@AB % E$5$5$B$B$Dt$V HHZ*T__,= >?II J  //;;CCMMO''"# '' $ 0 0/"4??3  II ,     . .y 9[ +  i&  + ""$1 affQi'$9:DDE   s6B&H88 I>8I99I>NFrm num_stepsautocastlrfsdp_cpu_offload save_modelmixed_precisionenable_sharded_grad_scaler use_pure_fp16sharded_grad_scaler_kwargsc |xr |j} t|jj} | i} t d d|i| } t j j|j|d}t|D]%}|jt jjt|5|jjt jt}| s|rMt|t s=t|t j"r|j%}nt'd|D}||}| rft|t rV|j(t*vrD|jD]1}|j-|jt jd3|jj/||j1| }ddd| j3}|s*| s(|j4t j6k7rt9d| r+|j-|j4t j:nnt|t r4| t9d|j-|j4|j<n*|j-|j4t j6|jj?|| rTt|t rD|jD]1}|j-|jt jd3| jA|| jC|s|jEjGDcic]\}}||jI}}}tK||jM|(t|t r|jOtPjRjUS#1swYxYwcc}}w) Nenabledg?)rMmomentum)rUc3<K|]}|jywr)r)rurs rJrvz9FSDPTestMixin._train_for_several_steps..s%>1affh%>rwr>zQloss data type should be float32, as the original parameter data type is float32.z'Expected mixed_precision to not be NonerH)+offload_paramsnextrrVrrhoptimSGDrz zero_gradamprLrr*rWrrrirrgrrrr\rscalerfloat32r|float16 param_dtyper`stepupdaterrclonerload_state_dict _assert_staterIDLErv)rUrmrKrLrMrNrOrPrQrRrScpu_offload_params model_devicesharded_grad_scalerrZrrZr[rEr_kvrs rJ_train_for_several_stepsz&FSDPTestMixin._train_for_several_stepss@.Q2B2Q2QE,,./66 % -)+ &/ . 2L    0 0 2rCHy!= 2A OO ##K#B M ..u||K/HI _Zt=T!%6 %  %%>%> >'"5$///>?#--/H((5<<3FGH||,,UF;>>|L- M.',,T2D"=::.(: !$$TZZ?t,&.,E$$TZZ1L1LM$$TZZ? LL % %d +!j&=))+DA$$QXXu||E/BCD $ $U +  & & (7<7G7G7I7O7O7QRtq!alR RE"%%j1{= 2~ eT "    2 2 3{{} M MnSs9DO(,O5(O2 r8T model_classr r ref_init_fn num_itersrrrr"use_orig_params init_kwargsc |tjk(r td|i}d}|jj }|j |jtjt jfddi|}|Ctrt|tgt}n-tdk(r t|}nt||g|}n||}|r|j}|j||| du||| | || }t|j}|j||| | | | d  |j |j|||fddi|}t'|t(st)||jfi|}|r|j}|t j*k(r|j-t}|duxr |j.}|xr|t j*k(}|xr|t j*k7}|rFt1j2d}|jD]}|j5|j2| |r|j7t8d tn t;}|5|j||d |||| | || } ddd|ry|r[t1j2d}|jD]}|j5|j2|  j-t} t=|}!t0j>jA| d | |s|j5||!ddyyy#t $r }t#d |d t%||d}~wwxYw#1swYxYw)a Tests FSDP training against a reference, which defaults to DDP but may be customized with ``ref_init_fn``. Args: model_class (Type[FSDPTestModel]): A model class that inherits from ``FSDPTestModel``, which defines the expected interface. fsdp_init_mode (FSDPInitMode): The mode to initialize the FSDP-wrapped model. This should not be ``NO_FSDP``. ref_init_fn (Optional[Callable]): A callable to invoke that wraps a non-wrapped model to construct the reference model, where this wrapper should provide data parallel semantics. If ``None``, then the callable defaults to the DDP constructor. z.Expects an FSDP init mode that wraps with FSDPN{Gz?rT)r3 output_devicer>)rLrMrNrPrQrRrS)rrrrPr"rqz Initializing z raised error zOAn FSDP-managed module with parameter CPU offloading enabled has parameters on F)rLrMrNrOrPrQrRrS) check_dtypezFSDP did not match DDP) exact_devicemsg)!rBrFr|rnrrdrLrMr3DDPrrrmrrrc ExceptionrrrrrNrrXrhrVrassertRaisesRegexrArrr< assert_close)"rUrnr rrorprOrrrrPr"rqrQrRrrrSr rMrrm ref_modelref_loss ddp_paramsrrHrXexpects_device_errorexpects_cpu_device cpu_devicercontext fsdp_lossfsdp_unsharded_paramss" rJ_test_fsdp_parityzFSDPTestMixin._test_fsdp_parityOsD \11 1 !QR R  K !!&&(        ( (       {m; %J 4&M #E*I !(I00  $D0(+'A''A1  )..01 *%6%6#2$4#2    Y)))""   #   J*d+j$*<*<L LJ #*J ~:: :#{3J$D0O[5O5O  N/>3N3NN   N/>3N3NN  e,J#..0 ;  z: ;$  " "%%0M3    55!,% /+E++E6 I   e,J#..0 ;  z: ;! [1I / ;  ""8YE"J  "=   %!,  ,9 "K Y}[MAxPQWX X YF  s$)"L,M, M5MMM!)rtNFNFFN)rCrDrErfrr r#r- classmethodrJrkrlrrfloatrrrrr rmrcrQrBrLrrrrrHrIrJrrs >JH34%4%v.2 15+0#<@YyyYY Y  Y %t+ YY($.Y%)YY%)cNT$9Y@(,",,595915!& %+0#-1<@#h-(h%h) h _ h  hh h,d2h,d2h($.hhh%)hh #s(^d*!h"%)cNT$9#hrIrcteZdZfdZedZedZedefdZedZ e dZ xZ S)FSDPTestcht|dtjd<|j y)N0TORCH_NCCL_DESYNC_DEBUG)rr osenviron_spawn_processesr s rJr zFSDPTest.setUps)  14 ,- rIctSrrrs rJrzFSDPTest.world_sizer rIc>tjjSr)rxdistributed_c10d_get_default_grouprs rJrnzFSDPTest.process_groups$$7799rIrRcyr7rHrs rJdestroy_pg_upon_exitzFSDPTest.destroy_pg_upon_exit srIc*t|jSr)r/r4rs rJr/zFSDPTest.init_methodst~~.//rIc ||}||_||_|jdd}td|jd|jt j j|jkr3tjtd|jj |r`t jjjjj!}t#j$d|j||nDt#j$|j&t(t+|j|jd} |jt0z} t2st4rt j j7| | g} t#j8| t j:j=t?|jA||t j:j=t#j8| t#jBy#t,$r=} d | j.d vr&tjtd jd} ~ wwxYw) Nr&Fr'r(r)r*r+r.r0rr1r2)"rr4r5r6rrhr7r8r9r:r.r;r<r=r\r&r>rxr?r/r@rrArarr2r5rBrrrr1rCrDrEs rJrJz FSDPTest._runs9~ "**Y. TYYKx/@AB    ) ) +doo = HHZ*T__,= >?II J  //;;CCMMO''"# '' $ 0 0/"4??3  II ,     . .y 9[ +  i&  + ""$1 affQi'$9:DDE   s-B&H// I588I00I5) rCrDrEr rrrnrrr/rrJrr s@rJrrss ::d004%4%rIrceZdZUdZeZeed<ede fdZ ede fdZ efdZ fdZ fdZed ZxZS) FSDPTestContinuousz FSDP test base class using MultiProcContinuousTest for faster test execution. This class reuses worker processes across tests, reducing process spawn overhead. Use this for tests that don't require fresh process state between tests. rrRctSr)r@rs rJ backend_strzFSDPTestContinuous.backend_strUs""rIctSr)rrs rJ device_typezFSDPTestContinuous.device_typeYsrIcJdtjd<tjj |kr)t j td|j|tz}tstrtjj|t|9|||y)Nrrr))rrrhr7r8r9r:r.r;rr2r5rBr_init_pg)rrr rdvz_filerIrs rJrzFSDPTestContinuous._init_pg]s} 14 ,-    ) ) +j 8 HHZ*ZL 9:DD E<'     . .y 9 z95rIct||j|jk7rt j t jjtyr) rr rMAIN_PROCESS_RANKrxrrhrrr1r s rJr zFSDPTestContinuous.setUpms>   99.. . LLN rIc|j|jk7rtjt|t jjyr) rrrxrrtearDownrhrrr s rJrzFSDPTestContinuous.tearDownws9 99.. . LLN  rIc.|jjSr)rrrs rJrnz FSDPTestContinuous.process_group~s~~   rI)rCrDrErfrrr__annotations__rrrrrr rrrnrr s@rJrrLsz #J"#C##C 6 6!!rIrcompile_compute_on_modulec@fdGddtfd}|S)Nctjjj|i|t |dr|dj yy)Nr)rhr\rrrrH)rarbrs rJ!fully_shard_with_compiled_computez=compiled_fsdp_test..fully_shard_with_compiled_computesN **D;F; $ , G.1  GOO 1 rIc(eZdZeZeZy)*compiled_fsdp_test..FullyShardModeN)rCrDrErEAGERCOMPILED_COMPUTErHrIrJFullyShardModers6rIrc4tfd}|S)Nctjjj}D]>}|jk7r"t st jdd5tjjj}tjj|jk(r|}n?|jk(r"dtjj_ }ntd||j|j<|i|tjj|j|j<|tjj_ Ay)Nz0Inductor on GPU needs Triton and recent GPU archr8) stacklevelr@z!Need to implement FullyShardMode=)rhr\rrrr6warningswarn _inductorconfigcompile_threadsrrNotImplementedError __globals__rC) rarboriginal_fully_shardmodeoriginal_compile_threadsfully_shard_patchrrfuncs rJwrapperz6compiled_fsdp_test..decorator..wrappers1(-(9(9(>(>(J(J & R>/// MMJWX+0??+A+A+Q+Q(!!))+>///(<%^<<<=>EOO**:(I%-;D6B CT  !5!>!>?d%f%!!))+BV  !5!>!>?9Q&&67 RrIr )rrrrs` rJ decoratorz%compiled_fsdp_test..decorators" t R  R>rI)r )rrrrs` @@rJcompiled_fsdp_testrs"""!F rIc&eZdZdfd ZdZxZS) SkipModulec\t|tjddd|_yN Fr)rrrkrlinr s rJrzSkipModule.__init__s" 99R%0rIc$|j|Sr)rr0s rJrzSkipModule.forwardsxx{rIrermr s@rJrrs 1rIrc$eZdZfdZdZxZS) NestedLinearct||r:ttjdddj t |_ytjdddj t |_yr)rrrrkrrr nested_linear)rU fsdp_wraprs rJrzNestedLinear.__init__sV  !%biiBU&C&F&F{&S!TD !#2r!>!A!A+!ND rIc$|j|Sr)rr0s rJrzNestedLinear.forwards!!!$$rIrmr s@rJrrsO%rIrc$eZdZfdZdZxZS) SkipModelct|tjdddj t |_tj t |_tt|t |_ y)NrFr)r)rI) rrrkrrrlinearr linear_skiprrr)rU double_nestrs rJrzSkipModel.__init__sW iiBU366{C %rs #$"++ 44 ?S IRRFF FH+q#a!8!8!:;299>d>DBIID$D.Q"-Q"h[J-[J|$7D]J.]J@J mJ Z ? .,ryy,JJ*JJZ/"))/@*r}}*Z'299'6 111  X  "  X  " 9H99 *X** .x.. .x.. :8:: >X>>0!00  0  0.+- "Tyy"TII"Tc3h "TJ45/6&LL^ O%}2O%d4!(?4!n/$+/d %299 %  rI