Ë
    ÛÓ^jhj  ã                   óN  — d Z ddlZddlmZ ddlmZ ddlZddlmc m	Z
 ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZmZ ddlmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z( ddl)m*Z* ddl+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 ddl4m5Z5m6Z6  e(jn                  e8«      Z9 e'«       rddl:m;Z;  e&d¬«      e G d„ de«      «       «       Z< G d„ de/«      Z= G d„ de0«      Z>	 	 dAd ej~                  d!ej€                  d"ej€                  d#ej€                  d$eej€                  d%f   d&eAdz  d'eAdz  d(eBej€                  ej€                  f   fd)„ZC e «       ZDeCeDd*<    G d+„ d,ej~                  «      ZE G d-„ d.e-«      ZF G d/„ d0ej~                  «      ZG G d1„ d2e«      ZH G d3„ d4e.«      ZI G d5„ d6e6«      ZJ	 	 	 	 dBd7ej€                  eBej€                     z  dz  d8eKdz  d9eKdz  d:eKd$ej€                  dz  d(ej€                  eKz  fd;„ZL G d<„ d=e5«      ZM G d>„ d?e,«      ZNg d@¢ZOy)CzPyTorch Doge model.é    N)ÚCallable)ÚUnion)Ústrict)Únné   )Úinitialization)ÚACT2FN)ÚCache)ÚPreTrainedConfig)Úcompile_friendly_flex_attention)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚRopeParameters)ÚAttentionInterfaceÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚis_torch_flex_attn_availableÚlogging)ÚOutputRecorderé   )ÚLlamaForSequenceClassificationÚLlamaMLPÚLlamaPreTrainedModelÚLlamaRMSNormÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forwardÚ	repeat_kv)ÚMixtralForCausalLMÚMixtralModel)Ú	BlockMaskzSmallDoge/Doge-320M)Ú
checkpointc                   ó:  ‡ — e Zd ZU dZdZdgZddddddddddddœZd	gd
gfddgdgfdgdgfdœZdZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZee	z  e
d<   dZee
d<   dZee
d<   dZee
d<   dZee
d<   d Zee
d!<   dZe	e
d"<   d#Zeez  d#z  e
d$<   d%Ze	e
d&<   d#Ze	d#z  e
d'<   d Zee
d(<   dZed#z  e
d)<   d Zee
d*<   d#Z e	d#z  e
d+<   dZ!e	e
d,<   d Z"ee
d-<   d.Z#e	e
d/<   d0Z$e	e
d1<   d Z%ee
d2<   d Z&ee
d3<   d4Z'ee
d5<   d#Z(e	d#z  e
d6<   d#Z)e	d#z  e
d7<   d#Z*e	e+e	   z  d#z  e
d8<   ˆ fd9„Z,ˆ xZ-S ):Ú
DogeConfigaè  
    keep_window_size (`int`, *optional*, defaults to 2048):
        The window size of tokens that are not dynamically masked, and dynamic masking is only performed when the sequence length exceeds this value.
    is_moe (`bool`, *optional*, defaults to `False`):
        Whether to use the Cross Domain Mixture of Experts, if `True`, the MoE will inherit the MLP to initialize.

    ```python
    >>> from transformers import DogeConfig, DogeModel

    >>> # Initializing a Doge-320M style configuration
    >>> configuration = DogeConfig()

    >>> # Initializing a model from the Doge-320M style configuration
    >>> model = DogeModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```ÚdogeÚpast_key_valuesÚcolwiseÚrowwiseÚcolwise_gather_outputÚrowwise_split_input)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.dt_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projzlayers.*.mlp.router_gatezlayers.*.mlp.down_embedzlayers.*.mlp.up_embedÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormi €  Ú
vocab_sizei   Úhidden_sizeé   Úintermediate_sizeé    Únum_hidden_layersç        Úhidden_dropoutÚsiluÚ
hidden_actg{®Gáz”?Úinitializer_rangeg�íµ ÷Æ°>Úrms_norm_epsTÚ	use_cacheFÚtie_word_embeddingsÚmax_position_embeddingsNÚrope_parametersé   Únum_attention_headsÚnum_key_value_headsÚattention_biasÚattention_dropoutÚmlp_biasÚsliding_windowÚkeep_window_sizeÚis_moei @  Únum_expertsé@   Únum_experts_per_tokÚnorm_topk_probÚoutput_router_logitsgü©ñÒMbP?Úrouter_aux_loss_coefÚpad_token_idÚbos_token_idÚeos_token_idc                 ó^   •— | j                   €| j                  | _         t        ‰| �  di |¤Ž y )N© )rG   rF   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úp/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/doge/modular_doge.pyrZ   zDogeConfig.__post_init__ƒ   s-   ø€ à×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$ä‰ÑÑ' Ó'ó    ).Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planr5   ÚintÚ__annotations__r6   r8   r:   r<   Úfloatr>   Ústrr?   r@   rA   ÚboolrB   rC   rD   r   ÚdictrF   rG   rH   rI   rJ   rK   rL   rM   rN   rP   rQ   rR   rS   rT   rU   rV   ÚlistrZ   Ú__classcell__©r]   s   @r^   r'   r'   :   sØ  ø… ñð& €JØ#4Ð"5Ðð &/Ø%.Ø%.Ø&/Ø%.Ø"+Ø )Ø"+Ø$;Ø#8Ø!6ñÐð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð €J�ÓØ€K�ÓØ!Ð�sÓ!ØÐ�sÓØ"%€N�E˜C‘KÓ%Ø€J�ÓØ#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ %Ð˜Ó%Ø#'Ð˜SÓ'Ø48€O�^ dÑ*¨TÑ1Ó8Ø Ð˜Ó Ø&*Ð˜˜t™Ó*Ø €N�DÓ Ø&)Ð�u˜t‘|Ó)Ø€HˆdÓØ!%€N�C˜$‘JÓ%Ø Ð�cÓ Ø€FˆDÓØ€K�ÓØ!Ð˜Ó!Ø €N�DÓ Ø!&Ð˜$Ó&Ø"'Ð˜%Ó'Ø#€L�#˜‘*Ó#Ø#€L�#˜‘*Ó#Ø+/€L�#˜˜S™	‘/ DÑ(Ó/÷(ð (r_   r'   c                   ó   — e Zd Zy)ÚDogeRMSNormN©r`   ra   rb   rX   r_   r^   rr   rr   ‹   ó   „ Ør_   rr   c                   ó   — e Zd Zy)ÚDogeRotaryEmbeddingNrs   rX   r_   r^   rv   rv   �   rt   r_   rv   ÚmoduleÚqueryÚkeyÚvaluer1   r$   ÚscalingÚsoftcapÚreturnc           
      ó.  ‡‡— d }d Št        |t        «      r|}n|Š‰�‰d d …d d …d d …d |j                  d   …f   Šˆˆfd„}	t        ||||	|d|d¬«      \  }
}|j	                  |j
                  «      }|
j                  dd«      j                  «       }
|
|fS )Néþÿÿÿc                 óh   •— ‰�‰t        j                  | ‰z  «      z  } ‰�| ‰|   |   |   |   z   } | S ©N)ÚtorchÚtanh)ÚscoreÚ	batch_idxÚhead_idxÚq_idxÚkv_idxÚcausal_maskr|   s        €€r^   Ú	score_modz)flex_attention_forward.<locals>.score_mod§   sI   ø€ ØÐØœeŸj™j¨°©Ó9Ñ9ˆEØÐ"Ø˜K¨	Ñ2°8Ñ<¸UÑCÀFÑKÑKˆEØˆr_   T)rŠ   Ú
block_maskÚ
enable_gqaÚscaleÚ
return_lseé   r   )Ú
isinstancer$   Úshaper   ÚtoÚdtypeÚ	transposeÚ
contiguous)rw   rx   ry   rz   r1   r{   r|   r\   r‹   rŠ   Úattn_outputÚattention_weightsr‰   s         `     @r^   Úflex_attention_forwardr˜   “   s¸   ù€ ð €JØ€KÜ�.¤)Ô,Ø#‰
à$ˆàÐØ!¢!¢Qª¨?¨S¯Y©Y°r©]¨?Ð":Ñ;ˆõô &EØØØØØØØð ô&Ñ"€KÐ"ð *×,Ñ,¨U¯[©[Ó9ÐØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ)Ð)Ð)r_   Údoge_flex_attentionc                   ór  ‡ — e Zd Zddededz  fˆ fd„Z	 	 ddej                  deej                  ej                  f   dej                  dz  de	dz  d	eej                  ej                  dz  eej                     dz  f   f
d
„Z
	 	 ddej                  dej                  dedej                  dz  fd„Zˆ xZS )ÚDogeAttentionNÚconfigÚ	layer_idxc                 ó  •— t         ‰| �  «        || _        || _        t	        |d|j
                  |j                  z  «      | _        |j                  |j                  z  | _	        | j                  dz  | _
        |j                  | _        |j                  | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j&                  t)        j*                  |j                  «      «      | _        t        j                  |j                  | j                  z  |j                  |j                  ¬«      | _        t        j                  |j                  | j                  z  |j
                  |j                  ¬«      | _        t3        | j                  |j4                  ¬«      | _        t3        | j                  |j4                  ¬«      | _        y )NÚhead_dimg      à¿©Úbias©Úeps)rY   Ú__init__rœ   r�   Úgetattrr6   rF   rŸ   rG   Únum_key_value_groupsr{   rI   rL   r   ÚLinearrH   Úq_projÚk_projÚv_projÚ	Parameterr‚   ÚzerosÚAÚdt_projÚo_projrr   r@   Úq_normÚk_norm©r[   rœ   r�   r]   s      €r^   r¤   zDogeAttention.__init__Æ   sÚ  ø€ Ü‰ÑÔØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØ &× 7Ñ 7ˆÔä—i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —‘œeŸk™k¨&×*DÑ*DÓEÓFˆŒÜ—y‘yØ×&Ñ&¨¯©Ñ6¸×8RÑ8RÐY_×YnÑYnô
ˆŒô —i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒô " $§-¡-°V×5HÑ5HÔIˆŒÜ! $§-¡-°V×5HÑ5HÔIˆ�r_   r0   Úposition_embeddingsr1   r)   r}   c                 óÒ  — |j                   d d }g |¢d‘| j                  ‘­}| j                  | j                  |«      j	                  |«      «      j                  dd«      }| j                  | j                  |«      j	                  |«      «      j                  dd«      }	| j                  |«      j	                  |«      j                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
| j                  |
j                  dd«      j                  |
j                   d   |
j                   d   d«      «      }t        j                  | j                   t#        j$                  |«      z  «      j                  dd«      }| j'                  ||| j(                  |¬«      }t+        || j,                  «      }t.        j1                  | j2                  j4                  t6        «      } || ||	|
f|| j8                  sdn| j:                  | j<                  dœ|¤Ž\  }} |j                  g |¢d‘­Ž j?                  «       }| jA                  |«      }||fS )	Néÿÿÿÿr�   r   r   r   )r0   Ú	dt_statesrL   r1   r;   )r1   Údropoutr{   )!r‘   rŸ   r°   r¨   Úviewr”   r±   r©   rª   r   Úupdater�   r®   Úreshaper‚   Úexpr­   ÚFÚsoftplusÚprepare_dynamic_maskrL   r!   r¦   ÚALL_ATTENTION_FUNCTIONSÚget_interfacerœ   Ú_attn_implementationr    ÚtrainingrI   r{   r•   r¯   )r[   r0   r³   r1   r)   r\   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinr¶   Ú	attn_maskÚattention_interfacer–   Úattn_weightss                     r^   ÚforwardzDogeAttention.forwardä   sV  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ 4§;¡;¨}Ó#=×#BÑ#BÀ<Ó#PÓQ×[Ñ[Ð\]Ð_`ÓaˆØ—[‘[ §¡¨]Ó!;×!@Ñ!@ÀÓ!NÓO×YÑYÐZ[Ð]^Ó_ˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ð —L‘LØ×"Ñ" 1 aÓ(×0Ñ0°×1CÑ1CÀAÑ1FÈ×HZÑHZÐ[]ÑH^Ð`bÓcó
ˆ	ô —I‘I˜dŸf™f¤q§z¡z°)Ó'<Ñ<Ó=×GÑGÈÈBÓOˆ	Ø×-Ñ-Ø'ØØ!×2Ñ2Ø)ð	 .ó 
ˆ	ô ˜i¨×)BÑ)BÓCˆ	ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØð		%
ð
 %Ø#Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r_   r¶   rL   c           	      ó  — t        j                  |j                  «      j                  }|j                  }|dd…dd…ddd…f   j	                  dd|j
                  d   d«      }|�¡t        |t        «      s‘|j                  t         j                  k(  rC|j                  }t        j                  |t        j                  d|j                  |¬«      |«      }|j                  |dd…dd…dd…d|j
                  d   …f   dk7  |«      }|j
                  d   |kD  rnt        j                  |||j                  ¬«      }t        j                  ||ddd	¬
«      j                  }	|j!                  d|	d«      }|j                  |dk(  |«      }|S )a8  
        The core idea of DMA is to calculate the dynamic attention mask to mask the tokens that should be masked, so as to form sparse attention.

        Combine `dt_states` with `attention_mask` to generate the final `attn_mask`.

        Args:
            hidden_states (`torch.Tensor`): The input hidden_states, used to determine the minimum value of the current input precision.
            dt_states (`torch.Tensor`): dt_states of shape `(batch_size, num_heads, key_sequence_length)`.
            keep_window_size (`int`): The window size of tokens that are not dynamically masked, and dynamic masking is only performed when the sequence length exceeds this value.
            attention_mask (`torch.Tensor`, *optional*): attention mask of shape `(batch_size, 1, query_sequence_length, key_sequence_length)`.
        Nrµ   r�   r;   )Údevicer“   r   ©r“   rÏ   TF)ÚdimÚlargestÚsortedg      ð?)r‚   Úfinfor“   ÚminÚexpandr‘   r�   r$   rl   ÚwhereÚtensorrÏ   Úmasked_fillÚ
zeros_likeÚtopkÚindicesÚscatter)
r[   r0   r¶   rL   r1   Ú	min_dtyper“   rÊ   Úactive_maskÚtopk_indicess
             r^   r¾   z"DogeAttention.prepare_dynamic_mask  sg  € ô$ —K‘K × 3Ñ 3Ó4×8Ñ8ˆ	Ø×#Ñ#ˆØša¢ Dª!˜mÑ,×3Ñ3Ø��M×'Ñ'¨Ñ*¨Bó
ˆ	ð Ð%¬j¸ÌÔ.SØ×#Ñ#¤u§z¡zÒ1Ø%×+Ñ+�Ü!&§¡Ø"¤E§L¡L°¸^×=RÑ=RÐZ_Ô$`Ðbkó"�ð "×-Ñ-¨nºQÂÂ1ÐF[È	ÏÉÐXZÑH[ÐF[Ð=[Ñ.\Ð`aÑ.aÐclÓmˆIØ�?‰?˜2ÑÐ!1Ò1Ü×*Ñ*¨9¸EÈ)×JZÑJZÔ[ˆKÜ Ÿ:™: iÐ1AÀrÐSWÐ`eÔf×nÑnˆLØ%×-Ñ-¨b°,ÀÓDˆKØ!×-Ñ-¨k¸SÑ.@À)ÓLˆIØÐr_   r�   ©NN)r7   N)r`   ra   rb   r'   rh   r¤   r‚   ÚTensorÚtupler
   rÍ   r¾   ro   rp   s   @r^   r›   r›   Å   sê   ø„ ñJ˜zð J°c¸D±jõ JðD /3Ø(,ñ3)à—|‘|ð3)ð # 5§<¡<°·±Ð#=Ñ>ð3)ð Ÿ™ tÑ+ð	3)ð
  ™ð3)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	Mó3)ðr !%Ø.2ñ#à—|‘|ð#ð —<‘<ð#ð ð	#ð
 Ÿ™ tÑ+÷#r_   r›   c                   ó   — e Zd Zy)ÚDogeMLPNrs   rX   r_   r^   rå   rå   ?  rt   r_   rå   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )Ú	DogeCDMoErœ   c                 óð  •— t         ‰| �  «        |j                  | _        |j                  | _        t        |j
                     | _        |j                  | _        t        j                  t        j                  | j                  «      «      | _        |j                  | _        |j                  | _        t        j                   | j                  | j                  |j"                  ¬«      | _        t        j                   | j                  | j                  |j"                  ¬«      | _        t        j                   | j                  | j                  |j"                  ¬«      | _        t        j                   | j                  | j                  dz  d¬«      | _        t        j,                  | j                  | j                  «      | _        t        j,                  | j                  | j                  «      | _        y )Nr    r   F)rY   r¤   r6   r8   r	   r>   Úact_fnrN   ÚmathÚfloorÚsqrtÚnum_keysrP   Útop_krQ   r   r§   rJ   Ú	gate_projÚup_projÚ	down_projÚrouter_gateÚ	EmbeddingÚ
down_embedÚup_embed©r[   rœ   r]   s     €r^   r¤   zDogeCDMoE.__init__D  s_  ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜ˜V×.Ñ.Ñ/ˆŒà!×-Ñ-ˆÔÜŸ
™
¤4§9¡9¨T×-=Ñ-=Ó#>Ó?ˆŒØ×/Ñ/ˆŒ
Ø$×3Ñ3ˆÔô Ÿ™ 4×#3Ñ#3°T×5KÑ5KÐRX×RaÑRaÔbˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPV×P_ÑP_Ô`ˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRX×RaÑRaÔbˆŒô Ÿ9™9 T×%5Ñ%5°t·}±}ÀqÑ7HÈuÔUˆÔô Ÿ,™, t×'7Ñ'7¸×9IÑ9IÓJˆŒÜŸ™ T×%5Ñ%5°t×7GÑ7GÓHˆ�r_   r0   r}   c                 ó¨  — |j                   \  }}}| j                  |«      j                  d||z  d«      }|j                  | j                  d¬«      \  \  }}\  }	}
|j                  d«      |j                  d«      z   }|	j                  d«      | j                  z  |
j                  d«      z   } |j                  g |j                   d d ¢d‘­Ž } |j                  g |j                   d d ¢d‘­Ž }|j                  | j                  d¬«      \  }}|j                  d|«      }t        j                  |d¬«      }| j                  r||j                  dd¬«      z  }| j                  |«      }| j                  |«      }t        j                  ||j                  ||z  dd«      «      j                  ||z  d«      }| j!                  |«      |z  }t        j                  |j                  ||z  dd«      |«      j                  ||d«      }| j#                  | j!                  | j%                  |«      «      | j'                  |«      z  «      }||z   }||fS )Nr   rµ   ©rÑ   r   T)rÑ   Úkeepdimr�   )r‘   rò   r¸   rÛ   rí   Ú	unsqueezerî   Úgatherr¼   ÚsoftmaxrQ   Úsumrô   rõ   r‚   Úmatmulré   rñ   rï   rð   )r[   r0   r\   ÚbszÚseq_lenÚ_Úrouter_logitsÚscores_xÚscores_yÚ	indices_xÚ	indices_yÚ
all_scoresÚall_indicesÚscoresÚposition_indicesrÜ   Úrouting_weightsrô   rõ   Úexperts_weightsÚexperts_statess                        r^   rÍ   zDogeCDMoE.forward[  sF  € ð
 (×-Ñ-‰ˆˆW�að ×(Ñ(¨Ó7×<Ñ<¸QÀÀgÁÈrÓRˆð 8E×7IÑ7IÈ$Ï-É-Ð]_Ð7IÓ7`Ñ4Ñˆ�8Ñ4˜y¨)Ø×'Ñ'¨Ó+¨h×.@Ñ.@ÀÓ.DÑDˆ
Ø×)Ñ)¨"Ó-°·±Ñ=À	×@SÑ@SÐTVÓ@WÑWˆØ$�Z—_‘_Ð@ j×&6Ñ&6°s¸Ð&;Ð@¸RÒ@ˆ
Ø&�k×&Ñ&ÐC¨×(9Ñ(9¸#¸2Ð(>ÐCÀÒCˆØ#-§?¡?°4·:±:À2 ?Ó#FÑ ˆÐ Ø×$Ñ$ RÐ)9Ó:ˆÜŸ)™) F°Ô3ˆØ×ÒØ˜×2Ñ2°rÀ4Ð2ÓHÑHˆOð —_‘_ WÓ-ˆ
Ø—=‘= Ó)ˆÜŸ,™, z°=×3EÑ3EÀcÈGÁmÐUWÐYZÓ3[Ó\×aÑaÐbeÐhoÑboÐqsÓtˆØŸ+™+ oÓ6¸ÑHˆÜŸ™ o×&:Ñ&:¸3À¹=È!ÈRÓ&PÐRZÓ[×`Ñ`ÐadÐfmÐoqÓrˆØŸ™ t§{¡{°4·>±>À-Ó3PÓ'QÐTX×T`ÑT`ÐanÓToÑ'oÓpˆØ%¨Ñ6ˆØ˜mÐ+Ð+r_   )	r`   ra   rb   r'   r¤   r‚   râ   rÍ   ro   rp   s   @r^   rç   rç   C  s0   ø„ ðI˜zõ Ið.,à—|‘|ð,ð 
�‰÷	,r_   rç   c                   óL  ‡ — e Zd Zddededz  fˆ fd„Z	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	e
dz  d
edz  dee   deej                  eej                  ej                  f   dz  f   fd„Zˆ xZS )ÚDogeDecoderLayerNrœ   r�   c                 ó*  •— t         ‰| �  «        |j                  | _        t        |j                  |j
                  ¬«      | _        t        ||¬«      | _        t        j                  t        j                  |j                  «      «      | _        t        |j                  |j
                  ¬«      | _        |j                  st!        |«      n
t#        |«      | _        t        j                  t        j                  |j                  «      «      | _        y )Nr¢   )rœ   r�   )rY   r¤   r<   rr   r6   r@   Úinput_layernormr›   Ú	self_attnr   r«   r‚   ÚonesÚinput_residualÚpost_attention_layernormrM   rå   rç   ÚmlpÚpost_attention_residualr²   s      €r^   r¤   zDogeDecoderLayer.__init__}  s¸   ø€ Ü‰ÑÔØ$×3Ñ3ˆÔä*¨6×+=Ñ+=À6×CVÑCVÔWˆÔÜ&¨fÀ	ÔJˆŒÜ Ÿl™l¬5¯:©:°f×6HÑ6HÓ+IÓJˆÔä(3°F×4FÑ4FÈF×L_ÑL_Ô(`ˆÔ%Ø*0¯-ª-”7˜6”?¼YÀvÓ=NˆŒÜ')§|¡|´E·J±J¸v×?QÑ?QÓ4RÓ'SˆÕ$r_   r0   r³   r1   Úposition_idsr)   rA   r\   r}   c           
      ó¤  — |}| j                  |«      } | j                  d||||||dœ|¤Ž\  }}	t        j                  || j                  | j
                  ¬«      }| j                  |z  |z   }|}| j                  |«      }| j                  |«      }t        j                  || j                  | j
                  ¬«      }| j                  |z  |z   }|S )N)r0   r³   r1   r  r)   rA   )ÚprÂ   rX   )
r  r  r¼   r·   r<   rÂ   r  r  r  r  )
r[   r0   r³   r1   r  r)   rA   r\   ÚresidualÚself_attn_weightss
             r^   rÍ   zDogeDecoderLayer.forward‰  så   € ð !ˆØ×,Ñ,¨]Ó;ˆØ+9¨4¯>©>ð ,
Ø'Ø 3Ø)Ø%Ø+Øñ,
ð ñ,
Ñ(ˆÐ(ô Ÿ	™	 -°4×3FÑ3FÐQU×Q^ÑQ^Ô_ˆØ×+Ñ+¨hÑ6¸ÑFˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆÜŸ	™	 -°4×3FÑ3FÐQU×Q^ÑQ^Ô_ˆØ×4Ñ4°xÑ?À-ÑOˆàÐr_   r�   )NNNNF)r`   ra   rb   r'   rh   r¤   r‚   râ   rã   Ú
LongTensorr
   rl   r   r   ÚFloatTensorrÍ   ro   rp   s   @r^   r  r  |  sì   ø„ ñ
T˜zð 
T°c¸D±jõ 
Tð IMØ.2Ø04Ø(,Ø!&ñ à—|‘|ð ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð ð Ÿ™ tÑ+ð	 ð
 ×&Ñ&¨Ñ-ð ð  ™ð ð ˜$‘;ð ð Ð+Ñ,ð ð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷ r_   r  c                   ó^   — e Zd ZdZdZ eed¬«      eedœZ	 e
j                  «       d„ «       Zy)ÚDogePreTrainedModelFr�   )Úindex)r  r0   Ú
attentionsc                 óz  — t        j                  | |«       t        |t        «      r-t	        |d«      r t        j                  |j                  «       yyt        |t        «      rXt	        |d«      rt        j                  |j                  «       t	        |d«      r t        j                  |j                  «       yyy)zInitialize the weightsr­   r  r  N)r   Ú_init_weightsr�   r›   ÚhasattrÚinitÚzeros_r­   r  Úones_r  r  )r[   rw   s     r^   r$  z!DogePreTrainedModel._init_weightsµ  sŒ   € ô 	×%Ñ% d¨FÔ3Ü�fœmÔ,Ü�v˜sÔ#Ü—‘˜FŸH™HÕ%ð $ä˜Ô 0Ô1Ü�vÐ/Ô0Ü—
‘
˜6×0Ñ0Ô1Ü�vÐ8Ô9Ü—
‘
˜6×9Ñ9Õ:ð :ð 2r_   N)r`   ra   rb   Ú_supports_flash_attnÚ_can_compile_fullgraphr   rç   r  r›   Ú_can_record_outputsr‚   Úno_gradr$  rX   r_   r^   r   r   ¬  sB   „ Ø ÐØ"Ðá'¨	¸Ô;Ø)Ø#ñÐð €U‡]�]ƒ_ñ
;ó ñ
;r_   r   c                   ó   — e Zd Zy)Ú	DogeModelNrs   rX   r_   r^   r.  r.  Ã  rt   r_   r.  Úgate_logitsrN   rí   rî   c                 óÌ  — | �t        | t        «      sy| d   j                  }| d   j                  }g }g }| D �]  }	|	j	                  |«      }	|	j                  |d¬«      \  \  }
}\  }}|
j                  d«      |j                  d«      z   }|j                  d«      |z  |j                  d«      z   } |j                  g |j                  dd ¢d‘­Ž } |j                  g |j                  dd ¢d‘­Ž }|j                  |d¬«      \  }}|j                  d|«      }t        j                  |d¬«      }|j                  |«       |j                  |«       �Œ t        j                  |d¬«      }t        j                  |d¬«      }|€}|j                  d«      }t        j                  |||¬«      }t        j                   |||¬«      }|j#                  d||«      |j                  d   z  }t        j$                  |d¬«      }�nD|j                  \  }}t'        | «      }|ddd…dd…df   j)                  ||||f«      j+                  d«      j	                  |«      }|j                  d«      |j-                  «          }t        j                  |||¬«      }t        j                   |||¬«      }|j#                  d||«      t        j.                  |«      z  }|ddd…dd…df   j)                  ||||f«      j+                  d|«      j	                  |«      }t        j.                  ||z  d¬«      t        j.                  |d¬«      z  }t        j.                  ||z  «      }||z  S )aø  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `router_gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [2, batch_size * sequence_length, num_keys].
        num_experts:
            Number of experts
        num_keys:
            Number of keys
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   rµ   rø   r   rÐ   )r�   rã   r“   rÏ   r’   rÛ   rú   r¸   r‘   rû   r¼   rü   Úappendr‚   Úcatr¬   Ú	ones_likeÚscatter_add_ÚmeanÚlenrÖ   rº   rl   rý   )r/  rN   rí   rî   r1   Úcompute_dtypeÚcompute_deviceÚall_expert_indicesÚall_routing_weightsÚlayer_gate_logitsr  r  r  r  r  r  r  r
  Úexpert_indicesr  Útokens_per_expertÚpadÚrouter_prob_per_expertÚ
batch_sizeÚsequence_lengthr:   Úexpert_attention_maskÚ router_per_expert_attention_maskÚoverall_losss                                r^   Úload_balancing_loss_funcrE  Ç  s{  € ð@ Ð¤*¨[¼%Ô"@Øà ‘N×(Ñ(€MØ  ‘^×*Ñ*€NØÐØÐà(ó 4ÐØ-×0Ñ0°Ó@Ðà7H×7MÑ7MÈhÐ\^Ð7MÓ7_Ñ4Ñˆ�8Ñ4˜y¨)à×'Ñ'¨Ó+¨h×.@Ñ.@ÀÓ.DÑDˆ
Ø×)Ñ)¨"Ó-°Ñ8¸9×;NÑ;NÈrÓ;RÑRˆØ$�Z—_‘_Ð@ j×&6Ñ&6°s¸Ð&;Ð@¸RÒ@ˆ
Ø&�k×&Ñ&ÐC¨×(9Ñ(9¸#¸2Ð(>ÐCÀÒCˆà(Ÿo™o¨e¸˜oÓ<ÑˆÐØ$×+Ñ+¨BÐ0@ÓAˆäŸ)™) J°BÔ7ˆà×!Ñ! .Ô1Ø×"Ñ" ?Ö3ð!4ô" Ÿ™Ð#5¸1Ô=ÐÜŸ)™)Ð$7¸QÔ?ÐàÐà/×4Ñ4°RÓ8ÐÜ!ŸK™K¨¸=ÐQ_Ô`ÐÜ�o‰oÐ0¸ÈnÔ]ˆØ-×:Ñ:¸1Ð>PÐRUÓVÐYk×YqÑYqÐrsÑYtÑtÐô "'§¡Ð,?ÀQÔ!GÒà&4×&:Ñ&:Ñ#ˆ
�OÜ Ó,Ðð ˜4¢¢A tÐ+Ñ,ß‰VÐ&¨
°OÀUÐKÓLß‰W�R‹[ß‰R�Óð	 	ð 0×4Ñ4°RÓ8Ð9N×9SÑ9SÓ9UÑVÐô "ŸK™K¨¸=ÐQ_Ô`ÐÜ�o‰oÐ0¸ÈnÔ]ˆØ-×:Ñ:¸1Ð>PÐRUÓVÔY^×YbÑYbØ!óZ
ñ 
Ðð ˜4¢¢A tÐ+Ñ,ß‰VÐ&¨
°OÀ[ÐQÓRß‰W�R˜Ó%ß‰R�Óð	 	)ô "'§¡Ð+>ÐAaÑ+aÐghÔ!iÔlq×luÑluØ,°!ôm
ñ "
Ðô —9‘9Ð.Ð1GÑGÓH€LØ˜+Ñ%Ð%r_   c                   ó  ‡ — e Zd Zˆ fd„Z	 	 	 	 	 	 	 	 	 ddej
                  dz  dej                  dz  dej
                  dz  dedz  dej                  dz  dej
                  dz  d	e	dz  d
e
ej                  z  de	dz  dee   defd„Zˆ xZS )ÚDogeForCausalLMc                 óf   •— t         ‰| �  |«       t        |«      | _        |j                  | _        y r�   )rY   r¤   r.  ÚmodelrN   rö   s     €r^   r¤   zDogeForCausalLM.__init__2  s*   ø€ Ü‰Ñ˜Ô Ü˜vÓ&ˆŒ
Ø!×-Ñ-ˆÕr_   Nr.   r1   r  r)   r/   ÚlabelsrA   Úlogits_to_keeprR   r\   r}   c
           
      óÊ  — |	�|	n| j                   j                  }	 | j                  d||||||dœ|
¤Ž}|j                  }t	        |t
        «      rt        | d«      n|}| j                  |dd…|dd…f   «      }d}|� | j                  ||| j                  fi |
¤Ž}d}|	rŠt        |j                  | j                  t        j                  t        j                  | j                  «      «      | j                   |«      }|�+|| j"                  |j%                  |j&                  «      z  z  }t)        ||||j*                  |j,                  |j.                  |j                  ¬«      S )ah  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, DogeForCausalLM

        >>> model = DogeForCausalLM.from_pretrained("SmallDoge/Doge-320M")
        >>> tokenizer = AutoTokenizer.from_pretrained("SmallDoge/Doge-320M")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r.   r1   r  r)   r/   rA   )ÚlossÚaux_lossÚlogitsr)   r0   r"  r  rX   )rœ   rR   rI  Úlast_hidden_stater�   rh   ÚsliceÚlm_headÚloss_functionr5   rE  r  rN   rê   rë   rì   rP   rS   r’   rÏ   r   r)   r0   r"  )r[   r.   r1   r  r)   r/   rJ  rA   rK  rR   r\   Úoutputsr0   Úslice_indicesrO  rM  rN  s                    r^   rÍ   zDogeForCausalLM.forward7  sm  € ðH %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð
 +5¨$¯*©*ð +
ØØ)Ø%Ø+Ø'Øñ+
ð ñ+
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ% f¨f°d·o±oÑPÈÑPˆDàˆÙÜ/Ø×%Ñ%Ø× Ñ Ü—
‘
œ4Ÿ9™9 T×%5Ñ%5Ó6Ó7Ø×(Ñ(ØóˆHð Ð!Ø˜×1Ñ1°H·K±KÀÇÁÓ4LÑLÑL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r_   )	NNNNNNNr   N)r`   ra   rb   r¤   r‚   r  râ   r
   r  rl   rh   r   r   r   rÍ   ro   rp   s   @r^   rG  rG  1  sò   ø„ ô.ð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.Ø,0ñO
à×#Ñ# dÑ*ðO
ð Ÿ™ tÑ+ðO
ð ×&Ñ&¨Ñ-ð	O
ð
  ™ðO
ð ×(Ñ(¨4Ñ/ðO
ð × Ñ  4Ñ'ðO
ð ˜$‘;ðO
ð ˜eŸl™lÑ*ðO
ð # T™kðO
ð Ð+Ñ,ðO
ð 
#÷O
r_   rG  c                   ó   — e Zd Zy)ÚDogeForSequenceClassificationNrs   rX   r_   r^   rW  rW  ‰  rt   r_   rW  )r'   rG  r.  r   rW  rá   )NNr   N)Prc   rê   Úcollections.abcr   Útypingr   r‚   Útorch.nn.functionalr   Ú
functionalr¼   Úhuggingface_hub.dataclassesr   Ú r   r&  Úactivationsr	   Úcache_utilsr
   Úconfiguration_utilsr   Úintegrations.flex_attentionr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.output_capturingr   Úllama.modeling_llamar   r   r   r   r   r   r    r!   Úmixtral.modeling_mixtralr"   r#   Ú
get_loggerr`   ÚloggerÚ!torch.nn.attention.flex_attentionr$   r'   rr   rv   ÚModulerâ   rj   rã   r˜   r¿   r›   rå   rç   r  r   r.  rh   rE  rG  rW  Ú__all__rX   r_   r^   ú<module>rp     sj  ðñ  ã Ý $Ý ã ß Ð Ý .Ý å &Ý !Ý  Ý 3Ý JÝ 9ß QÝ 1ß AÝ &ß ^Ó ^Ý 4÷	÷ 	ó 	÷ Hð 
ˆ×	Ñ	˜HÓ	%€áÔ!Ý;ñ Ð0Ô1ØôL(Ð!ó L(ó ó 2ðL(ô^	�,ô 	ô	Ð.ô 	ð !Ø ñ+*Ø�I‰Ið+*à�<‰<ð+*ð 
�‰ð+*ð �<‰<ð	+*ð
 ˜%Ÿ,™,¨Ð3Ñ4ð+*ð �T‰\ð+*ð �T‰\ð+*ð ˆ5�<‰<˜Ÿ™Ð%Ñ&ó+*ñ\ -Ó.Ð Ø1GÐ Ð-Ñ .ôw�B—I‘Iô wôt	ˆhô 	ô6,�—	‘	ô 6,ôr-Ð1ô -ô`;Ð.ô ;ô.	�ô 	ð #ØØØ*.ñg&Ø—‘  e§l¡lÑ 3Ñ3°dÑ:ðg&à�t‘ðg&ð �D‰jðg&ð ð	g&ð
 —L‘L 4Ñ'ðg&ð ‡\�\�CÑóg&ôTU
Ð(ô U
ôp	Ð$Bô 	ò�r_   