Ë
    ÛÓ^j]Ž  ã                   ój  — d dl Z d dlmZ d dlmZmZ d dlZd dlmc m	Z
 d dlmZ ddlmZ ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZmZ ddlmZmZ ddl m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z,m-Z-m.Z.m/Z/ ddl0m1Z1m2Z2 ddl3m4Z4m5Z5 ddl6m7Z7  e/«       rd dl8m9Z9  ed«       G d„ dejt                  «      «       Z; G d„ dejt                  «      Z<d„ Z= ed«      dGd„«       Z>d ej~                  d!e@d"ej~                  fd#„ZA	 dHd$ejt                  d%ej~                  d&ej~                  d'ej~                  d(ej~                  dz  d)eBd*eBd+e*e,   fd,„ZC	 	 dId$ejt                  d%ej~                  d&ej~                  d'ej~                  d(eej~                  d-f   d)eBdz  d.eBdz  d"eDej~                  ej~                  f   fd/„ZE e'«       ZFeEeFd0<    G d1„ d2ejt                  «      ZG G d3„ d4ejt                  «      ZH G d5„ d6ejt                  «      ZI G d7„ d8e«      ZJe- G d9„ d:e(«      «       ZKe- G d;„ d<eK«      «       ZL	 	 	 	 dJd=ej~                  eDej~                     z  dz  d>e@dz  d?e@dz  d@e@d(ej~                  dz  d"ej~                  e@z  fdA„ZMe- G dB„ dCeKe«      «       ZN G dD„ dEeeK«      ZOg dF¢ZPy)Ké    N)ÚCallable)ÚOptionalÚUnion)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernel_func_from_hub)Úcompile_friendly_flex_attention)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚAttentionInterfaceÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_torch_flex_attn_available)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú
DogeConfig)Ú	BlockMaskÚRMSNormc                   óh   ‡ — e Zd Zddeddfˆ fd„Zdej                  dej                  fd„Zd„ Zˆ xZ	S )	ÚDogeRMSNormÚepsÚreturnNc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)z:
        DogeRMSNorm is equivalent to T5LayerNorm
        N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer)   Ú	__class__s      €úq/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/doge/modeling_doge.pyr-   zDogeRMSNorm.__init__7   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕó    Úhidden_statesc                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S )Né   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtor/   Úfloat32ÚpowÚmeanÚrsqrtr2   r1   )r3   r8   Úinput_dtypeÚvariances       r6   ÚforwardzDogeRMSNorm.forward?   sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r7   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)Útupler1   Úshaper2   )r3   s    r6   Ú
extra_reprzDogeRMSNorm.extra_reprF   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr7   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Úfloatr-   r/   ÚTensorrE   rI   Ú__classcell__©r5   s   @r6   r(   r(   5   s7   ø„ ñ$¨ð $¸$õ $ð; U§\¡\ð ;°e·l±ló ;öJr7   r(   c                   óÄ   ‡ — e Zd ZU ej                  ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d   de
dz  ded	ef   fd
„«       Z ej                  «       ed„ «       «       Zˆ xZS )ÚDogeRotaryEmbeddingÚinv_freqNÚconfigc                 ó®  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |«      \  }| _
        | j                  d|d¬«       | j                  d|j                  «       d¬«       y )NÚ	rope_typeÚdefaultrS   F)Ú
persistentÚoriginal_inv_freq)r,   r-   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrT   Úrope_parametersrV   Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)r3   rT   ÚdeviceÚrope_init_fnrS   r5   s        €r6   r-   zDogeRotaryEmbedding.__init__M   s¶   ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àˆŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÒ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆ�$Ô(à×Ñ˜Z¨¸eÐÔDØ×ÑÐ0°(·.±.Ó2BÈuÐÕUr7   rb   ztorch.deviceÚseq_lenr*   ztorch.Tensorc                 ó  — | j                   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNç      ð?r   r:   ©r=   ©rb   r=   )	r]   Úgetattrr4   Únum_attention_headsr/   ÚarangeÚint64r>   rM   )rT   rb   rd   ÚbaseÚdimÚattention_factorrS   s          r6   r^   z3DogeRotaryEmbedding.compute_default_rope_parameters]   s�   € ð& ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r7   c                 óN  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  ||fd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j	                  |j                   ¬
«      	j	                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   r;   r#   ÚmpsÚcpuF)Údevice_typeÚenabledr:   ©rp   ri   )rS   rM   ÚexpandrH   r>   rb   Ú
isinstanceÚtypeÚstrr   Ú	transposer/   ÚcatÚcosr_   Úsinr=   )
r3   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedru   ÚfreqsÚembr~   r   s
             r6   rE   zDogeRotaryEmbedding.forward{   sR  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   ÃBFÆF$©N)NNN)rJ   rK   rL   r/   rN   Ú__annotations__r$   r-   Ústaticmethodr   ÚintrG   rM   r^   Úno_gradr   rE   rO   rP   s   @r6   rR   rR   J   s˜   ø… Ø�l‰lÓñV˜zõ Vð  à$(Ø+/Ø"ñ*Ø˜TÑ!ð*à˜Ñ(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ñ	%ò	*ó ð*ð: €U‡]�]ƒ_Øñ<ó ó ô<r7   rR   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Nr;   r:   rw   )rH   r/   r}   )r€   Úx1Úx2s      r6   Úrotate_halfrŽ   ‹   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r7   Úrotary_pos_embc                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezerŽ   )ÚqÚkr~   r   Úunsqueeze_dimÚq_embedÚk_embeds          r6   Úapply_rotary_pos_embr—   ’   sY   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐr7   r8   Ún_repr*   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r#   N)rH   rx   Úreshape)r8   r˜   ÚbatchÚnum_key_value_headsÚslenrg   s         r6   Ú	repeat_kvrž   ¬   so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr7   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t        || j                  «      }t        || j                  «      }	t        j                  ||j	                  dd«      «      |z  }
|�|
|z   }
t
        j                  j                  |
dt        j                  ¬«      j                  |j                  «      }
t
        j                  j                  |
|| j                  ¬«      }
t        j                  |
|	«      }|j	                  dd«      j                  «       }||
fS )Nr:   r   r;   )rp   r=   ©ÚpÚtrainingr#   )rž   Únum_key_value_groupsr/   Úmatmulr|   r   Ú
functionalÚsoftmaxr?   r>   r=   r¥   rª   Ú
contiguous)rŸ   r    r¡   r¢   r£   r¤   r¥   r¦   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r6   Úeager_attention_forwardr´   ¸   sà   € ô ˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r7   r%   Úsoftcapc           
      ó.  ‡‡— d }d Št        |t        «      r|}n|Š‰�‰d d …d d …d d …d |j                  d   …f   Šˆˆfd„}	t        ||||	|d|d¬«      \  }
}|j	                  |j
                  «      }|
j                  dd«      j                  «       }
|
|fS )Néþÿÿÿc                 óh   •— ‰�‰t        j                  | ‰z  «      z  } ‰�| ‰|   |   |   |   z   } | S r†   )r/   Útanh)ÚscoreÚ	batch_idxÚhead_idxÚq_idxÚkv_idxÚcausal_maskrµ   s        €€r6   Ú	score_modz)flex_attention_forward.<locals>.score_modå   sI   ø€ ØÐØœeŸj™j¨°©Ó9Ñ9ˆEØÐ"Ø˜K¨	Ñ2°8Ñ<¸UÑCÀFÑKÑKˆEØˆr7   T)rÀ   Ú
block_maskÚ
enable_gqaÚscaleÚ
return_lser#   r:   )ry   r%   rH   r   r>   r=   r|   r¯   )rŸ   r    r¡   r¢   r£   r¤   rµ   r¦   rÁ   rÀ   r³   Úattention_weightsr¿   s         `     @r6   Úflex_attention_forwardrÆ   Ñ   s¸   ù€ ð €JØ€KÜ�.¤)Ô,Ø#‰
à$ˆàÐØ!¢!¢Qª¨?¨S¯Y©Y°r©]¨?Ð":Ñ;ˆõô &EØØØØØØØð ô&Ñ"€KÐ"ð *×,Ñ,¨U¯[©[Ó9ÐØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ)Ð)Ð)r7   Údoge_flex_attentionc                   ór  ‡ — e Zd Zddededz  fˆ fd„Z	 	 ddej                  deej                  ej                  f   dej                  dz  de	dz  d	eej                  ej                  dz  eej                     dz  f   f
d
„Z
	 	 ddej                  dej                  dedej                  dz  fd„Zˆ xZS )ÚDogeAttentionNrT   Ú	layer_idxc                 ó  •— t         ‰| �  «        || _        || _        t	        |d|j
                  |j                  z  «      | _        |j                  |j                  z  | _	        | j                  dz  | _
        |j                  | _        |j                  | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j&                  t)        j*                  |j                  «      «      | _        t        j                  |j                  | j                  z  |j                  |j                  ¬«      | _        t        j                  |j                  | j                  z  |j
                  |j                  ¬«      | _        t3        | j                  |j4                  ¬«      | _        t3        | j                  |j4                  ¬«      | _        y )Nrg   g      à¿©Úbias©r)   )r,   r-   rT   rÊ   rk   r4   rl   rg   rœ   r«   r¤   Úattention_dropoutÚkeep_window_sizer   ÚLinearÚattention_biasÚq_projÚk_projÚv_projr.   r/   ÚzerosÚAÚdt_projÚo_projr(   Úrms_norm_epsÚq_normÚk_norm©r3   rT   rÊ   r5   s      €r6   r-   zDogeAttention.__init__  sÚ  ø€ Ü‰ÑÔØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØ &× 7Ñ 7ˆÔä—i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —‘œeŸk™k¨&×*DÑ*DÓEÓFˆŒÜ—y‘yØ×&Ñ&¨¯©Ñ6¸×8RÑ8RÐY_×YnÑYnô
ˆŒô —i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒô " $§-¡-°V×5HÑ5HÔIˆŒÜ! $§-¡-°V×5HÑ5HÔIˆ�r7   r8   Úposition_embeddingsr£   Úpast_key_valuesr*   c                 óÒ  — |j                   d d }g |¢d‘| j                  ‘­}| j                  | j                  |«      j	                  |«      «      j                  dd«      }| j                  | j                  |«      j	                  |«      «      j                  dd«      }	| j                  |«      j	                  |«      j                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
| j                  |
j                  dd«      j                  |
j                   d   |
j                   d   d«      «      }t        j                  | j                   t#        j$                  |«      z  «      j                  dd«      }| j'                  ||| j(                  |¬«      }t+        || j,                  «      }t.        j1                  | j2                  j4                  t6        «      } || ||	|
f|| j8                  sdn| j:                  | j<                  dœ|¤Ž\  }} |j                  g |¢d‘­Ž j?                  «       }| jA                  |«      }||fS )	Nr;   r#   r:   r   r·   )r8   Ú	dt_statesrÐ   r£   ç        )r£   r¥   r¤   )!rH   rg   rÛ   rÓ   Úviewr|   rÜ   rÔ   rÕ   r—   ÚupdaterÊ   rØ   rš   r/   Úexpr×   ÚFÚsoftplusÚprepare_dynamic_maskrÐ   rž   r«   ÚALL_ATTENTION_FUNCTIONSÚget_interfacerT   Ú_attn_implementationr´   rª   rÏ   r¤   r¯   rÙ   )r3   r8   rÞ   r£   rß   r¦   Úinput_shapeÚhidden_shapeÚquery_statesr°   r±   r~   r   rá   Ú	attn_maskÚattention_interfacer³   r²   s                     r6   rE   zDogeAttention.forward"  sV  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ 4§;¡;¨}Ó#=×#BÑ#BÀ<Ó#PÓQ×[Ñ[Ð\]Ð_`ÓaˆØ—[‘[ §¡¨]Ó!;×!@Ñ!@ÀÓ!NÓO×YÑYÐZ[Ð]^Ó_ˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ð —L‘LØ×"Ñ" 1 aÓ(×0Ñ0°×1CÑ1CÀAÑ1FÈ×HZÑHZÐ[]ÑH^Ð`bÓcó
ˆ	ô —I‘I˜dŸf™f¤q§z¡z°)Ó'<Ñ<Ó=×GÑGÈÈBÓOˆ	Ø×-Ñ-Ø'ØØ!×2Ñ2Ø)ð	 .ó 
ˆ	ô ˜i¨×)BÑ)BÓCˆ	ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØð		%
ð
 %Ø#Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r7   rá   rÐ   c           	      ó  — t        j                  |j                  «      j                  }|j                  }|dd…dd…ddd…f   j	                  dd|j
                  d   d«      }|�¡t        |t        «      s‘|j                  t         j                  k(  rC|j                  }t        j                  |t        j                  d|j                  |¬«      |«      }|j                  |dd…dd…dd…d|j
                  d   …f   dk7  |«      }|j
                  d   |kD  rnt        j                  |||j                  ¬«      }t        j                  ||ddd	¬
«      j                  }	|j!                  d|	d«      }|j                  |dk(  |«      }|S )a8  
        The core idea of DMA is to calculate the dynamic attention mask to mask the tokens that should be masked, so as to form sparse attention.

        Combine `dt_states` with `attention_mask` to generate the final `attn_mask`.

        Args:
            hidden_states (`torch.Tensor`): The input hidden_states, used to determine the minimum value of the current input precision.
            dt_states (`torch.Tensor`): dt_states of shape `(batch_size, num_heads, key_sequence_length)`.
            keep_window_size (`int`): The window size of tokens that are not dynamically masked, and dynamic masking is only performed when the sequence length exceeds this value.
            attention_mask (`torch.Tensor`, *optional*): attention mask of shape `(batch_size, 1, query_sequence_length, key_sequence_length)`.
        Nr;   r#   râ   rj   r   ©r=   rb   TF)rp   ÚlargestÚsortedrh   )r/   Úfinfor=   Úminrx   rH   ry   r%   ÚboolÚwhereÚtensorrb   Úmasked_fillÚ
zeros_likeÚtopkÚindicesÚscatter)
r3   r8   rá   rÐ   r£   Ú	min_dtyper=   rï   Úactive_maskÚtopk_indicess
             r6   rè   z"DogeAttention.prepare_dynamic_maskW  sg  € ô$ —K‘K × 3Ñ 3Ó4×8Ñ8ˆ	Ø×#Ñ#ˆØša¢ Dª!˜mÑ,×3Ñ3Ø��M×'Ñ'¨Ñ*¨Bó
ˆ	ð Ð%¬j¸ÌÔ.SØ×#Ñ#¤u§z¡zÒ1Ø%×+Ñ+�Ü!&§¡Ø"¤E§L¡L°¸^×=RÑ=RÐZ_Ô$`Ðbkó"�ð "×-Ñ-¨nºQÂÂ1ÐF[È	ÏÉÐXZÑH[ÐF[Ð=[Ñ.\Ð`aÑ.aÐclÓmˆIØ�?‰?˜2ÑÐ!1Ò1Ü×*Ñ*¨9¸EÈ)×JZÑJZÔ[ˆKÜ Ÿ:™: iÐ1AÀrÐSWÐ`eÔf×nÑnˆLØ%×-Ñ-¨b°,ÀÓDˆKØ!×-Ñ-¨k¸SÑ.@À)ÓLˆIØÐr7   r†   ©NN)i   N)rJ   rK   rL   r$   r‰   r-   r/   rN   rG   r
   rE   rè   rO   rP   s   @r6   rÉ   rÉ     sê   ø„ ñJ˜zð J°c¸D±jõ JðD /3Ø(,ñ3)à—|‘|ð3)ð # 5§<¡<°·±Ð#=Ñ>ð3)ð Ÿ™ tÑ+ð	3)ð
  ™ð3)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	Mó3)ðr !%Ø.2ñ#à—|‘|ð#ð —<‘<ð#ð ð	#ð
 Ÿ™ tÑ+÷#r7   rÉ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚDogeMLPc                 ó  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  |j                  ¬«      | _        t        j                  | j                  | j                  |j                  ¬«      | _	        t        j                  | j                  | j                  |j                  ¬«      | _
        t        |j                     | _        y )NrÌ   )r,   r-   rT   r4   Úintermediate_sizer   rÑ   Úmlp_biasÚ	gate_projÚup_projÚ	down_projr	   Ú
hidden_actÚact_fn©r3   rT   r5   s     €r6   r-   zDogeMLP.__init__~  s¸   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRX×RaÑRaÔbˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPV×P_ÑP_Ô`ˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRX×RaÑRaÔbˆŒÜ˜V×.Ñ.Ñ/ˆ�r7   c                 óˆ   — | j                  | j                  | j                  |«      «      | j                  |«      z  «      }|S r†   )r
  r  r  r	  )r3   r€   r
  s      r6   rE   zDogeMLP.forwardˆ  s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÑ#SÓTˆ	ØÐr7   )rJ   rK   rL   r-   rE   rO   rP   s   @r6   r  r  }  s   ø„ ô0ör7   r  c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )Ú	DogeCDMoErT   c                 óð  •— t         ‰| �  «        |j                  | _        |j                  | _        t        |j
                     | _        |j                  | _        t        j                  t        j                  | j                  «      «      | _        |j                  | _        |j                  | _        t        j                   | j                  | j                  |j"                  ¬«      | _        t        j                   | j                  | j                  |j"                  ¬«      | _        t        j                   | j                  | j                  |j"                  ¬«      | _        t        j                   | j                  | j                  dz  d¬«      | _        t        j,                  | j                  | j                  «      | _        t        j,                  | j                  | j                  «      | _        y )NrÌ   r:   F)r,   r-   r4   r  r	   r  r  Únum_expertsÚmathÚfloorÚsqrtÚnum_keysÚnum_experts_per_tokÚtop_kÚnorm_topk_probr   rÑ   r  r  r	  r
  Úrouter_gateÚ	EmbeddingÚ
down_embedÚup_embedr  s     €r6   r-   zDogeCDMoE.__init__Ž  s_  ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜ˜V×.Ñ.Ñ/ˆŒà!×-Ñ-ˆÔÜŸ
™
¤4§9¡9¨T×-=Ñ-=Ó#>Ó?ˆŒØ×/Ñ/ˆŒ
Ø$×3Ñ3ˆÔô Ÿ™ 4×#3Ñ#3°T×5KÑ5KÐRX×RaÑRaÔbˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPV×P_ÑP_Ô`ˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRX×RaÑRaÔbˆŒô Ÿ9™9 T×%5Ñ%5°t·}±}ÀqÑ7HÈuÔUˆÔô Ÿ,™, t×'7Ñ'7¸×9IÑ9IÓJˆŒÜŸ™ T×%5Ñ%5°t×7GÑ7GÓHˆ�r7   r8   r*   c                 ó¨  — |j                   \  }}}| j                  |«      j                  d||z  d«      }|j                  | j                  d¬«      \  \  }}\  }	}
|j                  d«      |j                  d«      z   }|	j                  d«      | j                  z  |
j                  d«      z   } |j                  g |j                   d d ¢d‘­Ž } |j                  g |j                   d d ¢d‘­Ž }|j                  | j                  d¬«      \  }}|j                  d|«      }t        j                  |d¬«      }| j                  r||j                  dd¬«      z  }| j                  |«      }| j                  |«      }t        j                  ||j                  ||z  dd«      «      j                  ||z  d«      }| j!                  |«      |z  }t        j                  |j                  ||z  dd«      |«      j                  ||d«      }| j#                  | j!                  | j%                  |«      «      | j'                  |«      z  «      }||z   }||fS )Nr:   r;   rw   r·   T)rp   r<   r#   )rH   r  rã   rü   r  r‘   r  Úgatherræ   r®   r  Úsumr  r  r/   r¬   r  r
  r  r	  )r3   r8   r¦   Úbszrd   Ú_Úrouter_logitsÚscores_xÚscores_yÚ	indices_xÚ	indices_yÚ
all_scoresÚall_indicesÚscoresÚposition_indicesrý   Úrouting_weightsr  r  Úexperts_weightsÚexperts_statess                        r6   rE   zDogeCDMoE.forward¥  sF  € ð
 (×-Ñ-‰ˆˆW�að ×(Ñ(¨Ó7×<Ñ<¸QÀÀgÁÈrÓRˆð 8E×7IÑ7IÈ$Ï-É-Ð]_Ð7IÓ7`Ñ4Ñˆ�8Ñ4˜y¨)Ø×'Ñ'¨Ó+¨h×.@Ñ.@ÀÓ.DÑDˆ
Ø×)Ñ)¨"Ó-°·±Ñ=À	×@SÑ@SÐTVÓ@WÑWˆØ$�Z—_‘_Ð@ j×&6Ñ&6°s¸Ð&;Ð@¸RÒ@ˆ
Ø&�k×&Ñ&ÐC¨×(9Ñ(9¸#¸2Ð(>ÐCÀÒCˆØ#-§?¡?°4·:±:À2 ?Ó#FÑ ˆÐ Ø×$Ñ$ RÐ)9Ó:ˆÜŸ)™) F°Ô3ˆØ×ÒØ˜×2Ñ2°rÀ4Ð2ÓHÑHˆOð —_‘_ WÓ-ˆ
Ø—=‘= Ó)ˆÜŸ,™, z°=×3EÑ3EÀcÈGÁmÐUWÐYZÓ3[Ó\×aÑaÐbeÐhoÑboÐqsÓtˆØŸ+™+ oÓ6¸ÑHˆÜŸ™ o×&:Ñ&:¸3À¹=È!ÈRÓ&PÐRZÓ[×`Ñ`ÐadÐfmÐoqÓrˆØŸ™ t§{¡{°4·>±>À-Ó3PÓ'QÐTX×T`ÑT`ÐanÓToÑ'oÓpˆØ%¨Ñ6ˆØ˜mÐ+Ð+r7   )	rJ   rK   rL   r$   r-   r/   rN   rE   rO   rP   s   @r6   r  r  �  s0   ø„ ðI˜zõ Ið.,à—|‘|ð,ð 
�‰÷	,r7   r  c                   óL  ‡ — e Zd Zddededz  fˆ fd„Z	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	e
dz  d
edz  dee   deej                  eej                  ej                  f   dz  f   fd„Zˆ xZS )ÚDogeDecoderLayerNrT   rÊ   c                 ó*  •— t         ‰| �  «        |j                  | _        t        |j                  |j
                  ¬«      | _        t        ||¬«      | _        t        j                  t        j                  |j                  «      «      | _        t        |j                  |j
                  ¬«      | _        |j                  st!        |«      n
t#        |«      | _        t        j                  t        j                  |j                  «      «      | _        y )NrÎ   )rT   rÊ   )r,   r-   Úhidden_dropoutr(   r4   rÚ   Úinput_layernormrÉ   Ú	self_attnr   r.   r/   r0   Úinput_residualÚpost_attention_layernormÚis_moer  r  ÚmlpÚpost_attention_residualrÝ   s      €r6   r-   zDogeDecoderLayer.__init__Ç  s¸   ø€ Ü‰ÑÔØ$×3Ñ3ˆÔä*¨6×+=Ñ+=À6×CVÑCVÔWˆÔÜ&¨fÀ	ÔJˆŒÜ Ÿl™l¬5¯:©:°f×6HÑ6HÓ+IÓJˆÔä(3°F×4FÑ4FÈF×L_ÑL_Ô(`ˆÔ%Ø*0¯-ª-”7˜6”?¼YÀvÓ=NˆŒÜ')§|¡|´E·J±J¸v×?QÑ?QÓ4RÓ'SˆÕ$r7   r8   rÞ   r£   r�   rß   Ú	use_cacher¦   r*   c           
      ó¤  — |}| j                  |«      } | j                  d||||||dœ|¤Ž\  }}	t        j                  || j                  | j
                  ¬«      }| j                  |z  |z   }|}| j                  |«      }| j                  |«      }t        j                  || j                  | j
                  ¬«      }| j                  |z  |z   }|S )N)r8   rÞ   r£   r�   rß   r:  r¨   © )
r3  r4  ræ   r¥   r2  rª   r5  r6  r8  r9  )
r3   r8   rÞ   r£   r�   rß   r:  r¦   ÚresidualÚself_attn_weightss
             r6   rE   zDogeDecoderLayer.forwardÓ  så   € ð !ˆØ×,Ñ,¨]Ó;ˆØ+9¨4¯>©>ð ,
Ø'Ø 3Ø)Ø%Ø+Øñ,
ð ñ,
Ñ(ˆÐ(ô Ÿ	™	 -°4×3FÑ3FÐQU×Q^ÑQ^Ô_ˆØ×+Ñ+¨hÑ6¸ÑFˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆÜŸ	™	 -°4×3FÑ3FÐQU×Q^ÑQ^Ô_ˆØ×4Ñ4°xÑ?À-ÑOˆàÐr7   r†   )NNNNF)rJ   rK   rL   r$   r‰   r-   r/   rN   rG   Ú
LongTensorr
   r÷   r   r   ÚFloatTensorrE   rO   rP   s   @r6   r0  r0  Æ  sì   ø„ ñ
T˜zð 
T°c¸D±jõ 
Tð IMØ.2Ø04Ø(,Ø!&ñ à—|‘|ð ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð ð Ÿ™ tÑ+ð	 ð
 ×&Ñ&¨Ñ-ð ð  ™ð ð ˜$‘;ð ð Ð+Ñ,ð ð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷ r7   r0  c                   ó–   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZ eed¬«      eed	œZ ej(                  «       ˆ fd
„«       Zˆ xZS )ÚDogePreTrainedModelrT   ÚmodelTr0  rß   Fr#   )Úindex)r#  r8   Ú
attentionsc                 ón  •— t         ‰| �  |«       t        |t        «      r-t	        |d«      r t        j                  |j                  «       yyt        |t        «      rXt	        |d«      rt        j                  |j                  «       t	        |d«      r t        j                  |j                  «       yyy)zInitialize the weightsr×   r5  r9  N)r,   Ú_init_weightsry   rÉ   ÚhasattrÚinitÚzeros_r×   r0  Úones_r5  r9  )r3   rŸ   r5   s     €r6   rG  z!DogePreTrainedModel._init_weights  sŠ   ø€ ô 	‰Ñ˜fÔ%Ü�fœmÔ,Ü�v˜sÔ#Ü—‘˜FŸH™HÕ%ð $ä˜Ô 0Ô1Ü�vÐ/Ô0Ü—
‘
˜6×0Ñ0Ô1Ü�vÐ8Ô9Ü—
‘
˜6×9Ñ9Õ:ð :ð 2r7   )rJ   rK   rL   r$   r‡   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr!   r  r0  rÉ   Ú_can_record_outputsr/   rŠ   rG  rO   rP   s   @r6   rB  rB  ö  sx   ø… àÓØÐØ&*Ð#Ø+Ð,ÐØ#4Ð"5ÐØ ÐØ€NØÐØ"ÐØ"&Ðá'¨	¸Ô;Ø)Ø#ñÐð €U‡]�]ƒ_ó
;ó ô
;r7   rB  c                   óò   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   defd„«       «       «       Zˆ xZS )Ú	DogeModelrT   c           	      ó  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                  ¬«      | _        t#        |¬«      | _        d| _        | j)                  «        y c c}w )NrÎ   ©rT   F)r,   r-   Úpad_token_idÚpadding_idxÚ
vocab_sizer   r  r4   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersr0  Úlayersr(   rÚ   ÚnormrR   Ú
rotary_embÚgradient_checkpointingÚ	post_initrÝ   s      €r6   r-   zDogeModel.__init__  sÆ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m‘mÜBGÈ×H`ÑH`ÓBaÖb°YÔ˜f iÕ0Òbó
ˆŒô   × 2Ñ 2¸×8KÑ8KÔLˆŒ	Ü-°VÔ<ˆŒØ&+ˆÔ#ð 	�‰Õùò cs   ÂDNÚ	input_idsr£   r�   rß   Úinputs_embedsr:  r¦   r*   c           
      óz  — |d u |d uz  rt        d«      ‚|r|€t        | j                  ¬«      }|€| j                  |«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }| j                  j                  €t        nt        }	 |	| j                  ||||¬«      }
|}| j                  ||¬«      }| j                  d | j                  j                   D ]  } ||f|
||||dœ|¤Ž}Œ | j!                  |«      }t#        ||¬	«      S )
Nz:You must specify exactly one of input_ids or inputs_embedsrY  r   r#   )rb   )rT   rg  r£   rß   r�   )r�   )r£   r�   rß   r:  rÞ   )Úlast_hidden_staterß   )Ú
ValueErrorr   rT   r]  Úget_seq_lengthr/   rm   rH   rb   r‘   Úsliding_windowr   r   rc  ra  r`  rb  r   )r3   rf  r£   r�   rß   rg  r:  r¦   Úpast_seen_tokensÚmask_functionr¿   r8   rÞ   Údecoder_layers                 r6   rE   zDogeModel.forward(  sd  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZá˜Ð0Ü*°$·+±+Ô>ˆOàÐ Ø ×-Ñ-¨iÓ8ˆMàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLà.2¯k©k×.HÑ.HÐ.PÕ*ÔVwˆÙ#Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨mÈ,˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÐIò 		ˆMÙ)Øðà*Ø)Ø /Ø#Ø$7ñð ñ‰Mð		ð Ÿ	™	 -Ó0ˆä%Ø+Ø+ô
ð 	
r7   )NNNNNN)rJ   rK   rL   r$   r-   r    r"   r   r/   r?  rN   r
   r@  r÷   r   r   r   rE   rO   rP   s   @r6   rW  rW    sÍ   ø„ ð˜zõ ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ4
à×#Ñ# dÑ*ð4
ð Ÿ™ tÑ+ð4
ð ×&Ñ&¨Ñ-ð	4
ð
  ™ð4
ð ×(Ñ(¨4Ñ/ð4
ð ˜$‘;ð4
ð Ð+Ñ,ð4
ð 
 ò4
ó ó ó  ô4
r7   rW  Úgate_logitsr  r  r  c                 óÌ  — | �t        | t        «      sy| d   j                  }| d   j                  }g }g }| D �]  }	|	j	                  |«      }	|	j                  |d¬«      \  \  }
}\  }}|
j                  d«      |j                  d«      z   }|j                  d«      |z  |j                  d«      z   } |j                  g |j                  dd ¢d‘­Ž } |j                  g |j                  dd ¢d‘­Ž }|j                  |d¬«      \  }}|j                  d|«      }t        j                  |d¬«      }|j                  |«       |j                  |«       �Œ t        j                  |d¬«      }t        j                  |d¬«      }|€}|j                  d«      }t        j                  |||¬«      }t        j                   |||¬«      }|j#                  d||«      |j                  d   z  }t        j$                  |d¬«      }�nD|j                  \  }}t'        | «      }|ddd…dd…df   j)                  ||||f«      j+                  d«      j	                  |«      }|j                  d«      |j-                  «          }t        j                  |||¬«      }t        j                   |||¬«      }|j#                  d||«      t        j.                  |«      z  }|ddd…dd…df   j)                  ||||f«      j+                  d|«      j	                  |«      }t        j.                  ||z  d¬«      t        j.                  |d¬«      z  }t        j.                  ||z  «      }||z  S )aø  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `router_gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [2, batch_size * sequence_length, num_keys].
        num_experts:
            Number of experts
        num_keys:
            Number of keys
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   r;   rw   r·   rò   )ry   rG   r=   rb   r>   rü   r‘   rã   rH   r  ræ   r®   Úappendr/   r}   rÖ   Ú	ones_likeÚscatter_add_rA   Úlenrx   rš   r÷   r   )rp  r  r  r  r£   Úcompute_dtypeÚcompute_deviceÚall_expert_indicesÚall_routing_weightsÚlayer_gate_logitsr$  r%  r&  r'  r(  r)  r"  r+  Úexpert_indicesr,  Útokens_per_expertÚpadÚrouter_prob_per_expertÚ
batch_sizeÚsequence_lengthr`  Úexpert_attention_maskÚ router_per_expert_attention_maskÚoverall_losss                                r6   Úload_balancing_loss_funcr„  b  s{  € ð@ Ð¤*¨[¼%Ô"@Øà ‘N×(Ñ(€MØ  ‘^×*Ñ*€NØÐØÐà(ó 4ÐØ-×0Ñ0°Ó@Ðà7H×7MÑ7MÈhÐ\^Ð7MÓ7_Ñ4Ñˆ�8Ñ4˜y¨)à×'Ñ'¨Ó+¨h×.@Ñ.@ÀÓ.DÑDˆ
Ø×)Ñ)¨"Ó-°Ñ8¸9×;NÑ;NÈrÓ;RÑRˆØ$�Z—_‘_Ð@ j×&6Ñ&6°s¸Ð&;Ð@¸RÒ@ˆ
Ø&�k×&Ñ&ÐC¨×(9Ñ(9¸#¸2Ð(>ÐCÀÒCˆà(Ÿo™o¨e¸˜oÓ<ÑˆÐØ$×+Ñ+¨BÐ0@ÓAˆäŸ)™) J°BÔ7ˆà×!Ñ! .Ô1Ø×"Ñ" ?Ö3ð!4ô" Ÿ™Ð#5¸1Ô=ÐÜŸ)™)Ð$7¸QÔ?ÐàÐà/×4Ñ4°RÓ8ÐÜ!ŸK™K¨¸=ÐQ_Ô`ÐÜ�o‰oÐ0¸ÈnÔ]ˆØ-×:Ñ:¸1Ð>PÐRUÓVÐYk×YqÑYqÐrsÑYtÑtÐô "'§¡Ð,?ÀQÔ!GÒà&4×&:Ñ&:Ñ#ˆ
�OÜ Ó,Ðð ˜4¢¢A tÐ+Ñ,ß‰VÐ&¨
°OÀUÐKÓLß‰W�R‹[ß‰R�Óð	 	ð 0×4Ñ4°RÓ8Ð9N×9SÑ9SÓ9UÑVÐô "ŸK™K¨¸=ÐQ_Ô`ÐÜ�o‰oÐ0¸ÈnÔ]ˆØ-×:Ñ:¸1Ð>PÐRUÓVÔY^×YbÑYbØ!óZ
ñ 
Ðð ˜4¢¢A tÐ+Ñ,ß‰VÐ&¨
°OÀ[ÐQÓRß‰W�R˜Ó%ß‰R�Óð	 	)ô "'§¡Ð+>ÐAaÑ+aÐghÔ!iÔlq×luÑluØ,°!ôm
ñ "
Ðô —9‘9Ð.Ð1GÑGÓH€LØ˜+Ñ%Ð%r7   c                   óN  ‡ — e Zd ZddiZddiZddgdgfiZˆ fd„Zee	 	 	 	 	 	 	 	 	 dd	e	j                  dz  d
e	j                  dz  de	j                  dz  dedz  de	j                  dz  de	j                  dz  dedz  dee	j                  z  dedz  dee   defd„«       «       Zˆ xZS )ÚDogeForCausalLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr8   Úlogitsc                 óN  •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  |j                  d¬«      | _        |j                  | _	        |j                  | _
        |j                  | _        | j                  «        y )NFrÌ   )r,   r-   rW  rC  r\  r   rÑ   r4   r‡  Úrouter_aux_loss_coefr  r  re  r  s     €r6   r-   zDogeForCausalLM.__init__Ò  s   ø€ Ü‰Ñ˜Ô Ü˜vÓ&ˆŒ
Ø ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ$*×$?Ñ$?ˆÔ!Ø!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ ð 	�‰Õr7   Nrf  r£   r�   rß   rg  Úlabelsr:  Úlogits_to_keepÚoutput_router_logitsr¦   r*   c
           
      óÊ  — |	�|	n| j                   j                  }	 | j                  d||||||dœ|
¤Ž}|j                  }t	        |t
        «      rt        | d«      n|}| j                  |dd…|dd…f   «      }d}|� | j                  ||| j                  fi |
¤Ž}d}|	rŠt        |j                  | j                  t        j                  t        j                  | j                  «      «      | j                   |«      }|�+|| j"                  |j%                  |j&                  «      z  z  }t)        ||||j*                  |j,                  |j.                  |j                  ¬«      S )ah  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, DogeForCausalLM

        >>> model = DogeForCausalLM.from_pretrained("SmallDoge/Doge-320M")
        >>> tokenizer = AutoTokenizer.from_pretrained("SmallDoge/Doge-320M")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)rf  r£   r�   rß   rg  r:  )ÚlossÚaux_lossr‰  rß   r8   rE  r#  r<  )rT   rŽ  rC  ri  ry   r‰   Úslicer‡  Úloss_functionr\  r„  r#  r  r  r  r  r  r‹  r>   rb   r   rß   r8   rE  )r3   rf  r£   r�   rß   rg  rŒ  r:  r�  rŽ  r¦   Úoutputsr8   Úslice_indicesr‰  r�  r‘  s                    r6   rE   zDogeForCausalLM.forwardÞ  sm  € ðL %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð
 +5¨$¯*©*ð +
ØØ)Ø%Ø+Ø'Øñ+
ð ñ+
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ% f¨f°d·o±oÑPÈÑPˆDàˆÙÜ/Ø×%Ñ%Ø× Ñ Ü—
‘
œ4Ÿ9™9 T×%5Ñ%5Ó6Ó7Ø×(Ñ(ØóˆHð Ð!Ø˜×1Ñ1°H·K±KÀÇÁÓ4LÑLÑL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r7   )	NNNNNNNr   N)rJ   rK   rL   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr-   r   r   r/   r?  rN   r
   r@  r÷   r‰   r   r   r   rE   rO   rP   s   @r6   r†  r†  Ì  s5  ø„ à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€Hô
ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.Ø,0ñO
à×#Ñ# dÑ*ðO
ð Ÿ™ tÑ+ðO
ð ×&Ñ&¨Ñ-ð	O
ð
  ™ðO
ð ×(Ñ(¨4Ñ/ðO
ð × Ñ  4Ñ'ðO
ð ˜$‘;ðO
ð ˜eŸl™lÑ*ðO
ð # T™kðO
ð Ð+Ñ,ðO
ð 
#òO
ó ó ôO
r7   r†  c                   ó   — e Zd Zy)ÚDogeForSequenceClassificationN)rJ   rK   rL   r<  r7   r6   rš  rš  2  s   „ Ør7   rš  )r†  rW  rB  rš  )r#   )râ   r  )NNr:   N)Qr  Úcollections.abcr   Útypingr   r   r/   Útorch.nn.functionalr   r­   ræ   Ú r   rI  Úactivationsr	   Úcache_utilsr
   r   Ú
generationr   Úintegrationsr   r   Úintegrations.flex_attentionr   Úmasking_utilsr   r   Úmodeling_layersr   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   r    Úutils.output_capturingr!   r"   Úconfiguration_doger$   Ú!torch.nn.attention.flex_attentionr%   ÚModuler(   rR   rŽ   r—   rN   r‰   rž   rM   r´   rG   rÆ   ré   rÉ   r  r  r0  rB  rW  r„  r†  rš  Ú__all__r<  r7   r6   ú<module>r±     s0  ðó. Ý $ß "ã ß Ð Ý å &Ý !ß .Ý )ß QÝ Jß Rß [ß Qß Kß AÝ &ß gÓ gß Gß EÝ *ñ  Ô!Ý;ñ ˜YÓ'ôJ�"—)‘)ó Jó (ðJô(><˜"Ÿ)™)ô ><òB(ñ Ð*Ó+òó ,ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(ó%ð> !Ø ñ+*Ø�I‰Ið+*à�<‰<ð+*ð 
�‰ð+*ð �<‰<ð	+*ð
 ˜%Ÿ,™,¨Ð3Ñ4ð+*ð �T‰\ð+*ð �T‰\ð+*ð ˆ5�<‰<˜Ÿ™Ð%Ñ&ó+*ñ\ -Ó.Ð Ø1GÐ Ð-Ñ .ôw�B—I‘Iô wôtˆb�i‰iô ô 6,�—	‘	ô 6,ôr-Ð1ô -ð` ô;˜/ó ;ó ð;ð> ôH
Ð#ó H
ó ðH
ðZ #ØØØ*.ñg&Ø—‘  e§l¡lÑ 3Ñ3°dÑ:ðg&à�t‘ðg&ð �D‰jðg&ð ð	g&ð
 —L‘L 4Ñ'ðg&ð ‡\�\�CÑóg&ðT ôb
Ð)¨?ó b
ó ðb
ôJ	Ð$DÐFYô 	ò c�r7   