Ë
    ÜÓ^jÊ  ã                   óâ  — d dl mZ d dlZd dlmZ ddlmZ ddlmZ ddl	m
Z
 ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZ ddlmZm Z   ejB                  e"«      Z# ed¬«      e G d„ de«      «       «       Z$ G d„ de «      Z% G d„ de«      Z& G d„ de«      Z' G d„ de«      Z( G d„ de«      Z) G d„ de«      Z* G d„ d e«      Z+ G d!„ d"e«      Z, G d#„ d$e«      Z-g d%¢Z.y)&é    )ÚCallableN)Ústricté   )ÚCache)ÚPreTrainedConfig)ÚFlashAttentionKwargs)ÚRopeParameters)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)Úauto_docstringÚloggingé   )	ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaForCausalLMÚLlamaForQuestionAnsweringÚLlamaForSequenceClassificationÚLlamaForTokenClassificationÚLlamaPreTrainedModelÚapply_rotary_pos_embÚeager_attention_forward)Ú
Qwen2ModelÚQwen2RotaryEmbeddingzHuggingFaceTB/SmolLM3-3B)Ú
checkpointc                   ó  ‡ — e Zd ZU dZdZdgZdZddddddddœZdgd	gfd
dgd
gfd
gd
gfdœZdZ	e
ed<   dZe
ed<   dZe
ed<   dZe
ed<   dZe
ed<   dZe
dz  ed<   dZeed<   dZe
ed<   dZeed<   d Zeed!<   d"Zeed#<   d$Ze
dz  ed%<   d&Ze
dz  ed'<   d(Ze
ee
   z  dz  ed)<   dZeez  dz  ed*<   d+Z eed,<   dZ!e
dz  ed-<   dZ"ee
   dz  ed.<   dZ#e
ed/<   dZ$ee   dz  ed0<   d+Z%eed1<   d2Z&ee
z  ed3<   d+Z'eed4<   d"Z(eed5<   ˆ fd6„Z)ˆ xZ*S )7ÚSmolLM3Configa>  
    no_rope_layers (`List[int]`, *optional*):
        List with at least the same length as the number of layers in the model.
        A `1` at an index position indicates that the corresponding layer will use RoPE,
        while a `0` indicates that it's a NoPE layer.
    no_rope_layer_interval (`int`, *optional*, defaults to 4):
        If `no_rope_layers` is `None`, it will be created using a NoPE layer every
        `no_rope_layer_interval` layers.

    ```python
    >>> from transformers import SmolLM3Model, SmolLM3Config

    >>> # Initializing a SmolLM3 style configuration
    >>> configuration = SmolLM3Config()

    >>> # Initializing a model from the SmolLM3 style configuration
    >>> model = SmolLM3Model(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Úsmollm3Úpast_key_valuesg    €„>AÚcolwiseÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormi õ Ú
vocab_sizei   Úhidden_sizei +  Úintermediate_sizeé$   Únum_hidden_layersé   Únum_attention_headsé   NÚnum_key_value_headsÚsiluÚ
hidden_acti €  Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeg�íµ ÷Æ°>Úrms_norm_epsTÚ	use_cacheiô Úpad_token_idi ô Úbos_token_idiô Úeos_token_idÚrope_parametersFÚuse_sliding_windowÚsliding_windowÚno_rope_layersÚno_rope_layer_intervalÚlayer_typesÚattention_biasç        Úattention_dropoutÚmlp_biasÚtie_word_embeddingsc                 ó"  •— | j                   €| j                  | _         | j                  €Dt        | j                  «      D �cg c]   }t        |dz   | j                  z  dk7  «      ‘Œ" c}| _        | j                  €�g | _        t        | j                  «      D ]b  }| j                  |   }| j                  r*| j                  �|s| j                  j                  d«       ŒH| j                  j                  d«       Œd t        ‰| �0  di |¤Ž y c c}w )Né   r   Úsliding_attentionÚfull_attention© )r0   r.   r=   Úranger,   Úintr>   r?   r;   r<   ÚappendÚsuperÚ__post_init__)ÚselfÚkwargsÚ	layer_idxÚhas_ropeÚ	__class__s       €úv/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/smollm3/modular_smollm3.pyrN   zSmolLM3Config.__post_init__q   sþ   ø€ Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$à×ÑÐ&äY^Ð_c×_uÑ_uÓYvö#ØLU”�Y ‘] d×&AÑ&AÑAÀQÑFÕGò#ˆDÔð ×ÑÐ#Ø!ˆDÔÜ" 4×#9Ñ#9Ó:ò >�	Ø×.Ñ.¨yÑ9�Ø×*Ò*¨t×/BÑ/BÐ/NÑW_Ø×$Ñ$×+Ñ+Ð,?Õ@à×$Ñ$×+Ñ+Ð,<Õ=ð>ô 	‰ÑÑ' Ó'ùò#s   Á%D)+Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚdefault_thetaÚbase_model_tp_planÚbase_model_pp_planr(   rK   Ú__annotations__r)   r*   r,   r.   r0   r2   Ústrr3   r4   Úfloatr5   r6   Úboolr7   r8   r9   Úlistr:   r	   Údictr;   r<   r=   r>   r?   r@   rB   rC   rD   rN   Ú__classcell__©rS   s   @rT   r   r   ,   sµ  ø… ñð, €JØ#4Ð"5ÐØ€Mð &/Ø%.Ø%.Ø%.Ø"+Ø )Ø"+ñÐð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð €J�ÓØ€K�ÓØ"Ð�sÓ"ØÐ�sÓØ!Ð˜Ó!Ø&'Ð˜˜t™Ó'Ø€J�ÓØ#(Ð˜SÓ(Ø#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ%€L�#˜‘*Ó%Ø%€L�#˜‘*Ó%Ø+1€L�#˜˜S™	‘/ DÑ(Ó1Ø48€O�^ dÑ*¨TÑ1Ó8Ø$Ð˜Ó$Ø!%€N�C˜$‘JÓ%Ø'+€N�D˜‘I Ñ$Ó+Ø"#Ð˜CÓ#Ø$(€K��c‘˜TÑ!Ó(Ø €N�DÓ Ø%(Ð�u˜s‘{Ó(Ø€HˆdÓØ $Ð˜Ó$÷(ð (ó    r   c                   ó   — e Zd Zy)ÚSmolLM3RotaryEmbeddingN©rU   rV   rW   rI   rf   rT   rh   rh   †   ó   „ Ørf   rh   c                   óð   ‡ — e Zd Zdedefˆ fd„Z	 ddej                  deej                  ej                  f   dej                  dz  de	dz  d	e
e   d
eej                  ej                  dz  f   fd„Zˆ xZS )ÚSmolLM3AttentionÚconfigrQ   c                 ó¼   •— t         ‰| �  ||«       |j                  |   | _        |j                  r$|j
                  |   dk(  r|j                  | _        y d | _        y )NrG   )rM   Ú__init__r=   Úuse_roper;   r?   r<   )rO   rm   rQ   rS   s      €rT   ro   zSmolLM3Attention.__init__‹   sb   ø€ Ü‰Ñ˜ Ô+à×-Ñ-¨iÑ8ˆŒð ×(Ò(¨V×-?Ñ-?À	Ñ-JÐNaÒ-að ×!Ñ!ð 	Õð ð 	Õrf   Nr#   Úposition_embeddingsr$   r   rP   Úreturnc                 ó8  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
| j                  r|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t        «      } || ||	|
|f| j                   sdn| j"                  | j$                  | j&                  dœ|¤Ž\  }} |j(                  g |¢d‘­Ž j+                  «       }| j-                  |«      }||fS )NéÿÿÿÿrF   r   rA   )ÚdropoutÚscalingr<   )ÚshapeÚhead_dimÚq_projÚviewÚ	transposeÚk_projÚv_projrp   r   ÚupdaterQ   r
   Úget_interfacerm   Ú_attn_implementationr   ÚtrainingrB   rv   r<   ÚreshapeÚ
contiguousÚo_proj)rO   r#   rq   r$   r   rP   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                   rT   ÚforwardzSmolLM3Attention.forward•   s¡  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà�=Š=Ø*‰HˆC�Ü';¸LÈ*ÐVYÐ[^Ó'_Ñ$ˆL˜*àÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(rf   )N)rU   rV   rW   r   rK   ro   ÚtorchÚTensorÚtupler   r   r   r�   rd   re   s   @rT   rl   rl   Š   sš   ø„ ð
˜}ð 
¸õ 
ð )-ñ()à—|‘|ð()ð # 5§<¡<°·±Ð#=Ñ>ð()ð Ÿ™ tÑ+ð	()ð
  ™ð()ð Ð-Ñ.ð()ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷()rf   rl   c                   ó   — e Zd Zy)ÚSmolLM3DecoderLayerNri   rI   rf   rT   r”   r”   À   rj   rf   r”   c                   ó   — e Zd Zy)ÚSmolLM3PreTrainedModelNri   rI   rf   rT   r–   r–   Ä   rj   rf   r–   c                   ó   — e Zd Zy)ÚSmolLM3ModelNri   rI   rf   rT   r˜   r˜   È   rj   rf   r˜   c                   ó   — e Zd Zy)ÚSmolLM3ForCausalLMNri   rI   rf   rT   rš   rš   Ì   rj   rf   rš   c                   ó   — e Zd Zy)Ú SmolLM3ForSequenceClassificationNri   rI   rf   rT   rœ   rœ   Ð   rj   rf   rœ   c                   ó   — e Zd Zy)ÚSmolLM3ForTokenClassificationNri   rI   rf   rT   rž   rž   Ô   rj   rf   rž   c                   ó   — e Zd Zy)ÚSmolLM3ForQuestionAnsweringNri   rI   rf   rT   r    r    Ø   rj   rf   r    )r   r–   r˜   rš   rœ   rž   r    )/Úcollections.abcr   r�   Úhuggingface_hub.dataclassesr   Úcache_utilsr   Úconfiguration_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_rope_utilsr	   Úmodeling_utilsr
   Úprocessing_utilsr   Úutilsr   r   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   Úqwen2.modeling_qwen2r   r   Ú
get_loggerrU   Úloggerr   rh   rl   r”   r–   r˜   rš   rœ   rž   r    Ú__all__rI   rf   rT   ú<module>r¯      s÷   ðõ %ã Ý .å  Ý 3Ý BÝ 1Ý 5Ý &ß ,÷
÷ 
õ 
÷ Dð 
ˆ×	Ñ	˜HÓ	%€ñ Ð5Ô6ØôU(Ð$ó U(ó ó 7ðU(ôp	Ð1ô 	ô3)�~ô 3)ôl	Ð+ô 	ô	Ð1ô 	ô	�:ô 	ô	Ð)ô 	ô	Ð'Eô 	ô	Ð$?ô 	ô	Ð";ô 	ò�rf   