Ë
    ÛÓ^jY  ã                   ó–  — d dl Z d dlmZ ddlmZmZ ddlmZmZ ddl	m
Z
 ddlmZ ddlmZmZmZ d	d
lmZ d	dlmZmZmZmZ d	dlmZmZ  ej6                  e«      Z ed¬«      e G d„ de«      «       «       Z G d„ de«      Z G d„ de«      Z  G d„ de«      Z! G d„ de«      Z" G d„ de
«      Z# G d„ de«      Z$ G d„ de«      Z%g d¢Z&y) é    N)Ústricté   )ÚCacheÚDynamicCache)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚBaseModelOutputWithPast)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚloggingé   )ÚLlamaConfig)ÚLlamaDecoderLayerÚLlamaForCausalLMÚ
LlamaModelÚLlamaPreTrainedModel)ÚQwen2AttentionÚQwen2RotaryEmbeddingzfacebook/cwm)Ú
checkpointc                   óš  ‡ — e Zd ZU dZdZdZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZedz  ed<   dZeee   z  dz  ed<   dZeed<   d Zeed!<   d"Zeez  ed#<   d$Zeed%<   d Zeed&<   dZedz  ed'<   d(Z eed)<   dZ!ee   dz  ed*<    e"«       Z#ˆ fd+„Z$ˆ xZ%S ),Ú	CwmConfigÚcwmç    €„.Ai õ Ú
vocab_sizei   Úhidden_sizei T  Úintermediate_sizeé@   Únum_hidden_layersé0   Únum_attention_headsé   Únum_key_value_headsé€   Úhead_dimÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsTÚ	use_cacheNÚpad_token_idÚeos_token_idi ô Úbos_token_idFÚtie_word_embeddingsg        Úattention_dropouté   Úpretraining_tpÚmlp_biasÚrope_parametersé    Úsliding_windowÚlayer_typesc                 ó¨  •— | j                   €dddddddœ| _         | j                  €4d}t        | j                  «      D �cg c]  }||z  d	k(  rd
nd‘Œ c}| _        | j                  rt        | j                  «      nd | _        t        | j                  «      | _        | j                  �| j                  ng d¢| _        t        ‰| �$  di |¤Ž y c c}w )Nr   g      0@g      @g      ð?r5   Úllama3)Ú
rope_thetaÚfactorÚhigh_freq_factorÚlow_freq_factorÚ original_max_position_embeddingsÚ	rope_typeé   r   Úfull_attentionÚsliding_attention)iô iô i	ô © )
r4   r7   Úranger   r6   ÚintÚlistr-   ÚsuperÚ__post_init__)ÚselfÚkwargsÚwindow_patternÚiÚ	__class__s       €ún/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/cwm/modular_cwm.pyrH   zCwmConfig.__post_init__D   sÝ   ø€ Ø×ÑÐ'à)ØØ$'Ø#&Ø48Ø%ñ$ˆDÔ ð ×ÑÐ#àˆNô ˜t×5Ñ5Ó6ö àð '(¨.Ñ&8¸AÒ&=Ñ!ÐDWÑWò ˆDÔð
 ;?×:MÒ:Mœc $×"5Ñ"5Ô6ÐSWˆÔÜ × 0Ñ 0Ó1ˆÔØ15×1BÑ1BÐ1N˜D×-Ò-ÒTlˆÔÜ‰ÑÑ' Ó'ùò s   ÁC)&Ú__name__Ú
__module__Ú__qualname__Ú
model_typeÚdefault_thetar   rE   Ú__annotations__r   r   r   r!   r#   r%   r'   Ústrr(   r)   Úfloatr*   r+   Úboolr,   r-   rF   r.   r/   r0   r2   r3   r4   Údictr6   r7   ÚAttributeErrorÚattention_biasrH   Ú__classcell__©rM   s   @rN   r   r   %   s'  ø… ð €JØ€Mà€J�ÓØ€K�ÓØ"Ð�sÓ"ØÐ�sÓØ!Ð˜Ó!Ø Ð˜Ó Ø€HˆcÓØ€J�ÓØ#)Ð˜SÓ)Ø#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ#€L�#˜‘*Ó#Ø+/€L�#˜˜S™	‘/ DÑ(Ó/Ø€L�#ÓØ %Ð˜Ó%Ø%(Ð�u˜s‘{Ó(Ø€N�CÓØ€HˆdÓØ#'€O�T˜D‘[Ó'Ø€N�CÓØ$(€K��c‘˜TÑ!Ó(á#Ó%€N÷(ð (ó    r   c                   ó   — e Zd Zy)ÚCwmRotaryEmbeddingN©rO   rP   rQ   rC   r]   rN   r_   r_   ]   ó   „ Ør]   r_   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚCwmAttentionÚconfigÚ	layer_idxc                 óØ  •— t         ‰| �  ||¬«       t        j                  j	                  |j
                  |j                  | j                  z  d¬«      | _        t        j                  j	                  |j
                  |j                  | j                  z  d¬«      | _
        t        j                  j	                  |j
                  |j                  | j                  z  d¬«      | _        y )N©rd   re   F)Úbias)rG   Ú__init__ÚtorchÚnnÚLinearr   r!   r%   Úq_projr#   Úk_projÚv_proj©rI   rd   re   rM   s      €rN   ri   zCwmAttention.__init__b   s®   ø€ Ü‰Ñ °)ÐÔ<Ü—h‘h—o‘o f×&8Ñ&8¸&×:TÑ:TÐW[×WdÑWdÑ:dÐkp�oÓqˆŒÜ—h‘h—o‘o f×&8Ñ&8¸&×:TÑ:TÐW[×WdÑWdÑ:dÐkp�oÓqˆŒÜ—h‘h—o‘o f×&8Ñ&8¸&×:TÑ:TÐW[×WdÑWdÑ:dÐkp�oÓqˆ�r]   ©rO   rP   rQ   r   rE   ri   r[   r\   s   @rN   rc   rc   a   s   ø„ ðr˜yð r°S÷ rñ rr]   rc   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚCwmDecoderLayerrd   re   c                 óL   •— t         ‰| �  ||¬«       t        ||¬«      | _        y )Nrg   )rG   ri   rc   Ú	self_attnrp   s      €rN   ri   zCwmDecoderLayer.__init__j   s#   ø€ Ü‰Ñ °)ÐÔ<Ü%¨V¸yÔIˆ�r]   rq   r\   s   @rN   rs   rs   i   s   ø„ ðJ˜yð J°S÷ Jñ Jr]   rs   c                   ó   — e Zd Zy)ÚCwmPreTrainedModelNr`   rC   r]   rN   rw   rw   o   ra   r]   rw   c                   ó   — e Zd Zy)ÚCwmModelOutputWithPastNr`   rC   r]   rN   ry   ry   s   ra   r]   ry   c                   óØ   ‡ — e Zd ZeZdefˆ fd„Z	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de	dz  dej                  dz  d	edz  d
ee   defd„Zˆ xZS )ÚCwmModelrd   c           	      óÌ   •— t         ‰| �  |«       t        j                  j	                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        y c c}w )N)	rG   ri   rj   rk   Ú
ModuleListrD   r   rs   Úlayersrp   s      €rN   ri   zCwmModel.__init__z   sI   ø€ Ü‰Ñ˜Ô Ü—h‘h×)Ñ)ÜAFÀv×G_ÑG_ÓA`Öa°IŒ_˜V YÕ/Òaó
ˆ�ùÚas   ÁA!NÚ	input_idsÚattention_maskÚposition_idsÚpast_key_valuesÚinputs_embedsr+   rJ   Úreturnc           	      óæ  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      s9| j                  ||||dœ}
|
j                  «       }t        d
i |
¤Žt        d
i |¤Ždœ}	|}| j                  ||«      }t!        | j"                  d | j                  j$                   «      D ]-  \  }} ||f|	| j                  j&                  |      |||dœ|¤Ž}Œ/ | j)                  |«      }t+        ||¬	«      S )Nz:You must specify exactly one of input_ids or inputs_embeds)rd   r   r1   )Údevice)rd   rƒ   r€   r‚   r�   )rA   rB   )r€   r�   r‚   Úposition_embeddings)Úlast_hidden_stater‚   rC   )Ú
ValueErrorÚembed_tokensr   rd   Úget_seq_lengthrj   ÚarangeÚshaper†   Ú	unsqueezeÚ
isinstancerX   Úcopyr   r   Ú
rotary_embÚ	enumerater~   r   r7   Únormry   )rI   r   r€   r�   r‚   rƒ   r+   rJ   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚsliding_mask_kwargsÚhidden_statesr‡   rL   Údecoder_layers                   rN   ÚforwardzCwmModel.forward€   s¤  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø*.×*;Ñ*;¸IÓ*FˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-ÄÔFàŸ+™+Ø!.Ø"0Ø#2Ø ,ñˆKð #.×"2Ñ"2Ó"4Ðô #5Ñ"C°{Ñ"CÜ%FÑ%]ÐI\Ñ%]ñ#Ðð
 &ˆØ"Ÿo™o¨m¸\ÓJÐä )¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÓ Wò 	ÑˆAˆ}Ù)Øðà2°4·;±;×3JÑ3JÈ1Ñ3MÑNØ)Ø /Ø$7ñð ñ‰Mð	ð Ÿ	™	 -Ó0ˆÜ%Ø+Ø+ô
ð 	
r]   )NNNNNN)rO   rP   rQ   r   Úconfig_classri   rj   Ú
LongTensorÚTensorr   ÚFloatTensorrW   r
   r   ry   rš   r[   r\   s   @rN   r{   r{   w   s³   ø„ Ø€Lð
˜yõ 
ð .2Ø.2Ø04Ø(,Ø26Ø!%ñ8
à×#Ñ# dÑ*ð8
ð Ÿ™ tÑ+ð8
ð ×&Ñ&¨Ñ-ð	8
ð
  ™ð8
ð ×(Ñ(¨4Ñ/ð8
ð ˜$‘;ð8
ð Ð+Ñ,ð8
ð 
 ÷8
r]   r{   c                   ó   — e Zd Zy)ÚCwmForCausalLMNr`   rC   r]   rN   r    r    »   ra   r]   r    )r   rw   r{   r    )'rj   Úhuggingface_hub.dataclassesr   Úcache_utilsr   r   Úmasking_utilsr   r   Úmodeling_outputsr	   Úprocessing_utilsr
   Úutilsr   r   r   Úllama.configuration_llamar   Úllama.modeling_llamar   r   r   r   Úqwen2.modeling_qwen2r   r   Ú
get_loggerrO   Úloggerr   r_   rc   rs   rw   ry   r{   r    Ú__all__rC   r]   rN   ú<module>r­      sÖ   ðó  Ý .ç .ß RÝ 7Ý &ß @Ñ @Ý 3÷ó ÷ Hð 
ˆ×	Ñ	˜HÓ	%€ñ ˜>Ô*Øô3(�ó 3(ó ó +ð3(ôl	Ð-ô 	ôr�>ô rôJÐ'ô Jô	Ð-ô 	ô	Ð4ô 	ôA
ˆzô A
ôH	Ð%ô 	ò�r]   