Ë
    ÛÓ^jÀ1  ã                   óâ  — d dl mZ d dlZd dlmZ d dlmZ ddlmZm	Z	 ddl
mZ ddlmZmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZmZmZ ddlmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& ddl'm(Z(  ejR                  e*«      Z+ ed¬«      e G d„ de«      «       «       Z, G d„ de$«      Z- G d„ de"«      Z. G d„ de«      Z/ G d„ de «      Z0 G d„ de#«      Z1 G d„ d e(«      Z2 G d!„ d"e!«      Z3g d#¢Z4y)$é    )ÚCallableN)Ústricté   )ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚBaseModelOutputWithPast)ÚRopeParametersÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úmaybe_autocasté   )ÚCohereAttentionÚCohereDecoderLayerÚCohereForCausalLMÚCohereLayerNormÚCoherePreTrainedModelÚCohereRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)ÚGemma2ModelzCohereForAI/c4ai-command-r-v01)Ú
checkpointc                   óÖ  ‡ — e Zd ZU dZdZdgZddddddddœZdgdgfd	d
gd	gfd	gd	gfdœZdZe	e
d<   dZe	e
d<   dZe	e
d<   dZee
d<   dZe	e
d<   dZe	e
d<   dZe	dz  e
d<   dZee
d<   dZe	e
d<   dZee
d<   dZee
d <   d!Zee
d"<   d#Ze	dz  e
d$<   d%Ze	dz  e
d&<   d'Ze	ee	   z  dz  e
d(<   d!Zee
d)<   dZee z  dz  e
d*<   d+Z!ee
d,<   d-Z"ee	z  e
d.<   d/Z#e	dz  e
d0<   dZ$ee   dz  e
d1<   ˆ fd2„Z%ˆ xZ&S )3ÚCohere2Configa  
    logit_scale (`float`, *optional*, defaults to 0.0625):
        The scaling factor for the output logits.

    ```python
    >>> from transformers import Cohere2Model, Cohere2Config

    >>> # Initializing a Cohere Nextmodel configuration
    >>> configuration = Cohere2Config()

    >>> # Initializing a model from the Cohere2 configuration
    >>> model = Cohere2Model(configuration) # doctest: +SKIP

    >>> # Accessing the model configuration
    >>> configuration = model.config # doctest: +SKIP
    ```
    Úcohere2Úpast_key_valuesÚcolwiseÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormi è Ú
vocab_sizei    Úhidden_sizei X  Úintermediate_sizeg      °?Úlogit_scaleé(   Únum_hidden_layersé@   Únum_attention_headsNÚnum_key_value_headsÚsiluÚ
hidden_actÚmax_position_embeddingsg{®Gáz”?Úinitializer_rangegñhãˆµøä>Úlayer_norm_epsTÚ	use_cacher   Úpad_token_idé   Úbos_token_idiä Úeos_token_idÚtie_word_embeddingsÚrope_parametersFÚattention_biasç        Úattention_dropouti   Úsliding_windowÚlayer_typesc                 óV  •— | j                   €| j                  | _         | j                  | j                  z  | _        | j                  €M|j                  dd«      }t        | j                  «      D �cg c]  }t        |dz   |z  «      rdnd‘Œ c}| _        t        ‰| �(  di |¤Ž y c c}w )NÚsliding_window_patterné   é   Úsliding_attentionÚfull_attention© )r4   r3   r-   Úhead_dimrE   ÚpopÚranger1   ÚboolÚsuperÚ__post_init__)ÚselfÚkwargsÚ_sliding_window_patternÚiÚ	__class__s       €úv/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/cohere2/modular_cohere2.pyrR   zCohere2Config.__post_init__m   s®   ø€ Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$ð ×(Ñ(¨D×,DÑ,DÑDˆŒð ×ÑÐ#à&,§j¡jÐ1IÈ1Ó&MÐ#ô ˜t×5Ñ5Ó6ö àô (,¨Q°©UÐ6MÑ,MÔ'NÑ#ÐTdÑdò ˆDÔô
 	‰ÑÑ' Ó'ùò s   Á2B&)'Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planr,   ÚintÚ__annotations__r-   r.   r/   Úfloatr1   r3   r4   r6   Ústrr7   r8   r9   r:   rP   r;   r=   r>   Úlistr?   r@   r   ÚdictrA   rC   rD   rE   rR   Ú__classcell__©rW   s   @rX   r    r    1   s…  ø… ñð$ €JØ#4Ð"5Ðà%.Ø%.Ø%.Ø%.Ø"+Ø )Ø"+ñÐð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð €J�ÓØ€K�ÓØ"Ð�sÓ"Ø€K�ÓØÐ�sÓØ!Ð˜Ó!Ø&*Ð˜˜t™Ó*Ø€J�ÓØ#'Ð˜SÓ'Ø#Ð�uÓ#Ø €N�EÓ Ø€IˆtÓØ €L�#˜‘*Ó Ø €L�#˜‘*Ó Ø+1€L�#˜˜S™	‘/ DÑ(Ó1Ø $Ð˜Ó$Ø48€O�^ dÑ*¨TÑ1Ó8Ø €N�DÓ Ø%(Ð�u˜s‘{Ó(Ø!%€N�C˜$‘JÓ%Ø$(€K��c‘˜TÑ!Ó(÷(ð (ó    r    c                   óD   — e Zd Z ej                  «       ed„ «       «       Zy)ÚCohere2RotaryEmbeddingc                 ó  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      }|d d …d d d …f   j                  «       }t	        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  |dd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j                  |j                   ¬
«      	j                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   éÿÿÿÿrI   ÚmpsÚcpuF)Údevice_typeÚenabledr   )Údim)Údtype)Úinv_freqrc   ÚexpandÚshapeÚ
isinstanceÚdeviceÚtyperd   r   Ú	transposeÚtorchÚrepeat_interleaveÚcosÚattention_scalingÚsinÚtors   )
rS   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrp   ÚfreqsÚembr}   r   s
             rX   ÚforwardzCohere2RotaryEmbedding.forward�   s@  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓeÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ×)Ñ)¨%°¸Ô;ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   Â=BFÆF
N)rY   rZ   r[   r{   Úno_gradr   r‡   rL   ri   rX   rk   rk   €   s$   „ Ø€U‡]�]ƒ_Øñ<ó ó ñ<ri   rk   c                   ó   — e Zd Zy)ÚCohere2LayerNormN©rY   rZ   r[   rL   ri   rX   rŠ   rŠ   ‘   ó   „ Øri   rŠ   c                   ó  — e Zd ZdZddededz  fd„Z	 ddej                  de	ej                  ej                  f   dej                  dz  d	e
dz  d
ee   de	ej                  ej                  dz  e	ej                     dz  f   fd„Zy)ÚCohere2Attentionz=Multi-headed attention from 'Attention Is All You Need' paperNÚconfigÚ	layer_idxc                 óÎ  — t         j                  j                  | «       || _        || _        t        |d|j                  |j                  z  «      | _        |j                  |j                  z  | _
        | j                  dz  | _        |j                  | _        d| _        |j                  |   dk(  r|j                  nd | _        t        j                   |j                  |j                  | j                  z  |j"                  ¬«      | _        t        j                   |j                  |j                  | j                  z  |j"                  ¬«      | _        t        j                   |j                  |j                  | j                  z  |j"                  ¬«      | _        t        j                   |j                  | j                  z  |j                  |j"                  ¬«      | _        y )NrM   g      à¿TrJ   )Úbias)ÚnnÚModuleÚ__init__r�   r�   Úgetattrr-   r3   rM   r4   Únum_key_value_groupsÚscalingrC   Ú	is_causalrE   rD   ÚLinearrA   Úq_projÚk_projÚv_projÚo_proj)rS   r�   r�   s      rX   r•   zCohere2Attention.__init__˜   sw  € Ü
�	‰	×Ñ˜4Ô ØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒØ7=×7IÑ7IÈ)Ñ7TÐXkÒ7k˜f×3Ò3ÐquˆÔä—i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆ�ri   r'   Úposition_embeddingsr(   r"   rT   Úreturnc                 ó8  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
|\  }}| j                  �t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t        «      } || ||	|
|f| j                   sdn| j"                  | j$                  | j                  dœ|¤Ž\  }} |j&                  g |¢d‘­Ž j)                  «       }| j+                  |«      }||fS )Nrm   rI   r   rB   )Údropoutr˜   rD   )rv   rM   r›   Úviewrz   rœ   r�   rD   r   Úupdater�   r   Úget_interfacer�   Ú_attn_implementationr   ÚtrainingrC   r˜   ÚreshapeÚ
contiguousrž   )rS   r'   rŸ   r(   r"   rT   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesr}   r   Úattention_interfaceÚattn_outputÚattn_weightss                   rX   r‡   zCohere2Attention.forward°   s¦  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSØ×ÑÐ*Ü';¸LÈ*ÐVYÐ[^Ó'_Ñ$ˆL˜*àÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(ri   ©N)rY   rZ   r[   r\   r    ra   r•   r{   ÚTensorÚtupler   r   r   r‡   rL   ri   rX   rŽ   rŽ   •   s°   „ ÙGñ
˜}ð 
¸¸t¹ó 
ð: )-ñ()à—|‘|ð()ð # 5§<¡<°·±Ð#=Ñ>ð()ð Ÿ™ tÑ+ð	()ð
  ™ð()ð Ð+Ñ,ð()ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	Mô()ri   rŽ   c                   ó$  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  de	dz  d	e
dz  d
ee   deej                  eej                  ej                  f   dz  f   fd„Zˆ xZS )ÚCohere2DecoderLayerr�   r�   c                 ó&   •— t         ‰| �  ||«       y r²   )rQ   r•   )rS   r�   r�   rW   s      €rX   r•   zCohere2DecoderLayer.__init__Ü   s   ø€ Ü‰Ñ˜ Õ+ri   Nr'   rŸ   r(   r"   r:   rT   r    c           	      ó”   — |}| j                  |«      } | j                  d|||||dœ|¤Ž\  }}	| j                  |«      }
||z   |
z   }|S )N)r'   rŸ   r(   r"   r:   rL   )Úinput_layernormÚ	self_attnÚmlp)rS   r'   rŸ   r(   r"   r:   rT   ÚresidualÚhidden_states_attentionÚ_Úhidden_states_mlps              rX   r‡   zCohere2DecoderLayer.forwardß   su   € ð !ˆØ×,Ñ,¨]Ó;ˆØ%3 T§^¡^ð &
Ø'Ø 3Ø)Ø+Øñ&
ð ñ&
Ñ"Ð ð !ŸH™H ]Ó3ÐØ Ð#:Ñ:Ð=NÑNˆØÐri   )NNNF)rY   rZ   r[   r    ra   r•   r{   r³   r´   r   rP   r   r   ÚFloatTensorr‡   rg   rh   s   @rX   r¶   r¶   Û   sÍ   ø„ ð,˜}ð ,¸õ ,ð IMØ.2Ø(,Ø!&ñà—|‘|ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ÿ™ tÑ+ð	ð
  ™ðð ˜$‘;ðð Ð+Ñ,ðð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷ri   r¶   c                   ó"   — e Zd ZU eed<   eedœZy)ÚCohere2PreTrainedModelr�   )r'   Ú
attentionsN)rY   rZ   r[   r    rb   r¶   rŽ   Ú_can_record_outputsrL   ri   rX   rÂ   rÂ   ø   s   … ØÓà,Ø&ñÑri   rÂ   c                   óÔ   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	e
dz  d
ee   defd„Zˆ xZS )ÚCohere2Modelr�   c                 óä   •— t         ‰| �  |«       t        |j                  |j                  ¬«      | _        t        j                  |j                  |j                  | j                  «      | _
        y )N)r-   Úeps)rQ   r•   rŠ   r-   r9   r+   r“   Ú	Embeddingr,   Úpadding_idxr)   )rS   r�   rW   s     €rX   r•   zCohere2Model.__init__  sR   ø€ Ü‰Ñ˜Ô Ü$°&×2DÑ2DÈ6×K`ÑK`ÔaˆŒ	ÜŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÕri   Nr%   r(   r‚   r"   r&   r:   rT   r    c           
      óš  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      s)| j                  ||||dœ}
t        d
i |
¤Žt        d
i |
¤Ždœ}	|}| j                  ||«      }t        | j                   «      D ].  \  }} ||f|	| j                  j"                  |      ||||dœ|¤Ž}Œ0 | j%                  |«      }t'        ||¬	«      S )Nz:You must specify exactly one of input_ids or inputs_embeds)r�   r   rI   )rx   )r�   r&   r(   r"   r‚   )rK   rJ   )r(   rŸ   r"   r:   r‚   )Úlast_hidden_stater"   rL   )Ú
ValueErrorr)   r   r�   Úget_seq_lengthr{   Úarangerv   rx   Ú	unsqueezerw   rf   r	   r
   Ú
rotary_embÚ	enumerater*   rE   r+   r   )rS   r%   r(   r‚   r"   r&   r:   rT   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr'   rŸ   rV   Údecoder_layers                  rX   r‡   zCohere2Model.forward  sƒ  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-ÄÔFàŸ+™+Ø!.Ø"0Ø#2Ø ,ñˆKô #5Ñ"C°{Ñ"CÜ%FÑ%UÈÑ%Uñ#Ðð
 &ˆØ"Ÿo™o¨m¸\ÓJÐä )¨$¯+©+Ó 6ò 		ÑˆAˆ}Ù)Øðà2°4·;±;×3JÑ3JÈ1Ñ3MÑNØ$7Ø /Ø#Ø)ñð ñ‰Mð		ð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ô
ð 	
ri   )NNNNNN)rY   rZ   r[   r    r•   r{   Ú
LongTensorr³   r   rÀ   rP   r   r   r   r‡   rg   rh   s   @rX   rÆ   rÆ      s°   ø„ ðb˜}õ bð .2Ø.2Ø04Ø(,Ø26Ø!%ñ7
à×#Ñ# dÑ*ð7
ð Ÿ™ tÑ+ð7
ð ×&Ñ&¨Ñ-ð	7
ð
  ™ð7
ð ×(Ñ(¨4Ñ/ð7
ð ˜$‘;ð7
ð Ð+Ñ,ð7
ð 
!÷7
ri   rÆ   c                   ó   — e Zd Zy)ÚCohere2ForCausalLMNr‹   rL   ri   rX   rÙ   rÙ   @  rŒ   ri   rÙ   )r    rÙ   rÆ   rÂ   )5Úcollections.abcr   r{   Útorch.nnr“   Úhuggingface_hub.dataclassesr   Úcache_utilsr   r   Úconfiguration_utilsr   Úmasking_utilsr	   r
   Úmodeling_outputsr   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úcohere.modeling_coherer   r   r   r   r   r   r   r   Úgemma2.modeling_gemma2r   Ú
get_loggerrY   Úloggerr    rk   rŠ   rŽ   r¶   rÂ   rÆ   rÙ   Ú__all__rL   ri   rX   ú<module>rë      sï   ðõ %ã Ý Ý .ç .Ý 3ß RÝ 7÷õ 6Ý &ß @Ñ @Ý +÷	÷ 	ó 	õ 1ð 
ˆ×	Ñ	˜HÓ	%€ñ Ð;Ô<ØôJ(Ð$ó J(ó ó =ðJ(ôZ<Ð2ô <ô"	�ô 	ôC)�ô C)ôLÐ,ô ô:Ð2ô ô=
�;ô =
ô@	Ð*ô 	ò \�ri   