Ë
    ÛÓ^j©0  ã                   ó  — d dl mZ d dlZd dlmZ ddlmZ ddlmZm	Z	 ddl
mZmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZmZ ddlmZ ddlmZ ddlmZ ddlmZm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z- ddl.m/Z/  ej`                  e1«      Z2 G d„ de%«      Z3 G d„ de«      Z4 G d„ de «      Z5 G d„ de(«      Z6 G d„ de"«      Z7 G d„ d e*«      Z8 G d!„ d"e&«      Z9 G d#„ d$e+«      Z: G d%„ d&e)«      Z; G d'„ d(e'«      Z<g d)¢Z=y)*é    )ÚCallableN)Únné   )Úinitialization)ÚCacheÚDynamicCache)Úcreate_causal_maskÚcreate_recurrent_attention_mask)ÚBaseModelOutputWithPastÚMoeModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚBambaConfig)Ú
BambaMixerÚBambaRMSNormGated)ÚGemma2RotaryEmbedding)
ÚGraniteFlashAttentionKwargsÚGraniteMoeSharedAttentionÚGraniteMoeSharedDecoderLayerÚGraniteMoeSharedForCausalLMÚGraniteMoeSharedMLPÚGraniteMoeSharedModelÚGraniteMoeSharedMoEÚGraniteMoeSharedPreTrainedModelÚapply_rotary_pos_embÚeager_attention_forwardé   )ÚGraniteMoeHybridConfigc                   óÚ   — e Zd ZdZ	 	 d
dej
                  dej
                  dz  dedz  deej
                  ej
                  f   dz  dee	   deej
                  ej
                  f   fd	„Z
y)ÚGraniteMoeHybridAttentionuæ   Hybrid variant that handles ``position_embeddings is None`` â€” granitemoe-hybrid configs can
    opt out of RoPE via ``position_embedding_type=None``, in which case the model passes ``None``
    instead of a ``(cos, sin)`` tuple.NÚhidden_statesÚattention_maskÚpast_key_valuesÚposition_embeddingsÚkwargsÚreturnc                 ó  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
|�|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t        «      } || ||	|
|f| j                  sdn| j                   | j"                  dœ|¤Ž\  }} |j$                  g |¢d‘­Ž j'                  «       }| j)                  |«      }||fS )Néÿÿÿÿr#   r   g        )ÚdropoutÚscaling)ÚshapeÚhead_dimÚq_projÚviewÚ	transposeÚk_projÚv_projr!   ÚupdateÚ	layer_idxr   Úget_interfaceÚconfigÚ_attn_implementationr"   ÚtrainingÚattention_dropoutr0   ÚreshapeÚ
contiguousÚo_proj)Úselfr'   r(   r)   r*   r+   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                   úˆ/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/granitemoehybrid/modular_granitemoehybrid.pyÚforwardz!GraniteMoeHybridAttention.forward7   s—  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆàÐ*Ø*‰HˆC�Ü';¸LÈ*ÐVYÐ[^Ó'_Ñ$ˆL˜*àÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(ó    )NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚtorchÚTensorr   Útupler   r   rN   © rO   rM   r&   r&   2   sŽ   „ ñ*ð )-ØHLñ%)à—|‘|ð%)ð Ÿ™ tÑ+ð%)ð  ™ð	%)ð
 # 5§<¡<°·±Ð#=Ñ>ÀÑEð%)ð Ð+Ñ,ð%)ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*ô%)rO   r&   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚGraniteMoeHybridMambaLayerr;   r9   c                 ó8   •— t         ‰| �  t        |«      |«       y ©N)ÚsuperÚ__init__r   ©rB   r;   r9   Ú	__class__s      €rM   r]   z#GraniteMoeHybridMambaLayer.__init__`   s   ø€ Ü‰Ñœ VÓ,¨iÕ8rO   )rP   rQ   rR   r$   Úintr]   Ú__classcell__©r_   s   @rM   rY   rY   _   s   ø„ ð9Ð5ð 9À#÷ 9ñ 9rO   rY   c                   ó    ‡ — e Zd Zdˆ fd„	Zˆ xZS )ÚGraniteMoeHybridRMSNormGatedc                 ó&   •— t         ‰| �  ||«       y r[   ©r\   r]   )rB   Úhidden_sizeÚepsr_   s      €rM   r]   z%GraniteMoeHybridRMSNormGated.__init__e   s   ø€ Ü‰Ñ˜ cÕ*rO   )g�íµ ÷Æ°>)rP   rQ   rR   r]   ra   rb   s   @rM   rd   rd   d   s   ø„ ÷+ñ +rO   rd   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚGraniteMoeHybridMLPr;   c                 ó$   •— t         ‰| �  |«       y r[   rf   ©rB   r;   r_   s     €rM   r]   zGraniteMoeHybridMLP.__init__j   s   ø€ Ü‰Ñ˜Õ rO   )rP   rQ   rR   r$   r]   ra   rb   s   @rM   rj   rj   i   s   ø„ ð!Ð5÷ !ñ !rO   rj   c                   ó   — e Zd Zy)ÚGraniteMoeHybridRotaryEmbeddingN©rP   rQ   rR   rW   rO   rM   rn   rn   n   ó   „ ØrO   rn   c                   ó   — e Zd Zy)ÚGraniteMoeHybridMoENro   rW   rO   rM   rr   rr   r   rp   rO   rr   c                   ó.  ‡ — e Zd Zdedefˆ fd„Ze	 	 	 	 ddej                  dej                  dz  de	dz  de
dz  d	eej                  ej                  f   dz  d
ee   deej                  eej                  ej                  f   dz  f   fd„«       Zˆ xZS )ÚGraniteMoeHybridDecoderLayerr;   r9   c                 ó`  •— t         ‰| �  ||«       t        |«      | _        d | _        d | _        |j                  |   dk(  rt        ||«      | _        nt        ||«      | _        |j                  |   | _	        |j                  dkD  rt        |«      nd | _        t        |dd«      dkD  | _        y )NÚlinear_attentionr   Únum_local_experts)r\   r]   rj   Ú
shared_mlpÚ	self_attnÚmambaÚlayers_block_typerY   r&   Ú
block_typerw   rr   Úblock_sparse_moeÚgetattrÚhas_expertsr^   s      €rM   r]   z%GraniteMoeHybridDecoderLayer.__init__w   sª   ø€ Ü‰Ñ˜ Ô+Ü-¨fÓ5ˆŒàˆŒØˆŒ
à×#Ñ# IÑ.Ð2DÒDÜ3°F¸IÓFˆD�Jä6°v¸yÓIˆDŒNØ ×2Ñ2°9Ñ=ˆŒð @F×?WÑ?WÐZ[Ò?[Ô 3°FÔ ;ÐaeˆÔô # 6Ð+>ÀÓBÀQÑFˆÕrO   Nr'   r(   r)   Ú	use_cacher*   r+   r,   c           	      óœ  — |}| j                  |«      }| j                  � | j                  d|||dœ|¤Ž}n | j                  d|||||dœ|¤Ž\  }}||| j                  z  z   }|}| j	                  |«      }| j
                  r&| j                  |«      }	|	| j                  |«      z   }n| j                  |«      }||| j                  z  z   }|S )N)r'   Úcache_paramsr(   )r'   r(   r)   r€   r*   rW   )Úinput_layernormrz   ry   Úresidual_multiplierÚpost_attention_layernormr   r}   rx   )
rB   r'   r(   r)   r€   r*   r+   ÚresidualÚ_Úmoe_hidden_statess
             rM   rN   z$GraniteMoeHybridDecoderLayer.forwardŠ   s  € ð !ˆØ×,Ñ,¨]Ó;ˆà�:‰:Ð!Ø&˜DŸJ™Jð Ø+Ø,Ø-ñð ñ	‰Mð  .˜tŸ~™~ð  Ø+Ø-Ø /Ø#Ø$7ñ ð ñ ÑˆM˜1ð ! =°4×3KÑ3KÑ#KÑKˆØ ˆØ×5Ñ5°mÓDˆà×ÒØ $× 5Ñ 5°mÓ DÐØ-°·±ÀÓ0NÑN‰Mà ŸO™O¨MÓ:ˆMà  =°4×3KÑ3KÑ#KÑKˆØÐrO   )NNFN)rP   rQ   rR   r$   r`   r]   r   rT   rU   r   ÚboolrV   r   r   ÚFloatTensorrN   ra   rb   s   @rM   rt   rt   v   sÞ   ø„ ðGÐ5ð GÀ#õ Gð& ð /3Ø(,Ø!&ØHLñ(à—|‘|ð(ð Ÿ™ tÑ+ð(ð  ™ð	(ð
 ˜$‘;ð(ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð(ð Ð4Ñ5ð(ð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	Uò(ó ô(rO   rt   c                   ó\   ‡ — e Zd ZU eed<   dgZdZ ej                  «       ˆ fd„«       Z	ˆ xZ
S )ÚGraniteMoeHybridPreTrainedModelr;   rt   Tc           
      óÊ  •— t         ‰| �  |«       t        |t        «      r“t	        j
                  |j                  «       t	        j                  |j                  t        j                  t        j                  d|j                  dz   «      «      «       t	        j
                  |j                  «       y t        |t        «      r t	        j
                  |j                  «       y y )Nr#   )r\   Ú_init_weightsÚ
isinstancerY   ÚinitÚones_Údt_biasÚcopy_ÚA_logrT   ÚlogÚarangeÚ	num_headsÚDrd   Úweight)rB   Úmoduler_   s     €rM   rŽ   z-GraniteMoeHybridPreTrainedModel._init_weights»   s�   ø€ ä‰Ñ˜fÔ%Ü�fÔ8Ô9Ü�J‰J�v—~‘~Ô&Ü�J‰J�v—|‘|¤U§Y¡Y¬u¯|©|¸A¸v×?OÑ?OÐRSÑ?SÓ/TÓ%UÔVÜ�J‰J�v—x‘xÕ Ü˜Ô <Ô=Ü�J‰J�v—}‘}Õ%ð >rO   )rP   rQ   rR   r$   Ú__annotations__Ú_no_split_modulesÚ_is_statefulrT   Úno_gradrŽ   ra   rb   s   @rM   rŒ   rŒ   ¶   s1   ø… Ø"Ó"Ø7Ð8ÐØ€Là€U‡]�]ƒ_ó&ó ô&rO   rŒ   c                   óø   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   deez  fd„«       «       «       Zˆ xZS )ÚGraniteMoeHybridModelr;   c           	      ó(  •— t         ‰| �  |«       t        j                  t	        |j
                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        |j                  | _        |j                  dk(  rt        |«      | _        y d | _        y c c}w )NÚrope)r\   r]   r   Ú
ModuleListÚrangeÚnum_hidden_layersrt   ÚlayersÚembedding_multiplierÚposition_embedding_typern   Ú
rotary_embr^   s      €rM   r]   zGraniteMoeHybridModel.__init__Ç   sz   ø€ Ü‰Ñ˜Ô Ü—m‘mÜNSÐTZ×TlÑTlÓNmÖnÀÔ)¨&°)Õ<Ònó
ˆŒð %+×$?Ñ$?ˆÔ!ØEK×EcÑEcÐgmÒEmÔ9¸&ÓAˆ�Ðswˆ�ùò os   ·BNÚ	input_idsr(   Úposition_idsr)   Úinputs_embedsr€   r+   r,   c           	      óÐ  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|| j                  z  }|r|€t        | j                  ¬«      }|€V|�|j                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      s(| j                  |||dœ}
t        d
i |
¤Žt        d
i |
¤Ždœ}	|}d }| j                  �| j                  ||«      }t!        | j"                  «      D ]-  \  }} ||f|	| j                  j$                  |      |||dœ|¤Ž}Œ/ | j'                  |«      }t)        ||¬	«      S )Nz:You must specify exactly one of input_ids or inputs_embeds)r;   r   r#   )Údevice)r;   r¬   r(   r)   )Úfull_attentionrv   )r(   r)   r€   r*   )Úlast_hidden_stater)   rW   )Ú
ValueErrorÚembed_tokensr§   r   r;   Úget_seq_lengthrT   r–   r1   r®   Ú	unsqueezer�   Údictr	   r
   r©   Ú	enumerater¦   r{   Únormr   )rB   rª   r(   r«   r)   r¬   r€   r+   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr'   r*   ÚiÚdecoder_layers                  rM   rN   zGraniteMoeHybridModel.forwardÏ   s   € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMà%¨×(AÑ(AÑAˆá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-ÄÔFð Ÿ+™+Ø!.Ø"0Ø#2ñ	ˆKô #5Ñ"C°{Ñ"CÜ$CÑ$RÀkÑ$Rñ#Ðð &ˆØ"ÐØ�?‰?Ð&Ø"&§/¡/°-ÀÓ"NÐä )¨$¯+©+Ó 6ò 	ÑˆAˆ}Ù)Øðà2°4·;±;×3PÑ3PÐQRÑ3SÑTØ /Ø#Ø$7ñð ñ‰Mð	ð Ÿ	™	 -Ó0ˆä%Ø+Ø+ô
ð 	
rO   )NNNNNN)rP   rQ   rR   r$   r]   r   r   r   rT   Ú
LongTensorrU   r   rŠ   r‰   r   r   rV   r   rN   ra   rb   s   @rM   r    r    Æ   sÖ   ø„ ðxÐ5õ xð ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ<
à×#Ñ# dÑ*ð<
ð Ÿ™ tÑ+ð<
ð ×&Ñ&¨Ñ-ð	<
ð
  ™ð<
ð ×(Ñ(¨4Ñ/ð<
ð ˜$‘;ð<
ð Ð4Ñ5ð<
ð 
Ð(Ñ	(ò<
ó ó  ó ô<
rO   r    c                   ó6   ‡ — e Zd ZddiZdefˆ fd„Zˆ fd„Zˆ xZS )ÚGraniteMoeHybridForCausalLMzlm_head.weightzmodel.embed_tokens.weightr;   c                 ód   •— t         ‰| �  |«       t        |«      | _        | j	                  «        y r[   )r\   r]   r    ÚmodelÚ	post_initrl   s     €rM   r]   z$GraniteMoeHybridForCausalLM.__init__  s&   ø€ Ü‰Ñ˜Ô Ü*¨6Ó2ˆŒ
à�‰ÕrO   c                 ó"   •— t        ‰| �  di |¤ŽS )a–  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, GraniteMoeHybridForCausalLM

        >>> model = GraniteMoeHybridForCausalLM.from_pretrained("ibm-granite/granite-4.0-h-tiny")
        >>> tokenizer = AutoTokenizer.from_pretrained("ibm-granite/granite-4.0-h-tiny")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```rW   )r\   rN   )rB   Úsuper_kwargsr_   s     €rM   rN   z#GraniteMoeHybridForCausalLM.forward  s   ø€ ô. ‰w‰Ñ. Ñ.Ð.rO   )rP   rQ   rR   Ú_tied_weights_keysr$   r]   rN   ra   rb   s   @rM   r¿   r¿     s&   ø„ Ø*Ð,GÐHÐðÐ5õ ÷/ð /rO   r¿   )r¿   r    rŒ   )>Úcollections.abcr   rT   r   Ú r   r�   Úcache_utilsr   r   Úmasking_utilsr	   r
   Úmodeling_outputsr   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   Úbamba.configuration_bambar   Úbamba.modeling_bambar   r   Úgemma2.modeling_gemma2r   Ú*granitemoeshared.modeling_granitemoesharedr   r   r   r   r   r   r   r    r!   r"   Úconfiguration_granitemoehybridr$   Ú
get_loggerrP   Úloggerr&   rY   rd   rj   rn   rr   rt   rŒ   r    r¿   Ú__all__rW   rO   rM   ú<module>rØ      sï   ðõ %ã Ý å &ß .ß Pß OÝ 5Ý &ß @Ñ @Ý 7Ý 5Ý 3ß @Ý :÷÷ ÷ õ Cð 
ˆ×	Ñ	˜HÓ	%€ô*)Ð 9ô *)ôZ9 ô 9ô
+Ð#4ô +ô
!Ð-ô !ô
	Ð&;ô 	ô	Ð-ô 	ô=Ð#?ô =ô@&Ð&Eô &ô H
Ð1ô H
ôV /Ð"=ô  /òF f�rO   