
    ^j0                        d dl mZ d dlZd dlmZ ddlmZ ddlmZm	Z	 ddl
mZmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZmZ ddlmZ ddlmZ ddlmZ ddlmZm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z- ddl.m/Z/  ej`                  e1      Z2 G d de%      Z3 G d de      Z4 G d de       Z5 G d de(      Z6 G d de"      Z7 G d d e*      Z8 G d! d"e&      Z9 G d# d$e+      Z: G d% d&e)      Z; G d' d(e'      Z<g d)Z=y)*    )CallableN)nn   )initialization)CacheDynamicCache)create_causal_maskcreate_recurrent_attention_mask)BaseModelOutputWithPastMoeModelOutputWithPast)ALL_ATTENTION_FUNCTIONS)Unpack)TransformersKwargsauto_docstringlogging)merge_with_config_defaults)capture_outputs   )BambaConfig)
BambaMixerBambaRMSNormGated)Gemma2RotaryEmbedding)
GraniteFlashAttentionKwargsGraniteMoeSharedAttentionGraniteMoeSharedDecoderLayerGraniteMoeSharedForCausalLMGraniteMoeSharedMLPGraniteMoeSharedModelGraniteMoeSharedMoEGraniteMoeSharedPreTrainedModelapply_rotary_pos_embeager_attention_forward   )GraniteMoeHybridConfigc                       e Zd ZdZ	 	 d
dej
                  dej
                  dz  dedz  deej
                  ej
                  f   dz  dee	   deej
                  ej
                  f   fd	Z
y)GraniteMoeHybridAttentionu   Hybrid variant that handles ``position_embeddings is None`` — granitemoe-hybrid configs can
    opt out of RoPE via ``position_embedding_type=None``, in which case the model passes ``None``
    instead of a ``(cos, sin)`` tuple.Nhidden_statesattention_maskpast_key_valuesposition_embeddingskwargsreturnc                    |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }	| j                  |      j                  |      j	                  dd      }
||\  }}t        ||	||      \  }}	| |j                  |	|
| j                        \  }	}
t        j                  | j                  j                  t              } || ||	|
|f| j                  sdn| j                   | j"                  d|\  }} |j$                  g |d j'                         }| j)                  |      }||fS )Nr#   r   g        )dropoutscaling)shapehead_dimq_projview	transposek_projv_projr!   update	layer_idxr   get_interfaceconfig_attn_implementationr"   trainingattention_dropoutr0   reshape
contiguouso_proj)selfr'   r(   r)   r*   r+   input_shapehidden_shapequery_states
key_statesvalue_statescossinattention_interfaceattn_outputattn_weightss                   /var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/granitemoehybrid/modular_granitemoehybrid.pyforwardz!GraniteMoeHybridAttention.forward7   s    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST**HC';L*VY[^'_$L*&'6'='=j,X\XfXf'g$J(?(M(MKK,,.E)
 %8	%
  $}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((    )NN)__name__
__module____qualname____doc__torchTensorr   tupler   r   rN    rO   rM   r&   r&   2   s    * )-HL%)||%) t+%) 	%)
 #5<<#=>E%) +,%) 
u||U\\)	*%)rO   r&   c                   (     e Zd Zdedef fdZ xZS )GraniteMoeHybridMambaLayerr;   r9   c                 8    t         |   t        |      |       y N)super__init__r   rB   r;   r9   	__class__s      rM   r]   z#GraniteMoeHybridMambaLayer.__init__`   s    V,i8rO   )rP   rQ   rR   r$   intr]   __classcell__r_   s   @rM   rY   rY   _   s    95 9# 9 9rO   rY   c                         e Zd Zd fd	Z xZS )GraniteMoeHybridRMSNormGatedc                 &    t         |   ||       y r[   r\   r]   )rB   hidden_sizeepsr_   s      rM   r]   z%GraniteMoeHybridRMSNormGated.__init__e   s    c*rO   )gư>)rP   rQ   rR   r]   ra   rb   s   @rM   rd   rd   d   s    + +rO   rd   c                   $     e Zd Zdef fdZ xZS )GraniteMoeHybridMLPr;   c                 $    t         |   |       y r[   rf   rB   r;   r_   s     rM   r]   zGraniteMoeHybridMLP.__init__j   s     rO   )rP   rQ   rR   r$   r]   ra   rb   s   @rM   rj   rj   i   s    !5 ! !rO   rj   c                       e Zd Zy)GraniteMoeHybridRotaryEmbeddingNrP   rQ   rR   rW   rO   rM   rn   rn   n       rO   rn   c                       e Zd Zy)GraniteMoeHybridMoENro   rW   rO   rM   rr   rr   r   rp   rO   rr   c                   .    e Zd Zdedef fdZe	 	 	 	 ddej                  dej                  dz  de	dz  de
dz  d	eej                  ej                  f   dz  d
ee   deej                  eej                  ej                  f   dz  f   fd       Z xZS )GraniteMoeHybridDecoderLayerr;   r9   c                 `   t         |   ||       t        |      | _        d | _        d | _        |j                  |   dk(  rt        ||      | _        nt        ||      | _        |j                  |   | _	        |j                  dkD  rt        |      nd | _        t        |dd      dkD  | _        y )Nlinear_attentionr   num_local_experts)r\   r]   rj   
shared_mlp	self_attnmambalayers_block_typerY   r&   
block_typerw   rr   block_sparse_moegetattrhas_expertsr^   s      rM   r]   z%GraniteMoeHybridDecoderLayer.__init__w   s    +-f5
##I.2DD3FIFDJ6vyIDN 229= @F?W?WZ[?[ 3F ;ae #6+>BQFrO   Nr'   r(   r)   	use_cacher*   r+   r,   c           	         |}| j                  |      }| j                   | j                  d|||d|}n | j                  d|||||d|\  }}||| j                  z  z   }|}| j	                  |      }| j
                  r&| j                  |      }	|	| j                  |      z   }n| j                  |      }||| j                  z  z   }|S )N)r'   cache_paramsr(   )r'   r(   r)   r   r*   rW   )input_layernormrz   ry   residual_multiplierpost_attention_layernormr   r}   rx   )
rB   r'   r(   r)   r   r*   r+   residual_moe_hidden_statess
             rM   rN   z$GraniteMoeHybridDecoderLayer.forward   s    !,,];::!&DJJ +,- 	M  .t~~  +- /#$7   M1 !=43K3K#KK 55mD $ 5 5m D-0NNM OOM:M =43K3K#KKrO   )NNFN)rP   rQ   rR   r$   r`   r]   r   rT   rU   r   boolrV   r   r   FloatTensorrN   ra   rb   s   @rM   rt   rt   v   s    G5 G# G&  /3(,!&HL(||( t+( 	(
 $;( #5<<#=>E( 45( 
u  %(9(95;L;L(L"MPT"TT	U( (rO   rt   c                   \     e Zd ZU eed<   dgZdZ ej                          fd       Z	 xZ
S )GraniteMoeHybridPreTrainedModelr;   rt   Tc           
         t         |   |       t        |t              rt	        j
                  |j                         t	        j                  |j                  t        j                  t        j                  d|j                  dz                      t	        j
                  |j                         y t        |t              r t	        j
                  |j                         y y )Nr#   )r\   _init_weights
isinstancerY   initones_dt_biascopy_A_logrT   logarange	num_headsDrd   weight)rB   moduler_   s     rM   r   z-GraniteMoeHybridPreTrainedModel._init_weights   s    f%f89JJv~~&JJv||UYYu||Av?O?ORS?S/T%UVJJvxx  <=JJv}}% >rO   )rP   rQ   rR   r$   __annotations___no_split_modules_is_statefulrT   no_gradr   ra   rb   s   @rM   r   r      s1    ""78LU]]_& &rO   r   c                        e Zd Zdef fdZeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   deez  fd                     Z xZS )GraniteMoeHybridModelr;   c           	      (   t         |   |       t        j                  t	        |j
                        D cg c]  }t        ||       c}      | _        |j                  | _        |j                  dk(  rt        |      | _        y d | _        y c c}w )Nrope)r\   r]   r   
ModuleListrangenum_hidden_layersrt   layersembedding_multiplierposition_embedding_typern   
rotary_embr^   s      rM   r]   zGraniteMoeHybridModel.__init__   sz     mmNSTZTlTlNmn)&)<n
 %+$?$?!EKEcEcgmEm9&Asw os   BN	input_idsr(   position_idsr)   inputs_embedsr   r+   r,   c           	         |d u |d uz  rt        d      || j                  |      }|| j                  z  }|r|t        | j                        }|V||j                         nd}t        j                  |j                  d   |j                        |z   }|j                  d      }t        |x}	t              s(| j                  |||d}
t        d
i |
t        d
i |
d}	|}d }| j                  | j                  ||      }t!        | j"                        D ]-  \  }} ||f|	| j                  j$                  |      |||d|}/ | j'                  |      }t)        ||	      S )Nz:You must specify exactly one of input_ids or inputs_embeds)r;   r   r#   )device)r;   r   r(   r)   )full_attentionrv   )r(   r)   r   r*   )last_hidden_stater)   rW   )
ValueErrorembed_tokensr   r   r;   get_seq_lengthrT   r   r1   r   	unsqueezer   dictr	   r
   r   	enumerater   r{   normr   )rB   r   r(   r   r)   r   r   r+   past_seen_tokenscausal_mask_mappingmask_kwargsr'   r*   idecoder_layers                  rM   rN   zGraniteMoeHybridModel.forward   s    -t";<YZZ  --i8M%(A(AA0*$++>OCRC^==?de <<(;(;A(>}G[G[\_ooL'11!4L?-F ++!."0#2	K #5"C{"C$C$Rk$R# &"??&"&//-"N )$++ 6 	A})24;;3P3PQR3ST /#$7 M	 		-0%++
 	
rO   )NNNNNN)rP   rQ   rR   r$   r]   r   r   r   rT   
LongTensorrU   r   r   r   r   r   rV   r   rN   ra   rb   s   @rM   r   r      s    x5 x  .2.204(,26!%<
##d*<
 t+<
 &&-	<

 <
 ((4/<
 $;<
 45<
 
(	(<
    <
rO   r   c                   6     e Zd ZddiZdef fdZ fdZ xZS )GraniteMoeHybridForCausalLMzlm_head.weightzmodel.embed_tokens.weightr;   c                 d    t         |   |       t        |      | _        | j	                          y r[   )r\   r]   r   model	post_initrl   s     rM   r]   z$GraniteMoeHybridForCausalLM.__init__  s&     *62
rO   c                 "    t        |   di |S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, GraniteMoeHybridForCausalLM

        >>> model = GraniteMoeHybridForCausalLM.from_pretrained("ibm-granite/granite-4.0-h-tiny")
        >>> tokenizer = AutoTokenizer.from_pretrained("ibm-granite/granite-4.0-h-tiny")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```rW   )r\   rN   )rB   super_kwargsr_   s     rM   rN   z#GraniteMoeHybridForCausalLM.forward  s    . w...rO   )rP   rQ   rR   _tied_weights_keysr$   r]   rN   ra   rb   s   @rM   r   r     s&    *,GH5 / /rO   r   )r   r   r   )>collections.abcr   rT   r    r   r   cache_utilsr   r   masking_utilsr	   r
   modeling_outputsr   r   modeling_utilsr   processing_utilsr   utilsr   r   r   utils.genericr   utils.output_capturingr   bamba.configuration_bambar   bamba.modeling_bambar   r   gemma2.modeling_gemma2r   *granitemoeshared.modeling_granitemoesharedr   r   r   r   r   r   r   r    r!   r"   configuration_granitemoehybridr$   
get_loggerrP   loggerr&   rY   rd   rj   rn   rr   rt   r   r   r   __all__rW   rO   rM   <module>r      s    %   & . P O 5 & @ @ 7 5 3 @ :   C 
		H	%*) 9 *)Z9 9
+#4 +
!- !
	&; 		- 	=#? =@&&E & H
1 H
V /"=  /F frO   