
    ^j9                     V   d dl mZ d dlZd dlmc mZ d dlmZ ddlmZm	Z	 ddl
mZ ddlmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZ ddlmZmZ ddlmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( ddl)m*Z*  e       r	d dl+m,Z,m-Z- nd\  Z,Z-e,e-fZ. e/e.      Z0 ejb                  e2      Z3 G d de&      Z4 G d de       Z5 G d dejl                        Z7 G d de"      Z8 G d dejl                        Z9 G d  d!e      Z: G d" d#e%      Z; G d$ d%e$      Z< G d& d'e#      Z=g d(Z>y))    )CallableN)nn   )CacheDynamicCache)force_accelerate_hooks)create_causal_maskcreate_recurrent_attention_mask)GradientCheckpointingLayer)BaseModelOutputWithPast)ALL_ATTENTION_FUNCTIONS)Unpack)TransformersKwargslogging)is_causal_conv1d_availableis_torchdynamo_compiling   )apply_mask_to_padding_states)Gemma2RotaryEmbedding)LlamaAttentionLlamaForCausalLM
LlamaModelLlamaPreTrainedModelLlamaRMSNormapply_rotary_pos_embeager_attention_forward   )
Lfm2Config)causal_conv1d_fncausal_conv1d_update)NNc                       e Zd Zy)Lfm2RMSNormN__name__
__module____qualname__     p/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/lfm2/modular_lfm2.pyr"   r"   8       r(   r"   c                       e Zd Zy)Lfm2RotaryEmbeddingNr#   r'   r(   r)   r,   r,   <   r*   r(   r,   c                   *     e Zd Zdef fdZd Z xZS )Lfm2MLPconfigc                    t         |           |j                  }|j                  rat	        d|z  dz        }|j
                  Dt	        |j
                  |z        }|j                  ||j                  z   dz
  |j                  z  z  }t        j                  |j                  |d      | _
        t        j                  |j                  |d      | _        t        j                  ||j                  d      | _        y )Nr   r   r   Fbias)super__init__intermediate_sizeblock_auto_adjust_ff_dimintblock_ffn_dim_multiplierblock_multiple_ofr   Linearhidden_sizew1w3w2)selfr/   r5   	__class__s      r)   r4   zLfm2MLP.__init__A   s    "44** #A(9$9A$= >..:$'(G(GJ[([$\!$*$<$<&)A)AAAE&JbJbb%! ))F..0AN))F..0AN))-v/A/ANr(   c                     | j                  t        j                  | j                  |            | j	                  |      z        S N)r>   Fsilur<   r=   )r?   xs     r)   forwardzLfm2MLP.forwardP   s/    wwqvvdggaj)DGGAJ677r(   )r$   r%   r&   r   r4   rF   __classcell__r@   s   @r)   r.   r.   @   s    Oz O8r(   r.   c                        e Zd Zdedef fdZ	 ddej                  deej                  ej                  f   dej                  dz  de	dz  d	eej                  ej                  dz  f   f
d
Z
 xZS )Lfm2Attentionr/   	layer_idxc                    t         |   ||       t        j                  |j                  |j
                  | j                  z  d      | _        t        j                  |j                  |j                  | j                  z  d      | _	        t        j                  |j                  |j                  | j                  z  d      | _
        t        j                  |j
                  | j                  z  |j                  d      | _        t        | j                  |j                        | _        t        | j                  |j                        | _        | `| `y )NFr1   eps)r3   r4   r   r:   r;   num_attention_headshead_dimq_projnum_key_value_headsk_projv_projout_projr"   norm_epsq_layernormk_layernormo_projattention_dropoutr?   r/   rK   r@   s      r)   r4   zLfm2Attention.__init__U   s    +ii 2 2F4N4NQUQ^Q^4^ejkii 2 2F4N4NQUQ^Q^4^ejkii 2 2F4N4NQUQ^Q^4^ejk		&"<"<t}}"LfN`N`glm&t}}&//J&t}}&//JK"r(   Nhidden_statesposition_embeddingsattention_maskpast_key_valuesreturnc                 
   |j                   d d }g |d| j                  }| j                   | j                  |      j                  |       j                  dd      }| j                   | j                  |      j                  |       j                  dd      }	 | j                  |      j                  | j                  dd      }
|\  }}t        ||	||      \  }}	| |j                  |	|
| j                        \  }	}
t        j                  | j                  j                  t               } || ||	|
|fd| j"                  d|\  }} |j$                  g |d j'                         }| j)                  |      }||fS )Nr   r   g        )dropoutscaling)shaperP   rW   rQ   view	transposerX   rS   rT   r   updaterK   r   get_interfacer/   _attn_implementationr   rd   reshape
contiguousrU   )r?   r\   r]   r^   r_   kwargsinput_shapehidden_shapequery_states
key_statesvalue_statescossinattention_interfaceattn_outputattn_weightsoutputs                    r)   rF   zLfm2Attention.forward`   s    $))#2.88b8$--8''(GM(B(G(G(VWaabcefg%%&Edkk-&@&E&E|&TU__`acde
6t{{=166EOOPQSTU&S#7jRUWZ#[ j&'6'='=j,X\XfXf'g$J(?(M(MKK,,.E)
 %8	%
 LL	%
 	%
!\ *k));;;;FFH{+|##r(   rB   )r$   r%   r&   r   r7   r4   torchTensortupler   rF   rG   rH   s   @r)   rJ   rJ   T   s    	#z 	#c 	#  )-%$||%$ #5<<#=>%$ t+	%$
 %$ 
u||U\\D00	1%$r(   rJ   c                       e Zd Zdedef fdZ	 	 	 ddej                  dedz  dej                  dz  dej                  dz  fd	Z
	 	 	 ddej                  dedz  dej                  dz  dej                  dz  fd
Z ed      	 	 	 ddej                  dedz  dej                  dz  dej                  dz  fd       Z xZS )Lfm2ShortConvr/   rK   c           	      Z   t         |           || _        || _        |j                  | _        |j                  | _        t        j                  |j                  |j                  | j
                  |j                  | j                  | j
                  dz
        | _        t        j                  |j                  d|j                  z  | j                        | _        t        j                  |j                  |j                  | j                        | _        |j                  |   | _        y )Nr   )in_channelsout_channelskernel_sizegroupsr2   paddingr   r1   )r3   r4   r/   rK   conv_L_cacheL_cache	conv_biasr2   r   Conv1dr;   convr:   in_projrU   layer_types
layer_typer[   s      r)   r4   zLfm2ShortConv.__init__   s    
 	"**$$	II**++%%LL1$
	 yy!3!3Q9K9K5KRVR[R[\		&"4"4f6H6HtyyY ,,Y7r(   NrE   r_   r^   seq_idxc                    t        ||      }| j                  |      j                  dd      }|j                  dd      \  }}}||z  }| j                  j
                  j                  | j                  j
                  j                  d      | j                  j
                  j                  d            }	||j                  | j                        rht        |j                  d      |j                  | j                     j                  d   |	| j                  j                  d       }
|
j                  d      }
n|jt         j"                  j%                  || j&                  |j(                  d   z
  df      }|j+                  || j                        d| j&                   d f   }t-        ||	| j                  j                  d |      }
||
z  }| j/                  |j                  dd      j1                               }|S )	Nrb   r   dimr   r   .)
activationr   )r   r   rg   chunkr   weightrf   sizehas_previous_staterK   r    squeezelayersconv_statesr2   	unsqueezer   
functionalpadr   re   update_conv_stater   rU   rl   )r?   rE   r_   r^   r   BCxBCBxconv_weightsconv_out
conv_stateys                r)   cuda_kernels_forwardz"Lfm2ShortConv.cuda_kernels_forward   s    )N;ll1o''B/))A2)&1aUyy'',,TYY-=-=-B-B1-EtyyGWGWG\G\]^G_`&?+M+Mdnn+]+

2&&t~~6BB1E		H  ))"-H*]]..rDLL288B<4OQR3ST
,>>z4>>Z[^aeamam`m`o[op
 (L$))..UYcjkHLMM!++b"-88:;r(   c           
         |j                   d   }t        ||      }| j                  |      j                  dd      }|j	                  dd      \  }}}||z  }	||j                  | j                        r|j                  |	| j                        d| j                   d f   }
t        j                  |
j                  |	j                        | j                  j                  d d dd d f   z  d      }| j                  r|| j                  j                  z  }|j!                  d      }n|w|j                   d   dk(  rd|jt"        j$                  j'                  |	| j                  |	j                   d   z
  df      }
|j                  |
| j                        d| j                   d f   }
|d   }|dd  |d d k7  j)                  d	      d   dz   }t        j*                  |j-                  d      ||j/                  d
|j1                               g      j3                         }g }t5        t7        |      dz
        D ]K  }||   ||dz      }}||kD  s|j9                  | j                  |	d d d d ||f         dd ||z
  f          M t        j*                  |d      }n|jt"        j$                  j'                  |	| j                  |	j                   d   z
  df      }
|j                  |
| j                        d| j                   d f   }
| j                  |	      dd |f   }||z  }|j                  dd      j;                         }| j=                  |      }|S )Nr   rb   r   r   r   .r   T)as_tuple)r   )re   r   r   rg   r   r   rK   r   r   ry   sumtodevicer   r   r2   r   r   r   r   nonzerocat	new_zerosnew_fullnumeltolistrangelenappendrl   rU   )r?   rE   r_   r^   r   seqlenr   r   r   r   r   r   sichangeboundspartsiser   s                       r)   slow_forwardzLfm2ShortConv.slow_forward   s    (N;ll1o''B/))A2)&1aU&?+M+Mdnn+](::2t~~NsUYUaUaTaTcOcdJyyryy!9DII<L<LQPQSTW<U!U[]^HyyDIINN*))"-H QWWQZ1_*]]..rDLL288B<4OQR3ST
,>>z4>>Z[^aeamam`m`o[op
Bf3B'00$0?BQFFYY 0 0 3VV__TSUS[S[S]=^_`ggiFE3v;?+ Iay&Q-1q5LL2aAaCi=!9#wQw,!GHI yyB/H*]]..rDLL288B<4OQR3ST
,>>z4>>Z[^aeamam`m`o[op
yy}S'6'\2HLKKB**,MM!r(   r   r\   c                     t         r7d|j                  j                  v rt               s| j	                  ||||      S | j                  ||||      S )Ncuda)r   )is_fast_path_availabler   typer   r   r   )r?   r\   r_   r^   r   s        r)   rF   zLfm2ShortConv.forward   sV     "f0D0D0I0I&IRjRl,,]O^el,mm  Y` aar(   )NNN)r$   r%   r&   r   r7   r4   ry   rz   r   	IntTensorr   r   r   rF   rG   rH   s   @r)   r}   r}      s&   88 86 )-.2*.!<<! ! t+	!
 4'!L )-.2*..<<. . t+	.
 4'.` F# )-.2*.	b||	b 	b t+		b
 4'	b $	br(   r}   c                        e Zd Zdedef fdZ	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	e
dz  d
ej                  fdZ xZS )Lfm2DecoderLayerr/   rK   c                 f   t         |           |j                  |   dk(  | _        | j                  rt	        ||      | _        nt        ||      | _        t        |      | _	        t        |j                  |j                        | _        t        |j                  |j                        | _        y )Nfull_attentionrM   )r3   r4   r   is_attention_layerrJ   	self_attnr}   r   r.   feed_forwardr"   r;   rV   operator_normffn_normr[   s      r)   r4   zLfm2DecoderLayer.__init__  s    "("4"4Y"?CS"S""*69=DN%fi8DI#FO(););Q#F$6$6FOOLr(   Nr\   r]   r^   position_idsr_   r`   c           	      .   |}| j                   r+ | j                  d| j                  |      ||||d|\  }}n3| j                  | j                  |      |||j	                  d            }||z   }|| j                  | j                  |            z   }|S )N)r\   r]   r^   r   r_   r   )r\   r_   r^   r   r'   )r   r   r   r   getr   r   )	r?   r\   r]   r^   r   r_   rm   residual_s	            r)   rF   zLfm2DecoderLayer.forward  s     !""-t~~  "00?$7-) /   M1 !II"00? /-

9-	 & M &0%(9(9$--:V(WWr(   )NNNN)r$   r%   r&   r   r7   r4   ry   rz   r{   
LongTensorr   rF   rG   rH   s   @r)   r   r     s    
Mz 
Mc 
M IM.204(,|| #5<<#=>E t+	
 &&-  
r(   r   c                       e Zd ZdZy)Lfm2PreTrainedModelTN)r$   r%   r&   _is_statefulr'   r(   r)   r   r   .  s    Lr(   r   c                        e Zd Zdef fdZ	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	e
dz  d
ee   defdZ xZS )	Lfm2Modelr/   c                 t    t         |   |       t        |j                  |j                        | _        | `y )NrM   )r3   r4   r"   r;   rV   embedding_normnorm)r?   r/   r@   s     r)   r4   zLfm2Model.__init__3  s.     )&*<*<&//RIr(   N	input_idsr^   r   r_   inputs_embeds	use_cacherm   r`   c           	         |d u |d uz  rt        d      || j                  |      }|r|t        | j                        }|V||j	                         nd}t        j                  |j                  d   |j                        |z   }|j                  d      }t        |x}	t              s)| j                  ||||d}
t        di |
t        di |
d}	|}| j                  ||      }t        | j                   d | j                  j"                         D ]-  \  }} ||f|	| j                  j$                  |      |||d	|}/ | j'                  |      }t)        ||
      S )Nz:You must specify exactly one of input_ids or inputs_embeds)r/   r   r   )r   )r/   r   r^   r_   r   )r   r   )r   )r^   r]   r   r_   )last_hidden_stater_   r'   )
ValueErrorembed_tokensr   r/   get_seq_lengthry   arangere   r   r   
isinstancedictr	   r
   
rotary_emb	enumerater   num_hidden_layersr   r   r   )r?   r   r^   r   r_   r   r   rm   past_seen_tokenscausal_mask_mappingmask_kwargsr\   r]   r   decoder_layers                  r)   rF   zLfm2Model.forward8  s    -t";<YZZ  --i8M0*$++>OCRC^==?de <<(;(;A(>}G[G[\_ooL'11!4L?-F++!."0#2 ,K #5"C{"C7F+F#
 &"oom,oW !*$++6U8U8U*V W 	A})24;;3J3J13MN$7) / M	 ++M:&++
 	
r(   )NNNNNN)r$   r%   r&   r   r4   ry   r   rz   r   FloatTensorboolr   r   r   rF   rG   rH   s   @r)   r   r   2  s    z  .2.204(,26!%8
##d*8
 t+8
 &&-	8

 8
 ((4/8
 $;8
 +,8
 
!8
r(   r   c                       e Zd Zy)Lfm2ForCausalLMNr#   r'   r(   r)   r   r   s  r*   r(   r   )r   r   r   )?collections.abcr   ry   torch.nn.functionalr   r   rC   cache_utilsr   r   integrations.accelerater   masking_utilsr	   r
   modeling_layersr   modeling_outputsr   modeling_utilsr   processing_utilsr   utilsr   r   utils.import_utilsr   r   bamba.modeling_bambar   gemma2.modeling_gemma2r   llama.modeling_llamar   r   r   r   r   r   r   configuration_lfm2r   causal_conv1dr   r    kernel_modulesallr   
get_loggerr$   loggerr"   r,   Moduler.   rJ   r}   r   r   r   r   __all__r'   r(   r)   <module>r     s   %     . = P 9 7 5 & 0 V ? :   + DD-7** #$89^,  
		H	%	, 		/ 	8bii 8(1$N 1$hvbBII vbr*1 *Z. >

 >
B	& 	 Br(   