Ë
    ÜÓ^jþ9  ã                   óV  — d dl mZ d dlZd dlmc mZ d dlmZ ddlmZm	Z	 ddl
mZ ddlmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZ ddlmZmZ ddlmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( ddl)m*Z*  e«       r	d dl+m,Z,m-Z- nd\  Z,Z-e,e-fZ. e/e.«      Z0 ejb                  e2«      Z3 G d„ de&«      Z4 G d„ de «      Z5 G d„ dejl                  «      Z7 G d„ de"«      Z8 G d„ dejl                  «      Z9 G d „ d!e«      Z: G d"„ d#e%«      Z; G d$„ d%e$«      Z< G d&„ d'e#«      Z=g d(¢Z>y))é    )ÚCallableN)Únné   )ÚCacheÚDynamicCache)Úforce_accelerate_hooks)Úcreate_causal_maskÚcreate_recurrent_attention_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚlogging)Úis_causal_conv1d_availableÚis_torchdynamo_compilingé   )Úapply_mask_to_padding_states)ÚGemma2RotaryEmbedding)ÚLlamaAttentionÚLlamaForCausalLMÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRMSNormÚapply_rotary_pos_embÚeager_attention_forwardé   )Ú
Lfm2Config)Úcausal_conv1d_fnÚcausal_conv1d_update)NNc                   ó   — e Zd Zy)ÚLfm2RMSNormN©Ú__name__Ú
__module__Ú__qualname__© ó    úp/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/lfm2/modular_lfm2.pyr"   r"   8   ó   „ Ør(   r"   c                   ó   — e Zd Zy)ÚLfm2RotaryEmbeddingNr#   r'   r(   r)   r,   r,   <   r*   r(   r,   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚLfm2MLPÚconfigc                 óþ  •— t         ‰| �  «        |j                  }|j                  rat	        d|z  dz  «      }|j
                  �Dt	        |j
                  |z  «      }|j                  ||j                  z   dz
  |j                  z  z  }t        j                  |j                  |d¬«      | _
        t        j                  |j                  |d¬«      | _        t        j                  ||j                  d¬«      | _        y )Nr   r   r   F©Úbias)ÚsuperÚ__init__Úintermediate_sizeÚblock_auto_adjust_ff_dimÚintÚblock_ffn_dim_multiplierÚblock_multiple_ofr   ÚLinearÚhidden_sizeÚw1Úw3Úw2)Úselfr/   r5   Ú	__class__s      €r)   r4   zLfm2MLP.__init__A   sÞ   ø€ Ü‰ÑÔØ"×4Ñ4ÐØ×*Ò*Ü # AÐ(9Ñ$9¸AÑ$=Ó >Ðà×.Ñ.Ð:Ü$'¨×(GÑ(GÐJ[Ñ([Ó$\Ð!Ø$*×$<Ñ$<Ø&¨×)AÑ)AÑAÀAÑEÈ&×JbÑJbÑbñ%Ð!ô —)‘)˜F×.Ñ.Ð0AÈÔNˆŒÜ—)‘)˜F×.Ñ.Ð0AÈÔNˆŒÜ—)‘)Ð-¨v×/AÑ/AÈÔNˆ�r(   c                 óŒ   — | j                  t        j                  | j                  |«      «      | j	                  |«      z  «      S ©N)r>   ÚFÚsilur<   r=   )r?   Úxs     r)   ÚforwardzLfm2MLP.forwardP   s/   € Ø�w‰w”q—v‘v˜dŸg™g a›jÓ)¨D¯G©G°A«JÑ6Ó7Ð7r(   )r$   r%   r&   r   r4   rF   Ú__classcell__©r@   s   @r)   r.   r.   @   s   ø„ ðO˜zõ Oö8r(   r.   c                   óæ   ‡ — e Zd Zdedefˆ fd„Z	 ddej                  deej                  ej                  f   dej                  dz  de	dz  d	eej                  ej                  dz  f   f
d
„Z
ˆ xZS )ÚLfm2Attentionr/   Ú	layer_idxc                 ó¶  •— t         ‰| �  ||«       t        j                  |j                  |j
                  | j                  z  d¬«      | _        t        j                  |j                  |j                  | j                  z  d¬«      | _	        t        j                  |j                  |j                  | j                  z  d¬«      | _
        t        j                  |j
                  | j                  z  |j                  d¬«      | _        t        | j                  |j                  ¬«      | _        t        | j                  |j                  ¬«      | _        | `| `y )NFr1   ©Úeps)r3   r4   r   r:   r;   Únum_attention_headsÚhead_dimÚq_projÚnum_key_value_headsÚk_projÚv_projÚout_projr"   Únorm_epsÚq_layernormÚk_layernormÚo_projÚattention_dropout©r?   r/   rK   r@   s      €r)   r4   zLfm2Attention.__init__U   sü   ø€ Ü‰Ñ˜ Ô+Ü—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆŒÜŸ	™	 &×"<Ñ"<¸t¿}¹}Ñ"LÈf×N`ÑN`ÐglÔmˆŒÜ& t§}¡}¸&¿/¹/ÔJˆÔÜ& t§}¡}¸&¿/¹/ÔJˆÔØˆKØÑ"r(   NÚhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valuesÚreturnc                 ó
  — |j                   d d }g |¢d‘| j                  ‘­}| j                   | j                  |«      j                  |Ž «      j                  dd«      }| j                   | j                  |«      j                  |Ž «      j                  dd«      }	 | j                  |«      j                  |Ž j                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t         «      } || ||	|
|fd| j"                  dœ|¤Ž\  }} |j$                  g |¢d‘­Ž j'                  «       }| j)                  |«      }||fS )Néÿÿÿÿr   r   g        )ÚdropoutÚscaling)ÚshaperP   rW   rQ   ÚviewÚ	transposerX   rS   rT   r   ÚupdaterK   r   Úget_interfacer/   Ú_attn_implementationr   rd   ÚreshapeÚ
contiguousrU   )r?   r\   r]   r^   r_   ÚkwargsÚinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightsÚoutputs                    r)   rF   zLfm2Attention.forward`   s¤  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà×'Ñ'Ð(G¨¯©°MÓ(B×(GÑ(GÈÐ(VÓW×aÑaÐbcÐefÓgˆØ×%Ñ%Ð&E d§k¡k°-Ó&@×&EÑ&EÀ|Ð&TÓU×_Ñ_Ð`aÐcdÓeˆ
Ø6�t—{‘{ =Ó1×6Ñ6¸ÐE×OÑOÐPQÐSTÓUˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð Ø—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—‘˜{Ó+ˆØ�|Ð#Ð#r(   rB   )r$   r%   r&   r   r7   r4   ÚtorchÚTensorÚtupler   rF   rG   rH   s   @r)   rJ   rJ   T   sŠ   ø„ ð	#˜zð 	#°cõ 	#ð  )-ñ%$à—|‘|ð%$ð # 5§<¡<°·±Ð#=Ñ>ð%$ð Ÿ™ tÑ+ð	%$ð
  ™ð%$ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷%$r(   rJ   c                   óˆ  ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej                  dedz  dej                  dz  dej                  dz  fd	„Z
	 	 	 ddej                  dedz  dej                  dz  dej                  dz  fd
„Z ed«      	 	 	 ddej                  dedz  dej                  dz  dej                  dz  fd„«       Zˆ xZS )ÚLfm2ShortConvr/   rK   c           	      óZ  •— t         ‰| �  «        || _        || _        |j                  | _        |j                  | _        t        j                  |j                  |j                  | j
                  |j                  | j                  | j
                  dz
  ¬«      | _        t        j                  |j                  d|j                  z  | j                  ¬«      | _        t        j                  |j                  |j                  | j                  ¬«      | _        |j                  |   | _        y )Nr   )Úin_channelsÚout_channelsÚkernel_sizeÚgroupsr2   Úpaddingr   r1   )r3   r4   r/   rK   Úconv_L_cacheÚL_cacheÚ	conv_biasr2   r   ÚConv1dr;   Úconvr:   Úin_projrU   Úlayer_typesÚ
layer_typer[   s      €r)   r4   zLfm2ShortConv.__init__‰   sá   ø€ ô
 	‰ÑÔØˆŒØ"ˆŒØ×*Ñ*ˆŒØ×$Ñ$ˆŒ	ä—I‘IØ×*Ñ*Ø×+Ñ+ØŸ™Ø×%Ñ%Ø—‘Ø—L‘L 1Ñ$ô
ˆŒ	ô —y‘y ×!3Ñ!3°Q¸×9KÑ9KÑ5KÐRV×R[ÑR[Ô\ˆŒÜŸ	™	 &×"4Ñ"4°f×6HÑ6HÈtÏyÉyÔYˆŒà ×,Ñ,¨YÑ7ˆ�r(   NrE   r_   r^   Úseq_idxc                 ó  — t        ||«      }| j                  |«      j                  dd«      }|j                  dd¬«      \  }}}||z  }| j                  j
                  j                  | j                  j
                  j                  d«      | j                  j
                  j                  d«      «      }	|�ƒ|j                  | j                  «      rht        |j                  d«      |j                  | j                     j                  d   |	| j                  j                  d «      }
|
j                  d«      }
n�|�jt         j"                  j%                  || j&                  |j(                  d   z
  df«      }|j+                  || j                  «      d| j&                   d …f   }t-        ||	| j                  j                  d |¬«      }
||
z  }| j/                  |j                  dd«      j1                  «       «      }|S )	Nrb   éþÿÿÿr   ©Údimr   r   .)Ú
activationrŒ   )r   r‰   rg   Úchunkrˆ   Úweightrf   ÚsizeÚhas_previous_staterK   r    ÚsqueezeÚlayersÚconv_statesr2   Ú	unsqueezer   Ú
functionalÚpadr…   re   Úupdate_conv_stater   rU   rl   )r?   rE   r_   r^   rŒ   ÚBCxÚBÚCÚBxÚconv_weightsÚconv_outÚ
conv_stateÚys                r)   Úcuda_kernels_forwardz"Lfm2ShortConv.cuda_kernels_forward¡   s­  € ô )¨¨NÓ;ˆØ�l‰l˜1‹o×'Ñ'¨¨BÓ/ˆØ—)‘)˜A 2�)Ó&‰ˆˆ1ˆaà�‰Uˆà—y‘y×'Ñ'×,Ñ,¨T¯Y©Y×-=Ñ-=×-BÑ-BÀ1Ó-EÀtÇyÁy×GWÑGW×G\ÑG\Ð]^ÓG_Ó`ˆØÐ&¨?×+MÑ+MÈdÏnÉnÔ+]Ü+Ø—
‘
˜2“Ø×&Ñ& t§~¡~Ñ6×BÑBÀ1ÑEØØ—	‘	—‘ØóˆHð  ×)Ñ)¨"Ó-‰HàÐ*ÜŸ]™]×.Ñ.¨r°D·L±LÀ2Ç8Á8ÈBÁ<Ñ4OÐQRÐ3SÓT�
Ø,×>Ñ>¸zÈ4Ï>É>ÓZÐ[^Ðae×amÑamÐ`mÑ`oÐ[oÑp�
ô (¨¨L¸$¿)¹)¿.¹.ÐUYÐcjÔkˆHà�‰LˆØ�M‰M˜!Ÿ+™+ b¨"Ó-×8Ñ8Ó:Ó;ˆØˆr(   c           
      óÆ  — |j                   d   }t        ||«      }| j                  |«      j                  dd«      }|j	                  dd¬«      \  }}}||z  }	|�Ò|j                  | j                  «      r·|j                  |	| j                  «      d| j                   d …f   }
t        j                  |
j                  |	j                  «      | j                  j                  d d …dd d …f   z  d¬«      }| j                  r|| j                  j                  z  }|j!                  d«      }�nþ|��w|j                   d   dk(  �rd|�jt"        j$                  j'                  |	| j                  |	j                   d   z
  df«      }
|j                  |
| j                  «      d| j                   d …f   }
|d   }|dd  |d d k7  j)                  d¬	«      d   dz   }t        j*                  |j-                  d«      ||j/                  d
|j1                  «       «      g«      j3                  «       }g }t5        t7        |«      dz
  «      D ]K  }||   ||dz      }}||kD  sŒ|j9                  | j                  |	d d …d d …||…f   «      dd ||z
  …f   «       ŒM t        j*                  |d¬«      }n„|�jt"        j$                  j'                  |	| j                  |	j                   d   z
  df«      }
|j                  |
| j                  «      d| j                   d …f   }
| j                  |	«      dd |…f   }||z  }|j                  dd«      j;                  «       }| j=                  |«      }|S )Nr   rb   rŽ   r   r�   .r   T)Úas_tuple)r   )re   r   r‰   rg   r’   r•   rK   rœ   r…   ry   ÚsumÚtoÚdevicerˆ   r“   r2   r™   r   rš   r›   ÚnonzeroÚcatÚ	new_zerosÚnew_fullÚnumelÚtolistÚrangeÚlenÚappendrl   rU   )r?   rE   r_   r^   rŒ   Úseqlenr�   rž   rŸ   r    r£   r¢   ÚsiÚchangeÚboundsÚpartsÚiÚsÚer¤   s                       r)   Úslow_forwardzLfm2ShortConv.slow_forwardÄ   s  € ð —‘˜‘ˆä(¨¨NÓ;ˆØ�l‰l˜1‹o×'Ñ'¨¨BÓ/ˆØ—)‘)˜A 2�)Ó&‰ˆˆ1ˆaà�‰UˆàÐ&¨?×+MÑ+MÈdÏnÉnÔ+]Ø(×:Ñ:¸2¸t¿~¹~ÓNÈsÐUY×UaÑUaÐTaÑTcÐOcÑdˆJÜ—y‘y §¡¨r¯y©yÓ!9¸D¿I¹I×<LÑ<LÊQÐPQÒSTÈWÑ<UÑ!UÐ[]Ô^ˆHØ�yŠyØ˜DŸI™IŸN™NÑ*�à×)Ñ)¨"Ó-ŠHØÑ  Q§W¡W¨Q¡Z°1£_àÐ*ÜŸ]™]×.Ñ.¨r°D·L±LÀ2Ç8Á8ÈBÁ<Ñ4OÐQRÐ3SÓT�
Ø,×>Ñ>¸zÈ4Ï>É>ÓZÐ[^Ðae×amÑamÐ`mÑ`oÐ[oÑp�
Ø˜‘ˆBØ˜˜�f  3 B Ñ'×0Ñ0¸$Ð0Ó?ÀÑBÀQÑFˆFÜ—Y‘Y × 0Ñ 0°Ó 3°V¸V¿_¹_ÈTÐSU×S[ÑS[ÓS]Ó=^Ð_Ó`×gÑgÓiˆFØˆEÜœ3˜v›;¨™?Ó+ò I�Ø˜a‘y &¨¨Q©¡-�1�Ø�q“5Ø—L‘L §¡¨2ªa²°A°a°C¨i©=Ó!9¸#¸wÀÀQÁ¸w¸,Ñ!GÕHðIô —y‘y ¨BÔ/‰HàÐ*ÜŸ]™]×.Ñ.¨r°D·L±LÀ2Ç8Á8ÈBÁ<Ñ4OÐQRÐ3SÓT�
Ø,×>Ñ>¸zÈ4Ï>É>ÓZÐ[^Ðae×amÑamÐ`mÑ`oÐ[oÑp�
à—y‘y “} S¨'¨6¨' \Ñ2ˆHà�‰LˆØ�K‰K˜˜BÓ×*Ñ*Ó,ˆØ�M‰M˜!ÓˆØˆr(   rˆ   r\   c                 ó¦   — t         r7d|j                  j                  v rt        «       s| j	                  ||||¬«      S | j                  ||||¬«      S )NÚcuda)rŒ   )Úis_fast_path_availablerª   Útyper   r¥   r¼   )r?   r\   r_   r^   rŒ   s        r)   rF   zLfm2ShortConv.forwardô   sV   € õ " f°×0DÑ0D×0IÑ0IÑ&IÔRjÔRlØ×,Ñ,¨]¸OÈ^ÐelÐ,ÓmÐmØ× Ñ  °ÀÐY`Ð ÓaÐar(   )NNN)r$   r%   r&   r   r7   r4   ry   rz   r   Ú	IntTensorr¥   r¼   r   rF   rG   rH   s   @r)   r}   r}   ˆ   s&  ø„ ð8àð8ð õ8ð6 )-Ø.2Ø*.ñ!à�<‰<ð!ð  ™ð!ð Ÿ™ tÑ+ð	!ð
 —‘ 4Ñ'ó!ðL )-Ø.2Ø*.ñ.à�<‰<ð.ð  ™ð.ð Ÿ™ tÑ+ð	.ð
 —‘ 4Ñ'ó.ñ` ˜FÓ#ð )-Ø.2Ø*.ñ	bà—|‘|ð	bð  ™ð	bð Ÿ™ tÑ+ð		bð
 —‘ 4Ñ'ò	bó $ô	br(   r}   c                   óì   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	e
dz  d
ej                  fd„Zˆ xZS )ÚLfm2DecoderLayerr/   rK   c                 óf  •— t         ‰| �  «        |j                  |   dk(  | _        | j                  rt	        ||«      | _        nt        ||«      | _        t        |«      | _	        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        y )NÚfull_attentionrM   )r3   r4   rŠ   Úis_attention_layerrJ   Ú	self_attnr}   rˆ   r.   Úfeed_forwardr"   r;   rV   Úoperator_normÚffn_normr[   s      €r)   r4   zLfm2DecoderLayer.__init__  s‰   ø€ Ü‰ÑÔØ"(×"4Ñ"4°YÑ"?ÐCSÑ"SˆÔà×"Ò"Ü*¨6°9Ó=ˆD�Nä% f¨iÓ8ˆDŒIÜ# F›OˆÔÜ(¨×);Ñ);ÀÇÁÔQˆÔÜ# F×$6Ñ$6¸F¿O¹OÔLˆ�r(   Nr\   r]   r^   Úposition_idsr_   r`   c           	      ó.  — |}| j                   r+ | j                  d| j                  |«      ||||dœ|¤Ž\  }}n3| j                  | j                  |«      |||j	                  d«      ¬«      }||z   }|| j                  | j                  |«      «      z   }|S )N)r\   r]   r^   rË   r_   rŒ   )r\   r_   r^   rŒ   r'   )rÆ   rÇ   rÉ   rˆ   ÚgetrÈ   rÊ   )	r?   r\   r]   r^   rË   r_   rm   ÚresidualÚ_s	            r)   rF   zLfm2DecoderLayer.forward  sµ   € ð !ˆØ×"Ò"Ø-˜tŸ~™~ð  Ø"×0Ñ0°Ó?Ø$7Ø-Ø)Ø /ñ ð ñ ÑˆM™1ð !ŸI™IØ"×0Ñ0°Ó?Ø /Ø-ØŸ
™
 9Ó-ð	 &ó ˆMð &¨Ñ0ˆØ%¨×(9Ñ(9¸$¿-¹-ÈÓ:VÓ(WÑWˆàÐr(   )NNNN)r$   r%   r&   r   r7   r4   ry   rz   r{   Ú
LongTensorr   rF   rG   rH   s   @r)   rÃ   rÃ     sŸ   ø„ ð
M˜zð 
M°cõ 
Mð IMØ.2Ø04Ø(,ñà—|‘|ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ÿ™ tÑ+ð	ð
 ×&Ñ&¨Ñ-ðð  ™ðð 
�‰÷r(   rÃ   c                   ó   — e Zd ZdZy)ÚLfm2PreTrainedModelTN)r$   r%   r&   Ú_is_statefulr'   r(   r)   rÒ   rÒ   .  s   „ Ø�Lr(   rÒ   c                   óÔ   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	e
dz  d
ee   defd„Zˆ xZS )Ú	Lfm2Modelr/   c                 ót   •— t         ‰| �  |«       t        |j                  |j                  ¬«      | _        | `y )NrM   )r3   r4   r"   r;   rV   Úembedding_normÚnorm)r?   r/   r@   s     €r)   r4   zLfm2Model.__init__3  s.   ø€ Ü‰Ñ˜Ô Ü)¨&×*<Ñ*<À&Ç/Á/ÔRˆÔØ‰Ir(   NÚ	input_idsr^   rË   r_   Úinputs_embedsÚ	use_cacherm   r`   c           	      óÈ  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      s)| j                  ||||dœ}
t        di |
¤Žt        di |
¤Ždœ}	|}| j                  ||¬«      }t        | j                   d | j                  j"                   «      D ]-  \  }} ||f|	| j                  j$                  |      |||d	œ|¤Ž}Œ/ | j'                  |«      }t)        ||¬
«      S )Nz:You must specify exactly one of input_ids or inputs_embeds)r/   r   r   )rª   )r/   rÚ   r^   r_   rË   )rÅ   rˆ   )rË   )r^   r]   rË   r_   )Úlast_hidden_stater_   r'   )Ú
ValueErrorÚembed_tokensr   r/   Úget_seq_lengthry   Úarangere   rª   r™   Ú
isinstanceÚdictr	   r
   Ú
rotary_embÚ	enumerater—   Únum_hidden_layersrŠ   r×   r   )r?   rÙ   r^   rË   r_   rÚ   rÛ   rm   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr\   r]   r¹   Údecoder_layers                  r)   rF   zLfm2Model.forward8  s˜  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-ÄÔFàŸ+™+Ø!.Ø"0Ø#2Ø ,ñˆKô #5Ñ"C°{Ñ"CÜ7ÑF¸+ÑFñ#Ðð
 &ˆØ"Ÿo™o¨mÈ,˜oÓWÐô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÓ Wò 	ÑˆAˆ}Ù)Øðà2°4·;±;×3JÑ3JÈ1Ñ3MÑNØ$7Ø)Ø /ñð ñ‰Mð	ð ×+Ñ+¨MÓ:ˆä&Ø+Ø+ô
ð 	
r(   )NNNNNN)r$   r%   r&   r   r4   ry   rÐ   rz   r   ÚFloatTensorÚboolr   r   r   rF   rG   rH   s   @r)   rÕ   rÕ   2  s®   ø„ ð˜zõ ð .2Ø.2Ø04Ø(,Ø26Ø!%ñ8
à×#Ñ# dÑ*ð8
ð Ÿ™ tÑ+ð8
ð ×&Ñ&¨Ñ-ð	8
ð
  ™ð8
ð ×(Ñ(¨4Ñ/ð8
ð ˜$‘;ð8
ð Ð+Ñ,ð8
ð 
!÷8
r(   rÕ   c                   ó   — e Zd Zy)ÚLfm2ForCausalLMNr#   r'   r(   r)   rî   rî   s  r*   r(   rî   )rî   rÕ   rÒ   )?Úcollections.abcr   ry   Útorch.nn.functionalr   rš   rC   Úcache_utilsr   r   Úintegrations.accelerater   Úmasking_utilsr	   r
   Úmodeling_layersr   Úmodeling_outputsr   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   Úutils.import_utilsr   r   Úbamba.modeling_bambar   Úgemma2.modeling_gemma2r   Úllama.modeling_llamar   r   r   r   r   r   r   Úconfiguration_lfm2r   Úcausal_conv1dr   r    Úkernel_modulesÚallr¿   Ú
get_loggerr$   Úloggerr"   r,   ÚModuler.   rJ   r}   rÃ   rÒ   rÕ   rî   Ú__all__r'   r(   r)   ú<module>r     s  ðõ %ã ß Ð Ý ç .Ý =ß PÝ 9Ý 7Ý 5Ý &ß 0ß VÝ ?Ý :÷÷ ñ õ +ñ ÔßDÐDà-7Ñ*ÐÐ*ð #Ð$8Ð9€Ù˜^Ó,Ð ð 
ˆ×	Ñ	˜HÓ	%€ô	�,ô 	ô	Ð/ô 	ô8ˆb�i‰iô 8ô(1$�Nô 1$ôhvb�B—I‘Iô vbôr*Ð1ô *ôZÐ.ô ô>
�
ô >
ôB	Ð&ô 	ò B�r(   