Ë
    ÜÓ^jŸ(  ã                   óÌ  — d Z ddlmZ ddlZddlmZ ddlmZ ddlmZ ddl	m
Z
 dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZmZmZmZmZ ddlmZ ddlmZ  ej>                  e «      Z!dZ"dZ# G d„ dejH                  «      Z% G d„ de«      Z&d&d„Z' G d„ dejH                  «      Z( G d„ de«      Z) G d„ de«      Z* G d„ d e«      Z+ G d!„ d"e«      Z, G d#„ d$e«      Z-g d%¢Z.y)'zPyTorch Phi-3 model.é    )ÚCallableN)Únné   )ÚACT2FN)ÚCache)ÚGenerationMixin)ÚFlashAttentionKwargs)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)Úloggingé   )ÚMistralDecoderLayerÚMistralForCausalLMÚ MistralForSequenceClassificationÚMistralForTokenClassificationÚMistralPreTrainedModelÚeager_attention_forwardÚrotate_half)ÚPhiRotaryEmbeddingé   )Ú
Phi3Configz microsoft/Phi-3-mini-4k-instructr   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚPhi3MLPc                 ó*  •— t         ‰| �  «        || _        t        j                  |j
                  d|j                  z  d¬«      | _        t        j                  |j                  |j
                  d¬«      | _        t        |j                     | _        y )Nr   F©Úbias)ÚsuperÚ__init__Úconfigr   ÚLinearÚhidden_sizeÚintermediate_sizeÚgate_up_projÚ	down_projr   Ú
hidden_actÚactivation_fn)Úselfr   Ú	__class__s     €úp/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/phi3/modular_phi3.pyr   zPhi3MLP.__init__1   sp   ø€ Ü‰ÑÔàˆŒÜŸI™I f×&8Ñ&8¸!¸f×>VÑ>VÑ:VÐ]bÔcˆÔÜŸ™ 6×#;Ñ#;¸V×=OÑ=OÐV[Ô\ˆŒÜ# F×$5Ñ$5Ñ6ˆÕó    Úhidden_statesÚreturnc                 óš   — | j                  |«      }|j                  dd¬«      \  }}|| j                  |«      z  }| j                  |«      S )Nr   éÿÿÿÿ©Údim)r#   Úchunkr&   r$   )r'   r+   Ú	up_statesÚgates       r)   ÚforwardzPhi3MLP.forward9   sL   € Ø×%Ñ% mÓ4ˆ	à#Ÿ/™/¨!°˜/Ó4‰ˆˆiØ × 2Ñ 2°4Ó 8Ñ8ˆ	à�~‰~˜iÓ(Ð(r*   )Ú__name__Ú
__module__Ú__qualname__r   ÚtorchÚFloatTensorr4   Ú__classcell__©r(   s   @r)   r   r   0   s'   ø„ ô7ð) U×%6Ñ%6ð )¸5×;LÑ;L÷ )r*   r   c                   ó   — e Zd Zy)ÚPhi3RotaryEmbeddingN©r5   r6   r7   © r*   r)   r=   r=   B   ó   „ Ør*   r=   c                 ó`  — |j                  |«      }|j                  |«      }|j                  d   }| dd|…f   | d|d…f   }}|dd|…f   |d|d…f   }	}t        j                  ||z  t	        |«      |z  z   |gd¬«      }
t        j                  ||z  t	        |«      |z  z   |	gd¬«      }|
|fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    r.   .Nr/   )Ú	unsqueezeÚshaper8   Úcatr   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds               r)   Úapply_rotary_pos_embrQ   F   sÐ   € ð$ �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cà—‘˜2‘€JØ�c˜;˜J˜;Ð&Ñ'¨¨3°
±Ð+;Ñ)<ˆ6€EØ�c˜;˜J˜;Ð&Ñ'¨¨3°
±Ð+;Ñ)<ˆ6€Eä�i‰i˜% #™+¬+°eÓ*<¸sÑ*BÑCÀVÐLÐRTÔU€GÜ�i‰i˜% #™+¬+°eÓ*<¸sÑ*BÑCÀVÐLÐRTÔU€GØ�GÐÐr*   c                   ó  ‡ — e Zd ZdZddededz  fˆ fd„Z	 ddej                  de	ej                  ej                  f   dej                  dz  d	e
dz  d
ee   de	ej                  ej                  dz  e	ej                     dz  f   fd„Zˆ xZS )ÚPhi3Attentionz=Multi-headed attention from 'Attention Is All You Need' paperNr   Ú	layer_idxc                 ó|  •— t         ‰| �  «        || _        || _        t	        |d|j
                  |j                  z  «      | _        |j                  |j                  z  | _	        |j                  | _        | j                  dz  | _
        |j                  | _        d| _        |j                  | j                  z  d|j                  | j                  z  z  z   }t        j                  |j                  | j                  z  |j
                  d¬«      | _        t        j                  |j
                  |d¬«      | _        y )NÚhead_dimg      à¿Tr   Fr   )r   r   r   rT   Úgetattrr!   Únum_attention_headsrV   Únum_key_value_headsÚnum_key_value_groupsÚscalingÚattention_dropoutÚ	is_causalr   r    Úo_projÚqkv_proj)r'   r   rT   Úop_sizer(   s       €r)   r   zPhi3Attention.__init__g   sý   ø€ Ü‰ÑÔØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø#)×#=Ñ#=ˆÔ Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒà×,Ñ,¨t¯}©}Ñ<¸qÀF×D^ÑD^Ðae×anÑanÑDnÑ?oÑoˆÜ—i‘i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^ÐejÔkˆŒÜŸ	™	 &×"4Ñ"4°gÀEÔJˆ�r*   r+   Úposition_embeddingsÚattention_maskÚpast_key_valuesÚkwargsr,   c           
      óâ  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      }| j                  j                  | j                  z  }	|dd |	…f   }
|d|	|	| j
                  | j                  z  z   …f   }|d|	| j
                  | j                  z  z   d …f   }|
j                  |«      j                  dd«      }
|j                  |«      j                  dd«      }|j                  |«      j                  dd«      }|\  }}t        |
|||«      \  }
}|� |j                  ||| j                  «      \  }}t        j                  | j                  j                  t        «      } || |
|||f| j                  sdn| j                   | j"                  t%        | j                  dd «      dœ|¤Ž\  }} |j&                  g |¢d‘­Ž j)                  «       }| j+                  |«      }||fS )Nr.   .r   r   g        Úsliding_window)Údropoutr[   rf   )rC   rV   r_   r   rX   rY   ÚviewÚ	transposerQ   ÚupdaterT   r
   Úget_interfaceÚ_attn_implementationr   Útrainingr\   r[   rW   ÚreshapeÚ
contiguousr^   )r'   r+   ra   rb   rc   rd   Úinput_shapeÚhidden_shapeÚqkvÚ	query_posÚquery_statesÚ
key_statesÚvalue_statesrG   rH   Úattention_interfaceÚattn_outputÚattn_weightss                     r)   r4   zPhi3Attention.forwardv   s  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà�m‰m˜MÓ*ˆØ—K‘K×3Ñ3°d·m±mÑCˆ	Ø˜3 
  
˜?Ñ+ˆØ˜˜i¨)°d×6NÑ6NÐQU×Q^ÑQ^Ñ6^Ñ*^Ð^Ð^Ñ_ˆ
Ø˜3 	¨D×,DÑ,DÀtÇ}Á}Ñ,TÑ TÑ VÐVÑWˆà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LÜ" 4§;¡;Ð0@À$ÓGñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r*   )N)r5   r6   r7   Ú__doc__r   Úintr   r8   ÚTensorÚtupler   r   r	   r4   r:   r;   s   @r)   rS   rS   d   s´   ø„ ÙGñK˜zð K°c¸D±jõ Kð( )-ñ-)à—|‘|ð-)ð # 5§<¡<°·±Ð#=Ñ>ð-)ð Ÿ™ tÑ+ð	-)ð
  ™ð-)ð Ð-Ñ.ð-)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷-)r*   rS   c                   óD  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
eej                  ej                  f   dz  dee   deej                  eej                  ej                  f   dz  f   fd„Zˆ xZS )ÚPhi3DecoderLayerr   rT   c                 ó  •— t         ‰| �  ||«       || _        t        ||¬«      | _        t        |«      | _        t        j                  |j                  «      | _
        t        j                  |j                  «      | _        y )N)r   rT   )r   r   r   rS   Ú	self_attnr   Úmlpr   ÚDropoutÚresid_pdropÚresid_attn_dropoutÚresid_mlp_dropout)r'   r   rT   r(   s      €r)   r   zPhi3DecoderLayer.__init__§   s`   ø€ Ü‰Ñ˜ Ô+ØˆŒÜ&¨fÀ	ÔJˆŒÜ˜6“?ˆŒÜ"$§*¡*¨V×-?Ñ-?Ó"@ˆÔÜ!#§¡¨F×,>Ñ,>Ó!?ˆÕr*   Nr+   rb   Úposition_idsrc   Ú	use_cachera   rd   r,   c           
      óü   — |}| j                  |«      } | j                  d||||||dœ|¤Ž\  }}	|| j                  |«      z   }|}| j                  |«      }| j	                  |«      }|| j                  |«      z   }|S )N)r+   rb   r‡   rc   rˆ   ra   r?   )Úinput_layernormr�   r…   Úpost_attention_layernormr‚   r†   )
r'   r+   rb   r‡   rc   rˆ   ra   rd   ÚresidualÚself_attn_weightss
             r)   r4   zPhi3DecoderLayer.forward¯   s¥   € ð !ˆØ×,Ñ,¨]Ó;ˆà+9¨4¯>©>ð ,
Ø'Ø)Ø%Ø+ØØ 3ñ,
ð ñ,
Ñ(ˆÐ(ð ! 4×#:Ñ#:¸=Ó#IÑIˆà ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ  4×#9Ñ#9¸-Ó#HÑHˆØÐr*   )NNNFN)r5   r6   r7   r   r{   r   r8   r|   Ú
LongTensorr   Úboolr}   r   r	   r9   r4   r:   r;   s   @r)   r   r   ¦   sæ   ø„ ð@˜zð @°cõ @ð /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð-Ñ.ðð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷r*   r   c                   ó   — e Zd ZdZy)ÚPhi3PreTrainedModelz0.0.5N)r5   r6   r7   Ú_versionr?   r*   r)   r‘   r‘   Î   s   „ Ø�Hr*   r‘   c                   ó    — e Zd Z	 	 	 	 	 	 dd„Zy)ÚPhi3ForCausalLMNc                 ó  — |rjt        | j                  d«      rT|j                  d   | j                  j                  dz   k\  r+|j	                  «       }	|	| j                  j                  k  rd }t        j                  | f|||||||dœ|¤Ž}
|
S )NÚ original_max_position_embeddingsr   )Ú	input_idsrc   rb   Úinputs_embedsr‡   rˆ   Úlogits_to_keep)Úhasattrr   rC   r–   Úget_seq_lengthr   Úprepare_inputs_for_generation)r'   r—   rc   rb   r˜   r‡   rˆ   r™   rd   Úpast_lengthÚmodel_inputss              r)   rœ   z-Phi3ForCausalLM.prepare_inputs_for_generationÓ   s›   € ñ" Ü˜Ÿ™Ð%GÔHØ—‘ Ñ" d§k¡k×&RÑ&RÐUVÑ&VÒVà)×8Ñ8Ó:ˆKØ˜dŸk™k×JÑJÒJØ"&�ä&×DÑDØð

àØ+Ø)Ø'Ø%ØØ)ñ

ð ñ

ˆð Ðr*   )NNNNTN)r5   r6   r7   rœ   r?   r*   r)   r”   r”   Ò   s   „ ð ØØØØØô$r*   r”   c                   ó   — e Zd Zy)ÚPhi3ForSequenceClassificationNr>   r?   r*   r)   r    r    ú   r@   r*   r    c                   ó   — e Zd Zy)ÚPhi3ForTokenClassificationNr>   r?   r*   r)   r¢   r¢   þ   r@   r*   r¢   )r‘   Ú	Phi3Modelr”   r    r¢   )r   )/rz   Úcollections.abcr   r8   r   Úactivationsr   Úcache_utilsr   Ú
generationr   Úmodeling_flash_attention_utilsr	   Úmodeling_utilsr
   Úprocessing_utilsr   Úutilsr   Úmistral.modeling_mistralr   r   r   r   r   r   r   Úphi.modeling_phir   Úconfiguration_phi3r   Ú
get_loggerr5   ÚloggerÚ_CHECKPOINT_FOR_DOCÚ_CONFIG_FOR_DOCÚModuler   r=   rQ   rS   r   r‘   r”   r    r¢   Ú__all__r?   r*   r)   ú<module>rµ      sÛ   ðñ å $ã Ý å !Ý  Ý )Ý BÝ 5Ý &Ý ÷÷ ñ õ 2Ý *ð 
ˆ×	Ñ	˜HÓ	%€à8Ð Ø€ô)ˆb�i‰iô )ô$	Ð,ô 	óô<?)�B—I‘Iô ?)ôD%Ð*ô %ôPÐ0ô ô%Ð(ô %ôP	Ð$Dô 	ô	Ð!>ô 	ò�r*   