Ë
    ÜÓ^j7*  ã                   óæ  — d dl mZ d dlmZ d dlZd dlmZ ddlmZm	Z	 ddl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZ ddlmZ ddlmZ ddlmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z' ddl(m)Z)  ejT                  e+«      Z,dZ-dZ. G d„ de%«      Z/ G d„ de«      Z0 G d„ de«      Z1 G d„ de«      Z2 G d„ de$«      Z3 G d„ d e#«      Z4 G d!„ d"e «      Z5 G d#„ d$e!«      Z6 G d%„ d&e"«      Z7g d'¢Z8y)(é    )ÚCallable)ÚOptionalNé   )ÚCacheÚDynamicCache)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚCLIPMLP)	ÚLlamaAttentionÚLlamaForCausalLMÚLlamaForSequenceClassificationÚLlamaForTokenClassificationÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forwardé   )Ú	PhiConfigzmicrosoft/phi-1r   c                   óR   — e Zd Ze	 	 	 d	dedz  ded   dedz  dedef   fd„«       Z	y)
ÚPhiRotaryEmbeddingNÚconfigÚdeviceztorch.deviceÚseq_lenÚreturnztorch.Tensorc                 ón  — | j                   d   }| j                   j                  dd«      }t        | dd«      xs | j                  | j                  z  }t        ||z  «      }d}d|t        j                  d|dt        j                  ¬«      j                  |t        j                  ¬	«      |z  z  z  }||fS )
a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimNr   r   )Údtype)r"   r)   )Úrope_parametersÚgetÚgetattrÚhidden_sizeÚnum_attention_headsÚintÚtorchÚarangeÚint64ÚtoÚfloat)	r!   r"   r#   Úbaser'   r(   ÚdimÚattention_factorÚinv_freqs	            ún/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/phi/modular_phi.pyÚcompute_default_rope_parametersz2PhiRotaryEmbedding.compute_default_rope_parameters(   s»   € ð& ×%Ñ% lÑ3ˆØ &× 6Ñ 6× :Ñ :Ð;RÐTWÓ XÐÜ˜6 :¨tÓ4Òh¸×8JÑ8JÈf×NhÑNhÑ8hˆÜ�(Ð2Ñ2Ó3ˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)ó    )NNN)
Ú__name__Ú
__module__Ú__qualname__Ústaticmethodr   r   r/   Útupler4   r:   © r;   r9   r    r    '   sY   „ Øà#'Ø+/Ø"ñ*Ø˜DÑ ð*à˜Ñ(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ñ	%ò	*ó ñ*r;   r    c                   óæ   ‡ — e Zd Zdedefˆ fd„Z	 ddej                  deej                  ej                  f   dej                  dz  de	dz  d	eej                  ej                  dz  f   f
d
„Z
ˆ xZS )ÚPhiAttentionr!   Ú	layer_idxc                 ó¢  •— t         ‰| �  ||«       t        j                  |j                  |j
                  | j                  z  d¬«      | _        t        j                  |j                  |j                  | j                  z  d¬«      | _	        t        j                  |j                  |j                  | j                  z  d¬«      | _
        t        j                  |j
                  | j                  z  |j                  d¬«      | _        | `t        | j                  |j                  d   z  «      | _        |j                   | _        | j                   r}t        j"                  |j                  |j
                  z  |j$                  d¬«      | _        t        j"                  |j                  |j
                  z  |j$                  d¬«      | _        y y )NT©Úbiasr'   )ÚepsÚelementwise_affine)ÚsuperÚ__init__ÚnnÚLinearr-   r.   r(   Úq_projÚnum_key_value_headsÚk_projÚv_projÚdenseÚo_projr/   r*   Úrotary_ndimsÚqk_layernormÚ	LayerNormÚlayer_norm_epsÚq_layernormÚk_layernorm©Úselfr!   rD   Ú	__class__s      €r9   rK   zPhiAttention.__init__J   sh  ø€ Ü‰Ñ˜ Ô+Ü—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐeiÔjˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐeiÔjˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐeiÔjˆŒÜ—Y‘Y˜v×9Ñ9¸D¿M¹MÑIÈ6×K]ÑK]ÐdhÔiˆŒ
ØˆKÜ §¡°×0FÑ0FÐG^Ñ0_Ñ _Ó`ˆÔØ"×/Ñ/ˆÔØ×ÒÜ!Ÿ|™|Ø×"Ñ" f×&@Ñ&@Ñ@Àf×F[ÑF[Ðptô ˆDÔô  "Ÿ|™|Ø×"Ñ" f×&@Ñ&@Ñ@Àf×F[ÑF[Ðptô ˆDÕð	 r;   NÚhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valuesr$   c                 ób  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
| j                  r"| j                  |«      }| j                  |	«      }	|\  }}|dd | j                  …f   |d| j                  d …f   }}|	dd | j                  …f   |	d| j                  d …f   }}t        ||||«      \  }}t        j                  ||fd¬«      }t        j                  ||fd¬«      }	|� |j                  |	|
| j                  «      \  }	}
t!        j"                  | j$                  j&                  t(        «      } || ||	|
|f| j*                  sdn| j,                  | j.                  dœ|¤Ž\  }} |j0                  g |¢d‘­Ž j3                  «       }| j5                  |«      }||fS )Néÿÿÿÿr   r   .)r6   g        )ÚdropoutÚscaling)Úshaper(   rN   ÚviewÚ	transposerP   rQ   rU   rX   rY   rT   r   r0   ÚcatÚupdaterD   r   Úget_interfacer!   Ú_attn_implementationr   ÚtrainingÚattention_dropoutrd   ÚreshapeÚ
contiguousrR   )r[   r]   r^   r_   r`   ÚkwargsÚinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚ	query_rotÚ
query_passÚkey_rotÚkey_passÚattention_interfaceÚattn_outputÚattn_weightss                       r9   ÚforwardzPhiAttention.forward[   sV  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà×ÒØ×+Ñ+¨LÓ9ˆLØ×)Ñ)¨*Ó5ˆJà&‰ˆˆSð ˜Ð1 × 1Ñ 1Ð1Ð1Ñ2Ø˜˜d×/Ñ/Ñ1Ð1Ñ2ð ˆ	ð
 �sÐ/˜d×/Ñ/Ð/Ð/Ñ0Ø�s˜D×-Ñ-Ñ/Ð/Ñ0ð ˆô
 2°)¸WÀcÈ3ÓOÑˆ	�7ô —y‘y )¨ZÐ!8¸bÔAˆÜ—Y‘Y ¨Ð2¸Ô;ˆ
àÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—j‘j Ó-ˆØ˜LÐ(Ð(r;   )N)r<   r=   r>   r   r/   rK   r0   ÚTensorr@   r   r   Ú__classcell__©r\   s   @r9   rC   rC   I   sŠ   ø„ ð˜yð °Sõ ð, )-ñ8)à—|‘|ð8)ð # 5§<¡<°·±Ð#=Ñ>ð8)ð Ÿ™ tÑ+ð	8)ð
  ™ð8)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷8)r;   rC   c                   ó   — e Zd Zy)ÚPhiMLPN©r<   r=   r>   rA   r;   r9   r„   r„   –   ó   „ Ør;   r„   c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
eej                  ej                  f   dz  dee   dej                  fd„Zˆ xZS )ÚPhiDecoderLayerr!   rD   c                 ó  •— t         ‰| �  «        t        ||¬«      | _        t	        |«      | _        t        j                  |j                  |j                  ¬«      | _
        t        j                  |j                  «      | _        y )N)rD   ©rH   )rJ   rK   rC   Ú	self_attnr„   ÚmlprL   rV   r-   rW   Úinput_layernormÚDropoutÚresid_pdropÚresid_dropoutrZ   s      €r9   rK   zPhiDecoderLayer.__init__›   s]   ø€ Ü‰ÑÔÜ% f¸	ÔBˆŒÜ˜&“>ˆŒÜ!Ÿ|™|¨F×,>Ñ,>ÀF×DYÑDYÔZˆÔÜŸZ™Z¨×(:Ñ(:Ó;ˆÕr;   Nr]   r_   Úposition_idsr`   Ú	use_cacher^   rp   r$   c           
      óÖ   — |}| j                  |«      } | j                  d||||||dœ|¤Ž\  }	}
| j                  |	«      }	| j                  | j                  |«      «      }|	|z   |z   }|S )N)r]   r_   r‘   r`   r’   r^   rA   )r�   r‹   r�   rŒ   )r[   r]   r_   r‘   r`   r’   r^   rp   ÚresidualÚattn_outputsÚ_Úfeed_forward_hidden_statess               r9   r   zPhiDecoderLayer.forward¢   s�   € ð !ˆà×,Ñ,¨]Ó;ˆà(˜$Ÿ.™.ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
‰ˆ�að ×)Ñ)¨,Ó7ˆà%)×%7Ñ%7¸¿¹ÀÓ8OÓ%PÐ"Ø$Ð'AÑAÀHÑLˆàÐr;   )NNNFN)r<   r=   r>   r   r/   rK   r0   r€   Ú
LongTensorr   Úboolr@   r   r   r   r�   r‚   s   @r9   rˆ   rˆ   š   s»   ø„ ð<˜yð <°Sõ <ð /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð+Ñ,ðð 
�‰÷r;   rˆ   c                   ó   — e Zd ZeedœZy)ÚPhiPreTrainedModel)r]   Ú
attentionsN)r<   r=   r>   rˆ   rC   Ú_can_record_outputsrA   r;   r9   r›   r›   Á   s   „ à(Ø"ñÑr;   r›   c                   óò   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   defd„«       «       «       Zˆ xZS )ÚPhiModelr!   c           	      ód  •— t         ‰| �  |«       t        j                  t	        |j
                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        j                  |j                  «      | _
        t        j                  |j                  |j                  ¬«      | _        | `y c c}w )NrŠ   )rJ   rK   rL   Ú
ModuleListÚrangeÚnum_hidden_layersrˆ   ÚlayersrŽ   Ú
embd_pdropÚembed_dropoutrV   r-   rW   Úfinal_layernormÚnormrZ   s      €r9   rK   zPhiModel.__init__É   sƒ   ø€ Ü‰Ñ˜Ô Ü—m‘mÜAFÀv×G_ÑG_ÓA`Öa°IŒ_˜V YÕ/Òaó
ˆŒô  ŸZ™Z¨×(9Ñ(9Ó:ˆÔÜ!Ÿ|™|¨F×,>Ñ,>ÀF×DYÑDYÔZˆÔØ‰Iùò	 bs   ·B-NÚ	input_idsr_   r‘   r`   Úinputs_embedsr’   rp   r$   c           
      ó^  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        | j                  ||||¬«      }	| j                  |«      }|}
| j                  |
|¬«      }| j                  d | j                  j                   D ]  } ||
f|	||||dœ|¤Ž}
Œ | j                  |
«      }
t!        |
|¬	«      S )
Nz:You must specify exactly one of input_ids or inputs_embeds)r!   r   r   )r"   )r!   rª   r_   r`   r‘   )r‘   )r_   r‘   r`   r’   r^   )Úlast_hidden_stater`   )Ú
ValueErrorÚembed_tokensr   r!   Úget_seq_lengthr0   r1   re   r"   Ú	unsqueezer   r¦   Ú
rotary_embr¤   r£   r§   r
   )r[   r©   r_   r‘   r`   rª   r’   rp   Úpast_seen_tokensÚcausal_maskr]   r^   Údecoder_layers                r9   r   zPhiModel.forwardÒ   s^  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð ×*Ñ*¨=Ó9ˆØ%ˆØ"Ÿo™o¨mÈ,˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÐIò 		ˆMÙ)Øðà*Ø)Ø /Ø#Ø$7ñð ñ‰Mð		ð ×,Ñ,¨]Ó;ˆä&Ø+Ø+ô
ð 	
r;   )NNNNNN)r<   r=   r>   r   rK   r   r   r   r0   r˜   r€   r   ÚFloatTensorr™   r   r   r
   r   r�   r‚   s   @r9   rŸ   rŸ   È   sÍ   ø„ ð˜yõ ð  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ4
à×#Ñ# dÑ*ð4
ð Ÿ™ tÑ+ð4
ð ×&Ñ&¨Ñ-ð	4
ð
  ™ð4
ð ×(Ñ(¨4Ñ/ð4
ð ˜$‘;ð4
ð Ð+Ñ,ð4
ð 
!ò4
ó ó ó  ô4
r;   rŸ   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚPhiForCausalLMc                 ó†   •— t         ‰| �  |«       t        j                  |j                  |j
                  d¬«      | _        y )NTrF   )rJ   rK   rL   rM   r-   Ú
vocab_sizeÚlm_head)r[   r!   r\   s     €r9   rK   zPhiForCausalLM.__init__  s0   ø€ Ü‰Ñ˜Ô Ü—y‘y ×!3Ñ!3°V×5FÑ5FÈTÔRˆ�r;   )r<   r=   r>   rK   r�   r‚   s   @r9   r·   r·     s   ø„ ÷Sð Sr;   r·   c                   ó   — e Zd Zy)ÚPhiForSequenceClassificationNr…   rA   r;   r9   r¼   r¼     r†   r;   r¼   c                   ó   — e Zd Zy)ÚPhiForTokenClassificationNr…   rA   r;   r9   r¾   r¾     r†   r;   r¾   )r›   rŸ   r·   r¼   r¾   )9Úcollections.abcr   Útypingr   r0   Útorch.nnrL   Úcache_utilsr   r   Úmasking_utilsr   Úmodeling_layersr	   Úmodeling_outputsr
   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   Úclip.modeling_clipr   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   Úconfiguration_phir   Ú
get_loggerr<   ÚloggerÚ_CHECKPOINT_FOR_DOCÚ_CONFIG_FOR_DOCr    rC   r„   rˆ   r›   rŸ   r·   r¼   r¾   Ú__all__rA   r;   r9   ú<module>rÓ      sì   ðÝ $Ý ã Ý ç .Ý /Ý 9õõ 6Ý &ß @Ñ @Ý 7Ý 5Ý (÷
÷ 
õ 
õ )ð 
ˆ×	Ñ	˜HÓ	%€à'Ð Ø€ô*Ð-ô *ôDJ)�>ô J)ôZ	ˆWô 	ô$Ð0ô $ôNÐ-ô ôA
ˆzô A
ôHSÐ%ô Sô	Ð#Aô 	ô	Ð ;ô 	ò�r;   