Ë
    ÜÓ^jÐV  ã                   ó„  — d Z ddlZddlZddlmZ ddlZddlmc mZ	 ddlmZ ddl
mZ ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ  ej2                  e«      Z G d„ dej8                  «      Z G d„ dej8                  «      Z G d„ dej8                  «      Z G d„ dej8                  «      Z  G d„ dej8                  «      Z! G d„ dej8                  «      Z" G d„ dej8                  «      Z# G d„ dej8                  «      Z$e G d„ de«      «       Z%e G d„ d e%«      «       Z& ed!¬"«       G d#„ d$e%«      «       Z'g d%¢Z(y)&zPyTorch PVT model.é    N)ÚIterable)Únné   )Úinitialization)ÚACT2FN)ÚBaseModelOutputÚImageClassifierOutput)ÚPreTrainedModel)Úauto_docstringÚloggingé   )Ú	PvtConfigc                   óà   ‡ — e Zd ZdZ	 ddedeee   z  deee   z  dedededefˆ fd	„Zd
e	j                  dedede	j                  fd„Zde	j                  dee	j                  eef   fd„Zˆ xZS )ÚPvtPatchEmbeddingszì
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    ÚconfigÚ
image_sizeÚ
patch_sizeÚstrideÚnum_channelsÚhidden_sizeÚ	cls_tokenc                 óâ  •— t         ‰	| �  «        || _        t        |t        j
                  j                  «      r|n||f}t        |t        j
                  j                  «      r|n||f}|d   |d   z  |d   |d   z  z  }|| _        || _        || _	        || _
        t        j                  t        j                  d|r|dz   n||«      «      | _        |r*t        j                  t        j                   dd|«      «      nd | _        t        j$                  ||||¬«      | _        t        j(                  ||j*                  ¬«      | _        t        j.                  |j0                  ¬«      | _        y )Nr   r   ©Úkernel_sizer   ©Úeps)Úp)ÚsuperÚ__init__r   Ú
isinstanceÚcollectionsÚabcr   r   r   r   Únum_patchesr   Ú	ParameterÚtorchÚrandnÚposition_embeddingsÚzerosr   ÚConv2dÚ
projectionÚ	LayerNormÚlayer_norm_epsÚ
layer_normÚDropoutÚhidden_dropout_probÚdropout)
Úselfr   r   r   r   r   r   r   r#   Ú	__class__s
            €úo/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/pvt/modeling_pvt.pyr   zPvtPatchEmbeddings.__init__,   s0  ø€ ô 	‰ÑÔØˆŒÜ#-¨j¼+¿/¹/×:RÑ:RÔ#S‘ZÐZdÐfpÐYqˆ
Ü#-¨j¼+¿/¹/×:RÑ:RÔ#S‘ZÐZdÐfpÐYqˆ
Ø! !‘}¨
°1©Ñ5¸*ÀQ¹-È:ÐVWÉ=Ñ:XÑYˆØ$ˆŒØ$ˆŒØ(ˆÔØ&ˆÔä#%§<¡<Ü�K‰K˜©i˜;¨š?¸[È+ÓVó$
ˆÔ ñ JSœŸ™¤e§k¡k°!°Q¸Ó&DÔEÐX\ˆŒÜŸ)™) L°+È6ÐZdÔeˆŒÜŸ,™, {¸×8MÑ8MÔNˆŒÜ—z‘z F×$>Ñ$>Ô?ˆ�ó    Ú
embeddingsÚheightÚwidthÚreturnc                 ó’  — ||z  }t         j                  j                  «       s<|| j                  j                  | j                  j                  z  k(  r| j
                  S |j                  d||d«      j                  dddd«      }t        j                  |||fd¬«      }|j                  dd||z  «      j                  ddd«      }|S )Nr   éÿÿÿÿr   r   é   Úbilinear)ÚsizeÚmode)
r%   ÚjitÚ
is_tracingr   r   r'   ÚreshapeÚpermuteÚFÚinterpolate)r1   r5   r6   r7   r#   Úinterpolated_embeddingss         r3   Úinterpolate_pos_encodingz+PvtPatchEmbeddings.interpolate_pos_encodingH   s½   € Ø˜u‘nˆô �y‰y×#Ñ#Ô%¨+¸¿¹×9OÑ9OÐRV×R]ÑR]×RhÑRhÑ9hÒ*hØ×+Ñ+Ð+Ø×'Ñ'¨¨6°5¸"Ó=×EÑEÀaÈÈAÈqÓQˆ
Ü"#§-¡-°
À&È%ÀÐWaÔ"bÐØ"9×"AÑ"AÀ!ÀRÈÐRWÉÓ"X×"`Ñ"`ÐabÐdeÐghÓ"iÐØ&Ð&r4   Úpixel_valuesc                 ó”  — |j                   \  }}}}|| j                  k7  rt        d«      ‚| j                  |«      }|j                   �^ }}}|j	                  d«      j                  dd«      }| j                  |«      }| j                  �‰| j                  j                  |dd«      }	t        j                  |	|fd¬«      }| j                  | j                  d d …dd …f   ||«      }
t        j                  | j                  d d …d d…f   |
fd¬«      }
n| j                  | j                  ||«      }
| j                  ||
z   «      }|||fS )NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.r;   r   r:   ©Údim)Úshaper   Ú
ValueErrorr*   ÚflattenÚ	transposer-   r   Úexpandr%   ÚcatrF   r'   r0   )r1   rG   Ú
batch_sizer   r6   r7   Úpatch_embedÚ_r5   r   r'   s              r3   ÚforwardzPvtPatchEmbeddings.forwardS   sM  € Ø2>×2DÑ2DÑ/ˆ
�L &¨%Ø˜4×,Ñ,Ò,ÜØwóð ð —o‘o lÓ3ˆØ'×-Ñ-ÑˆˆF�EØ!×)Ñ)¨!Ó,×6Ñ6°q¸!Ó<ˆØ—_‘_ [Ó1ˆ
Ø�>‰>Ð%ØŸ™×-Ñ-¨j¸"¸bÓAˆIÜŸ™ I¨zÐ#:ÀÔBˆJØ"&×"?Ñ"?À×@XÑ@XÒYZÐ\]Ñ\^ÐY^Ñ@_ÐagÐinÓ"oÐÜ"'§)¡)¨T×-EÑ-EÂaÈÈ!ÈÀeÑ-LÐNaÐ,bÐhiÔ"jÑà"&×"?Ñ"?À×@XÑ@XÐZ`ÐbgÓ"hÐØ—\‘\ *Ð/BÑ"BÓCˆ
à˜6 5Ð(Ð(r4   ©F)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úintr   Úboolr   r%   ÚTensorrF   ÚtuplerT   Ú__classcell__©r2   s   @r3   r   r   %   sÄ   ø„ ñð  ñ@àð@ð ˜( 3™-Ñ'ð@ð ˜( 3™-Ñ'ð	@ð
 ð@ð ð@ð ð@ð õ@ð8	'°5·<±<ð 	'Èð 	'ÐUXð 	'Ð]b×]iÑ]ió 	'ð) E§L¡Lð )°U¸5¿<¹<ÈÈcÐ;QÑ5R÷ )r4   r   c                   ó`   ‡ — e Zd Zdedefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚPvtSelfOutputr   r   c                 ó    •— t         ‰| �  «        t        j                  ||«      | _        t        j
                  |j                  «      | _        y ©N)r   r   r   ÚLinearÚdenser.   r/   r0   )r1   r   r   r2   s      €r3   r   zPvtSelfOutput.__init__j   s6   ø€ Ü‰ÑÔÜ—Y‘Y˜{¨KÓ8ˆŒ
Ü—z‘z &×"<Ñ"<Ó=ˆ�r4   Úhidden_statesr8   c                 óJ   — | j                  |«      }| j                  |«      }|S rc   )re   r0   ©r1   rf   s     r3   rT   zPvtSelfOutput.forwardo   s$   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØÐr4   ©
rV   rW   rX   r   rZ   r   r%   r\   rT   r^   r_   s   @r3   ra   ra   i   s1   ø„ ð>˜yð >°sõ >ð
 U§\¡\ð °e·l±l÷ r4   ra   c                   ó¦   ‡ — e Zd ZdZdedededefˆ fd„Zdedej                  fd	„Z
	 ddej                  d
edededeej                     f
d„Zˆ xZS )ÚPvtEfficientSelfAttentionzxEfficient self-attention mechanism with reduction of the sequence [PvT paper](https://huggingface.co/papers/2102.12122).r   r   Únum_attention_headsÚsequences_reduction_ratioc                 ó˜  •— t         ‰| �  «        || _        || _        | j                  | j                  z  dk7  r&t	        d| j                  › d| j                  › d�«      ‚t        | j                  | j                  z  «      | _        | j                  | j                  z  | _        t        j                  | j                  | j                  |j                  ¬«      | _        t        j                  | j                  | j                  |j                  ¬«      | _        t        j                  | j                  | j                  |j                  ¬«      | _        t        j                  |j                  «      | _        || _        |dkD  rEt        j$                  ||||¬«      | _        t        j(                  ||j*                  ¬«      | _        y y )	Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (ú))Úbiasr   r   r   )r   r   r   rl   rL   rZ   Úattention_head_sizeÚall_head_sizer   rd   Úqkv_biasÚqueryÚkeyÚvaluer.   Úattention_probs_dropout_probr0   rm   r)   Úsequence_reductionr+   r,   r-   ©r1   r   r   rl   rm   r2   s        €r3   r   z"PvtEfficientSelfAttention.__init__x   sr  ø€ ô 	‰ÑÔØ&ˆÔØ#6ˆÔ à×Ñ˜d×6Ñ6Ñ6¸!Ò;ÜØ# D×$4Ñ$4Ð#5ð 6Ø×2Ñ2Ð3°1ð6óð ô
 $' t×'7Ñ'7¸$×:RÑ:RÑ'RÓ#SˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä—Y‘Y˜t×/Ñ/°×1CÑ1CÈ&Ï/É/ÔZˆŒ
Ü—9‘9˜T×-Ñ-¨t×/AÑ/AÈÏÉÔXˆŒÜ—Y‘Y˜t×/Ñ/°×1CÑ1CÈ&Ï/É/ÔZˆŒ
ä—z‘z &×"EÑ"EÓFˆŒà)BˆÔ&Ø$ qÒ(Ü&(§i¡iØ˜[Ð6OÐXqô'ˆDÔ#ô !Ÿl™l¨;¸F×<QÑ<QÔRˆD�Oð	 )r4   rf   r8   c                 ó¤   — |j                  «       d d | j                  | j                  fz   }|j                  |«      }|j	                  dddd«      S )Nr:   r   r;   r   r   )r=   rl   rq   ÚviewrB   )r1   rf   Ú	new_shapes      r3   Útranspose_for_scoresz.PvtEfficientSelfAttention.transpose_for_scores•   sT   € Ø!×&Ñ&Ó(¨¨"Ð-°×1IÑ1IÈ4×KcÑKcÐ0dÑdˆ	Ø%×*Ñ*¨9Ó5ˆØ×$Ñ$ Q¨¨1¨aÓ0Ð0r4   r6   r7   Úoutput_attentionsc                 ó¸  — | j                  | j                  |«      «      }| j                  dkD  r{|j                  \  }}}|j	                  ddd«      j                  ||||«      }| j                  |«      }|j                  ||d«      j	                  ddd«      }| j                  |«      }| j                  | j                  |«      «      }	| j                  | j                  |«      «      }
t        j                  ||	j                  dd«      «      }|t        j                  | j                  «      z  }t         j"                  j%                  |d¬«      }| j'                  |«      }t        j                  ||
«      }|j	                  dddd«      j)                  «       }|j+                  «       d d | j,                  fz   }|j/                  |«      }|r||f}|S |f}|S )Nr   r   r;   r:   éþÿÿÿrI   r   )r}   rt   rm   rK   rB   rA   rx   r-   ru   rv   r%   ÚmatmulrN   ÚmathÚsqrtrq   r   Ú
functionalÚsoftmaxr0   Ú
contiguousr=   rr   r{   )r1   rf   r6   r7   r~   Úquery_layerrQ   Úseq_lenr   Ú	key_layerÚvalue_layerÚattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapeÚoutputss                   r3   rT   z!PvtEfficientSelfAttention.forwardš   sÆ  € ð ×/Ñ/°·
±
¸=Ó0IÓJˆà×)Ñ)¨AÒ-Ø0=×0CÑ0CÑ-ˆJ˜ à)×1Ñ1°!°Q¸Ó:×BÑBÀ:È|Ð]cÐejÓkˆMà ×3Ñ3°MÓBˆMà)×1Ñ1°*¸lÈBÓO×WÑWÐXYÐ[\Ð^_Ó`ˆMØ ŸO™O¨MÓ:ˆMà×-Ñ-¨d¯h©h°}Ó.EÓFˆ	Ø×/Ñ/°·
±
¸=Ó0IÓJˆô !Ÿ<™<¨°Y×5HÑ5HÈÈRÓ5PÓQÐà+¬d¯i©i¸×8PÑ8PÓ.QÑQÐô Ÿ-™-×/Ñ/Ð0@ÀbÐ/ÓIˆð Ÿ,™, Ó7ˆäŸ™ _°kÓBˆà%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ%×*Ñ*Ð+BÓCˆá6G�= /Ð2ˆàˆð O\ÐM]ˆàˆr4   rU   )rV   rW   rX   rY   r   rZ   Úfloatr   r%   r\   r}   r[   r]   rT   r^   r_   s   @r3   rk   rk   u   s‘   ø„ ñ CðSØðSØ.1ðSØHKðSØhmõSð:1°#ð 1¸%¿,¹,ó 1ð #(ñ*à—|‘|ð*ð ð*ð ð	*ð
  ð*ð 
ˆu�|‰|Ñ	÷*r4   rk   c                   ó~   ‡ — e Zd Zdedededefˆ fd„Z	 ddej                  deded	e	d
e
ej                     f
d„Zˆ xZS )ÚPvtAttentionr   r   rl   rm   c                 ón   •— t         ‰| �  «        t        ||||¬«      | _        t	        ||¬«      | _        y )N)r   rl   rm   )r   )r   r   rk   r1   ra   Úoutputry   s        €r3   r   zPvtAttention.__init__È   s8   ø€ ô 	‰ÑÔÜ-ØØ#Ø 3Ø&?ô	
ˆŒ	ô $ F¸ÔDˆ�r4   rf   r6   r7   r~   r8   c                 óh   — | j                  ||||«      }| j                  |d   «      }|f|dd  z   }|S )Nr   r   )r1   r”   )r1   rf   r6   r7   r~   Úself_outputsÚattention_outputr�   s           r3   rT   zPvtAttention.forwardÔ   sE   € ð —y‘y °¸Ð?PÓQˆàŸ;™; |°A¡Ó7ÐØ#Ð%¨°Q°RÐ(8Ñ8ˆØˆr4   rU   )rV   rW   rX   r   rZ   r�   r   r%   r\   r[   r]   rT   r^   r_   s   @r3   r’   r’   Ç   sn   ø„ ð
EØð
EØ.1ð
EØHKð
EØhmõ
Eð _dñØ"Ÿ\™\ðØ36ðØ?BðØW[ðà	ˆu�|‰|Ñ	÷r4   r’   c            
       óz   ‡ — e Zd Z	 	 d
dedededz  dedz  fˆ fd„Zdej                  dej                  fd	„Zˆ xZ	S )ÚPvtFFNNr   Úin_featuresÚhidden_featuresÚout_featuresc                 ój  •— t         ‰| �  «        |�|n|}t        j                  ||«      | _        t        |j                  t        «      rt        |j                     | _	        n|j                  | _	        t        j                  ||«      | _
        t        j                  |j                  «      | _        y rc   )r   r   r   rd   Údense1r    Ú
hidden_actÚstrr   Úintermediate_act_fnÚdense2r.   r/   r0   )r1   r   rš   r›   rœ   r2   s        €r3   r   zPvtFFN.__init__ß   s‡   ø€ ô 	‰ÑÔØ'3Ð'?‘|À[ˆÜ—i‘i ¨_Ó=ˆŒÜ�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÔ$Ü—i‘i °Ó>ˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r4   rf   r8   c                 ó°   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|S rc   )rž   r¡   r0   r¢   rh   s     r3   rT   zPvtFFN.forwardð   sP   € ØŸ™ MÓ2ˆØ×0Ñ0°Ó?ˆØŸ™ ]Ó3ˆØŸ™ MÓ2ˆØŸ™ ]Ó3ˆØÐr4   )NNri   r_   s   @r3   r™   r™   Þ   sY   ø„ ð
 '+Ø#'ñ>àð>ð ð>ð ˜t™ð	>ð
 ˜D‘jõ>ð" U§\¡\ð °e·l±l÷ r4   r™   c                   ór   ‡ — e Zd ZdZd	deddfˆ fd„Zdej                  dej                  fd„Zde	fd„Z
ˆ xZS )
ÚPvtDropPathzÏStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    Ú	drop_probr8   Nc                 ó0   •— t         ‰| �  «        || _        y rc   )r   r   r¦   )r1   r¦   r2   s     €r3   r   zPvtDropPath.__init__  s   ø€ Ü‰ÑÔØ"ˆ�r4   rf   c                 óP  — | j                   dk(  s| j                  s|S d| j                   z
  }|j                  d   fd|j                  dz
  z  z   }t	        j
                  ||j                  |j                  ¬«      }t	        j                  ||z   «      }|j                  |«      |z  S )Nç        r   r   )r   )ÚdtypeÚdevice)
r¦   ÚtrainingrK   Úndimr%   Úrandrª   r«   ÚfloorÚdiv)r1   rf   Ú	keep_probrK   Úrandom_tensors        r3   rT   zPvtDropPath.forward  s—   € Ø�>‰>˜SÒ ¨¯ªØ Ð Ø˜Ÿ™Ñ&ˆ	Ø×$Ñ$ QÑ'Ð)¨D°M×4FÑ4FÈÑ4JÑ,KÑKˆÜŸ
™
 5°×0CÑ0CÈM×L`ÑL`ÔaˆÜŸ™ M°IÑ$=Ó>ˆØ× Ñ  Ó+¨mÑ;Ð;r4   c                 ó    — d| j                   › �S )Nzp=)r¦   )r1   s    r3   Ú
extra_reprzPvtDropPath.extra_repr  s   € Ø�D—N‘NÐ#Ð$Ð$r4   )r©   )rV   rW   rX   rY   r�   r   r%   r\   rT   r    r´   r^   r_   s   @r3   r¥   r¥   ú   sB   ø„ ññ# %ð #°$õ #ð< U§\¡\ð <°e·l±ló <ð%˜C÷ %r4   r¥   c                   óf   ‡ — e Zd Zdedededededefˆ fd„Zddej                  d	ed
ede	fd„Z
ˆ xZS )ÚPvtLayerr   r   rl   Ú	drop_pathrm   Ú	mlp_ratioc                 óv  •— t         ‰| �  «        t        j                  ||j                  ¬«      | _        t        ||||¬«      | _        |dkD  rt        |«      nt        j                  «       | _
        t        j                  ||j                  ¬«      | _        t        ||z  «      }t        |||¬«      | _        y )Nr   )r   r   rl   rm   r©   )r   rš   r›   )r   r   r   r+   r,   Úlayer_norm_1r’   Ú	attentionr¥   ÚIdentityr·   Úlayer_norm_2rZ   r™   Úmlp)	r1   r   r   rl   r·   rm   r¸   Úmlp_hidden_sizer2   s	           €r3   r   zPvtLayer.__init__  s–   ø€ ô 	‰ÑÔÜŸL™L¨¸&×:OÑ:OÔPˆÔÜ%ØØ#Ø 3Ø&?ô	
ˆŒð 4=¸s²?œ YÔ/ÌÏÉËˆŒÜŸL™L¨¸&×:OÑ:OÔPˆÔÜ˜k¨IÑ5Ó6ˆÜ °[ÐRaÔbˆ�r4   rf   r6   r7   r~   c                 ó  — | j                  | j                  |«      |||¬«      }|d   }|dd  }| j                  |«      }||z   }| j                  | j	                  |«      «      }| j                  |«      }||z   }	|	f|z   }|S )N)rf   r6   r7   r~   r   r   )r»   rº   r·   r¾   r½   )
r1   rf   r6   r7   r~   Úself_attention_outputsr—   r�   Ú
mlp_outputÚlayer_outputs
             r3   rT   zPvtLayer.forward)  s¢   € Ø!%§¡Ø×+Ñ+¨MÓ:ØØØ/ð	 "0ó "
Ðð 2°!Ñ4ÐØ(¨¨Ð,ˆàŸ>™>Ð*:Ó;ÐØ(¨=Ñ8ˆà—X‘X˜d×/Ñ/°Ó>Ó?ˆ
à—^‘^ JÓ/ˆ
Ø$ zÑ1ˆà�/ GÑ+ˆàˆr4   rU   )rV   rW   rX   r   rZ   r�   r   r%   r\   r[   rT   r^   r_   s   @r3   r¶   r¶     so   ø„ ðcàðcð ðcð !ð	cð
 ðcð $)ðcð õcñ, U§\¡\ð ¸3ð Àsð Ð_c÷ r4   r¶   c                   ót   ‡ — e Zd Zdefˆ fd„Z	 	 	 d
dej                  dedz  dedz  dedz  dee	z  f
d	„Z
ˆ xZS )Ú
PvtEncoderr   c                 ó¶  •— t         ‰	| �  «        || _        t        j                  d|j
                  t        |j                  «      d¬«      j                  «       }g }t        |j                  «      D ]©  }|j                  t        ||dk(  r|j                  n| j                  j                  d|dz   z  z  |j                  |   |j                  |   |dk(  r|j                   n|j"                  |dz
     |j"                  |   ||j                  dz
  k(  ¬«      «       Œ« t%        j&                  |«      | _        g }d}t        |j                  «      D ]¹  }g }|dk7  r||j                  |dz
     z  }t        |j                  |   «      D ]\  }|j                  t+        ||j"                  |   |j,                  |   |||z      |j.                  |   |j0                  |   ¬«      «       Œ^ |j                  t%        j&                  |«      «       Œ» t%        j&                  |«      | _        t%        j4                  |j"                  d   |j6                  ¬	«      | _        y )
Nr   Úcpu)r«   r;   r   )r   r   r   r   r   r   r   )r   r   rl   r·   rm   r¸   r:   r   )r   r   r   r%   ÚlinspaceÚdrop_path_rateÚsumÚdepthsÚtolistÚrangeÚnum_encoder_blocksÚappendr   r   Úpatch_sizesÚstridesr   Úhidden_sizesr   Ú
ModuleListÚpatch_embeddingsr¶   rl   Úsequence_reduction_ratiosÚ
mlp_ratiosÚblockr+   r,   r-   )
r1   r   Údrop_path_decaysr5   ÚiÚblocksÚcurÚlayersÚjr2   s
            €r3   r   zPvtEncoder.__init__A  s-  ø€ Ü‰ÑÔØˆŒô !Ÿ>™>¨!¨V×-BÑ-BÄCÈÏÉÓDVÐ_dÔe×lÑlÓnÐð ˆ
ä�v×0Ñ0Ó1ò 	ˆAØ×ÑÜ"Ø!Ø45¸²F˜v×0Ò0ÀÇÁ×@VÑ@VÐ[\ÐabÐefÑafÑ[gÑ@hØ%×1Ñ1°!Ñ4Ø!Ÿ>™>¨!Ñ,Ø89¸Qº ×!4Ò!4ÀF×DWÑDWÐXYÐ\]ÑX]ÑD^Ø &× 3Ñ 3°AÑ 6Ø 6×#<Ñ#<¸qÑ#@Ñ@ôõ
ð	ô !#§¡¨jÓ 9ˆÔð ˆØˆÜ�v×0Ñ0Ó1ò 	1ˆAàˆFØ�AŠvØ�v—}‘} Q¨¡UÑ+Ñ+�Ü˜6Ÿ=™=¨Ñ+Ó,ò 
�Ø—‘ÜØ%Ø$*×$7Ñ$7¸Ñ$:Ø,2×,FÑ,FÀqÑ,IØ"2°3¸±7Ñ";Ø28×2RÑ2RÐSTÑ2UØ"(×"3Ñ"3°AÑ"6ôõ	ð
ð �M‰Mœ"Ÿ-™-¨Ó/Õ0ð!	1ô$ —]‘] 6Ó*ˆŒ
ô Ÿ,™, v×':Ñ':¸2Ñ'>ÀF×DYÑDYÔZˆ�r4   rG   r~   NÚoutput_hidden_statesÚreturn_dictr8   c                 ó2  — |rdnd }|rdnd }|j                   d   }t        | j                  «      }|}	t        t	        | j
                  | j                  «      «      D ]|  \  }
\  }} ||	«      \  }	}}|D ]&  } ||	|||«      }|d   }	|r	||d   fz   }|sŒ!||	fz   }Œ( |
|dz
  k7  sŒI|	j                  |||d«      j                  dddd«      j                  «       }	Œ~ | j                  |	«      }	|r||	fz   }|st        d„ |	||fD «       «      S t        |	||¬«      S )	N© r   r   r:   r   r;   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrc   rá   )Ú.0Úvs     r3   ú	<genexpr>z%PvtEncoder.forward.<locals>.<genexpr>‘  s   è ø€ Òm˜qÐ_`Ñ_lœÑmùs   ‚Š©Úlast_hidden_staterf   Ú
attentions)rK   Úlenr×   Ú	enumerateÚziprÔ   rA   rB   r†   r-   r]   r   )r1   rG   r~   rÞ   rß   Úall_hidden_statesÚall_self_attentionsrQ   Ú
num_blocksrf   ÚidxÚembedding_layerÚblock_layerr6   r7   r×   Úlayer_outputss                    r3   rT   zPvtEncoder.forwards  si  € ñ #7™B¸DÐÙ$5™b¸4Ðà!×'Ñ'¨Ñ*ˆ
Ü˜Ÿ™“_ˆ
Ø$ˆÜ3<¼SÀ×AVÑAVÐX\×XbÑXbÓ=cÓ3dò 	vÑ/ˆCÑ/�/ ;á+:¸=Ó+IÑ(ˆM˜6 5à$ò M�Ù % m°V¸UÐDUÓ V�Ø -¨aÑ 0�Ù$Ø*=ÀÈqÑAQÐ@SÑ*SÐ'Ú'Ø(9¸]Ð<LÑ(LÑ%ðMð �j 1‘nÓ$Ø -× 5Ñ 5°jÀ&È%ÐQSÓ T× \Ñ \Ð]^Ð`aÐcdÐfgÓ h× sÑ sÓ u‘ð	vð Ÿ™¨Ó6ˆÙØ 1°]Ð4DÑ DÐÙÜÑm ]Ð4EÐGZÐ$[ÔmÓmÐmÜØ+Ø+Ø*ô
ð 	
r4   )FFT)rV   rW   rX   r   r   r%   ÚFloatTensorr[   r]   r   rT   r^   r_   s   @r3   rÅ   rÅ   @  si   ø„ ð0[˜yõ 0[ðj */Ø,1Ø#'ñ#
à×'Ñ'ð#
ð   $™;ð#
ð # T™kð	#
ð
 ˜D‘[ð#
ð 
�Ñ	 ÷#
r4   rÅ   c                   ó€   ‡ — e Zd ZU eed<   dZdZdZg Z e	j                  «       dej                  ddfˆ fd„«       Zˆ xZS )	ÚPvtPreTrainedModelr   ÚpvtrG   )ÚimageÚmoduler8   Nc                 ó  •— t         ‰| �  |«       | j                  j                  }t	        |t
        j                  t
        j                  f«      rOt        j                  |j                  d|¬«       |j                  � t        j                  |j                  «       yyt	        |t        «      rRt        j                  |j                  d|¬«       |j                  �#t        j                  |j                  d|¬«       yyy)zInitialize the weightsr©   )ÚmeanÚstdN)r   Ú_init_weightsr   Úinitializer_ranger    r   rd   r)   ÚinitÚtrunc_normal_Úweightrp   Úzeros_r   r'   r   )r1   rø   rû   r2   s      €r3   rü   z PvtPreTrainedModel._init_weights¡  s¾   ø€ ô 	‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fœrŸy™y¬"¯)©)Ð4Ô5Ü×Ñ˜vŸ}™}°3¸CÕ@Ø�{‰{Ð&Ü—‘˜FŸK™KÕ(ð 'ä˜Ô 2Ô3Ü×Ñ˜v×9Ñ9ÀÈÕMØ×ÑÐ+Ü×"Ñ" 6×#3Ñ#3¸#À3ÖGð ,ð 4r4   )rV   rW   rX   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚ_no_split_modulesr%   Úno_gradr   ÚModulerü   r^   r_   s   @r3   rõ   rõ   ™  sP   ø… àÓØÐØ$€OØ!ÐØÐà€U‡]�]ƒ_ðH B§I¡Ið H°$ô Hó ôHr4   rõ   c                   ó~   ‡ — e Zd Zdefˆ fd„Ze	 	 	 d
dej                  dedz  dedz  dedz  de	e
z  f
d	„«       Zˆ xZS )ÚPvtModelr   c                 ór   •— t         ‰| �  |«       || _        t        |«      | _        | j                  «        y rc   )r   r   r   rÅ   ÚencoderÚ	post_init©r1   r   r2   s     €r3   r   zPvtModel.__init__²  s1   ø€ Ü‰Ñ˜Ô ØˆŒô " &Ó)ˆŒð 	�‰Õr4   NrG   r~   rÞ   rß   r8   c                 ó,  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }| j	                  ||||¬«      }|d   }|s	|f|dd  z   S t        ||j                  |j                  ¬«      S )N©rG   r~   rÞ   rß   r   r   ræ   )r   r~   rÞ   rß   r  r   rf   rè   )r1   rG   r~   rÞ   rß   ÚkwargsÚencoder_outputsÚsequence_outputs           r3   rT   zPvtModel.forward¼  s·   € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàŸ,™,Ø%Ø/Ø!5Ø#ð	 'ó 
ˆð *¨!Ñ,ˆáØ#Ð%¨¸¸Ð(;Ñ;Ð;äØ-Ø)×7Ñ7Ø&×1Ñ1ô
ð 	
r4   )NNN)rV   rW   rX   r   r   r   r%   ró   r[   r]   r   rT   r^   r_   s   @r3   r
  r
  °  su   ø„ ð˜yõ ð ð *.Ø,0Ø#'ñ
à×'Ñ'ð
ð   $™;ð
ð # T™kð	
ð
 ˜D‘[ð
ð 
�Ñ	 ò
ó ô
r4   r
  z¤
    Pvt Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.
    )Úcustom_introc                   ó¨   ‡ — e Zd Zdeddfˆ fd„Ze	 	 	 	 ddej                  dz  dej                  dz  dedz  dedz  d	edz  de	e
z  fd
„«       Zˆ xZS )ÚPvtForImageClassificationr   r8   Nc                 ó0  •— t         ‰| �  |«       |j                  | _        t        |«      | _        |j                  dkD  r-t        j                  |j                  d   |j                  «      nt        j                  «       | _	        | j                  «        y )Nr   r:   )r   r   Ú
num_labelsr
  rö   r   rd   rÒ   r¼   Ú
classifierr  r  s     €r3   r   z"PvtForImageClassification.__init__ä  sy   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ˜FÓ#ˆŒð FL×EVÑEVÐYZÒEZŒB�I‰I�f×)Ñ)¨"Ñ-¨v×/@Ñ/@ÔAÔ`b×`kÑ`kÓ`mð 	Œð
 	�‰Õr4   rG   Úlabelsr~   rÞ   rß   c                 óR  — |�|n| j                   j                  }| j                  ||||¬«      }|d   }| j                  |dd…ddd…f   «      }	d}
|�| j	                  ||	| j                   «      }
|s|	f|dd z   }|
�|
f|z   S |S t        |
|	|j                  |j                  ¬«      S )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr  r   r   )ÚlossÚlogitsrf   rè   )r   rß   rö   r  Úloss_functionr	   rf   rè   )r1   rG   r  r~   rÞ   rß   r  r�   r  r  r  r”   s               r3   rT   z!PvtForImageClassification.forwardò  sÑ   € ð  &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—(‘(Ø%Ø/Ø!5Ø#ð	 ó 
ˆð " !™*ˆà—‘ ²°A²q°Ñ!9Ó:ˆàˆØÐØ×%Ñ% f¨f°d·k±kÓBˆDáØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r4   )NNNN)rV   rW   rX   r   r   r   r%   r\   r[   r]   r	   rT   r^   r_   s   @r3   r  r  Ý  s–   ø„ ð˜yð ¨Tõ ð ð '+Ø)-Ø,0Ø#'ñ)
à—l‘l TÑ)ð)
ð —‘˜tÑ#ð)
ð   $™;ð	)
ð
 # T™kð)
ð ˜D‘[ð)
ð 
Ð&Ñ	&ò)
ó ô)
r4   r  )r  r
  rõ   ))rY   r!   r‚   Úcollections.abcr   r%   Útorch.nn.functionalr   r„   rC   Ú r   rþ   Úactivationsr   Úmodeling_outputsr   r	   Úmodeling_utilsr
   Úutilsr   r   Úconfiguration_pvtr   Ú
get_loggerrV   Úloggerr  r   ra   rk   r’   r™   r¥   r¶   rÅ   rõ   r
  r  Ú__all__rá   r4   r3   ú<module>r*     s9  ðñ  ã Û Ý $ã ß Ð Ý å &Ý !ß FÝ -ß ,Ý (ð 
ˆ×	Ñ	˜HÓ	%€ôA)˜Ÿ™ô A)ôH	�B—I‘Iô 	ôO §	¡	ô Oôd�2—9‘9ô ô.ˆR�Y‰Yô ô8%�"—)‘)ô %ô0+ˆr�y‰yô +ô\V
�—‘ô V
ðr ôH˜ó Hó ðHð, ô)
Ð!ó )
ó ð)
ñX ðôô9
Ð 2ó 9
óð9
òx J�r4   