Ë
    ÛÓ^jP\  ã                   ó¦  — d Z ddlZddlmZ ddlZddlmZ ddlmZm	Z	m
Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ  ej0                  e«      Z ed¬«      e G d„ de«      «       «       Z G d„ dej8                  «      Z G d„ dej8                  «      Z G d„ dej8                  «      Z G d„ dej8                  «      Z  G d„ dej8                  «      Z! G d„ dej8                  «      Z" G d„ dej8                  «      Z# G d„ d ej8                  «      Z$ G d!„ d"ej8                  «      Z% G d#„ d$ej8                  «      Z& G d%„ d&ej8                  «      Z' G d'„ d(ej8                  «      Z( G d)„ d*ej8                  «      Z) G d+„ d,ej8                  «      Z*e G d-„ d.e«      «       Z+e G d/„ d0e+«      «       Z, ed1¬«       G d2„ d3e+«      «       Z-g d4¢Z.y)5zPyTorch CvT model.é    N)Ú	dataclass)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)Ú$ImageClassifierOutputWithNoAttentionÚModelOutput)ÚPreTrainedModel)Úauto_docstringÚloggingé   )Ú	CvtConfigzV
    Base class for model's outputs, with potential hidden states and attentions.
    )Úcustom_introc                   ó”   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   y)ÚBaseModelOutputWithCLSTokenz£
    cls_token_value (`torch.FloatTensor` of shape `(batch_size, 1, hidden_size)`):
        Classification token at the output of the last layer of the model.
    NÚlast_hidden_stateÚcls_token_value.Úhidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__r   r   Útuple© ó    úo/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/cvt/modeling_cvt.pyr   r   !   sS   … ñð
 37Ð�u×(Ñ(¨4Ñ/Ó6Ø04€O�U×&Ñ&¨Ñ-Ó4Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ô>r    r   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚCvtEmbeddingsz'
    Construct the CvT embeddings.
    c                 ó€   •— t         ‰| �  «        t        |||||¬«      | _        t	        j
                  |«      | _        y )N)Ú
patch_sizeÚnum_channelsÚ	embed_dimÚstrideÚpadding)ÚsuperÚ__init__ÚCvtConvEmbeddingsÚconvolution_embeddingsr   ÚDropoutÚdropout)Úselfr%   r&   r'   r(   r)   Údropout_rateÚ	__class__s          €r!   r+   zCvtEmbeddings.__init__7   s:   ø€ Ü‰ÑÔÜ&7Ø!°È	ÐZ`Ðjqô'
ˆÔ#ô —z‘z ,Ó/ˆ�r    c                 óJ   — | j                  |«      }| j                  |«      }|S ©N)r-   r/   )r0   Úpixel_valuesÚhidden_states      r!   ÚforwardzCvtEmbeddings.forward>   s&   € Ø×2Ñ2°<Ó@ˆØ—|‘| LÓ1ˆØÐr    ©r   r   r   r   r+   r7   Ú__classcell__©r2   s   @r!   r#   r#   2   s   ø„ ñô0ör    r#   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )r,   z"
    Image to Conv Embedding.
    c                 óö   •— t         ‰| �  «        t        |t        j                  j
                  «      r|n||f}|| _        t        j                  |||||¬«      | _	        t        j                  |«      | _        y )N)Úkernel_sizer(   r)   )r*   r+   Ú
isinstanceÚcollectionsÚabcÚIterabler%   r   ÚConv2dÚ
projectionÚ	LayerNormÚnormalization)r0   r%   r&   r'   r(   r)   r2   s         €r!   r+   zCvtConvEmbeddings.__init__I   sa   ø€ Ü‰ÑÔÜ#-¨j¼+¿/¹/×:RÑ:RÔ#S‘ZÐZdÐfpÐYqˆ
Ø$ˆŒÜŸ)™) L°)ÈÐ\bÐlsÔtˆŒÜŸ\™\¨)Ó4ˆÕr    c                 ó   — | j                  |«      }|j                  \  }}}}||z  }|j                  |||«      j                  ddd«      }| j                  r| j	                  |«      }|j                  ddd«      j                  ||||«      }|S ©Nr   é   r   )rC   ÚshapeÚviewÚpermuterE   )r0   r5   Ú
batch_sizer&   ÚheightÚwidthÚhidden_sizes          r!   r7   zCvtConvEmbeddings.forwardP   s™   € Ø—‘ |Ó4ˆØ2>×2DÑ2DÑ/ˆ
�L &¨%Ø˜u‘nˆà#×(Ñ(¨°\À;ÓO×WÑWÐXYÐ[\Ð^_Ó`ˆØ×ÒØ×-Ñ-¨lÓ;ˆLà#×+Ñ+¨A¨q°!Ó4×9Ñ9¸*ÀlÐTZÐ\aÓbˆØÐr    r8   r:   s   @r!   r,   r,   D   s   ø„ ñô5ö
r    r,   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚCvtSelfAttentionConvProjectionc           	      ó˜   •— t         ‰| �  «        t        j                  |||||d|¬«      | _        t        j
                  |«      | _        y )NF)r=   r)   r(   ÚbiasÚgroups)r*   r+   r   rB   ÚconvolutionÚBatchNorm2drE   )r0   r'   r=   r)   r(   r2   s        €r!   r+   z'CvtSelfAttentionConvProjection.__init__^   sG   ø€ Ü‰ÑÔÜŸ9™9ØØØ#ØØØØô
ˆÔô  Ÿ^™^¨IÓ6ˆÕr    c                 óJ   — | j                  |«      }| j                  |«      }|S r4   )rU   rE   ©r0   r6   s     r!   r7   z&CvtSelfAttentionConvProjection.forwardk   s(   € Ø×'Ñ'¨Ó5ˆØ×)Ñ)¨,Ó7ˆØÐr    ©r   r   r   r+   r7   r9   r:   s   @r!   rQ   rQ   ]   s   ø„ ô7ör    rQ   c                   ó   — e Zd Zd„ Zy)Ú CvtSelfAttentionLinearProjectionc                 óz   — |j                   \  }}}}||z  }|j                  |||«      j                  ddd«      }|S rG   )rI   rJ   rK   )r0   r6   rL   r&   rM   rN   rO   s          r!   r7   z(CvtSelfAttentionLinearProjection.forwardr   sK   € Ø2>×2DÑ2DÑ/ˆ
�L &¨%Ø˜u‘nˆà#×(Ñ(¨°\À;ÓO×WÑWÐXYÐ[\Ð^_Ó`ˆØÐr    N)r   r   r   r7   r   r    r!   r[   r[   q   s   „ ór    r[   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚCvtSelfAttentionProjectionc                 óp   •— t         ‰| �  «        |dk(  rt        ||||«      | _        t	        «       | _        y )NÚdw_bn)r*   r+   rQ   Úconvolution_projectionr[   Úlinear_projection)r0   r'   r=   r)   r(   Úprojection_methodr2   s         €r!   r+   z#CvtSelfAttentionProjection.__init__{   s7   ø€ Ü‰ÑÔØ Ò'Ü*HÈÐT_ÐahÐjpÓ*qˆDÔ'Ü!AÓ!CˆÕr    c                 óJ   — | j                  |«      }| j                  |«      }|S r4   )ra   rb   rX   s     r!   r7   z"CvtSelfAttentionProjection.forward�   s(   € Ø×2Ñ2°<Ó@ˆØ×-Ñ-¨lÓ;ˆØÐr    )r`   rY   r:   s   @r!   r^   r^   z   s   ø„ õDör    r^   c                   ó.   ‡ — e Zd Z	 dˆ fd„	Zd„ Zd„ Zˆ xZS )ÚCvtSelfAttentionc                 óÎ  •— t         ‰| �  «        |dz  | _        || _        || _        || _        t        |||||dk(  rdn|¬«      | _        t        |||||¬«      | _        t        |||||¬«      | _	        t        j                  |||	¬«      | _        t        j                  |||	¬«      | _        t        j                  |||	¬«      | _        t        j                  |
«      | _        y )Ng      à¿ÚavgÚlinear)rc   )rS   )r*   r+   ÚscaleÚwith_cls_tokenr'   Ú	num_headsr^   Úconvolution_projection_queryÚconvolution_projection_keyÚconvolution_projection_valuer   ÚLinearÚprojection_queryÚprojection_keyÚprojection_valuer.   r/   )r0   rl   r'   r=   Ú	padding_qÚ
padding_kvÚstride_qÚ	stride_kvÚqkv_projection_methodÚqkv_biasÚattention_drop_raterk   Úkwargsr2   s                €r!   r+   zCvtSelfAttention.__init__ˆ   sá   ø€ ô 	‰ÑÔØ ‘_ˆŒ
Ø,ˆÔØ"ˆŒØ"ˆŒä,FØØØØØ*?À5Ò*H™hÐNcô-
ˆÔ)ô +EØ�{ J°	ÐMbô+
ˆÔ'ô -GØ�{ J°	ÐMbô-
ˆÔ)ô !#§	¡	¨)°YÀXÔ NˆÔÜ Ÿi™i¨	°9À8ÔLˆÔÜ "§	¡	¨)°YÀXÔ NˆÔä—z‘zÐ"5Ó6ˆ�r    c                 ó´   — |j                   \  }}}| j                  | j                  z  }|j                  ||| j                  |«      j	                  dddd«      S )Nr   rH   r   r   )rI   r'   rl   rJ   rK   )r0   r6   rL   rO   Ú_Úhead_dims         r!   Ú"rearrange_for_multi_head_attentionz3CvtSelfAttention.rearrange_for_multi_head_attention±   sV   € Ø%1×%7Ñ%7Ñ"ˆ
�K Ø—>‘> T§^¡^Ñ3ˆà× Ñ  ¨[¸$¿.¹.È(ÓS×[Ñ[Ð\]Ð_`ÐbcÐefÓgÐgr    c                 ó`  — | j                   rt        j                  |d||z  gd«      \  }}|j                  \  }}}|j	                  ddd«      j                  ||||«      }| j                  |«      }| j                  |«      }	| j                  |«      }
| j                   rKt        j                  |	fd¬«      }	t        j                  ||fd¬«      }t        j                  ||
fd¬«      }
| j                  | j                  z  }| j                  | j                  |	«      «      }	| j                  | j                  |«      «      }| j                  | j                  |
«      «      }
t        j                   d|	|g«      | j"                  z  }t        j$                  j&                  j)                  |d¬«      }| j+                  |«      }t        j                   d||
g«      }|j                  \  }}}}|j	                  dddd«      j-                  «       j                  ||| j                  |z  «      }|S )	Nr   r   rH   ©Údimzbhlk,bhtk->bhltéÿÿÿÿzbhlt,bhtv->bhlvr   )rk   r   ÚsplitrI   rK   rJ   rn   rm   ro   Úcatr'   rl   r   rq   rr   rs   Úeinsumrj   r   Ú
functionalÚsoftmaxr/   Ú
contiguous)r0   r6   rM   rN   Ú	cls_tokenrL   rO   r&   ÚkeyÚqueryÚvaluer~   Úattention_scoreÚattention_probsÚcontextr}   s                   r!   r7   zCvtSelfAttention.forward·   sñ  € Ø×ÒÜ&+§k¡k°,ÀÀFÈUÁNÐ@SÐUVÓ&WÑ#ˆI�|Ø0<×0BÑ0BÑ-ˆ
�K à#×+Ñ+¨A¨q°!Ó4×9Ñ9¸*ÀlÐTZÐ\aÓbˆà×-Ñ-¨lÓ;ˆØ×1Ñ1°,Ó?ˆØ×1Ñ1°,Ó?ˆà×ÒÜ—I‘I˜y¨%Ð0°aÔ8ˆEÜ—)‘)˜Y¨Ð,°!Ô4ˆCÜ—I‘I˜y¨%Ð0°aÔ8ˆEà—>‘> T§^¡^Ñ3ˆà×7Ñ7¸×8MÑ8MÈeÓ8TÓUˆØ×5Ñ5°d×6IÑ6IÈ#Ó6NÓOˆØ×7Ñ7¸×8MÑ8MÈeÓ8TÓUˆäŸ,™,Ð'8¸5À#¸,ÓGÈ$Ï*É*ÑTˆÜŸ(™(×-Ñ-×5Ñ5°oÈ2Ð5ÓNˆØŸ,™, Ó7ˆä—,‘,Ð0°?ÀEÐ2JÓKˆà&Ÿ}™}Ñˆˆ1ˆk˜1Ø—/‘/ ! Q¨¨1Ó-×8Ñ8Ó:×?Ñ?À
ÈKÐY]×YgÑYgÐjrÑYrÓsˆØˆr    ©T)r   r   r   r+   r   r7   r9   r:   s   @r!   rf   rf   ‡   s   ø„ ð õ'7òRhör    rf   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚCvtSelfOutputz 
    The residual connection is defined in CvtLayer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    c                 óŒ   •— t         ‰| �  «        t        j                  ||«      | _        t        j
                  |«      | _        y r4   )r*   r+   r   rp   Údenser.   r/   )r0   r'   Ú	drop_rater2   s      €r!   r+   zCvtSelfOutput.__init__Þ   s0   ø€ Ü‰ÑÔÜ—Y‘Y˜y¨)Ó4ˆŒ
Ü—z‘z )Ó,ˆ�r    c                 óJ   — | j                  |«      }| j                  |«      }|S r4   ©r•   r/   ©r0   r6   Úinput_tensors      r!   r7   zCvtSelfOutput.forwardã   s$   € Ø—z‘z ,Ó/ˆØ—|‘| LÓ1ˆØÐr    r8   r:   s   @r!   r“   r“   Ø   s   ø„ ñô
-ö
r    r“   c                   ó(   ‡ — e Zd Z	 dˆ fd„	Zd„ Zˆ xZS )ÚCvtAttentionc                 óx   •— t         ‰| �  «        t        |||||||||	|
|«      | _        t	        ||«      | _        y r4   )r*   r+   rf   Ú	attentionr“   Úoutput)r0   rl   r'   r=   rt   ru   rv   rw   rx   ry   rz   r–   rk   r2   s                €r!   r+   zCvtAttention.__init__ê   sM   ø€ ô 	‰ÑÔÜ)ØØØØØØØØ!ØØØó
ˆŒô $ I¨yÓ9ˆ�r    c                 óP   — | j                  |||«      }| j                  ||«      }|S r4   )rž   rŸ   )r0   r6   rM   rN   Úself_outputÚattention_outputs         r!   r7   zCvtAttention.forward	  s+   € Ø—n‘n \°6¸5ÓAˆØŸ;™; {°LÓAÐØÐr    r‘   rY   r:   s   @r!   rœ   rœ   é   s   ø„ ð õ:ö> r    rœ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚCvtIntermediatec                 ó¢   •— t         ‰| �  «        t        j                  |t	        ||z  «      «      | _        t        j                  «       | _        y r4   )r*   r+   r   rp   Úintr•   ÚGELUÚ
activation)r0   r'   Ú	mlp_ratior2   s      €r!   r+   zCvtIntermediate.__init__  s7   ø€ Ü‰ÑÔÜ—Y‘Y˜y¬#¨i¸)Ñ.CÓ*DÓEˆŒ
ÜŸ'™'›)ˆ�r    c                 óJ   — | j                  |«      }| j                  |«      }|S r4   )r•   r¨   rX   s     r!   r7   zCvtIntermediate.forward  s$   € Ø—z‘z ,Ó/ˆØ—‘ |Ó4ˆØÐr    rY   r:   s   @r!   r¤   r¤     s   ø„ ô$ö
r    r¤   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú	CvtOutputc                 ó¤   •— t         ‰| �  «        t        j                  t	        ||z  «      |«      | _        t        j                  |«      | _        y r4   )r*   r+   r   rp   r¦   r•   r.   r/   )r0   r'   r©   r–   r2   s       €r!   r+   zCvtOutput.__init__  s:   ø€ Ü‰ÑÔÜ—Y‘Yœs 9¨yÑ#8Ó9¸9ÓEˆŒ
Ü—z‘z )Ó,ˆ�r    c                 óT   — | j                  |«      }| j                  |«      }||z   }|S r4   r˜   r™   s      r!   r7   zCvtOutput.forward!  s.   € Ø—z‘z ,Ó/ˆØ—|‘| LÓ1ˆØ# lÑ2ˆØÐr    rY   r:   s   @r!   r¬   r¬     s   ø„ ô-ö
r    r¬   c                   ór   ‡ — e Zd ZdZd	deddfˆ fd„Zdej                  dej                  fd„Zde	fd„Z
ˆ xZS )
ÚCvtDropPathzÏStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    Ú	drop_probÚreturnNc                 ó0   •— t         ‰| �  «        || _        y r4   )r*   r+   r±   )r0   r±   r2   s     €r!   r+   zCvtDropPath.__init__0  s   ø€ Ü‰ÑÔØ"ˆ�r    r   c                 óP  — | j                   dk(  s| j                  s|S d| j                   z
  }|j                  d   fd|j                  dz
  z  z   }t	        j
                  ||j                  |j                  ¬«      }t	        j                  ||z   «      }|j                  |«      |z  S )Nç        r   r   )r   )ÚdtypeÚdevice)
r±   ÚtrainingrI   Úndimr   Úrandr¶   r·   ÚfloorÚdiv)r0   r   Ú	keep_probrI   Úrandom_tensors        r!   r7   zCvtDropPath.forward4  s—   € Ø�>‰>˜SÒ ¨¯ªØ Ð Ø˜Ÿ™Ñ&ˆ	Ø×$Ñ$ QÑ'Ð)¨D°M×4FÑ4FÈÑ4JÑ,KÑKˆÜŸ
™
 5°×0CÑ0CÈM×L`ÑL`ÔaˆÜŸ™ M°IÑ$=Ó>ˆØ× Ñ  Ó+¨mÑ;Ð;r    c                 ó    — d| j                   › �S )Nzp=)r±   )r0   s    r!   Ú
extra_reprzCvtDropPath.extra_repr=  s   € Ø�D—N‘NÐ#Ð$Ð$r    )rµ   )r   r   r   r   Úfloatr+   r   ÚTensorr7   ÚstrrÀ   r9   r:   s   @r!   r°   r°   )  sB   ø„ ññ# %ð #°$õ #ð< U§\¡\ð <°e·l±ló <ð%˜C÷ %r    r°   c                   ó,   ‡ — e Zd ZdZ	 dˆ fd„	Zd„ Zˆ xZS )ÚCvtLayerzb
    CvtLayer composed by attention layers, normalization and multi-layer perceptrons (mlps).
    c                 óX  •— t         ‰| �  «        t        |||||||||	|
||«      | _        t	        ||«      | _        t        |||«      | _        |dkD  rt        |«      nt        j                  «       | _        t        j                  |«      | _        t        j                  |«      | _        y )Nrµ   )r*   r+   rœ   rž   r¤   Úintermediater¬   rŸ   r°   r   ÚIdentityÚ	drop_pathrD   Úlayernorm_beforeÚlayernorm_after)r0   rl   r'   r=   rt   ru   rv   rw   rx   ry   rz   r–   r©   Údrop_path_raterk   r2   s                  €r!   r+   zCvtLayer.__init__F  s¢   ø€ ô" 	‰ÑÔÜ%ØØØØØØØØ!ØØØØó
ˆŒô ,¨I°yÓAˆÔÜ 	¨9°iÓ@ˆŒØ8FÈÒ8Lœ ^Ô4ÔRT×R]ÑR]ÓR_ˆŒÜ "§¡¨YÓ 7ˆÔÜ!Ÿ|™|¨IÓ6ˆÕr    c                 ó  — | j                  | j                  |«      ||«      }|}| j                  |«      }||z   }| j                  |«      }| j	                  |«      }| j                  ||«      }| j                  |«      }|S r4   )rž   rÊ   rÉ   rË   rÇ   rŸ   )r0   r6   rM   rN   Úself_attention_outputr¢   Úlayer_outputs          r!   r7   zCvtLayer.forwardm  s‘   € Ø $§¡Ø×!Ñ! ,Ó/ØØó!
Ðð
 1ÐØŸ>™>Ð*:Ó;Ðð (¨,Ñ6ˆð ×+Ñ+¨LÓ9ˆØ×(Ñ(¨Ó6ˆð —{‘{ <°Ó>ˆØ—~‘~ lÓ3ˆØÐr    r‘   r8   r:   s   @r!   rÅ   rÅ   A  s   ø„ ñð& õ%7öNr    rÅ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚCvtStagec                 ó|  •— t         ‰| �  «        || _        || _        | j                  j                  | j                     rFt        j                  t        j                  dd| j                  j                  d   «      «      | _        t        |j                  | j                     |j                  | j                     | j                  dk(  r|j                  n|j                  | j                  dz
     |j                  | j                     |j                  | j                     |j                  | j                     ¬«      | _        t        j"                  d|j$                  | j                     |j&                  |   d¬«      D �cg c]  }|j)                  «       ‘Œ }}t        j*                  t-        |j&                  | j                     «      D �cg c�]T  }t/        |j0                  | j                     |j                  | j                     |j2                  | j                     |j4                  | j                     |j6                  | j                     |j8                  | j                     |j:                  | j                     |j<                  | j                     |j>                  | j                     |j@                  | j                     |j                  | j                     || j                     |jB                  | j                     |j                  | j                     ¬«      ‘�ŒW c}Ž | _"        y c c}w c c}w )Nr   rƒ   r   )r%   r(   r&   r'   r)   r1   Úcpu)r·   )rl   r'   r=   rt   ru   rw   rv   rx   ry   rz   r–   rÌ   r©   rk   )#r*   r+   ÚconfigÚstagerŠ   r   Ú	Parameterr   Úrandnr'   r#   Úpatch_sizesÚpatch_strider&   Úpatch_paddingr–   Ú	embeddingÚlinspacerÌ   ÚdepthÚitemÚ
SequentialÚrangerÅ   rl   Ú
kernel_qkvrt   ru   rw   rv   rx   ry   rz   r©   Úlayers)r0   rÔ   rÕ   ÚxÚdrop_path_ratesr}   r2   s         €r!   r+   zCvtStage.__init__„  s  ø€ Ü‰ÑÔØˆŒØˆŒ
Ø�;‰;× Ñ  §¡Ò,ÜŸ\™\¬%¯+©+°a¸¸D¿K¹K×<QÑ<QÐRTÑ<UÓ*VÓWˆDŒNä&Ø×)Ñ)¨$¯*©*Ñ5Ø×&Ñ& t§z¡zÑ2Ø04·
±
¸a²˜×,Ò,ÀV×EUÑEUÐVZ×V`ÑV`ÐcdÑVdÑEeØ×&Ñ& t§z¡zÑ2Ø×(Ñ(¨¯©Ñ4Ø×)Ñ)¨$¯*©*Ñ5ô
ˆŒô $Ÿn™n¨Q°×0EÑ0EÀdÇjÁjÑ0QÐSY×S_ÑS_Ð`eÑSfÐotÔuö
ØˆA�F‰F�Hð
ˆð 
ô —m‘mô$ ˜vŸ|™|¨D¯J©JÑ7Ó8÷#ð" ô! Ø$×.Ñ.¨t¯z©zÑ:Ø$×.Ñ.¨t¯z©zÑ:Ø &× 1Ñ 1°$·*±*Ñ =Ø$×.Ñ.¨t¯z©zÑ:Ø%×0Ñ0°·±Ñ<Ø$×.Ñ.¨t¯z©zÑ:Ø#Ÿ_™_¨T¯Z©ZÑ8Ø*0×*FÑ*FÀtÇzÁzÑ*RØ#Ÿ_™_¨T¯Z©ZÑ8Ø(.×(BÑ(BÀ4Ç:Á:Ñ(NØ$×.Ñ.¨t¯z©zÑ:Ø#2°4·:±:Ñ#>Ø$×.Ñ.¨t¯z©zÑ:Ø#)×#3Ñ#3°D·J±JÑ#?÷òð
ˆ�ùò	
ùò
s   ÆL4ÇEL9c                 óZ  — d }| j                  |«      }|j                  \  }}}}|j                  ||||z  «      j                  ddd«      }| j                  j
                  | j                     r6| j
                  j                  |dd«      }t        j                  ||fd¬«      }| j                  D ]  } ||||«      }|}Œ | j                  j
                  | j                     rt        j                  |d||z  gd«      \  }}|j                  ddd«      j                  ||||«      }||fS )Nr   rH   r   rƒ   r�   )rÛ   rI   rJ   rK   rÔ   rŠ   rÕ   Úexpandr   r…   râ   r„   )	r0   r6   rŠ   rL   r&   rM   rN   ÚlayerÚlayer_outputss	            r!   r7   zCvtStage.forward®  s'  € Øˆ	Ø—~‘~ lÓ3ˆØ2>×2DÑ2DÑ/ˆ
�L &¨%à#×(Ñ(¨°\À6ÈEÁ>ÓR×ZÑZÐ[\Ð^_ÐabÓcˆØ�;‰;× Ñ  §¡Ò,ØŸ™×-Ñ-¨j¸"¸bÓAˆIÜ Ÿ9™9 i°Ð%>ÀAÔFˆLà—[‘[ò 	)ˆEÙ! ,°¸Ó>ˆMØ(‰Lð	)ð �;‰;× Ñ  §¡Ò,Ü&+§k¡k°,ÀÀFÈUÁNÐ@SÐUVÓ&WÑ#ˆI�|Ø#×+Ñ+¨A¨q°!Ó4×9Ñ9¸*ÀlÐTZÐ\aÓbˆØ˜YÐ&Ð&r    rY   r:   s   @r!   rÑ   rÑ   ƒ  s   ø„ ô(
öT'r    rÑ   c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )Ú
CvtEncoderc                 óô   •— t         ‰| �  «        || _        t        j                  g «      | _        t        t        |j                  «      «      D ]'  }| j
                  j                  t        ||«      «       Œ) y r4   )r*   r+   rÔ   r   Ú
ModuleListÚstagesrà   ÚlenrÝ   ÚappendrÑ   )r0   rÔ   Ú	stage_idxr2   s      €r!   r+   zCvtEncoder.__init__Ã  s[   ø€ Ü‰ÑÔØˆŒÜ—m‘m BÓ'ˆŒÜœs 6§<¡<Ó0Ó1ò 	<ˆIØ�K‰K×Ñœx¨°	Ó:Õ;ñ	<r    c                 óÂ   — |rdnd }|}d }t        | j                  «      D ]  \  }} ||«      \  }}|sŒ||fz   }Œ |st        d„ |||fD «       «      S t        |||¬«      S )Nr   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr4   r   )Ú.0Úvs     r!   ú	<genexpr>z%CvtEncoder.forward.<locals>.<genexpr>Õ  s   è ø€ Òb˜qÐTUÑTaœÑbùs   ‚Š©r   r   r   )Ú	enumeraterí   r   r   )	r0   r5   Úoutput_hidden_statesÚreturn_dictÚall_hidden_statesr6   rŠ   r}   Ústage_modules	            r!   r7   zCvtEncoder.forwardÊ  sˆ   € Ù"6™B¸DÐØ#ˆàˆ	Ü!*¨4¯;©;Ó!7ò 	HÑˆA�Ù&2°<Ó&@Ñ#ˆL˜)Ú#Ø$5¸¸Ñ$GÑ!ð	Hñ
 ÜÑb \°9Ð>OÐ$PÔbÓbÐbä*Ø*Ø%Ø+ô
ð 	
r    )FTrY   r:   s   @r!   rê   rê   Â  s   ø„ ô<÷
r    rê   c                   ó`   ‡ — e Zd ZU eed<   dZdZdgZ ej                  «       ˆ fd„«       Z
ˆ xZS )ÚCvtPreTrainedModelrÔ   Úcvtr5   rÅ   c                 ó  •— t         ‰| �  |«       t        |t        j                  t        j
                  f«      rct        j                  |j                  d| j                  j                  ¬«       |j                  � t        j                  |j                  «       yyt        |t        «      r[| j                  j                  |j                     r7t        j                  |j                  d| j                  j                  ¬«       yyy)zInitialize the weightsrµ   )ÚmeanÚstdN)r*   Ú_init_weightsr>   r   rp   rB   ÚinitÚtrunc_normal_ÚweightrÔ   Úinitializer_rangerS   Úzeros_rÑ   rŠ   rÕ   )r0   Úmoduler2   s     €r!   r  z CvtPreTrainedModel._init_weightså  s´   ø€ ô 	‰Ñ˜fÔ%Ü�fœrŸy™y¬"¯)©)Ð4Ô5Ü×Ñ˜vŸ}™}°3¸D¿K¹K×<YÑ<YÕZØ�{‰{Ð&Ü—‘˜FŸK™KÕ(ð 'ä˜¤Ô)Ø�{‰{×$Ñ$ V§\¡\Ò2Ü×"Ñ" 6×#3Ñ#3¸#À4Ç;Á;×C`ÑC`Öað 3ð *r    )r   r   r   r   r   Úbase_model_prefixÚmain_input_nameÚ_no_split_modulesr   Úno_gradr  r9   r:   s   @r!   rý   rý   Þ  s8   ø… àÓØÐØ$€OØ#˜Ðà€U‡]�]ƒ_ó	bó ô	br    rý   c                   óv   ‡ — e Zd Zdˆ fd„	Ze	 	 	 d	dej                  dz  dedz  dedz  dee	z  fd„«       Z
ˆ xZS )
ÚCvtModelc                 ór   •— t         ‰| �  |«       || _        t        |«      | _        | j                  «        y)zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        N)r*   r+   rÔ   rê   ÚencoderÚ	post_init)r0   rÔ   Úadd_pooling_layerr2   s      €r!   r+   zCvtModel.__init__ô  s/   ø€ ô
 	‰Ñ˜Ô ØˆŒÜ! &Ó)ˆŒØ�‰Õr    Nr5   rø   rù   r²   c                 ó  — |�|n| j                   j                  }|�|n| j                   j                  }|€t        d«      ‚| j	                  |||¬«      }|d   }|s	|f|dd  z   S t        ||j                  |j                  ¬«      S )Nz You have to specify pixel_values©rø   rù   r   r   rö   )rÔ   rø   rù   Ú
ValueErrorr  r   r   r   )r0   r5   rø   rù   r{   Úencoder_outputsÚsequence_outputs          r!   r7   zCvtModel.forwardþ  sª   € ð %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐÜÐ?Ó@Ð@àŸ,™,ØØ!5Ø#ð 'ó 
ˆð
 *¨!Ñ,ˆáØ#Ð%¨¸¸Ð(;Ñ;Ð;ä*Ø-Ø+×;Ñ;Ø)×7Ñ7ô
ð 	
r    r‘   )NNN)r   r   r   r+   r   r   rÂ   Úboolr   r   r7   r9   r:   s   @r!   r  r  ò  sd   ø„ õð ð -1Ø,0Ø#'ñ	
à—l‘l TÑ)ð
ð # T™kð
ð ˜D‘[ð	
ð 
Ð,Ñ	,ò
ó ô
r    r  z¤
    Cvt Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.
    c                   ó”   ‡ — e Zd Zˆ fd„Ze	 	 	 	 d	dej                  dz  dej                  dz  dedz  dedz  dee	z  f
d„«       Z
ˆ xZS )
ÚCvtForImageClassificationc                 ó‚  •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        t        j                  |j                  d   «      | _        |j                  dkD  r-t        j                  |j                  d   |j                  «      nt        j                  «       | _        | j                  «        y )NF)r  rƒ   r   )r*   r+   Ú
num_labelsr  rþ   r   rD   r'   Ú	layernormrp   rÈ   Ú
classifierr  )r0   rÔ   r2   s     €r!   r+   z"CvtForImageClassification.__init__&  s–   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ˜F°eÔ<ˆŒÜŸ™ f×&6Ñ&6°rÑ&:Ó;ˆŒð CI×BSÑBSÐVWÒBWŒB�I‰I�f×&Ñ& rÑ*¨F×,=Ñ,=Ô>Ô]_×]hÑ]hÓ]jð 	Œð
 	�‰Õr    Nr5   Úlabelsrø   rù   r²   c                 ób  — |�|n| j                   j                  }| j                  |||¬«      }|d   }|d   }| j                   j                  d   r| j	                  |«      }nI|j
                  \  }	}
}}|j                  |	|
||z  «      j                  ddd«      }| j	                  |«      }|j                  d¬«      }| j                  |«      }d}|��¯| j                   j                  €¡| j                   j                  dk(  rd| j                   _
        nv| j                   j                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd	| j                   _
        nd
| j                   _
        | j                   j                  dk(  rSt!        «       }| j                   j                  dk(  r& ||j#                  «       |j#                  «       «      }n– |||«      }nŒ| j                   j                  d	k(  rGt%        «       } ||j                  d| j                   j                  «      |j                  d«      «      }n,| j                   j                  d
k(  rt'        «       } |||«      }|s|f|dd z   }|�|f|z   S |S t)        |||j*                  ¬«      S )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr  r   r   rƒ   rH   r�   Ú
regressionÚsingle_label_classificationÚmulti_label_classification)ÚlossÚlogitsr   )rÔ   rù   rþ   rŠ   r  rI   rJ   rK   r   r  Úproblem_typer  r¶   r   Úlongr¦   r   Úsqueezer   r   r
   r   )r0   r5   r  rø   rù   r{   Úoutputsr  rŠ   rL   r&   rM   rN   Úsequence_output_meanr%  r$  Úloss_fctrŸ   s                     r!   r7   z!CvtForImageClassification.forward4  s`  € ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆØ—(‘(ØØ!5Ø#ð ó 
ˆð " !™*ˆØ˜A‘Jˆ	Ø�;‰;× Ñ  Ò$Ø"Ÿn™n¨YÓ7‰Oà6E×6KÑ6KÑ3ˆJ˜ f¨eà-×2Ñ2°:¸|ÈVÐV[É^Ó\×dÑdÐefÐhiÐklÓmˆOØ"Ÿn™n¨_Ó=ˆOà.×3Ñ3¸Ð3Ó:ÐØ—‘Ð!5Ó6ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—;‘;×)Ñ)¨QÒ.Ø/;�D—K‘KÕ,Ø—[‘[×+Ñ+¨aÒ/°V·\±\ÄUÇZÁZÒ5OÐSY×S_ÑS_Ôch×clÑclÒSlØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—;‘;×)Ñ)¨QÒ.Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±×0FÑ0FÓ GÈÏÉÐUWËÓY‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�áØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä3¸ÀfÐ\c×\qÑ\qÔrÐrr    )NNNN)r   r   r   r+   r   r   rÂ   r  r   r
   r7   r9   r:   s   @r!   r  r    s�   ø„ ôð ð -1Ø&*Ø,0Ø#'ñ=sà—l‘l TÑ)ð=sð —‘˜tÑ#ð=sð # T™kð	=sð
 ˜D‘[ð=sð 
Ð5Ñ	5ò=só ô=sr    r  )r  r  rý   )/r   Úcollections.abcr?   Údataclassesr   r   r   Útorch.nnr   r   r   Ú r	   r  Úmodeling_outputsr
   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_cvtr   Ú
get_loggerr   Úloggerr   ÚModuler#   r,   rQ   r[   r^   rf   r“   rœ   r¤   r¬   r°   rÅ   rÑ   rê   rý   r  r  Ú__all__r   r    r!   ú<module>r8     sÄ  ðñ ã Ý !ã Ý ß AÑ Aå &ß QÝ -ß ,Ý (ð 
ˆ×	Ñ	˜HÓ	%€ñ ðôð
 ô? +ó ?ó óð?ô�B—I‘Iô ô$˜Ÿ	™	ô ô2 R§Y¡Yô ô( r§y¡yô ô
 §¡ô 
ôN�r—y‘yô Nôb�B—I‘Iô ô"# �2—9‘9ô # ôL	�b—i‘iô 	ô
�—	‘	ô 
ô%�"—)‘)ô %ô0?ˆr�y‰yô ?ôD<'ˆr�y‰yô <'ô~
�—‘ô 
ð8 ôb˜ó bó ðbð& ô)
Ð!ó )
ó ð)
ñX ðôôMsÐ 2ó MsóðMsò` J�r    