Ë
    ÜÓ^jÿb  ã                   óJ  — d Z ddlZddlmZ ddlZddlmZ ddlmZ ddl	m
Z
mZmZmZ ddlmZ dd	lmZmZ d
dlmZ  ej*                  e«      Z ed¬«      e G d„ de«      «       «       Z G d„ dej2                  «      Z G d„ dej2                  «      Z G d„ dej2                  «      Z G d„ dej2                  «      Z G d„ dej2                  «      Z G d„ dej2                  «      Z G d„ dej2                  «      Z  G d„ dej2                  «      Z! G d „ d!ej2                  «      Z" G d"„ d#ej2                  «      Z# G d$„ d%ej2                  «      Z$e G d&„ d'e«      «       Z%e G d(„ d)e%«      «       Z& ed*¬«       G d+„ d,e%«      «       Z' ed-¬«       G d.„ d/e%«      «       Z(g d0¢Z)y)1zPyTorch LeViT model.é    N)Ú	dataclass)Únné   )Úinitialization)ÚBaseModelOutputWithNoAttentionÚ(BaseModelOutputWithPoolingAndNoAttentionÚ$ImageClassifierOutputWithNoAttentionÚModelOutput)ÚPreTrainedModel)Úauto_docstringÚloggingé   )ÚLevitConfigzD
    Output type of [`LevitForImageClassificationWithTeacher`].
    )Úcustom_introc                   ó¸   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
eej                     dz  ed<   y)Ú,LevitForImageClassificationWithTeacherOutputan  
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Prediction scores as the average of the `cls_logits` and `distillation_logits`.
    cls_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Prediction scores of the classification head (i.e. the linear layer on top of the final hidden state of the
        class token).
    distillation_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Prediction scores of the distillation head (i.e. the linear layer on top of the final hidden state of the
        distillation token).
    NÚlogitsÚ
cls_logitsÚdistillation_logitsÚhidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__r   r   r   Útuple© ó    ús/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/levit/modeling_levit.pyr   r   %   sc   … ñ	ð (,€FˆE×Ñ Ñ$Ó+Ø+/€J�×!Ñ! DÑ(Ó/Ø48Ð˜×*Ñ*¨TÑ1Ó8Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ô9r    r   c                   ó,   ‡ — e Zd ZdZ	 dˆ fd„	Zd„ Zˆ xZS )ÚLevitConvEmbeddingsz[
    LeViT Conv Embeddings with Batch Norm, used in the initial patch embedding layer.
    c	           
      óš   •— t         ‰	| �  «        t        j                  |||||||d¬«      | _        t        j
                  |«      | _        y )NF)ÚdilationÚgroupsÚbias)ÚsuperÚ__init__r   ÚConv2dÚconvolutionÚBatchNorm2dÚ
batch_norm)
ÚselfÚin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingr%   r&   Úbn_weight_initÚ	__class__s
            €r!   r)   zLevitConvEmbeddings.__init__B   sF   ø€ ô 	‰ÑÔÜŸ9™9Ø˜ {°F¸GÈhÐ_eÐlqô
ˆÔô Ÿ.™.¨Ó6ˆ�r    c                 óJ   — | j                  |«      }| j                  |«      }|S ©N)r+   r-   )r.   Ú
embeddingss     r!   ÚforwardzLevitConvEmbeddings.forwardK   s&   € Ø×%Ñ% jÓ1ˆ
Ø—_‘_ ZÓ0ˆ
ØÐr    )r   r   r   ©r   r   r   r   r)   r9   Ú__classcell__©r5   s   @r!   r#   r#   =   s   ø„ ñð
 mnõ7ör    r#   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚLevitPatchEmbeddingsz�
    LeViT patch embeddings, for final embeddings to be passed to transformer blocks. It consists of multiple
    `LevitConvEmbeddings`.
    c                 óX  •— t         ‰| �  «        t        |j                  |j                  d   dz  |j
                  |j                  |j                  «      | _        t        j                  «       | _        t        |j                  d   dz  |j                  d   dz  |j
                  |j                  |j                  «      | _        t        j                  «       | _        t        |j                  d   dz  |j                  d   dz  |j
                  |j                  |j                  «      | _        t        j                  «       | _        t        |j                  d   dz  |j                  d   |j
                  |j                  |j                  «      | _        |j                  | _        y )Nr   é   é   é   )r(   r)   r#   Únum_channelsÚhidden_sizesr1   r2   r3   Úembedding_layer_1r   Ú	HardswishÚactivation_layer_1Úembedding_layer_2Úactivation_layer_2Úembedding_layer_3Úactivation_layer_3Úembedding_layer_4©r.   Úconfigr5   s     €r!   r)   zLevitPatchEmbeddings.__init__W   so  ø€ Ü‰ÑÔÜ!4Ø×Ñ ×!4Ñ!4°QÑ!7¸1Ñ!<¸f×>PÑ>PÐRX×R_ÑR_Ðag×aoÑaoó"
ˆÔô #%§,¡,£.ˆÔä!4Ø×Ñ Ñ" aÑ'¨×)<Ñ)<¸QÑ)?À1Ñ)DÀf×FXÑFXÐZ`×ZgÑZgÐio×iwÑiwó"
ˆÔô #%§,¡,£.ˆÔä!4Ø×Ñ Ñ" aÑ'¨×)<Ñ)<¸QÑ)?À1Ñ)DÀf×FXÑFXÐZ`×ZgÑZgÐio×iwÑiwó"
ˆÔô #%§,¡,£.ˆÔä!4Ø×Ñ Ñ" aÑ'¨×)<Ñ)<¸QÑ)?À×ASÑASÐU[×UbÑUbÐdj×drÑdró"
ˆÔð #×/Ñ/ˆÕr    c                 ó„  — |j                   d   }|| j                  k7  rt        d«      ‚| j                  |«      }| j	                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|j                  d«      j                  dd«      S )Nr   zeMake sure that the channel dimension of the pixel values match with the one set in the configuration.rB   )ÚshaperC   Ú
ValueErrorrE   rG   rH   rI   rJ   rK   rL   ÚflattenÚ	transpose)r.   Úpixel_valuesrC   r8   s       r!   r9   zLevitPatchEmbeddings.forwardm   sÀ   € Ø#×)Ñ)¨!Ñ,ˆØ˜4×,Ñ,Ò,ÜØwóð ð ×+Ñ+¨LÓ9ˆ
Ø×,Ñ,¨ZÓ8ˆ
Ø×+Ñ+¨JÓ7ˆ
Ø×,Ñ,¨ZÓ8ˆ
Ø×+Ñ+¨JÓ7ˆ
Ø×,Ñ,¨ZÓ8ˆ
Ø×+Ñ+¨JÓ7ˆ
Ø×!Ñ! !Ó$×.Ñ.¨q°!Ó4Ð4r    r:   r<   s   @r!   r>   r>   Q   s   ø„ ñô
0ö,5r    r>   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚMLPLayerWithBNc                 ó�   •— t         ‰| �  «        t        j                  ||d¬«      | _        t        j
                  |«      | _        y )NF)Úin_featuresÚout_featuresr'   )r(   r)   r   ÚLinearÚlinearÚBatchNorm1dr-   )r.   Ú	input_dimÚ
output_dimr4   r5   s       €r!   r)   zMLPLayerWithBN.__init__~   s3   ø€ Ü‰ÑÔÜ—i‘i¨IÀJÐUZÔ[ˆŒÜŸ.™.¨Ó4ˆ�r    c                 óˆ   — | j                  |«      }| j                  |j                  dd«      «      j                  |«      }|S )Nr   r   )r[   r-   rR   Ú
reshape_as©r.   Úhidden_states     r!   r9   zMLPLayerWithBN.forwardƒ   s<   € Ø—{‘{ <Ó0ˆØ—‘ |×';Ñ';¸A¸qÓ'AÓB×MÑMÈlÓ[ˆØÐr    )r   ©r   r   r   r)   r9   r;   r<   s   @r!   rV   rV   }   s   ø„ õ5ö
r    rV   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚLevitSubsamplec                 ó>   •— t         ‰| �  «        || _        || _        y r7   )r(   r)   r2   Ú
resolution)r.   r2   rg   r5   s      €r!   r)   zLevitSubsample.__init__Š   s   ø€ Ü‰ÑÔØˆŒØ$ˆ�r    c                 óÜ   — |j                   \  }}}|j                  || j                  | j                  |«      d d …d d | j                  …d d | j                  …f   j	                  |d|«      }|S )Néÿÿÿÿ)rP   Úviewrg   r2   Úreshape)r.   rb   Ú
batch_sizeÚ_Úchannelss        r!   r9   zLevitSubsample.forward�   sk   € Ø".×"4Ñ"4Ñˆ
�A�xØ#×(Ñ(¨°T·_±_ÀdÇoÁoÐW_Ó`Ú‰~�$—+‘+ˆ~™~ $§+¡+˜~Ð-ñ
ç
‰'�*˜b (Ó
+ð 	ð Ðr    rc   r<   s   @r!   re   re   ‰   s   ø„ ô%ö
r    re   c                   ó^   ‡ — e Zd Zˆ fd„Z ej
                  «       dˆ fd„	«       Zd„ Zd„ Zˆ xZ	S )ÚLevitAttentionc                 óš  •— t         ‰| �  «        || _        |dz  | _        || _        || _        ||z  |z  ||z  dz  z   | _        ||z  |z  | _        t        || j                  «      | _	        t        j                  «       | _        t        | j                  |d¬«      | _        t        t        j                   t#        |«      t#        |«      «      «      }t%        |«      }|| _        i g }	}|D ]W  }
|D ]P  }t)        |
d   |d   z
  «      t)        |
d   |d   z
  «      f}||vrt%        |«      ||<   |	j+                  ||   «       ŒR ŒY |	| _        i | _        t0        j                  j3                  t1        j4                  |t%        |«      «      «      | _        | j9                  dt1        j:                  |	«      j=                  ||«      d¬«       y )	Nç      à¿rB   r   )r4   r   Úattention_bias_idxsF©Ú
persistent)r(   r)   Únum_attention_headsÚscaleÚkey_dimÚattention_ratioÚout_dim_keys_valuesÚout_dim_projectionrV   Úqueries_keys_valuesr   rF   Ú
activationÚ
projectionÚlistÚ	itertoolsÚproductÚrangeÚlenÚ
len_pointsÚabsÚappendÚindicesÚattention_bias_cacher   Ú	ParameterÚzerosÚattention_biasesÚregister_bufferÚ
LongTensorrj   )r.   rD   rx   rv   ry   rg   Úpointsr„   Úattention_offsetsr‡   Úp1Úp2Úoffsetr5   s                €r!   r)   zLevitAttention.__init__˜   sÀ  ø€ Ü‰ÑÔØ#6ˆÔ Ø˜d‘]ˆŒ
ØˆŒØ.ˆÔØ#2°WÑ#<Ð?RÑ#RÐU\Ð_rÑUrÐuvÑUvÑ#vˆÔ Ø"1°GÑ";Ð>QÑ"QˆÔä#1°,À×@XÑ@XÓ#YˆÔ ÜŸ,™,›.ˆŒÜ(¨×)@Ñ)@À,Ð_`ÔaˆŒä”i×'Ñ'¬¨jÓ(9¼5ÀÓ;LÓMÓNˆÜ˜“[ˆ
Ø$ˆŒØ%'¨˜7ÐØò 	:ˆBØò :�Ü˜b ™e b¨¡e™mÓ,¬c°"°Q±%¸"¸Q¹%±-Ó.@ÐA�ØÐ!2Ñ2Ü03Ð4EÓ0FÐ% fÑ-Ø—‘Ð0°Ñ8Õ9ñ	:ð	:ð ˆŒà$&ˆÔ!Ü %§¡× 2Ñ 2´5·;±;Ð?RÔTWÐXiÓTjÓ3kÓ lˆÔØ×ÑØ!¤5×#3Ñ#3°GÓ#<×#AÑ#AÀ*ÈjÓ#YÐfkð 	õ 	
r    c                 óR   •— t         ‰| �  |«       |r| j                  ri | _        y y y r7   ©r(   Útrainrˆ   ©r.   Úmoder5   s     €r!   r•   zLevitAttention.train·   ó)   ø€ ä‰‰�dÔÙ�D×-Ò-Ø(*ˆDÕ%ð .ˆ4r    c                 óø   — | j                   r| j                  d d …| j                  f   S t        |«      }|| j                  vr*| j                  d d …| j                  f   | j                  |<   | j                  |   S r7   ©Útrainingr‹   rs   Ústrrˆ   ©r.   ÚdeviceÚ
device_keys      r!   Úget_attention_biasesz#LevitAttention.get_attention_biases½   ót   € Ø�=Š=Ø×(Ñ(ª¨D×,DÑ,DÐ)DÑEÐEä˜V›ˆJØ ×!:Ñ!:Ñ:Ø8<×8MÑ8MÊaÐQU×QiÑQiÐNiÑ8j�×)Ñ)¨*Ñ5Ø×,Ñ,¨ZÑ8Ð8r    c                 óÂ  — |j                   \  }}}| j                  |«      }|j                  ||| j                  d«      j	                  | j
                  | j
                  | j                  | j
                  z  gd¬«      \  }}}|j                  dddd«      }|j                  dddd«      }|j                  dddd«      }||j                  dd«      z  | j                  z  | j                  |j                  «      z   }	|	j                  d¬«      }	|	|z  j                  dd«      j                  ||| j                  «      }| j                  | j!                  |«      «      }|S ©Nri   r   ©Údimr   rB   r   éþÿÿÿ)rP   r|   rj   rv   Úsplitrx   ry   ÚpermuterS   rw   r    rž   Úsoftmaxrk   r{   r~   r}   )
r.   rb   rl   Ú
seq_lengthrm   r|   ÚqueryÚkeyÚvalueÚ	attentions
             r!   r9   zLevitAttention.forwardÆ   sN  € Ø$0×$6Ñ$6Ñ!ˆ
�J Ø"×6Ñ6°|ÓDÐØ/×4Ñ4°ZÀÈT×MeÑMeÐgiÓj×pÑpØ�\‰\˜4Ÿ<™<¨×)=Ñ)=ÀÇÁÑ)LÐMÐSTð qó 
Ñˆˆs�Eð —‘˜a  A qÓ)ˆØ�k‰k˜!˜Q  1Ó%ˆØ—‘˜a  A qÓ)ˆà˜CŸM™M¨"¨bÓ1Ñ1°D·J±JÑ>À×AZÑAZÐ[g×[nÑ[nÓAoÑoˆ	Ø×%Ñ%¨"Ð%Ó-ˆ	Ø! EÑ)×4Ñ4°Q¸Ó:×BÑBÀ:ÈzÐ[_×[rÑ[rÓsˆØ—‘ t§¡°|Ó'DÓEˆØÐr    ©T©
r   r   r   r)   r   Úno_gradr•   r    r9   r;   r<   s   @r!   rp   rp   —   s.   ø„ ô
ð> €U‡]�]ƒ_ô+ó ð+ò
9ör    rp   c                   ó^   ‡ — e Zd Zˆ fd„Z ej
                  «       dˆ fd„	«       Zd„ Zd„ Zˆ xZ	S )ÚLevitAttentionSubsamplec	                 ó¢  •— t         ‰| �  «        || _        |dz  | _        || _        || _        ||z  |z  ||z  z   | _        ||z  |z  | _        || _        t        || j                  «      | _
        t        ||«      | _        t        |||z  «      | _        t        j                  «       | _        t        | j                  |«      | _        i | _        t'        t)        j*                  t-        |«      t-        |«      «      «      }	t'        t)        j*                  t-        |«      t-        |«      «      «      }
t/        |	«      t/        |
«      }}|| _        || _        i g }}|
D ]q  }|	D ]j  }d}t5        |d   |z  |d   z
  |dz
  dz  z   «      t5        |d   |z  |d   z
  |dz
  dz  z   «      f}||vrt/        |«      ||<   |j7                  ||   «       Œl Œs || _        t:        j                  j=                  t;        j>                  |t/        |«      «      «      | _         | jC                  dt;        jD                  |«      jG                  ||«      d¬«       y )Nrr   r   r   rB   rs   Frt   )$r(   r)   rv   rw   rx   ry   rz   r{   Úresolution_outrV   Úkeys_valuesre   Úqueries_subsampleÚqueriesr   rF   r}   r~   rˆ   r   r€   r�   r‚   rƒ   Úlen_points_r„   r…   r†   r‡   r   r‰   rŠ   r‹   rŒ   r�   rj   )r.   r]   r^   rx   rv   ry   r2   Úresolution_inrµ   rŽ   Úpoints_r„   r¹   r�   r‡   r�   r‘   Úsizer’   r5   s                      €r!   r)   z LevitAttentionSubsample.__init__Ø   sG  ø€ ô 	‰ÑÔØ#6ˆÔ Ø˜d‘]ˆŒ
ØˆŒØ.ˆÔØ#2°WÑ#<Ð?RÑ#RÐU\Ð_rÑUrÑ#rˆÔ Ø"1°GÑ";Ð>QÑ"QˆÔØ,ˆÔä)¨)°T×5MÑ5MÓNˆÔÜ!/°¸Ó!FˆÔÜ% i°Ð;NÑ1NÓOˆŒÜŸ,™,›.ˆŒÜ(¨×)@Ñ)@À*ÓMˆŒà$&ˆÔ!ä”i×'Ñ'¬¨mÓ(<¼eÀMÓ>RÓSÓTˆÜ”y×(Ñ(¬¨~Ó)>ÄÀnÓ@UÓVÓWˆÜ"% f£+¬s°7«|�Kˆ
Ø&ˆÔØ$ˆŒØ%'¨˜7ÐØò 	:ˆBØò :�Ø�Ü˜b ™e f™n¨r°!©uÑ4¸¸q¹ÀA±~ÑEÓFÌÈBÈqÉEÐTZÉNÐ]_Ð`aÑ]bÑLbÐfjÐmnÑfnÐrsÑesÑLsÓHtÐu�ØÐ!2Ñ2Ü03Ð4EÓ0FÐ% fÑ-Ø—‘Ð0°Ñ8Õ9ñ:ð	:ð ˆŒä %§¡× 2Ñ 2´5·;±;Ð?RÔTWÐXiÓTjÓ3kÓ lˆÔØ×ÑØ!¤5×#3Ñ#3°GÓ#<×#AÑ#AÀ+ÈzÓ#ZÐglð 	õ 	
r    c                 óR   •— t         ‰| �  |«       |r| j                  ri | _        y y y r7   r”   r–   s     €r!   r•   zLevitAttentionSubsample.train  r˜   r    c                 óø   — | j                   r| j                  d d …| j                  f   S t        |«      }|| j                  vr*| j                  d d …| j                  f   | j                  |<   | j                  |   S r7   rš   r�   s      r!   r    z,LevitAttentionSubsample.get_attention_biases  r¡   r    c                 óL  — |j                   \  }}}| j                  |«      j                  ||| j                  d«      j	                  | j
                  | j                  | j
                  z  gd¬«      \  }}|j                  dddd«      }|j                  dddd«      }| j                  | j                  |«      «      }|j                  || j                  dz  | j                  | j
                  «      j                  dddd«      }||j                  dd«      z  | j                  z  | j                  |j                  «      z   }|j                  d¬«      }||z  j                  dd«      j!                  |d| j"                  «      }| j%                  | j'                  |«      «      }|S r£   )rP   r¶   rj   rv   r§   rx   ry   r¨   r¸   r·   rµ   rS   rw   r    rž   r©   rk   r{   r~   r}   )	r.   rb   rl   rª   rm   r¬   r­   r«   r®   s	            r!   r9   zLevitAttentionSubsample.forward  s~  € Ø$0×$6Ñ$6Ñ!ˆ
�J à×Ñ˜\Ó*ß‰T�*˜j¨$×*BÑ*BÀBÓGß‰U�D—L‘L $×"6Ñ"6¸¿¹Ñ"EÐFÈAˆUÓNñ 	ˆˆUð
 �k‰k˜!˜Q  1Ó%ˆØ—‘˜a  A qÓ)ˆà—‘˜T×3Ñ3°LÓAÓBˆØ—
‘
˜: t×':Ñ':¸AÑ'=¸t×?WÑ?WÐY]×YeÑYeÓf×nÑnØˆq�!�Qó
ˆð ˜CŸM™M¨"¨bÓ1Ñ1°D·J±JÑ>À×AZÑAZÐ[g×[nÑ[nÓAoÑoˆ	Ø×%Ñ%¨"Ð%Ó-ˆ	Ø! EÑ)×4Ñ4°Q¸Ó:×BÑBÀ:ÈrÐSW×SjÑSjÓkˆØ—‘ t§¡°|Ó'DÓEˆØÐr    r¯   r°   r<   s   @r!   r³   r³   ×   s/   ø„ ô.
ð` €U‡]�]ƒ_ô+ó ð+ò
9ör    r³   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚLevitMLPLayerzE
    MLP Layer with `2X` expansion in contrast to ViT with `4X`.
    c                 ó˜   •— t         ‰| �  «        t        ||«      | _        t	        j
                  «       | _        t        ||«      | _        y r7   )r(   r)   rV   Ú	linear_upr   rF   r}   Úlinear_down)r.   r]   Ú
hidden_dimr5   s      €r!   r)   zLevitMLPLayer.__init__2  s8   ø€ Ü‰ÑÔÜ'¨	°:Ó>ˆŒÜŸ,™,›.ˆŒÜ)¨*°iÓ@ˆÕr    c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r7   )rÃ   r}   rÄ   ra   s     r!   r9   zLevitMLPLayer.forward8  s4   € Ø—~‘~ lÓ3ˆØ—‘ |Ó4ˆØ×'Ñ'¨Ó5ˆØÐr    r:   r<   s   @r!   rÁ   rÁ   -  s   ø„ ñôAör    rÁ   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚLevitResidualLayerz"
    Residual Block for LeViT
    c                 ó>   •— t         ‰| �  «        || _        || _        y r7   )r(   r)   ÚmoduleÚ	drop_rate)r.   rÊ   rË   r5   s      €r!   r)   zLevitResidualLayer.__init__D  s   ø€ Ü‰ÑÔØˆŒØ"ˆ�r    c                 ó„  — | j                   rŸ| j                  dkD  r�t        j                  |j	                  d«      dd|j
                  ¬«      }|j                  | j                  «      j                  d| j                  z
  «      j                  «       }|| j                  |«      |z  z   }|S || j                  |«      z   }|S )Nr   r   )rž   )
r›   rË   r   Úrandr¼   rž   Úge_ÚdivÚdetachrÊ   )r.   rb   Úrnds      r!   r9   zLevitResidualLayer.forwardI  sŸ   € Ø�=Š=˜TŸ^™^¨aÒ/Ü—*‘*˜\×.Ñ.¨qÓ1°1°aÀ×@SÑ@SÔTˆCØ—'‘'˜$Ÿ.™.Ó)×-Ñ-¨a°$·.±.Ñ.@ÓA×HÑHÓJˆCØ'¨$¯+©+°lÓ*CÀcÑ*IÑIˆLØÐà'¨$¯+©+°lÓ*CÑCˆLØÐr    r:   r<   s   @r!   rÈ   rÈ   ?  s   ø„ ñô#ö
 r    rÈ   c                   ó.   ‡ — e Zd ZdZˆ fd„Zd„ Zd„ Zˆ xZS )Ú
LevitStagezP
    LeViT Stage consisting of `LevitMLPLayer` and `LevitAttention` layers.
    c                 ó
  •— t         ‰| �  «        g | _        || _        |
| _        t        |«      D ]–  }| j                  j                  t        t        |||||
«      | j                  j                  «      «       |dkD  sŒO||z  }| j                  j                  t        t        ||«      | j                  j                  «      «       Œ˜ |	d   dk(  �r| j                  dz
  |	d   z  dz   | _        | j                  j                  t        | j                  j                  ||dz    |	d   |	d   |	d   |	d   |
| j                  dœŽ«       | j                  | _        |	d   dkD  r| j                  j                  |dz      |	d   z  }| j                  j                  t        t        | j                  j                  |dz      |«      | j                  j                  «      «       t        j                  | j                  «      | _        y )	Nr   Ú	Subsampler   é   rB   r   )rx   rv   ry   r2   rº   rµ   rA   )r(   r)   ÚlayersrN   rº   r‚   r†   rÈ   rp   Údrop_path_raterÁ   rµ   r³   rD   r   Ú
ModuleList)r.   rN   ÚidxrD   rx   Údepthsrv   ry   Ú	mlp_ratioÚdown_opsrº   rm   rÅ   r5   s                €r!   r)   zLevitStage.__init__Y  sÖ  ø€ ô 	‰ÑÔØˆŒØˆŒØ*ˆÔä�v“ò 	ˆAØ�K‰K×ÑÜ"Ü" <°Ð:MÈÐ`mÓnØ—K‘K×.Ñ.óôð ˜1‹}Ø)¨IÑ5�
Ø—‘×"Ñ"Ü&¤}°\À:Ó'NÐPT×P[ÑP[×PjÑPjÓkõð	ð �A‰;˜+Ó%Ø#'×#5Ñ#5¸Ñ#9¸hÀq¹kÑ"IÈAÑ"MˆDÔØ�K‰K×ÑÜ'Ø—[‘[×-Ñ-¨c°C¸!±GÐ<Ø$ Q™KØ(0°©Ø$,¨Q¡KØ# A™;Ø"/Ø#'×#6Ñ#6òô
ð "&×!4Ñ!4ˆDÔØ˜‰{˜QŠØ!Ÿ[™[×5Ñ5°c¸A±gÑ>ÀÈ!ÁÑL�
Ø—‘×"Ñ"Ü&Ü% d§k¡k×&>Ñ&>¸sÀQ¹wÑ&GÈÓTÐVZ×VaÑVa×VpÑVpóôô —m‘m D§K¡KÓ0ˆ�r    c                 ó   — | j                   S r7   )rº   )r.   s    r!   Úget_resolutionzLevitStage.get_resolution�  s   € Ø×!Ñ!Ð!r    c                 ó8   — | j                   D ]
  } ||«      }Œ |S r7   )r×   )r.   rb   Úlayers      r!   r9   zLevitStage.forward“  s%   € Ø—[‘[ò 	/ˆEÙ  Ó.‰Lð	/àÐr    )r   r   r   r   r)   rß   r9   r;   r<   s   @r!   rÓ   rÓ   T  s   ø„ ñô51òn"ör    rÓ   c                   ó*   ‡ — e Zd ZdZˆ fd„Zdd„Zˆ xZS )ÚLevitEncoderzC
    LeViT Encoder consisting of multiple `LevitStage` stages.
    c                 ó¦  •— t         ‰| �  «        || _        | j                  j                  | j                  j                  z  }g | _        | j                  j                  j                  dg«       t        t        |j                  «      «      D ]œ  }t        |||j                  |   |j                  |   |j                  |   |j                  |   |j                  |   |j                   |   |j                  |   |«
      }|j#                  «       }| j
                  j                  |«       Œž t%        j&                  | j
                  «      | _        y )NÚ )r(   r)   rN   Ú
image_sizeÚ
patch_sizeÚstagesrÝ   r†   r‚   rƒ   rÛ   rÓ   rD   rx   rv   ry   rÜ   rß   r   rÙ   )r.   rN   rg   Ú	stage_idxÚstager5   s        €r!   r)   zLevitEncoder.__init__ž  s  ø€ Ü‰ÑÔØˆŒØ—[‘[×+Ñ+¨t¯{©{×/EÑ/EÑEˆ
ØˆŒØ�‰×Ñ×#Ñ# R DÔ)äœs 6§=¡=Ó1Ó2ò 	&ˆIÜØØØ×#Ñ# IÑ.Ø—‘˜yÑ)Ø—‘˜iÑ(Ø×*Ñ*¨9Ñ5Ø×&Ñ& yÑ1Ø× Ñ  Ñ+Ø—‘ 	Ñ*ØóˆEð ×-Ñ-Ó/ˆJØ�K‰K×Ñ˜uÕ%ð	&ô  —m‘m D§K¡KÓ0ˆ�r    c                 ó¦   — |rdnd }| j                   D ]  }|r||fz   } ||«      }Œ |r||fz   }|st        d„ ||fD «       «      S t        ||¬«      S )Nr   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr7   r   )Ú.0Úvs     r!   ú	<genexpr>z'LevitEncoder.forward.<locals>.<genexpr>Â  s   è ø€ ÒW˜qÈÉœÑWùs   ‚Š)Úlast_hidden_stater   )rè   r   r   )r.   rb   Úoutput_hidden_statesÚreturn_dictÚall_hidden_statesrê   s         r!   r9   zLevitEncoder.forward·  ss   € Ù"6™B¸DÐà—[‘[ò 	/ˆEÙ#Ø$5¸¸Ñ$GÐ!Ù  Ó.‰Lð	/ñ
  Ø 1°\°OÑ CÐÙÜÑW \Ð3DÐ$EÔWÓWÐWä-ÀÐ\mÔnÐnr    )FTr:   r<   s   @r!   rã   rã   ™  s   ø„ ñô1÷2or    rã   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚLevitClassificationLayerz$
    LeViT Classification Layer
    c                 óŒ   •— t         ‰| �  «        t        j                  |«      | _        t        j
                  ||«      | _        y r7   )r(   r)   r   r\   r-   rZ   r[   )r.   r]   r^   r5   s      €r!   r)   z!LevitClassificationLayer.__init__Ì  s0   ø€ Ü‰ÑÔÜŸ.™.¨Ó3ˆŒÜ—i‘i 	¨:Ó6ˆ�r    c                 óJ   — | j                  |«      }| j                  |«      }|S r7   )r-   r[   )r.   rb   r   s      r!   r9   z LevitClassificationLayer.forwardÑ  s#   € Ø—‘ |Ó4ˆØ—‘˜\Ó*ˆØˆr    r:   r<   s   @r!   rõ   rõ   Ç  s   ø„ ñô7ö
r    rõ   c                   ó<   ‡ — e Zd ZU eed<   dZdZdZdgZˆ fd„Z	ˆ xZ
S )ÚLevitPreTrainedModelrN   ÚlevitrT   )ÚimagerÈ   c                 óì  •— t         ‰| �  |«       t        |t        «      rbt	        j
                  |j                  t        j                  |j                  «      j                  |j                  |j                  «      «       y t        |t        «      rbt	        j
                  |j                  t        j                  |j                  «      j                  |j                  |j                  «      «       y y r7   )r(   Ú_init_weightsÚ
isinstancerp   ÚinitÚcopy_rs   r   r�   r‡   rj   r„   r³   r¹   )r.   rÊ   r5   s     €r!   rý   z"LevitPreTrainedModel._init_weightsß  s­   ø€ Ü‰Ñ˜fÔ%Ü�fœnÔ-Ü�J‰JØ×*Ñ*¬E×,<Ñ,<¸V¿^¹^Ó,L×,QÑ,QÐRX×RcÑRcÐek×evÑevÓ,wõô ˜Ô 7Ô8Ü�J‰JØ×*Ñ*Ü× Ñ  §¡Ó0×5Ñ5°f×6HÑ6HÈ&×J[ÑJ[Ó\õð 9r    )r   r   r   r   r   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚ_no_split_modulesrý   r;   r<   s   @r!   rù   rù   ×  s-   ø… àÓØÐØ$€OØ!ÐØ-Ð.Ð÷
ð 
r    rù   c                   ót   ‡ — e Zd Zˆ fd„Ze	 	 	 ddej                  dz  dedz  dedz  dee	z  fd„«       Z
ˆ xZS )	Ú
LevitModelc                 ó’   •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        | j                  «        y r7   )r(   r)   rN   r>   Úpatch_embeddingsrã   ÚencoderÚ	post_initrM   s     €r!   r)   zLevitModel.__init__î  s:   ø€ Ü‰Ñ˜Ô ØˆŒÜ 4°VÓ <ˆÔÜ# FÓ+ˆŒà�‰Õr    NrT   rñ   rò   Úreturnc                 óD  — |�|n| j                   j                  }|�|n| j                   j                  }|€t        d«      ‚| j	                  |«      }| j                  |||¬«      }|d   }|j                  d¬«      }|s
||f|dd  z   S t        |||j                  ¬«      S )Nz You have to specify pixel_values©rñ   rò   r   r   r¤   )rð   Úpooler_outputr   )	rN   rñ   rò   rQ   r  r	  Úmeanr   r   )	r.   rT   rñ   rò   Úkwargsr8   Úencoder_outputsrð   Úpooled_outputs	            r!   r9   zLevitModel.forwardö  sÌ   € ð %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐÜÐ?Ó@Ð@à×*Ñ*¨<Ó8ˆ
ØŸ,™,ØØ!5Ø#ð 'ó 
ˆð ,¨AÑ.Ðð *×.Ñ.°1Ð.Ó5ˆáØ% }Ð5¸ÈÈÐ8KÑKÐKä7Ø/Ø'Ø)×7Ñ7ô
ð 	
r    ©NNN)r   r   r   r)   r   r   r   Úboolr   r   r9   r;   r<   s   @r!   r  r  ì  sf   ø„ ôð ð 26Ø,0Ø#'ñ	"
à×'Ñ'¨$Ñ.ð"
ð # T™kð"
ð ˜D‘[ð	"
ð 
Ð9Ñ	9ò"
ó ô"
r    r  z…
    Levit Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for
    ImageNet.
    c                   ó”   ‡ — e Zd Zˆ fd„Ze	 	 	 	 d	dej                  dz  dej                  dz  dedz  dedz  de	e
z  f
d„«       Zˆ xZS )
ÚLevitForImageClassificationc                 ó>  •— t         ‰| �  |«       || _        |j                  | _        t	        |«      | _        |j                  dkD  r#t        |j                  d   |j                  «      nt        j                  j                  «       | _        | j                  «        y ©Nr   ri   )r(   r)   rN   Ú
num_labelsr  rú   rõ   rD   r   r   ÚIdentityÚ
classifierr
  rM   s     €r!   r)   z$LevitForImageClassification.__init__#  s€   ø€ Ü‰Ñ˜Ô ØˆŒØ ×+Ñ+ˆŒÜ Ó'ˆŒ
ð
 × Ñ  1Ò$ô % V×%8Ñ%8¸Ñ%<¸f×>OÑ>OÔPä—‘×"Ñ"Ó$ð 	Œð 	�‰Õr    NrT   Úlabelsrñ   rò   r  c                 óH  — |�|n| j                   j                  }| j                  |||¬«      }|d   }|j                  d«      }| j	                  |«      }d}	|�| j                  ||| j                   «      }	|s|f|dd z   }
|	�|	f|
z   S |
S t        |	||j                  ¬«      S )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr  r   r   rB   )Úlossr   r   )rN   rò   rú   r  r  Úloss_functionr	   r   )r.   rT   r  rñ   rò   r  ÚoutputsÚsequence_outputr   r  Úoutputs              r!   r9   z#LevitForImageClassification.forward3  sÂ   € ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—*‘*˜\Ð@TÐbm�*Ónˆà! !™*ˆØ)×.Ñ.¨qÓ1ˆØ—‘ Ó1ˆàˆØÐØ×%Ñ% f¨f°d·k±kÓBˆDáØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä3ØØØ!×/Ñ/ô
ð 	
r    )NNNN)r   r   r   r)   r   r   r   r�   r  r   r	   r9   r;   r<   s   @r!   r  r    s~   ø„ ôð  ð 26Ø*.Ø,0Ø#'ñ"
à×'Ñ'¨$Ñ.ð"
ð × Ñ  4Ñ'ð"
ð # T™kð	"
ð
 ˜D‘[ð"
ð 
Ð5Ñ	5ò"
ó ô"
r    r  ap  
    LeViT Model transformer with image classification heads on top (a linear layer on top of the final hidden state and
    a linear layer on top of the final hidden state of the distillation token) e.g. for ImageNet. .. warning::
           This model supports inference-only. Fine-tuning with distillation (i.e. with a teacher) is not yet
           supported.
    c                   ót   ‡ — e Zd Zˆ fd„Ze	 	 	 ddej                  dz  dedz  dedz  dee	z  fd„«       Z
ˆ xZS )	Ú&LevitForImageClassificationWithTeacherc                 óè  •— t         ‰| �  |«       || _        |j                  | _        t	        |«      | _        |j                  dkD  r#t        |j                  d   |j                  «      nt        j                  j                  «       | _        |j                  dkD  r#t        |j                  d   |j                  «      nt        j                  j                  «       | _        | j                  «        y r  )r(   r)   rN   r  r  rú   rõ   rD   r   r   r  r  Úclassifier_distillr
  rM   s     €r!   r)   z/LevitForImageClassificationWithTeacher.__init__b  sÅ   ø€ Ü‰Ñ˜Ô ØˆŒØ ×+Ñ+ˆŒÜ Ó'ˆŒ
ð
 × Ñ  1Ò$ô % V×%8Ñ%8¸Ñ%<¸f×>OÑ>OÔPä—‘×"Ñ"Ó$ð 	Œð × Ñ  1Ò$ô % V×%8Ñ%8¸Ñ%<¸f×>OÑ>OÔPä—‘×"Ñ"Ó$ð 	Ôð 	�‰Õr    NrT   rñ   rò   r  c                 ó.  — |�|n| j                   j                  }| j                  |||¬«      }|d   }|j                  d«      }| j	                  |«      | j                  |«      }}||z   dz  }	|s|	||f|dd  z   }
|
S t        |	|||j                  ¬«      S )Nr  r   r   rB   )r   r   r   r   )rN   rò   rú   r  r  r&  r   r   )r.   rT   rñ   rò   r  r   r!  r   Údistill_logitsr   r"  s              r!   r9   z.LevitForImageClassificationWithTeacher.forwardw  s´   € ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—*‘*˜\Ð@TÐbm�*Ónˆà! !™*ˆØ)×.Ñ.¨qÓ1ˆØ%)§_¡_°_Ó%EÀt×G^ÑG^Ð_nÓGo�Nˆ
Ø˜~Ñ-°Ñ2ˆáØ˜j¨.Ð9¸GÀAÀB¸KÑGˆFØˆMä;ØØ!Ø .Ø!×/Ñ/ô	
ð 	
r    r  )r   r   r   r)   r   r   r   r  r   r   r9   r;   r<   s   @r!   r$  r$  Y  sf   ø„ ôð* ð 26Ø,0Ø#'ñ	
à×'Ñ'¨$Ñ.ð
ð # T™kð
ð ˜D‘[ð	
ð 
Ð=Ñ	=ò
ó ô
r    r$  )r  r$  r  rù   )*r   r€   Údataclassesr   r   r   rå   r   rÿ   Úmodeling_outputsr   r   r	   r
   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_levitr   Ú
get_loggerr   Úloggerr   ÚModuler#   r>   rV   re   rp   r³   rÁ   rÈ   rÓ   rã   rõ   rù   r  r  r$  Ú__all__r   r    r!   ú<module>r2     s¹  ðñ ã Ý !ã Ý å &÷ó õ .ß ,Ý ,ð 
ˆ×	Ñ	˜HÓ	%€ñ ðôð
 ô:°;ó :ó óð:ô$˜"Ÿ)™)ô ô()5˜2Ÿ9™9ô )5ôX	�R—Y‘Yô 	ô�R—Y‘Yô ô=�R—Y‘Yô =ô@S˜bŸi™iô Sôl�B—I‘Iô ô$ ˜Ÿ™ô  ô*B�—‘ô BôJ+o�2—9‘9ô +oô\˜rŸy™yô ð  ô˜?ó ó ðð( ô,
Ð%ó ,
ó ð,
ñ^ ðôô4
Ð"6ó 4
óð4
ñn ðôô0
Ð-Aó 0
óð0
òf�r    