
    ^jP\                        d Z ddlZddlmZ ddlZddlmZ ddlmZm	Z	m
Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ  ej0                  e      Z ed      e G d de                    Z G d dej8                        Z G d dej8                        Z G d dej8                        Z G d dej8                        Z  G d dej8                        Z! G d dej8                        Z" G d dej8                        Z# G d d ej8                        Z$ G d! d"ej8                        Z% G d# d$ej8                        Z& G d% d&ej8                        Z' G d' d(ej8                        Z( G d) d*ej8                        Z) G d+ d,ej8                        Z*e G d- d.e             Z+e G d/ d0e+             Z, ed1       G d2 d3e+             Z-g d4Z.y)5zPyTorch CvT model.    N)	dataclass)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )initialization)$ImageClassifierOutputWithNoAttentionModelOutput)PreTrainedModel)auto_docstringlogging   )	CvtConfigzV
    Base class for model's outputs, with potential hidden states and attentions.
    )custom_introc                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   y)BaseModelOutputWithCLSTokenz
    cls_token_value (`torch.FloatTensor` of shape `(batch_size, 1, hidden_size)`):
        Classification token at the output of the last layer of the model.
    Nlast_hidden_statecls_token_value.hidden_states)__name__
__module____qualname____doc__r   torchFloatTensor__annotations__r   r   tuple     o/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/cvt/modeling_cvt.pyr   r   !   sS    
 37u((4/604OU&&-4:>M5**C/047>r    r   c                   (     e Zd ZdZ fdZd Z xZS )CvtEmbeddingsz'
    Construct the CvT embeddings.
    c                     t         |           t        |||||      | _        t	        j
                  |      | _        y )N)
patch_sizenum_channels	embed_dimstridepadding)super__init__CvtConvEmbeddingsconvolution_embeddingsr   Dropoutdropout)selfr%   r&   r'   r(   r)   dropout_rate	__class__s          r!   r+   zCvtEmbeddings.__init__7   s:    &7!	Z`jq'
# zz,/r    c                 J    | j                  |      }| j                  |      }|S N)r-   r/   )r0   pixel_valueshidden_states      r!   forwardzCvtEmbeddings.forward>   s&    22<@||L1r    r   r   r   r   r+   r7   __classcell__r2   s   @r!   r#   r#   2   s    0r    r#   c                   (     e Zd ZdZ fdZd Z xZS )r,   z"
    Image to Conv Embedding.
    c                     t         |           t        |t        j                  j
                        r|n||f}|| _        t        j                  |||||      | _	        t        j                  |      | _        y )N)kernel_sizer(   r)   )r*   r+   
isinstancecollectionsabcIterabler%   r   Conv2d
projection	LayerNormnormalization)r0   r%   r&   r'   r(   r)   r2   s         r!   r+   zCvtConvEmbeddings.__init__I   sa    #-j+//:R:R#SZZdfpYq
$))L)\blst\\)4r    c                     | j                  |      }|j                  \  }}}}||z  }|j                  |||      j                  ddd      }| j                  r| j	                  |      }|j                  ddd      j                  ||||      }|S Nr      r   )rC   shapeviewpermuterE   )r0   r5   
batch_sizer&   heightwidthhidden_sizes          r!   r7   zCvtConvEmbeddings.forwardP   s    |42>2D2D/
L&%un#((\;OWWXY[\^_`--l;L#++Aq!499*lTZ\abr    r8   r:   s   @r!   r,   r,   D   s    5
r    r,   c                   $     e Zd Z fdZd Z xZS )CvtSelfAttentionConvProjectionc           	          t         |           t        j                  |||||d|      | _        t        j
                  |      | _        y )NF)r=   r)   r(   biasgroups)r*   r+   r   rB   convolutionBatchNorm2drE   )r0   r'   r=   r)   r(   r2   s        r!   r+   z'CvtSelfAttentionConvProjection.__init__^   sG    99#
  ^^I6r    c                 J    | j                  |      }| j                  |      }|S r4   )rU   rE   r0   r6   s     r!   r7   z&CvtSelfAttentionConvProjection.forwardk   s(    ''5)),7r    r   r   r   r+   r7   r9   r:   s   @r!   rQ   rQ   ]   s    7r    rQ   c                       e Zd Zd Zy) CvtSelfAttentionLinearProjectionc                 z    |j                   \  }}}}||z  }|j                  |||      j                  ddd      }|S rG   )rI   rJ   rK   )r0   r6   rL   r&   rM   rN   rO   s          r!   r7   z(CvtSelfAttentionLinearProjection.forwardr   sK    2>2D2D/
L&%un#((\;OWWXY[\^_`r    N)r   r   r   r7   r   r    r!   r[   r[   q   s    r    r[   c                   &     e Zd Zd fd	Zd Z xZS )CvtSelfAttentionProjectionc                 p    t         |           |dk(  rt        ||||      | _        t	               | _        y )Ndw_bn)r*   r+   rQ   convolution_projectionr[   linear_projection)r0   r'   r=   r)   r(   projection_methodr2   s         r!   r+   z#CvtSelfAttentionProjection.__init__{   s7    '*HT_ahjp*qD'!A!Cr    c                 J    | j                  |      }| j                  |      }|S r4   )ra   rb   rX   s     r!   r7   z"CvtSelfAttentionProjection.forward   s(    22<@--l;r    )r`   rY   r:   s   @r!   r^   r^   z   s    Dr    r^   c                   .     e Zd Z	 d fd	Zd Zd Z xZS )CvtSelfAttentionc                    t         |           |dz  | _        || _        || _        || _        t        |||||dk(  rdn|      | _        t        |||||      | _        t        |||||      | _	        t        j                  |||	      | _        t        j                  |||	      | _        t        j                  |||	      | _        t        j                  |
      | _        y )Ng      avglinear)rc   )rS   )r*   r+   scalewith_cls_tokenr'   	num_headsr^   convolution_projection_queryconvolution_projection_keyconvolution_projection_valuer   Linearprojection_queryprojection_keyprojection_valuer.   r/   )r0   rl   r'   r=   	padding_q
padding_kvstride_q	stride_kvqkv_projection_methodqkv_biasattention_drop_raterk   kwargsr2   s                r!   r+   zCvtSelfAttention.__init__   s     	_
,"",F*?5*HhNc-
) +E{J	Mb+
' -G{J	Mb-
) !#		)YX N ii	98L "		)YX Nzz"56r    c                     |j                   \  }}}| j                  | j                  z  }|j                  ||| j                  |      j	                  dddd      S )Nr   rH   r   r   )rI   r'   rl   rJ   rK   )r0   r6   rL   rO   _head_dims         r!   "rearrange_for_multi_head_attentionz3CvtSelfAttention.rearrange_for_multi_head_attention   sV    %1%7%7"
K>>T^^3  [$..(S[[\]_`bcefggr    c                 `   | j                   rt        j                  |d||z  gd      \  }}|j                  \  }}}|j	                  ddd      j                  ||||      }| j                  |      }| j                  |      }	| j                  |      }
| j                   rKt        j                  |	fd      }	t        j                  ||fd      }t        j                  ||
fd      }
| j                  | j                  z  }| j                  | j                  |	            }	| j                  | j                  |            }| j                  | j                  |
            }
t        j                   d|	|g      | j"                  z  }t        j$                  j&                  j)                  |d      }| j+                  |      }t        j                   d||
g      }|j                  \  }}}}|j	                  dddd      j-                         j                  ||| j                  |z        }|S )	Nr   r   rH   dimzbhlk,bhtk->bhltzbhlt,bhtv->bhlvr   )rk   r   splitrI   rK   rJ   rn   rm   ro   catr'   rl   r   rq   rr   rs   einsumrj   r   
functionalsoftmaxr/   
contiguous)r0   r6   rM   rN   	cls_tokenrL   rO   r&   keyqueryvaluer~   attention_scoreattention_probscontextr}   s                   r!   r7   zCvtSelfAttention.forward   s   &+kk,FUN@SUV&W#I|0<0B0B-
K#++Aq!499*lTZ\ab--l;11,?11,?IIy%0a8E))Y,!4CIIy%0a8E>>T^^3778M8Me8TU55d6I6I#6NO778M8Me8TU,,'85#,G$**T((--55o25N,,7,,0?E2JK&}}1k1//!Q1-88:??
KY]YgYgjrYrsr    T)r   r   r   r+   r   r7   r9   r:   s   @r!   rf   rf      s     '7Rhr    rf   c                   (     e Zd ZdZ fdZd Z xZS )CvtSelfOutputz
    The residual connection is defined in CvtLayer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    c                     t         |           t        j                  ||      | _        t        j
                  |      | _        y r4   )r*   r+   r   rp   denser.   r/   )r0   r'   	drop_rater2   s      r!   r+   zCvtSelfOutput.__init__   s0    YYy)4
zz),r    c                 J    | j                  |      }| j                  |      }|S r4   r   r/   r0   r6   input_tensors      r!   r7   zCvtSelfOutput.forward   s$    zz,/||L1r    r8   r:   s   @r!   r   r      s    
-
r    r   c                   (     e Zd Z	 d fd	Zd Z xZS )CvtAttentionc                 x    t         |           t        |||||||||	|
|      | _        t	        ||      | _        y r4   )r*   r+   rf   	attentionr   output)r0   rl   r'   r=   rt   ru   rv   rw   rx   ry   rz   r   rk   r2   s                r!   r+   zCvtAttention.__init__   sM     	)!
 $Iy9r    c                 P    | j                  |||      }| j                  ||      }|S r4   )r   r   )r0   r6   rM   rN   self_outputattention_outputs         r!   r7   zCvtAttention.forward	  s+    nn\65A;;{LAr    r   rY   r:   s   @r!   r   r      s     :> r    r   c                   $     e Zd Z fdZd Z xZS )CvtIntermediatec                     t         |           t        j                  |t	        ||z              | _        t        j                         | _        y r4   )r*   r+   r   rp   intr   GELU
activation)r0   r'   	mlp_ratior2   s      r!   r+   zCvtIntermediate.__init__  s7    YYy#i).C*DE
'')r    c                 J    | j                  |      }| j                  |      }|S r4   )r   r   rX   s     r!   r7   zCvtIntermediate.forward  s$    zz,/|4r    rY   r:   s   @r!   r   r     s    $
r    r   c                   $     e Zd Z fdZd Z xZS )	CvtOutputc                     t         |           t        j                  t	        ||z        |      | _        t        j                  |      | _        y r4   )r*   r+   r   rp   r   r   r.   r/   )r0   r'   r   r   r2   s       r!   r+   zCvtOutput.__init__  s:    YYs9y#899E
zz),r    c                 T    | j                  |      }| j                  |      }||z   }|S r4   r   r   s      r!   r7   zCvtOutput.forward!  s.    zz,/||L1#l2r    rY   r:   s   @r!   r   r     s    -
r    r   c                   r     e Zd ZdZd	deddf fdZdej                  dej                  fdZde	fdZ
 xZS )
CvtDropPathzStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    	drop_probreturnNc                 0    t         |           || _        y r4   )r*   r+   r   )r0   r   r2   s     r!   r+   zCvtDropPath.__init__0  s    "r    r   c                 P   | j                   dk(  s| j                  s|S d| j                   z
  }|j                  d   fd|j                  dz
  z  z   }t	        j
                  ||j                  |j                        }t	        j                  ||z         }|j                  |      |z  S )N        r   r   )r   )dtypedevice)
r   trainingrI   ndimr   randr   r   floordiv)r0   r   	keep_probrI   random_tensors        r!   r7   zCvtDropPath.forward4  s    >>S   &	$$Q')DM4F4F4J,KK

50C0CML`L`aMI$=>  +m;;r    c                      d| j                    S )Nzp=)r   )r0   s    r!   
extra_reprzCvtDropPath.extra_repr=  s    DNN#$$r    )r   )r   r   r   r   floatr+   r   Tensorr7   strr   r9   r:   s   @r!   r   r   )  sB    #% #$ #<U\\ <ell <%C %r    r   c                   ,     e Zd ZdZ	 d fd	Zd Z xZS )CvtLayerzb
    CvtLayer composed by attention layers, normalization and multi-layer perceptrons (mlps).
    c                 X   t         |           t        |||||||||	|
||      | _        t	        ||      | _        t        |||      | _        |dkD  rt        |      nt        j                         | _        t        j                  |      | _        t        j                  |      | _        y )Nr   )r*   r+   r   r   r   intermediater   r   r   r   Identity	drop_pathrD   layernorm_beforelayernorm_after)r0   rl   r'   r=   rt   ru   rv   rw   rx   ry   rz   r   r   drop_path_raterk   r2   s                  r!   r+   zCvtLayer.__init__F  s    " 	%!
 ,IyA	9i@8F8L^4RTR]R]R_ "Y 7!||I6r    c                    | j                  | j                  |      ||      }|}| j                  |      }||z   }| j                  |      }| j	                  |      }| j                  ||      }| j                  |      }|S r4   )r   r   r   r   r   r   )r0   r6   rM   rN   self_attention_outputr   layer_outputs          r!   r7   zCvtLayer.forwardm  s     $!!,/!

 1>>*:; (,6 ++L9((6 {{<>~~l3r    r   r8   r:   s   @r!   r   r   A  s    & %7Nr    r   c                   $     e Zd Z fdZd Z xZS )CvtStagec                 |   t         |           || _        || _        | j                  j                  | j                     rFt        j                  t        j                  dd| j                  j                  d               | _        t        |j                  | j                     |j                  | j                     | j                  dk(  r|j                  n|j                  | j                  dz
     |j                  | j                     |j                  | j                     |j                  | j                           | _        t        j"                  d|j$                  | j                     |j&                  |   d      D cg c]  }|j)                          }}t        j*                  t-        |j&                  | j                           D cg c]T  }t/        |j0                  | j                     |j                  | j                     |j2                  | j                     |j4                  | j                     |j6                  | j                     |j8                  | j                     |j:                  | j                     |j<                  | j                     |j>                  | j                     |j@                  | j                     |j                  | j                     || j                     |jB                  | j                     |j                  | j                           W c} | _"        y c c}w c c}w )Nr   r   r   )r%   r(   r&   r'   r)   r1   cpu)r   )rl   r'   r=   rt   ru   rw   rv   rx   ry   rz   r   r   r   rk   )#r*   r+   configstager   r   	Parameterr   randnr'   r#   patch_sizespatch_strider&   patch_paddingr   	embeddinglinspacer   depthitem
Sequentialranger   rl   
kernel_qkvrt   ru   rw   rv   rx   ry   rz   r   layers)r0   r   r   xdrop_path_ratesr}   r2   s         r!   r+   zCvtStage.__init__  s   
;;  ,\\%++aDKK<Q<QRT<U*VWDN&))$**5&&tzz204

a,,VEUEUVZV`V`cdVdEe&&tzz2((4))$**5
 $nnQ0E0Edjj0QSYS_S_`eSfotu
AFFH
 
 mm$ v||DJJ78#" ! $..tzz:$..tzz: & 1 1$** =$..tzz:%00<$..tzz:#__TZZ8*0*F*Ftzz*R#__TZZ8(.(B(B4::(N$..tzz:#24::#>$..tzz:#)#3#3DJJ#?
	

s   L4EL9c                 Z   d }| j                  |      }|j                  \  }}}}|j                  ||||z        j                  ddd      }| j                  j
                  | j                     r6| j
                  j                  |dd      }t        j                  ||fd      }| j                  D ]  } ||||      }|} | j                  j
                  | j                     rt        j                  |d||z  gd      \  }}|j                  ddd      j                  ||||      }||fS )Nr   rH   r   r   r   )r   rI   rJ   rK   r   r   r   expandr   r   r   r   )	r0   r6   r   rL   r&   rM   rN   layerlayer_outputss	            r!   r7   zCvtStage.forward  s'   	~~l32>2D2D/
L&%#((\6E>RZZ[\^_abc;;  ,--j"bAI 99i%>AFL[[ 	)E!,>M(L	) ;;  ,&+kk,FUN@SUV&W#I|#++Aq!499*lTZ\abY&&r    rY   r:   s   @r!   r   r     s    (
T'r    r   c                   &     e Zd Z fdZddZ xZS )
CvtEncoderc                     t         |           || _        t        j                  g       | _        t        t        |j                              D ]'  }| j
                  j                  t        ||             ) y r4   )r*   r+   r   r   
ModuleListstagesr   lenr   appendr   )r0   r   	stage_idxr2   s      r!   r+   zCvtEncoder.__init__  s[    mmB's6<<01 	<IKKx	:;	<r    c                     |rdnd }|}d }t        | j                        D ]  \  }} ||      \  }}|s||fz   } |st        d |||fD              S t        |||      S )Nr   c              3   &   K   | ]	  }||  y wr4   r   ).0vs     r!   	<genexpr>z%CvtEncoder.forward.<locals>.<genexpr>  s     bqTUTabs   r   r   r   )	enumerater   r   r   )	r0   r5   output_hidden_statesreturn_dictall_hidden_statesr6   r   r}   stage_modules	            r!   r7   zCvtEncoder.forward  s    "6BD#	!*4;;!7 	HA&2<&@#L)#$5$G!	H
 b\9>O$Pbbb**%+
 	
r    )FTrY   r:   s   @r!   r   r     s    <
r    r   c                   `     e Zd ZU eed<   dZdZdgZ ej                          fd       Z
 xZS )CvtPreTrainedModelr   cvtr5   r   c                    t         |   |       t        |t        j                  t        j
                  f      rct        j                  |j                  d| j                  j                         |j                   t        j                  |j                         yyt        |t              r[| j                  j                  |j                     r7t        j                  |j                  d| j                  j                         yyy)zInitialize the weightsr   )meanstdN)r*   _init_weightsr>   r   rp   rB   inittrunc_normal_weightr   initializer_rangerS   zeros_r   r   r   )r0   moduler2   s     r!   r  z CvtPreTrainedModel._init_weights  s     	f%fryy"))45v}}3DKK<Y<YZ{{&FKK( '){{$$V\\2""6#3#3#4;;C`C`a 3 *r    )r   r   r   r   r   base_model_prefixmain_input_name_no_split_modulesr   no_gradr  r9   r:   s   @r!   r   r     s8    $O#U]]_	b 	br    r   c                   v     e Zd Zd fd	Ze	 	 	 d	dej                  dz  dedz  dedz  dee	z  fd       Z
 xZS )
CvtModelc                 r    t         |   |       || _        t        |      | _        | j                          y)zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        N)r*   r+   r   r   encoder	post_init)r0   r   add_pooling_layerr2   s      r!   r+   zCvtModel.__init__  s/    
 	 !&)r    Nr5   r   r   r   c                    ||n| j                   j                  }||n| j                   j                  }|t        d      | j	                  |||      }|d   }|s	|f|dd  z   S t        ||j                  |j                        S )Nz You have to specify pixel_valuesr   r   r   r   r   )r   r   r   
ValueErrorr  r   r   r   )r0   r5   r   r   r{   encoder_outputssequence_outputs          r!   r7   zCvtModel.forward  s     %9$D $++JjJj 	 &1%<k$++BYBY?@@,,!5# ' 

 *!,#%(;;;*-+;;)77
 	
r    r   )NNN)r   r   r   r+   r   r   r   boolr   r   r7   r9   r:   s   @r!   r  r    sd      -1,0#'	
llT)
 #Tk
 D[	
 
,	,
 
r    r  z
    Cvt Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.
    c                        e Zd Z fdZe	 	 	 	 d	dej                  dz  dej                  dz  dedz  dedz  dee	z  f
d       Z
 xZS )
CvtForImageClassificationc                    t         |   |       |j                  | _        t        |d      | _        t        j                  |j                  d         | _        |j                  dkD  r-t        j                  |j                  d   |j                        nt        j                         | _        | j                          y )NF)r  r   r   )r*   r+   
num_labelsr  r   r   rD   r'   	layernormrp   r   
classifierr  )r0   r   r2   s     r!   r+   z"CvtForImageClassification.__init__&  s      ++Fe<f&6&6r&:; CIBSBSVWBWBIIf&&r*F,=,=>]_]h]h]j 	
 	r    Nr5   labelsr   r   r   c                 b   ||n| j                   j                  }| j                  |||      }|d   }|d   }| j                   j                  d   r| j	                  |      }nI|j
                  \  }	}
}}|j                  |	|
||z        j                  ddd      }| j	                  |      }|j                  d      }| j                  |      }d}|| j                   j                  | j                   j                  dk(  rd| j                   _
        nv| j                   j                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd	| j                   _
        nd
| j                   _
        | j                   j                  dk(  rSt!               }| j                   j                  dk(  r& ||j#                         |j#                               }n |||      }n| j                   j                  d	k(  rGt%               } ||j                  d| j                   j                        |j                  d            }n,| j                   j                  d
k(  rt'               } |||      }|s|f|dd z   }||f|z   S |S t)        |||j*                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr  r   r   r   rH   r   
regressionsingle_label_classificationmulti_label_classification)losslogitsr   )r   r   r   r   r  rI   rJ   rK   r   r  problem_typer  r   r   longr   r   squeezer   r   r
   r   )r0   r5   r  r   r   r{   outputsr  r   rL   r&   rM   rN   sequence_output_meanr%  r$  loss_fctr   s                     r!   r7   z!CvtForImageClassification.forward4  s`    &1%<k$++BYBY((!5#  
 "!*AJ	;;  $"nnY7O6E6K6K3Jfe-22:|VV[^\ddefhiklmO"nn_=O.333:!56{{''/;;))Q./;DKK,[[++a/V\\UZZ5OSYS_S_chclclSl/LDKK,/KDKK,{{''<7"9;;))Q.#FNN$4fnn6FGD#FF3D))-JJ+-B0F0F GUWY))-II,./Y,F)-)9TGf$EvE3f\c\q\qrrr    )NNNN)r   r   r   r+   r   r   r   r  r   r
   r7   r9   r:   s   @r!   r  r    s      -1&*,0#'=sllT)=s t#=s #Tk	=s
 D[=s 
5	5=s =sr    r  )r  r  r   )/r   collections.abcr?   dataclassesr   r   r   torch.nnr   r   r    r	   r  modeling_outputsr
   r   modeling_utilsr   utilsr   r   configuration_cvtr   
get_loggerr   loggerr   Moduler#   r,   rQ   r[   r^   rf   r   r   r   r   r   r   r   r   r   r  r  __all__r   r    r!   <module>r8     s     !   A A & Q - , ( 
		H	% 
 ?+ ? ?BII $		 2RYY (ryy 
 
Nryy NbBII "# 299 # L	bii 	
		 
%")) %0?ryy ?D<'ryy <'~
 
8 b b b& )
! )
 )
X Ms 2 MsMs` Jr    