
    ^jV                        d Z ddlZddlZddlmZ ddlZddlmc mZ	 ddlmZ ddl
mZ ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ  ej2                  e      Z G d dej8                        Z G d dej8                        Z G d dej8                        Z G d dej8                        Z  G d dej8                        Z! G d dej8                        Z" G d dej8                        Z# G d dej8                        Z$e G d de             Z%e G d d e%             Z& ed!"       G d# d$e%             Z'g d%Z(y)&zPyTorch PVT model.    N)Iterable)nn   )initialization)ACT2FN)BaseModelOutputImageClassifierOutput)PreTrainedModel)auto_docstringlogging   )	PvtConfigc                        e Zd ZdZ	 ddedeee   z  deee   z  dedededef fd	Zd
e	j                  dedede	j                  fdZde	j                  dee	j                  eef   fdZ xZS )PvtPatchEmbeddingsz
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    config
image_size
patch_sizestridenum_channelshidden_size	cls_tokenc                    t         	|           || _        t        |t        j
                  j                        r|n||f}t        |t        j
                  j                        r|n||f}|d   |d   z  |d   |d   z  z  }|| _        || _        || _	        || _
        t        j                  t        j                  d|r|dz   n||            | _        |r*t        j                  t        j                   dd|            nd | _        t        j$                  ||||      | _        t        j(                  ||j*                        | _        t        j.                  |j0                        | _        y )Nr   r   kernel_sizer   eps)p)super__init__r   
isinstancecollectionsabcr   r   r   r   num_patchesr   	Parametertorchrandnposition_embeddingszerosr   Conv2d
projection	LayerNormlayer_norm_eps
layer_normDropouthidden_dropout_probdropout)
selfr   r   r   r   r   r   r   r#   	__class__s
            o/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/pvt/modeling_pvt.pyr   zPvtPatchEmbeddings.__init__,   s0    	#-j+//:R:R#SZZdfpYq
#-j+//:R:R#SZZdfpYq
!!}
15*Q-:VW=:XY$$(&#%<<KKi;?[+V$
  JSekk!Q&DEX\))L+6Zde,,{8M8MNzzF$>$>?    
embeddingsheightwidthreturnc                    ||z  }t         j                  j                         s<|| j                  j                  | j                  j                  z  k(  r| j
                  S |j                  d||d      j                  dddd      }t        j                  |||fd      }|j                  dd||z        j                  ddd      }|S )Nr   r   r      bilinear)sizemode)
r%   jit
is_tracingr   r   r'   reshapepermuteFinterpolate)r1   r5   r6   r7   r#   interpolated_embeddingss         r3   interpolate_pos_encodingz+PvtPatchEmbeddings.interpolate_pos_encodingH   s    un yy##%+9O9ORVR]R]RhRh9h*h+++''65"=EEaAqQ
"#--
&%Wa"b"9"A"A!RRW"X"`"`abdegh"i&&r4   pixel_valuesc                    |j                   \  }}}}|| j                  k7  rt        d      | j                  |      }|j                   ^ }}}|j	                  d      j                  dd      }| j                  |      }| j                  | j                  j                  |dd      }	t        j                  |	|fd      }| j                  | j                  d d dd f   ||      }
t        j                  | j                  d d d df   |
fd      }
n| j                  | j                  ||      }
| j                  ||
z         }|||fS )NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.r;   r   r:   dim)shaper   
ValueErrorr*   flatten	transposer-   r   expandr%   catrF   r'   r0   )r1   rG   
batch_sizer   r6   r7   patch_embed_r5   r   r'   s              r3   forwardzPvtPatchEmbeddings.forwardS   sM   2>2D2D/
L&%4,,,w  ool3'--FE!))!,66q!<__[1
>>%--j"bAIIz#:BJ"&"?"?@X@XYZ\]\^Y^@_agin"o"'))T-E-Ea!e-LNa,bhi"j"&"?"?@X@XZ`bg"h\\*/B"BC
65((r4   F)__name__
__module____qualname____doc__r   intr   boolr   r%   TensorrF   tuplerT   __classcell__r2   s   @r3   r   r   %   s      @@ (3-'@ (3-'	@
 @ @ @ @8	'5<< 	' 	'UX 	']b]i]i 	')ELL )U5<<c;Q5R )r4   r   c                   `     e Zd Zdedef fdZdej                  dej                  fdZ xZ	S )PvtSelfOutputr   r   c                     t         |           t        j                  ||      | _        t        j
                  |j                        | _        y N)r   r   r   Lineardenser.   r/   r0   )r1   r   r   r2   s      r3   r   zPvtSelfOutput.__init__j   s6    YY{K8
zz&"<"<=r4   hidden_statesr8   c                 J    | j                  |      }| j                  |      }|S rc   )re   r0   r1   rf   s     r3   rT   zPvtSelfOutput.forwardo   s$    

=1]3r4   
rV   rW   rX   r   rZ   r   r%   r\   rT   r^   r_   s   @r3   ra   ra   i   s1    >y >s >
U\\ ell r4   ra   c                        e Zd ZdZdedededef fdZdedej                  fd	Z
	 ddej                  d
edededeej                     f
dZ xZS )PvtEfficientSelfAttentionzxEfficient self-attention mechanism with reduction of the sequence [PvT paper](https://huggingface.co/papers/2102.12122).r   r   num_attention_headssequences_reduction_ratioc                    t         |           || _        || _        | j                  | j                  z  dk7  r&t	        d| j                   d| j                   d      t        | j                  | j                  z        | _        | j                  | j                  z  | _        t        j                  | j                  | j                  |j                        | _        t        j                  | j                  | j                  |j                        | _        t        j                  | j                  | j                  |j                        | _        t        j                  |j                        | _        || _        |dkD  rEt        j$                  ||||      | _        t        j(                  ||j*                        | _        y y )	Nr   zThe hidden size (z6) is not a multiple of the number of attention heads ())biasr   r   r   )r   r   r   rl   rL   rZ   attention_head_sizeall_head_sizer   rd   qkv_biasquerykeyvaluer.   attention_probs_dropout_probr0   rm   r)   sequence_reductionr+   r,   r-   r1   r   r   rl   rm   r2   s        r3   r   z"PvtEfficientSelfAttention.__init__x   sr    	&#6 d666!;#D$4$4#5 622316 
 $'t'7'7$:R:R'R#S !558P8PPYYt//1C1C&//Z
99T--t/A/AXYYt//1C1C&//Z
zz&"E"EF)B&$q(&(ii[6OXq'D# !ll;F<Q<QRDO	 )r4   rf   r8   c                     |j                         d d | j                  | j                  fz   }|j                  |      }|j	                  dddd      S )Nr:   r   r;   r   r   )r=   rl   rq   viewrB   )r1   rf   	new_shapes      r3   transpose_for_scoresz.PvtEfficientSelfAttention.transpose_for_scores   sT    !&&("-1I1I4KcKc0dd	%**95$$Q1a00r4   r6   r7   output_attentionsc                    | j                  | j                  |            }| j                  dkD  r{|j                  \  }}}|j	                  ddd      j                  ||||      }| j                  |      }|j                  ||d      j	                  ddd      }| j                  |      }| j                  | j                  |            }	| j                  | j                  |            }
t        j                  ||	j                  dd            }|t        j                  | j                        z  }t         j"                  j%                  |d      }| j'                  |      }t        j                  ||
      }|j	                  dddd      j)                         }|j+                         d d | j,                  fz   }|j/                  |      }|r||f}|S |f}|S )Nr   r   r;   r:   rI   r   )r}   rt   rm   rK   rB   rA   rx   r-   ru   rv   r%   matmulrN   mathsqrtrq   r   
functionalsoftmaxr0   
contiguousr=   rr   r{   )r1   rf   r6   r7   r~   query_layerrQ   seq_lenr   	key_layervalue_layerattention_scoresattention_probscontext_layernew_context_layer_shapeoutputss                   r3   rT   z!PvtEfficientSelfAttention.forward   s    //

=0IJ))A-0=0C0C-J)11!Q:BB:|]cejkM 33MBM)11*lBOWWXY[\^_`M OOM:M--dhh}.EF	//

=0IJ !<<Y5H5HR5PQ+dii8P8P.QQ --//0@b/I ,,7_kB%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S%**+BC6G=/2 O\M]r4   rU   )rV   rW   rX   rY   r   rZ   floatr   r%   r\   r}   r[   r]   rT   r^   r_   s   @r3   rk   rk   u   s     CSS.1SHKShmS:1# 1%,, 1 #(*||* * 	*
  * 
u||	*r4   rk   c                   ~     e Zd Zdedededef fdZ	 ddej                  deded	e	d
e
ej                     f
dZ xZS )PvtAttentionr   r   rl   rm   c                 n    t         |           t        ||||      | _        t	        ||      | _        y )N)r   rl   rm   )r   )r   r   rk   r1   ra   outputry   s        r3   r   zPvtAttention.__init__   s8     	-# 3&?	
	 $FDr4   rf   r6   r7   r~   r8   c                 h    | j                  ||||      }| j                  |d         }|f|dd  z   }|S )Nr   r   )r1   r   )r1   rf   r6   r7   r~   self_outputsattention_outputr   s           r3   rT   zPvtAttention.forward   sE     yy?PQ;;|A7#%QR(88r4   rU   )rV   rW   rX   r   rZ   r   r   r%   r\   r[   r]   rT   r^   r_   s   @r3   r   r      sn    
E
E.1
EHK
Ehm
E _d"\\36?BW[	u||	r4   r   c            
       z     e Zd Z	 	 d
dedededz  dedz  f fdZdej                  dej                  fd	Z xZ	S )PvtFFNNr   in_featureshidden_featuresout_featuresc                 j   t         |           ||n|}t        j                  ||      | _        t        |j                  t              rt        |j                     | _	        n|j                  | _	        t        j                  ||      | _
        t        j                  |j                        | _        y rc   )r   r   r   rd   dense1r    
hidden_actstrr   intermediate_act_fndense2r.   r/   r0   )r1   r   r   r   r   r2   s        r3   r   zPvtFFN.__init__   s     	'3'?|[ii_=f''-'-f.?.?'@D$'-'8'8D$ii>zz&"<"<=r4   rf   r8   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }|S rc   )r   r   r0   r   rh   s     r3   rT   zPvtFFN.forward   sP    M200?]3M2]3r4   )NNri   r_   s   @r3   r   r      sY    
 '+#'>> > t	>
 Dj>"U\\ ell r4   r   c                   r     e Zd ZdZd	deddf fdZdej                  dej                  fdZde	fdZ
 xZS )
PvtDropPathzStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    	drop_probr8   Nc                 0    t         |           || _        y rc   )r   r   r   )r1   r   r2   s     r3   r   zPvtDropPath.__init__  s    "r4   rf   c                 P   | j                   dk(  s| j                  s|S d| j                   z
  }|j                  d   fd|j                  dz
  z  z   }t	        j
                  ||j                  |j                        }t	        j                  ||z         }|j                  |      |z  S )N        r   r   )r   )dtypedevice)
r   trainingrK   ndimr%   randr   r   floordiv)r1   rf   	keep_probrK   random_tensors        r3   rT   zPvtDropPath.forward  s    >>S   &	$$Q')DM4F4F4J,KK

50C0CML`L`aMI$=>  +m;;r4   c                      d| j                    S )Nzp=)r   )r1   s    r3   
extra_reprzPvtDropPath.extra_repr  s    DNN#$$r4   )r   )rV   rW   rX   rY   r   r   r%   r\   rT   r   r   r^   r_   s   @r3   r   r      sB    #% #$ #<U\\ <ell <%C %r4   r   c                   f     e Zd Zdedededededef fdZddej                  d	ed
ede	fdZ
 xZS )PvtLayerr   r   rl   	drop_pathrm   	mlp_ratioc                 v   t         |           t        j                  ||j                        | _        t        ||||      | _        |dkD  rt        |      nt        j                         | _
        t        j                  ||j                        | _        t        ||z        }t        |||      | _        y )Nr   )r   r   rl   rm   r   )r   r   r   )r   r   r   r+   r,   layer_norm_1r   	attentionr   Identityr   layer_norm_2rZ   r   mlp)	r1   r   r   rl   r   rm   r   mlp_hidden_sizer2   s	           r3   r   zPvtLayer.__init__  s     	LL&:O:OP%# 3&?	
 4=s?Y/LL&:O:OPkI56[Rabr4   rf   r6   r7   r~   c                    | j                  | j                  |      |||      }|d   }|dd  }| j                  |      }||z   }| j                  | j	                  |            }| j                  |      }||z   }	|	f|z   }|S )N)rf   r6   r7   r~   r   r   )r   r   r   r   r   )
r1   rf   r6   r7   r~   self_attention_outputsr   r   
mlp_outputlayer_outputs
             r3   rT   zPvtLayer.forward)  s    !%++M:/	 "0 "
 2!4(,>>*:;(=8XXd//>?
^^J/
$z1/G+r4   rU   )rV   rW   rX   r   rZ   r   r   r%   r\   r[   rT   r^   r_   s   @r3   r   r     so    cc c !	c
 c $)c c,U\\ 3 s _c r4   r   c                   t     e Zd Zdef fdZ	 	 	 d
dej                  dedz  dedz  dedz  dee	z  f
d	Z
 xZS )
PvtEncoderr   c                    t         	|           || _        t        j                  d|j
                  t        |j                        d      j                         }g }t        |j                        D ]  }|j                  t        ||dk(  r|j                  n| j                  j                  d|dz   z  z  |j                  |   |j                  |   |dk(  r|j                   n|j"                  |dz
     |j"                  |   ||j                  dz
  k(                t%        j&                  |      | _        g }d}t        |j                        D ]  }g }|dk7  r||j                  |dz
     z  }t        |j                  |         D ]\  }|j                  t+        ||j"                  |   |j,                  |   |||z      |j.                  |   |j0                  |                ^ |j                  t%        j&                  |              t%        j&                  |      | _        t%        j4                  |j"                  d   |j6                  	      | _        y )
Nr   cpu)r   r;   r   )r   r   r   r   r   r   r   )r   r   rl   r   rm   r   r:   r   )r   r   r   r%   linspacedrop_path_ratesumdepthstolistrangenum_encoder_blocksappendr   r   patch_sizesstridesr   hidden_sizesr   
ModuleListpatch_embeddingsr   rl   sequence_reduction_ratios
mlp_ratiosblockr+   r,   r-   )
r1   r   drop_path_decaysr5   iblockscurlayersjr2   s
            r3   r   zPvtEncoder.__init__A  s-    !>>!V-B-BCDV_delln 
v001 	A"!45Fv00@V@V[\abefaf[g@h%11!4!>>!,89Q!4!4FDWDWXY\]X]D^ & 3 3A 66#<#<q#@@
	 !#j 9 v001 	1AFAvv}}QU++6==+, 
%$*$7$7$:,2,F,Fq,I"237";282R2RST2U"("3"3A"6	
 MM"--/0!	1$ ]]6*
 ,,v':':2'>FDYDYZr4   rG   r~   Noutput_hidden_statesreturn_dictr8   c                 2   |rdnd }|rdnd }|j                   d   }t        | j                        }|}	t        t	        | j
                  | j                              D ]|  \  }
\  }} ||	      \  }	}}|D ]&  } ||	|||      }|d   }	|r	||d   fz   }|s!||	fz   }( |
|dz
  k7  sI|	j                  |||d      j                  dddd      j                         }	~ | j                  |	      }	|r||	fz   }|st        d |	||fD              S t        |	||      S )	N r   r   r:   r   r;   c              3   &   K   | ]	  }||  y wrc   r   ).0vs     r3   	<genexpr>z%PvtEncoder.forward.<locals>.<genexpr>  s     mq_`_lms   last_hidden_staterf   
attentions)rK   lenr   	enumeratezipr   rA   rB   r   r-   r]   r   )r1   rG   r~   r   r   all_hidden_statesall_self_attentionsrQ   
num_blocksrf   idxembedding_layerblock_layerr6   r7   r   layer_outputss                    r3   rT   zPvtEncoder.forwards  si    #7BD$5b4!''*
_
$3<SAVAVX\XbXb=c3d 	v/C//;+:=+I(M65$ M %mVUDU V -a 0$*=qAQ@S*S''(9]<L(L%M j1n$ - 5 5j&%QS T \ \]^`acdfg h s s u	v 6 1]4D Dm]4EGZ$[mmm++*
 	
r4   )FFT)rV   rW   rX   r   r   r%   FloatTensorr[   r]   r   rT   r^   r_   s   @r3   r   r   @  si    0[y 0[j */,1#'#
''#
  $;#
 #Tk	#

 D[#
 
	 #
r4   r   c                        e Zd ZU eed<   dZdZdZg Z e	j                         dej                  ddf fd       Z xZS )	PvtPreTrainedModelr   pvtrG   )imagemoduler8   Nc                    t         |   |       | j                  j                  }t	        |t
        j                  t
        j                  f      rOt        j                  |j                  d|       |j                   t        j                  |j                         yyt	        |t              rRt        j                  |j                  d|       |j                  #t        j                  |j                  d|       yyy)zInitialize the weightsr   )meanstdN)r   _init_weightsr   initializer_ranger    r   rd   r)   inittrunc_normal_weightrp   zeros_r   r'   r   )r1   r   r   r2   s      r3   r   z PvtPreTrainedModel._init_weights  s     	f%kk++fryy"))45v}}3C@{{&FKK( ' 23v99M+""6#3#3#3G , 4r4   )rV   rW   rX   r   __annotations__base_model_prefixmain_input_nameinput_modalities_no_split_modulesr%   no_gradr   Moduler   r^   r_   s   @r3   r   r     sP    $O!U]]_HBII H$ H Hr4   r   c                   ~     e Zd Zdef fdZe	 	 	 d
dej                  dedz  dedz  dedz  de	e
z  f
d	       Z xZS )PvtModelr   c                 r    t         |   |       || _        t        |      | _        | j                          y rc   )r   r   r   r   encoder	post_initr1   r   r2   s     r3   r   zPvtModel.__init__  s1      "&) 	r4   NrG   r~   r   r   r8   c                 ,   ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }| j	                  ||||      }|d   }|s	|f|dd  z   S t        ||j                  |j                        S )NrG   r~   r   r   r   r   r   )r   r~   r   r   r  r   rf   r   )r1   rG   r~   r   r   kwargsencoder_outputssequence_outputs           r3   rT   zPvtModel.forward  s     2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++BYBY,,%/!5#	 ' 
 *!,#%(;;;-)77&11
 	
r4   )NNN)rV   rW   rX   r   r   r   r%   r   r[   r]   r   rT   r^   r_   s   @r3   r
  r
    su    y   *.,0#'
''
  $;
 #Tk	

 D[
 
	 
 
r4   r
  z
    Pvt Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.
    )custom_introc                        e Zd Zdeddf fdZe	 	 	 	 ddej                  dz  dej                  dz  dedz  dedz  d	edz  de	e
z  fd
       Z xZS )PvtForImageClassificationr   r8   Nc                 0   t         |   |       |j                  | _        t        |      | _        |j                  dkD  r-t        j                  |j                  d   |j                        nt        j                         | _	        | j                          y )Nr   r:   )r   r   
num_labelsr
  r   r   rd   r   r   
classifierr  r  s     r3   r   z"PvtForImageClassification.__init__  sy      ++F# FLEVEVYZEZBIIf))"-v/@/@A`b`k`k`m 	
 	r4   rG   labelsr~   r   r   c                 R   ||n| j                   j                  }| j                  ||||      }|d   }| j                  |dddddf         }	d}
|| j	                  ||	| j                         }
|s|	f|dd z   }|
|
f|z   S |S t        |
|	|j                  |j                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr  r   r   )losslogitsrf   r   )r   r   r   r  loss_functionr	   rf   r   )r1   rG   r  r~   r   r   r  r   r  r  r  r   s               r3   rT   z!PvtForImageClassification.forward  s      &1%<k$++BYBY((%/!5#	  
 "!*Aq!9:%%ffdkkBDY,F)-)9TGf$EvE$!//))	
 	
r4   )NNNN)rV   rW   rX   r   r   r   r%   r\   r[   r]   r	   rT   r^   r_   s   @r3   r  r    s    y T   '+)-,0#')
llT))
 t#)
  $;	)

 #Tk)
 D[)
 
&	&)
 )
r4   r  )r  r
  r   ))rY   r!   r   collections.abcr   r%   torch.nn.functionalr   r   rC    r   r   activationsr   modeling_outputsr   r	   modeling_utilsr
   utilsr   r   configuration_pvtr   
get_loggerrV   loggerr  r   ra   rk   r   r   r   r   r   r   r
  r  __all__r   r4   r3   <module>r*     s9       $     & ! F - , ( 
		H	%A) A)H	BII 	O		 Od299 .RYY 8%")) %0+ryy +\V
 V
r H H H, )
! )
 )
X 9
 2 9
9
x Jr4   