Ë
    éÓ^jWR  ã                   ó  — d Z ddlZddlmZmZmZmZmZmZm	Z	 ddl
Z
ddlmZ ddlmc mZ ddlmZmZ ddlmZmZmZmZmZmZmZ ddlmZ ddlmZ dd	lm Z  dd
l!m"Z"m#Z# dgZ$ G d„ dejJ                  «      Z& G d„ dejJ                  «      Z' G d„ dejJ                  «      Z( G d„ dejJ                  «      Z) G d„ dejJ                  «      Z* G d„ dejJ                  «      Z+d„ Z,d%d„Z-d&d„Z. e# e.d¬«       e.d¬«       e.d¬«       e.d¬«       e.d¬«       e.d¬«       e.d¬«      dœ«      Z/e"d%de+fd„«       Z0e"d%de+fd„«       Z1e"d%de+fd „«       Z2e"d%de+fd!„«       Z3e"d%de+fd"„«       Z4e"d%de+fd#„«       Z5e"d%de+fd$„«       Z6y)'a   Pyramid Vision Transformer v2

@misc{wang2021pvtv2,
      title={PVTv2: Improved Baselines with Pyramid Vision Transformer},
      author={Wenhai Wang and Enze Xie and Xiang Li and Deng-Ping Fan and Kaitao Song and Ding Liang and
        Tong Lu and Ping Luo and Ling Shao},
      year={2021},
      eprint={2106.13797},
      archivePrefix={arXiv},
      primaryClass={cs.CV}
}

Based on Apache 2.0 licensed code at https://github.com/whai362/PVT

Modifications and timm support by / Copyright 2022, Ross Wightman
é    N)ÚCallableÚListÚOptionalÚTupleÚUnionÚTypeÚAny©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚDropPathÚcalculate_drop_path_ratesÚ	to_2tupleÚ	to_ntupleÚtrunc_normal_Ú	LayerNormÚuse_fused_attné   )Úbuild_model_with_cfg)Úfeature_take_indices)Ú
checkpoint)Úregister_modelÚgenerate_default_cfgsÚPyramidVisionTransformerV2c                   ó”   ‡ — e Zd Zddej                  ddddfdedee   dee   deej                     de	d	e
fˆ fd
„Zdee   fd„Zˆ xZS )ÚMlpWithDepthwiseConvNç        FÚin_featuresÚhidden_featuresÚout_featuresÚ	act_layerÚdropÚ
extra_reluc	                 ó¢  •— t         ‰
| �  «        ||dœ}	|xs |}|xs |}t        j                  ||fi |	¤Ž| _        |rt        j
                  «       nt        j                  «       | _        t        j                  ||dddfd|dœ|	¤Ž| _	         |«       | _
        t        j                  ||fi |	¤Ž| _        t        j                  |«      | _        y )N©ÚdeviceÚdtypeé   r   T)ÚbiasÚgroups)ÚsuperÚ__init__ÚnnÚLinearÚfc1ÚReLUÚIdentityÚreluÚConv2dÚdwconvÚactÚfc2ÚDropoutr"   )Úselfr   r   r    r!   r"   r#   r&   r'   ÚddÚ	__class__s             €ú]/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/pvt_v2.pyr,   zMlpWithDepthwiseConv.__init__$   s³   ø€ ô 	‰ÑÔØ¨Ñ/ˆØ#Ò2 {ˆØ)Ò8¨[ˆÜ—9‘9˜[¨/Ñ@¸RÑ@ˆŒÙ!+”B—G‘G”I´·±³ˆŒ	Ü—i‘i °À!ÀQÈÐsÐPTÐ]lÑsÐprÑsˆŒÙ“;ˆŒÜ—9‘9˜_¨lÑA¸bÑAˆŒÜ—J‘J˜tÓ$ˆ�	ó    Ú	feat_sizec                 óª  — | j                  |«      }|j                  \  }}}|j                  dd«      j                  |||d   |d   «      }| j	                  |«      }| j                  |«      }|j                  d«      j                  dd«      }| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|S )Nr   é   r   )
r/   ÚshapeÚ	transposeÚviewr2   r4   Úflattenr5   r"   r6   )r8   Úxr=   ÚBÚNÚCs         r;   ÚforwardzMlpWithDepthwiseConv.forward:   s´   € Ø�H‰H�Q‹KˆØ—'‘'‰ˆˆ1ˆaØ�K‰K˜˜1Ó×"Ñ" 1 a¨°1©°yÀ±|ÓDˆØ�I‰I�a‹LˆØ�K‰K˜‹NˆØ�I‰I�a‹L×"Ñ" 1 aÓ(ˆØ�H‰H�Q‹KˆØ�I‰I�a‹LˆØ�H‰H�Q‹KˆØ�I‰I�a‹LˆØˆr<   )Ú__name__Ú
__module__Ú__qualname__r-   ÚGELUÚintr   r   ÚModuleÚfloatÚboolr,   r   rH   Ú__classcell__©r:   s   @r;   r   r   #   s}   ø„ ð .2Ø*.Ø)+¯©ØØ$ØØñ%àð%ð & c™]ð%ð # 3™-ð	%ð
 ˜BŸI™I‘ð%ð ð%ð õ%ð, D¨¡I÷ r<   r   c                   óš   ‡ — e Zd ZU ej                  j
                  e   ed<   	 	 	 	 	 	 	 	 ddededededede	de	fˆ fd	„Z
d
ee   fd„Zˆ xZS )Ú	AttentionÚ
fused_attnÚdimÚ	num_headsÚsr_ratioÚlinear_attnÚqkv_biasÚ	attn_dropÚ	proj_dropc
                 ó’  •— t         ‰| �  «        ||	dœ}
||z  dk(  sJ d|› d|› d�«       ‚|| _        || _        ||z  | _        | j                  dz  | _        t        «       | _        t        j                  ||fd|i|
¤Ž| _
        t        j                  ||dz  fd|i|
¤Ž| _        t        j                  |«      | _        t        j                  ||fi |
¤Ž| _        t        j                  |«      | _        |sdd | _        |d	kD  rBt        j"                  ||f||d
œ|
¤Ž| _        t        j&                  |fi |
¤Ž| _        d | _        y d | _        d | _        d | _        y t        j,                  d«      | _        t        j"                  ||fd	d	d
œ|
¤Ž| _        t        j&                  |fi |
¤Ž| _        t        j.                  «       | _        y )Nr%   r   zdim z  should be divided by num_heads ú.g      à¿r)   r?   r   )Úkernel_sizeÚstrideé   )r+   r,   rV   rW   Úhead_dimÚscaler   rU   r-   r.   ÚqÚkvr7   r[   Úprojr\   Úpoolr3   Úsrr   Únormr5   ÚAdaptiveAvgPool2drL   )r8   rV   rW   rX   rY   rZ   r[   r\   r&   r'   r9   r:   s              €r;   r,   zAttention.__init__K   s¢  ø€ ô 	‰ÑÔØ¨Ñ/ˆØ�Y‰ !Ò#Ð] t¨C¨5Ð0PÐQZÐP[Ð[\Ð%]Ó]Ð#àˆŒØ"ˆŒØ˜yÑ(ˆŒØ—]‘] dÑ*ˆŒ
Ü(Ó*ˆŒä—‘˜3 Ñ9¨(Ð9°bÑ9ˆŒÜ—)‘)˜C  q¡Ñ>¨xÐ>¸2Ñ>ˆŒÜŸ™ IÓ.ˆŒÜ—I‘I˜c 3Ñ-¨"Ñ-ˆŒ	ÜŸ™ IÓ.ˆŒáØˆDŒIØ˜!Š|ÜŸ)™) C¨ÐZ¸(È8ÑZÐWYÑZ�”ÜŸL™L¨Ñ3°Ñ3�”	ð ˆD�Hð �”Ø �”	ØˆD�Hä×,Ñ,¨QÓ/ˆDŒIÜ—i‘i  SÐH°aÀÑHÀRÑHˆDŒGÜŸ™ SÑ/¨BÑ/ˆDŒIÜ—w‘w“yˆD�Hr<   r=   c                 ód  — |j                   \  }}}|\  }}| j                  |«      j                  ||| j                  d«      j	                  dddd«      }| j
                  �Ö|j	                  ddd«      j                  ||||«      }| j                  | j                  |«      «      j                  ||d«      j	                  ddd«      }| j                  |«      }| j                  |«      }| j                  |«      j                  |dd| j                  | j                  «      j	                  ddddd«      }	�n| j                  �µ|j	                  ddd«      j                  ||||«      }| j                  |«      j                  ||d«      j	                  ddd«      }| j                  |«      }| j                  |«      j                  |dd| j                  | j                  «      j	                  ddddd«      }	nK| j                  |«      j                  |dd| j                  | j                  «      j	                  ddddd«      }	|	j                  d«      \  }
}| j                  r<t        j                  ||
|| j                  r| j                   j"                  nd¬«      }nL|| j$                  z  }||
j'                  d	d«      z  }|j)                  d¬
«      }| j!                  |«      }||z  }|j'                  dd«      j                  |||«      }| j+                  |«      }| j-                  |«      }|S )Néÿÿÿÿr   r?   r   r(   é   r   )Ú	dropout_péþÿÿÿ©rV   )r@   rd   ÚreshaperW   Úpermuterg   rh   ri   r5   re   rb   ÚunbindrU   ÚFÚscaled_dot_product_attentionÚtrainingr[   Úprc   rA   Úsoftmaxrf   r\   )r8   rD   r=   rE   rF   rG   ÚHÚWrd   re   ÚkÚvÚattns                r;   rH   zAttention.forwardv   s¹  € Ø—'‘'‰ˆˆ1ˆaØ‰ˆˆ1Ø�F‰F�1‹I×Ñ˜a  D§N¡N°BÓ7×?Ñ?ÀÀ1ÀaÈÓKˆà�9‰9Ð Ø—	‘	˜!˜Q Ó"×*Ñ*¨1¨a°°AÓ6ˆAØ—‘˜Ÿ	™	 !›Ó%×-Ñ-¨a°°BÓ7×?Ñ?ÀÀ1ÀaÓHˆAØ—	‘	˜!“ˆAØ—‘˜“ˆAØ—‘˜“×#Ñ# A r¨1¨d¯n©n¸d¿m¹mÓL×TÑTÐUVÐXYÐ[\Ð^_ÐabÓcŠBà�w‰wÐ"Ø—I‘I˜a  AÓ&×.Ñ.¨q°!°Q¸Ó:�Ø—G‘G˜A“J×&Ñ& q¨!¨RÓ0×8Ñ8¸¸A¸qÓA�Ø—I‘I˜a“L�Ø—W‘W˜Q“Z×'Ñ'¨¨2¨q°$·.±.À$Ç-Á-ÓP×XÑXÐYZÐ\]Ð_`ÐbcÐefÓg‘à—W‘W˜Q“Z×'Ñ'¨¨2¨q°$·.±.À$Ç-Á-ÓP×XÑXÐYZÐ\]Ð_`ÐbcÐefÓg�Ø�y‰y˜‹|‰ˆˆ1à�?Š?Ü×.Ñ.¨q°!°QÐVZ×VcÒVcÀ$Ç.Á.×BRÒBRÐikÔl‰Aà�D—J‘J‘ˆAØ�q—{‘{ 2 rÓ*Ñ*ˆDØ—<‘< B�<Ó'ˆDØ—>‘> $Ó'ˆDØ�q‘ˆAà�K‰K˜˜1Ó×%Ñ% a¨¨AÓ.ˆØ�I‰I�a‹LˆØ�N‰N˜1ÓˆØˆr<   )é   r   FTr   r   NN)rI   rJ   rK   ÚtorchÚjitÚFinalrP   Ú__annotations__rM   rO   r,   r   rH   rQ   rR   s   @r;   rT   rT   H   sŠ   ø… Ø—	‘	—‘ Ñ%Ó%ð
 ØØ %Ø!Ø!Ø!ØØñ)!àð)!ð ð)!ð ð	)!ð
 ð)!ð ð)!ð ð)!ð õ)!ðV! D¨¡I÷ !r<   rT   c                   ó¾   ‡ — e Zd Zdddddddej                  eddfdededed	ed
ededededede	ej                     de	ej                     fˆ fd„Zdee   fd„Zˆ xZS )ÚBlockç      @r   Fr   NrV   rW   Ú	mlp_ratiorX   rY   rZ   r\   r[   Ú	drop_pathr!   Ú
norm_layerc           
      óˆ  •— t         ‰| �  «        ||dœ} ||fi |¤Ž| _        t        |f||||||dœ|¤Ž| _        |	dkD  rt        |	«      nt        j                  «       | _         ||fi |¤Ž| _	        t        d|t        ||z  «      |
||dœ|¤Ž| _        |	dkD  rt        |	«      | _        y t        j                  «       | _        y )Nr%   )rW   rX   rY   rZ   r[   r\   r   )r   r   r!   r"   r#   © )r+   r,   Únorm1rT   r}   r   r-   r1   Ú
drop_path1Únorm2r   rM   ÚmlpÚ
drop_path2)r8   rV   rW   r†   rX   rY   rZ   r\   r[   r‡   r!   rˆ   r&   r'   r9   r:   s                  €r;   r,   zBlock.__init__œ   sÛ   ø€ ô  	‰ÑÔØ¨Ñ/ˆÙ Ñ* rÑ*ˆŒ
ÜØð	
àØØ#ØØØñ	
ð ñ	
ˆŒ	ð 2;¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒá Ñ* rÑ*ˆŒ
Ü'ð 
ØÜ  i¡Ó0ØØØ"ñ
ð ñ
ˆŒð 2;¸R²œ( 9Ó-ˆ�ÄRÇ[Á[Ã]ˆ�r<   r=   c                 óÒ   — || j                  | j                  | j                  |«      |«      «      z   }|| j                  | j	                  | j                  |«      |«      «      z   }|S ©N)rŒ   r}   r‹   r�   rŽ   r�   )r8   rD   r=   s      r;   rH   zBlock.forwardÆ   sS   € Ø�—‘ §	¡	¨$¯*©*°Q«-¸Ó CÓDÑDˆØ�—‘ §¡¨¯©°A«¸	Ó BÓCÑCˆàˆr<   )rI   rJ   rK   r-   rL   r   rM   rO   rP   r   rN   r,   r   rH   rQ   rR   s   @r;   r„   r„   š   sÈ   ø„ ð  "ØØ %Ø"Ø!Ø!Ø!Ø)+¯©Ø*3ØØñ(Sàð(Sð ð(Sð ð	(Sð
 ð(Sð ð(Sð ð(Sð ð(Sð ð(Sð ð(Sð ˜BŸI™I‘ð(Sð ˜RŸY™Y™õ(SðT D¨¡I÷ r<   r„   c            	       ó\   ‡ — e Zd ZdZ	 	 	 	 	 	 ddeeeeef   f   dededefˆ fd„Zd„ Zˆ xZ	S )	ÚOverlapPatchEmbedz Image to Patch Embedding
    Ú
patch_sizer`   Úin_chansÚ	embed_dimc                 ó  •— t         ‰| �  «        ||dœ}t        |«      }t        |«      |kD  sJ d«       ‚|| _        t        j                  |||f||d   dz  |d   dz  fdœ|¤Ž| _        t        j                  |fi |¤Ž| _	        y )Nr%   z!Set larger patch_size than strider   r?   r   )r`   Úpadding)
r+   r,   r   Úmaxr”   r-   r3   rf   r   ri   )	r8   r”   r`   r•   r–   r&   r'   r9   r:   s	           €r;   r,   zOverlapPatchEmbed.__init__Ð   s�   ø€ ô 	‰ÑÔØ¨Ñ/ˆÜ˜zÓ*ˆ
Ü�:‹ Ò'ÐLÐ)LÓLÐ'Ø$ˆŒÜ—I‘IØ�i ðSà J¨q¡M°QÑ$6¸
À1¹ÈÑ8JÐ#KñSàOQñSˆŒ	ô —L‘L Ñ1¨bÑ1ˆ�	r<   c                 ór   — | j                  |«      }|j                  dddd«      }| j                  |«      }|S )Nr   r?   r(   r   )rf   rr   ri   ©r8   rD   s     r;   rH   zOverlapPatchEmbed.forwardã   s5   € Ø�I‰I�a‹LˆØ�I‰I�a˜˜A˜qÓ!ˆØ�I‰I�a‹LˆØˆr<   )ra   rm   r(   i   NN)
rI   rJ   rK   Ú__doc__r   rM   r   r,   rH   rQ   rR   s   @r;   r“   r“   Í   s^   ø„ ñð 78ØØØ ØØñ2à˜c 5¨¨c¨¡?Ð2Ñ3ð2ð ð2ð ð	2ð
 õ2ö&r<   r“   c                   ó„   ‡ — e Zd Zdddddddddeddfded	ed
edededededededededeee   ef   de	fˆ fd„Z
d„ Zˆ xZS )ÚPyramidVisionTransformerStageTr~   r   Fr…   r   NrV   Údim_outÚdepthÚ
downsamplerW   rX   rY   r†   rZ   r\   r[   r‡   rˆ   c                 ó`  •— t         ‰| �  «        ||dœ}d| _        |rt        ddd||dœ|¤Ž| _        n||k(  sJ ‚d | _        t        j                  t        |«      D �cg c]-  }t        d||||||	|
|t        |t        «      r||   n||dœ
|¤Ž‘Œ/ c}«      | _         ||fi |¤Ž| _        y c c}w )Nr%   Fr(   r?   ©r”   r`   r•   r–   )
rV   rW   rX   rY   r†   rZ   r\   r[   r‡   rˆ   rŠ   )r+   r,   Úgrad_checkpointingr“   r¡   r-   Ú
ModuleListÚranger„   Ú
isinstanceÚlistÚblocksri   )r8   rV   rŸ   r    r¡   rW   rX   rY   r†   rZ   r\   r[   r‡   rˆ   r&   r'   r9   Úir:   s                     €r;   r,   z&PyramidVisionTransformerStage.__init__ë   sè   ø€ ô$ 	‰ÑÔØ¨Ñ/ˆØ"'ˆÔáÜ/ð ØØØØ!ñ	ð
 ñˆD�Oð ˜'’>Ð!�>Ø"ˆDŒOä—m‘mô ˜“<ö%!ð ô &+ð &
ØØØØ#ØØØØÜ&0°¼DÔ&A�i ’lÀyØ!ñ&
ð ó&
ò %!ó "ˆŒñ ˜wÑ-¨"Ñ-ˆ�	ùò%!s   Á2B+c                 óÊ  — | j                   �| j                  |«      }|j                  \  }}}}||f}|j                  |d|«      }| j                  D ]C  }| j                  r,t
        j                  j                  «       st        |||«      }Œ; |||«      }ŒE | j                  |«      }|j                  ||d   |d   d«      j                  dddd«      j                  «       }|S )Nrl   r   r   r(   r?   )r¡   r@   rq   r©   r¤   r   r€   Úis_scriptingr   ri   rr   Ú
contiguous)r8   rD   rE   ry   rz   rG   r=   Úblks           r;   rH   z%PyramidVisionTransformerStage.forward  sÕ   € à�?‰?Ð&à—‘ Ó"ˆAØ—W‘W‰
ˆˆ1ˆa�Ø˜�Fˆ	Ø�I‰I�a˜˜QÓˆØ—;‘;ò 	&ˆCØ×&Ò&¬u¯y©y×/EÑ/EÔ/GÜ˜s A yÓ1‘á˜˜9Ó%‘ð		&ð
 �I‰I�a‹LˆØ�I‰I�a˜ 1™ y°¡|°RÓ8×@Ñ@ÀÀAÀqÈ!ÓL×WÑWÓYˆØˆr<   )rI   rJ   rK   r   rM   rP   rO   r   r   r   r,   rH   rQ   rR   s   @r;   rž   rž   ê   sÂ   ø„ ð  $ØØØ %Ø"Ø!Ø!Ø!Ø36Ø#,ØØñ!0.àð0.ð ð0.ð ð	0.ð
 ð0.ð ð0.ð ð0.ð ð0.ð ð0.ð ð0.ð ð0.ð ð0.ð ˜T %™[¨%Ð/Ñ0ð0.ð !õ0.ödr<   rž   c                    ó¸  ‡ — e Zd Zddddddddd	d
ddddeddfdedededeedf   deedf   deedf   deedf   deedf   dedededededede	e
j                     fˆ fd„Zd„ Zd„ Zej                   j"                  d „ «       Zej                   j"                  d4d!„«       Zej                   j"                  d5d"„«       Zd#e
j                  fd$„Zd6dedee   fd%„Z	 	 	 	 	 d7d&ej0                  d'eeeee   f      d(ed)ed*ed+ed#eeej0                     eej0                  eej0                     f   f   fd,„Z	 	 	 d8d'eeee   f   d-ed.efd/„Zd0„ Zd4d1efd2„Zd3„ Zˆ xZ S )9r   r(   éè  Úavg©r(   rm   é   r(   )é@   é€   é   é   )r   r?   rm   r~   )r~   rm   r?   r   )ç       @r¸   r…   r…   TFr   Nr•   Únum_classesÚglobal_poolÚdepths.Ú
embed_dimsrW   Ú	sr_ratiosÚ
mlp_ratiosrZ   ÚlinearÚ	drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_raterˆ   c                 ó–  •— t         ‰| �  «        ||dœ}|| _        || _        |dv sJ ‚|| _        || _        t        |«      } t        |«      |«      } t        |«      |«      } t        |«      |«      }t        |«      |k(  sJ ‚g | _        t        ddd||d   dœ|¤Ž| _
        t        ||d¬«      }d}|d   }g }t        |«      D ]l  }|t        d|||   ||   |dkD  ||   ||   ||   |
|	||||   |d	œ|¤Žgz  }||   }|||   z  }| xj                  t        |dd
|z  z  d|› �¬«      gz  c_        Œn t        j                   |Ž | _        |d   x| _        | _        t        j(                  |«      | _        |dkD  rt        j,                  |d   |fi |¤Žnt        j.                  «       | _        | j3                  | j4                  «       y )Nr%   ©r±   Ú ra   rm   r   r£   T)Ú	stagewise)rV   rŸ   r    r¡   rW   rX   r†   rY   rZ   r\   r[   r‡   rˆ   r?   ústages.)Únum_chsÚ	reductionÚmodulerl   rŠ   )r+   r,   r¹   r•   rº   r»   Úlenr   Úfeature_infor“   Úpatch_embedr   r¦   rž   Údictr-   Ú
SequentialÚstagesÚnum_featuresÚhead_hidden_sizer7   Ú	head_dropr.   r1   ÚheadÚapplyÚ_init_weights)r8   r•   r¹   rº   r»   r¼   rW   r½   r¾   rZ   r¿   rÀ   rÁ   rÂ   rÃ   rˆ   r&   r'   r9   Ú
num_stagesÚdprÚcurÚprev_dimrÑ   rª   r:   s                            €r;   r,   z#PyramidVisionTransformerV2.__init__0  s*  ø€ ô( 	‰ÑÔØ¨Ñ/ˆØ&ˆÔØ ˆŒØ˜kÑ)Ð)Ð)Ø&ˆÔØˆŒÜ˜“[ˆ
Ø*”Y˜zÓ*¨:Ó6ˆ
Ø)”I˜jÓ)¨)Ó4ˆ	Ø)”I˜jÓ)¨)Ó4ˆ	Ü�:‹ :Ò-Ð-Ð-ØˆÔä,ð 
ØØØØ  ‘mñ	
ð
 ñ
ˆÔô (¨¸È$ÔOˆØˆØ˜a‘=ˆØˆÜ�zÓ"ò 	dˆAØÔ4ð ØØ" 1™Ø˜Q‘iØ˜q™5Ø# A™,Ø" 1™Ø$ Q™-Ø"Ø!Ø(Ø(Ø˜a™&Ø%ñð ñð ñ ˆFð  " !‘}ˆHØ�6˜!‘9ÑˆCØ×Ò¤$¨xÀ1ÀqÈ!ÁtÁ8ÐV]Ð^_Ð]`ÐTaÔ"bÐ!cÑcÖð'	dô( —m‘m VÐ,ˆŒð 5?¸r±NÐBˆÔ˜DÔ1ÜŸ™ IÓ.ˆŒØDOÐRSÂO”B—I‘I˜j¨™n¨kÑ@¸RÒ@ÔY[×YdÑYdÓYfˆŒ	à�
‰
�4×%Ñ%Õ&r<   c                 ó’  — t        |t        j                  «      rjt        |j                  d¬«       t        |t        j                  «      r8|j
                  �+t        j                  j                  |j
                  d«       y y y t        |t        j                  «      r©|j                  d   |j                  d   z  |j                  z  }||j                  z  }|j                  j                  j                  dt        j                  d|z  «      «       |j
                  �%|j
                  j                  j!                  «        y y y )Ng{®Gáz”?)Ústdr   r   g       @)r§   r-   r.   r   Úweightr)   ÚinitÚ	constant_r3   r_   Úout_channelsr*   ÚdataÚnormal_ÚmathÚsqrtÚzero_)r8   ÚmÚfan_outs      r;   r×   z(PyramidVisionTransformerV2._init_weights{  sã   € Ü�aœŸ™Ô#Ü˜!Ÿ(™(¨Õ,Ü˜!œRŸY™YÔ'¨A¯F©FÐ,>Ü—‘×!Ñ! !§&¡&¨!Õ,ð -?Ð'ä˜œ2Ÿ9™9Ô%Ø—m‘m AÑ&¨¯©°qÑ)9Ñ9¸A¿N¹NÑJˆGØ˜Ÿ™Ñ ˆGØ�H‰H�M‰M×!Ñ! !¤T§Y¡Y¨s°W©}Ó%=Ô>Ø�v‰vÐ!Ø—‘—‘×!Ñ!Õ#ð "ð	 &r<   c                 ó&   — d| j                   _        y )NF)rÎ   Úrequires_grad©r8   s    r;   Úfreeze_patch_embz+PyramidVisionTransformerV2.freeze_patch_emb‡  s   € Ø).ˆ×ÑÕ&r<   c                 ó   — i S r‘   rŠ   rë   s    r;   Úno_weight_decayz*PyramidVisionTransformerV2.no_weight_decayŠ  s   € àˆ	r<   c                 ó    — t        dd¬«      }|S )Nz^patch_embedz^stages\.(\d+))Ústemr©   )rÏ   )r8   ÚcoarseÚmatchers      r;   Úgroup_matcherz(PyramidVisionTransformerV2.group_matcherŽ  s   € äØ Ø$ô
ˆð ˆr<   c                 ó4   — | j                   D ]	  }||_        Œ y r‘   )rÑ   r¤   )r8   ÚenableÚss      r;   Úset_grad_checkpointingz1PyramidVisionTransformerV2.set_grad_checkpointing–  s   € à—‘ò 	*ˆAØ#)ˆAÕ ñ	*r<   Úreturnc                 ó   — | j                   S r‘   )rÕ   rë   s    r;   Úget_classifierz)PyramidVisionTransformerV2.get_classifier›  s   € Ø�y‰yÐr<   c                 óž  — || _         |�|dv sJ ‚|| _        t        | j                  d«      r | j                  j                  j
                  nd }t        | j                  d«      r | j                  j                  j                  nd }|dkD  r)t        j                  | j                  |||¬«      | _        y t        j                  «       | _        y )NrÅ   rÞ   r   r%   )r¹   rº   ÚhasattrrÕ   rÞ   r&   r'   r-   r.   rÒ   r1   )r8   r¹   rº   r&   r'   s        r;   Úreset_classifierz+PyramidVisionTransformerV2.reset_classifierž  s£   € Ø&ˆÔØÐ"Ø +Ñ-Ð-Ð-Ø*ˆDÔÜ,3°D·I±I¸xÔ,H�—‘×!Ñ!×(Ò(ÈdˆÜ*1°$·)±)¸XÔ*F�—	‘	× Ñ ×&Ò&ÈDˆØ]hÐklÒ]l”B—I‘I˜d×/Ñ/°ÀVÐSXÔYˆ�	Ôrt×r}Ñr}Órˆ�	r<   rD   Úindicesri   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 ór  — |dv sJ d«       ‚g }t        t        | j                  «      |«      \  }}	| j                  |«      }t        j
                  j                  «       s|s| j                  }
n| j                  d|	dz    }
t        |
«      D ]#  \  }} ||«      }||v sŒ|j                  |«       Œ% |r|S ||fS )aï   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )ÚNCHWzOutput shape must be NCHW.Nr   )	r   rÌ   rÑ   rÎ   r   r€   r¬   Ú	enumerateÚappend)r8   rD   rþ   ri   rÿ   r   r  ÚintermediatesÚtake_indicesÚ	max_indexrÑ   Úfeat_idxÚstages                r;   Úforward_intermediatesz0PyramidVisionTransformerV2.forward_intermediates§  sÅ   € ð* ˜YÑ&ÐDÐ(DÓDÐ&ØˆÜ"6´s¸4¿;¹;Ó7GÈÓ"QÑˆ�ið ×Ñ˜QÓˆÜ�9‰9×!Ñ!Ô#©:Ø—[‘[‰Fà—[‘[  )¨a¡-Ð0ˆFä(¨Ó0ò 	(‰OˆH�eÙ�a“ˆAØ˜<Ò'Ø×$Ñ$ QÕ'ð	(ñ
 Ø Ð à�-ÐÐr<   Ú
prune_normÚ
prune_headc                 ó    — t        t        | j                  «      |«      \  }}| j                  d|dz    | _        |r| j                  dd«       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   rÆ   )r   rÌ   rÑ   rý   )r8   rþ   r  r  r  r  s         r;   Úprune_intermediate_layersz4PyramidVisionTransformerV2.prune_intermediate_layersÑ  sM   € ô #7´s¸4¿;¹;Ó7GÈÓ"QÑˆ�iØ—k‘k . 9¨q¡=Ð1ˆŒÙØ×!Ñ! ! RÔ(ØÐr<   c                 óJ   — | j                  |«      }| j                  |«      }|S r‘   )rÎ   rÑ   r›   s     r;   Úforward_featuresz+PyramidVisionTransformerV2.forward_featuresß  s$   € Ø×Ñ˜QÓˆØ�K‰K˜‹NˆØˆr<   Ú
pre_logitsc                 óŠ   — | j                   r|j                  d¬«      }| j                  |«      }|r|S | j                  |«      S )N)rl   ro   rp   )rº   ÚmeanrÔ   rÕ   )r8   rD   r  s      r;   Úforward_headz'PyramidVisionTransformerV2.forward_headä  s>   € Ø×ÒØ—‘˜8�Ó$ˆAØ�N‰N˜1ÓˆÙˆqÐ0 D§I¡I¨a£LÐ0r<   c                 óJ   — | j                  |«      }| j                  |«      }|S r‘   )r  r  r›   s     r;   rH   z"PyramidVisionTransformerV2.forwardê  s'   € Ø×!Ñ! !Ó$ˆØ×Ñ˜aÓ ˆØˆr<   ©F)Tr‘   )NFFr  F)r   FT)!rI   rJ   rK   r   rM   Ústrr   rO   rP   r   r-   rN   r,   r×   rì   r   r€   Úignorerî   ró   r÷   rú   r   rý   ÚTensorr   r   r  r  r  r  rH   rQ   rR   s   @r;   r   r   /  s‹  ø„ ð Ø#Ø$Ø&2Ø*=Ø)5Ø)5Ø,<Ø!Ø Ø!Ø$&Ø$&Ø$&Ø*3ØØñ%I'àðI'ð ðI'ð ð	I'ð
 ˜#˜s˜(‘OðI'ð ˜c 3˜h™ðI'ð ˜S #˜X‘ðI'ð ˜S #˜X‘ðI'ð ˜e S˜jÑ)ðI'ð ðI'ð ðI'ð ðI'ð "ðI'ð "ðI'ð "ðI'ð  ˜RŸY™Y™õ!I'òV
$ò/ð ‡Y�Y×Ññó ðð ‡Y�Y×Ñòó ðð ‡Y�Y×Ñò*ó ð*ð §	¡	ó ñ@¨Cð @¸hÀs¹mó @ð 8<ØØ$Ø$Ø',ñ( à�|‰|ð( ð ˜e C¨¨c© NÑ3Ñ4ð( ð ð	( ð
 ð( ð ð( ð !%ð( ð 
ˆt�E—L‘LÑ! 5¨¯©°t¸E¿L¹LÑ7IÐ)IÑ#JÐJÑ	Kó( ðX ./Ø$Ø#ñ	à˜3  S¡	˜>Ñ*ðð ðð ó	òñ
1¨$ó 1ör<   c                 ó~  — d| v r| S i }ddl }| j                  «       D ]�  \  }}|j                  d«      rH|j                  dd«      }|j                  dd«      }|j                  dd	«      }|j                  d
d«      }|j                  dd«      }|j	                  dd„ |«      }|j	                  dd„ |«      }|||<   ŒŸ |S )z$ Remap original checkpoints -> timm zpatch_embed.proj.weightr   NrÎ   Úpatch_embed1Úpatch_embed2zstages.1.downsampleÚpatch_embed3zstages.2.downsampleÚpatch_embed4zstages.3.downsamplezdwconv.dwconvr4   zblock(\d+).(\d+)c                 óf   — dt        | j                  d«      «      dz
  › d| j                  d«      › �S )NrÈ   r   z.blocks.r?   ©rM   Úgroup©rD   s    r;   ú<lambda>z&checkpoint_filter_fn.<locals>.<lambda>þ  s4   € °G¼CÀÇÁÈÃ
»OÈaÑ<OÐ;PÐPXÐYZ×Y`ÑY`ÐabÓYcÐXdÐ2e€ r<   z
^norm(\d+)c                 óD   — dt        | j                  d«      «      dz
  › d�S )NrÈ   r   z.normr!  r#  s    r;   r$  z&checkpoint_filter_fn.<locals>.<lambda>ÿ  s!   € ¨g´c¸!¿'¹'À!»*³oÈÑ6IÐ5JÈ%Ð,P€ r<   )ÚreÚitemsÚ
startswithÚreplaceÚsub)Ú
state_dictÚmodelÚout_dictr&  r{   r|   s         r;   Úcheckpoint_filter_fnr.  ð  sÌ   € à  JÑ.ØÐà€HÛØ× Ñ Ó"ò 	‰ˆˆ1Ø�<‰<˜Ô&Ø—	‘	˜.¨-Ó8ˆAØ—	‘	˜.Ð*?Ó@ˆAØ—	‘	˜.Ð*?Ó@ˆAØ—	‘	˜.Ð*?Ó@ˆAØ�I‰I�o xÓ0ˆØ�F‰FÐ&Ñ(eÐghÓiˆØ�F‰F�=Ñ"PÐRSÓTˆØˆ�Šð	ð €Or<   c                 óš   — t        t        d«      «      }|j                  d|«      }t        t        | |ft
        t        d|¬«      dœ|¤Ž}|S )Nrm   Úout_indicesT)Úflatten_sequentialr0  )Úpretrained_filter_fnÚfeature_cfg)Útupler¦   Úpopr   r   r.  rÏ   )ÚvariantÚ
pretrainedÚkwargsÚdefault_out_indicesr0  r,  s         r;   Ú_create_pvt2r:    sY   € Ü¤ a£›/ÐØ—*‘*˜]Ð,?Ó@€KÜ Ü"ØØðô 2Ü¨D¸kÔJñð ñ€Eð €Lr<   c                 ó4   — | dddddt         t        dddd	d
œ|¥S )Nr°   )r(   éà   r<  )ra   ra   gÍÌÌÌÌÌì?Úbicubiczpatch_embed.projrÕ   Fz
apache-2.0)Úurlr¹   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationr  rÝ   Ú
first_convÚ
classifierÚfixed_input_sizeÚlicenser
   )r>  r8  s     r;   Ú_cfgrG    s6   € à 4°}ÐSYØ¨)Ü%Ô.BØ(¸ÐTYØñð ðð r<   ztimm/)Ú	hf_hub_id)zpvt_v2_b0.in1kzpvt_v2_b1.in1kzpvt_v2_b2.in1kzpvt_v2_b3.in1kzpvt_v2_b4.in1kzpvt_v2_b5.in1kzpvt_v2_b2_li.in1krø   c           	      óL   — t        ddd¬«      }t        dd| it        |fi |¤Ž¤ŽS )N©r?   r?   r?   r?   )é    r´   é    r¶   ©r   r?   é   r~   ©r»   r¼   rW   r7  )Ú	pvt_v2_b0©rÏ   r:  ©r7  r8  Ú
model_argss      r;   rP  rP  (  s0   € ä˜\Ð6HÐT`Ôa€JÜÑY°
ÐY¼dÀ:Ñ>XÐQWÑ>XÑYÐYr<   c           	      óL   — t        ddd¬«      }t        dd| it        |fi |¤Ž¤ŽS )NrJ  ©r´   rµ   i@  r·   rM  rO  r7  )Ú	pvt_v2_b1rQ  rR  s      r;   rV  rV  .  ó0   € ä˜\Ð6IÐUaÔb€JÜÑY°
ÐY¼dÀ:Ñ>XÐQWÑ>XÑYÐYr<   c           	      óL   — t        ddd¬«      }t        dd| it        |fi |¤Ž¤ŽS )Nr²   rU  rM  rO  r7  )Ú	pvt_v2_b2rQ  rR  s      r;   rY  rY  4  rW  r<   c           	      óL   — t        ddd¬«      }t        dd| it        |fi |¤Ž¤ŽS )N)r(   rm   é   r(   rU  rM  rO  r7  )Ú	pvt_v2_b3rQ  rR  s      r;   r\  r\  :  ó0   € ä˜]Ð7JÐVbÔc€JÜÑY°
ÐY¼dÀ:Ñ>XÐQWÑ>XÑYÐYr<   c           	      óL   — t        ddd¬«      }t        dd| it        |fi |¤Ž¤ŽS )N)r(   r~   é   r(   rU  rM  rO  r7  )Ú	pvt_v2_b4rQ  rR  s      r;   r`  r`  @  r]  r<   c           	      óN   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤ŽS )N)r(   r³   é(   r(   rU  rM  )rm   rm   rm   rm   )r»   r¼   rW   r¾   r7  )Ú	pvt_v2_b5rQ  rR  s      r;   rc  rc  F  s6   € äØÐ)<ÈÐamôo€JäÑY°
ÐY¼dÀ:Ñ>XÐQWÑ>XÑYÐYr<   c           	      óN   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤ŽS )Nr²   rU  rM  T)r»   r¼   rW   r¿   r7  )Úpvt_v2_b2_lirQ  rR  s      r;   re  re  M  s6   € äØÐ(;À|Ð\`ôb€JäÑ\°:Ð\ÄÀjÑA[ÐTZÑA[Ñ\Ð\r<   r  )rÆ   )7rœ   rä   Útypingr   r   r   r   r   r   r	   r   Útorch.nnr-   Útorch.nn.functionalÚ
functionalrt   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_manipulater   Ú	_registryr   r   Ú__all__rN   r   rT   r„   r“   rž   r   r.  r:  rG  Údefault_cfgsrP  rV  rY  r\  r`  rc  re  rŠ   r<   r;   ú<module>rr     sà  ðñó" ß D× DÑ Dã Ý ß Ð ç Aß {× {Ñ {Ý *Ý +Ý #ß <à'Ð
(€ô"˜2Ÿ9™9ô "ôJO�—	‘	ô Oôd0ˆB�I‰Iô 0ôf˜Ÿ	™	ô ô:B B§I¡Iô BôJ~ §¡ô ~òBó(óñ %Ù WÔ-Ù WÔ-Ù WÔ-Ù WÔ-Ù WÔ-Ù WÔ-Ù¨Ô0ñ&ó €ð ñZÐ-Gò Zó ðZð
 ñZÐ-Gò Zó ðZð
 ñZÐ-Gò Zó ðZð
 ñZÐ-Gò Zó ðZð
 ñZÐ-Gò Zó ðZð
 ñZÐ-Gò Zó ðZð ñ]Ð0Jò ]ó ñ]r<   