Ë
    éÓ^jße  ã                   ó”  — d Z ddlmZ ddlmZmZmZmZmZ ddl	Z	ddl
mZ ddlmZmZ ddlmZmZmZmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ dgZ G d„ dej>                  «      Z  G d„ dej>                  «      Z! G d„ dej>                  «      Z" G d„ dej>                  «      Z#d„ Z$ed0dee%e%f   de&fd„«       Z' G d„ dej>                  «      Z(d0d„Z)d1d„Z* e e*d¬«       e*dd¬«       e*dd dd!¬"«       e*d¬«       e*dd¬«       e*dd dd!¬"«       e*d¬«       e*dd¬«       e*d¬«       e*d¬«       e*d¬«      d#œ«      Z+ed0d$e(fd%„«       Z,ed0d$e(fd&„«       Z-ed0d$e(fd'„«       Z.ed0d$e(fd(„«       Z/ed0d$e(fd)„«       Z0ed0d$e(fd*„«       Z1ed0d$e(fd+„«       Z2ed0d$e(fd,„«       Z3ed0d$e(fd-„«       Z4ed0d$e(fd.„«       Z5ed0d$e(fd/„«       Z6y)2aô   CrossViT Model

@inproceedings{
    chen2021crossvit,
    title={{CrossViT: Cross-Attention Multi-Scale Vision Transformer for Image Classification}},
    author={Chun-Fu (Richard) Chen and Quanfu Fan and Rameswar Panda},
    booktitle={International Conference on Computer Vision (ICCV)},
    year={2021}
}

Paper link: https://arxiv.org/abs/2103.14899
Original code: https://github.com/IBM/CrossViT/blob/main/models/crossvit.py

NOTE: model names have been renamed from originals to represent actual input res all *_224 -> *_240 and *_384 -> *_408

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
Modified from Timm. https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py
é    )Úpartial)ÚListÚOptionalÚTupleÚTypeÚUnionN©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚDropPathÚcalculate_drop_path_ratesÚ	to_2tupleÚtrunc_normal_Ú_asserté   )Úbuild_model_with_cfg)Úregister_notrace_function)Úregister_modelÚgenerate_default_cfgs)ÚBlockÚCrossVitc                   ób   ‡ — e Zd ZdZ	 	 	 	 	 	 	 d	deeeeef   f   dedededef
ˆ fd„Zd„ Z	ˆ xZ
S )
Ú
PatchEmbedz Image to Patch Embedding
    Úimg_sizeÚ
patch_sizeÚin_chansÚ	embed_dimÚ
multi_convc                 óŽ  •— ||dœ}t         ‰
| �  «        t        |«      }t        |«      }|d   |d   z  |d   |d   z  z  }	|| _        || _        |	| _        |�rM|d   dk(  ržt        j                  t        j                  ||dz  fddddœ|¤Žt        j                  d	¬
«      t        j                  |dz  |dz  fddddœ|¤Žt        j                  d	¬
«      t        j                  |dz  |fddddœ|¤Ž«      | _
        y |d   dk(  ržt        j                  t        j                  ||dz  fddddœ|¤Žt        j                  d	¬
«      t        j                  |dz  |dz  fddddœ|¤Žt        j                  d	¬
«      t        j                  |dz  |fddddœ|¤Ž«      | _
        y y t        j                  ||f||dœ|¤Ž| _
        y )N©ÚdeviceÚdtyper   r   é   é   é   é   )Úkernel_sizeÚstrideÚpaddingT)Úinplaceé   é   )r'   r(   )ÚsuperÚ__init__r   r   r   Únum_patchesÚnnÚ
SequentialÚConv2dÚReLUÚproj)Úselfr   r   r   r   r   r!   r"   Úddr/   Ú	__class__s             €ú_/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/crossvit.pyr.   zPatchEmbed.__init__+   sÓ  ø€ ð ¨Ñ/ˆÜ‰ÑÔÜ˜XÓ&ˆÜ˜zÓ*ˆ
Ø ‘{ j°¡mÑ3¸À¹ÀzÐRSÁ}Ñ8TÑUˆØ ˆŒØ$ˆŒØ&ˆÔÚØ˜!‰} Ò"ÜŸM™MÜ—I‘I˜h¨	°Q©ÐaÀAÈaÐYZÑaÐ^`ÑaÜ—G‘G DÔ)Ü—I‘I˜i¨1™n¨i¸1©nÐgÈ!ÐTUÐ_`ÑgÐdfÑgÜ—G‘G DÔ)Ü—I‘I˜i¨1™n¨iÐbÀQÈqÐZ[ÑbÐ_aÑbó�•	ð ˜A‘ "Ò$ÜŸM™MÜ—I‘I˜h¨	°Q©ÐaÀAÈaÐYZÑaÐ^`ÑaÜ—G‘G DÔ)Ü—I‘I˜i¨1™n¨i¸1©nÐgÈ!ÐTUÐ_`ÑgÐdfÑgÜ—G‘G DÔ)Ü—I‘I˜i¨1™n¨iÐbÀQÈqÐZ[ÑbÐ_aÑbó�•	ð %ô Ÿ	™	 (¨IÐgÀ:ÐV`ÑgÐdfÑgˆD�Ió    c                 ó”  — |j                   \  }}}}t        || j                  d   k(  d|› d|› d| j                  d   › d| j                  d   › d�	«       t        || j                  d   k(  d|› d|› d| j                  d   › d| j                  d   › d�	«       | j                  |«      j	                  d«      j                  dd«      }|S )Nr   zInput image size (Ú*z) doesn't match model (r   z).r+   )Úshaper   r   r4   ÚflattenÚ	transpose)r5   ÚxÚBÚCÚHÚWs         r8   ÚforwardzPatchEmbed.forwardQ   sã   € Ø—W‘W‰
ˆˆ1ˆa�ä��T—]‘] 1Ñ%Ñ%Ø$ Q C q¨¨Ð+BÀ4Ç=Á=ÐQRÑCSÐBTÐTUÐVZ×VcÑVcÐdeÑVfÐUgÐgiÐjô	lä��T—]‘] 1Ñ%Ñ%Ø$ Q C q¨¨Ð+BÀ4Ç=Á=ÐQRÑCSÐBTÐTUÐVZ×VcÑVcÐdeÑVfÐUgÐgiÐjô	là�I‰I�a‹L× Ñ  Ó#×-Ñ-¨a°Ó3ˆØˆr9   )éà   r,   r&   é   FNN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úintr   Úboolr.   rD   Ú__classcell__©r7   s   @r8   r   r   '   sr   ø„ ñð
 58Ø ØØ Ø$ØØñ$hà˜C  s¨C x¡Ð0Ñ1ð$hð ð$hð ð	$hð
 ð$hð õ$höLr9   r   c                   óH   ‡ — e Zd Z	 	 	 	 	 	 ddededededef
ˆ fd„Zd„ Zˆ xZS )	ÚCrossAttentionÚdimÚ	num_headsÚqkv_biasÚ	attn_dropÚ	proj_dropc                 ó¬  •— ||dœ}t         ‰
| �  «        || _        ||z  }	|	dz  | _        t	        j
                  ||fd|i|¤Ž| _        t	        j
                  ||fd|i|¤Ž| _        t	        j
                  ||fd|i|¤Ž| _        t	        j                  |«      | _
        t	        j
                  ||fi |¤Ž| _        t	        j                  |«      | _        y )Nr    g      à¿Úbias)r-   r.   rR   Úscaler0   ÚLinearÚwqÚwkÚwvÚDropoutrT   r4   rU   )r5   rQ   rR   rS   rT   rU   r!   r"   r6   Úhead_dimr7   s             €r8   r.   zCrossAttention.__init__]   sÂ   ø€ ð ¨Ñ/ˆÜ‰ÑÔØ"ˆŒØ˜)Ñ#ˆà Ñ%ˆŒ
ä—)‘)˜C Ñ:¨8Ð:°rÑ:ˆŒÜ—)‘)˜C Ñ:¨8Ð:°rÑ:ˆŒÜ—)‘)˜C Ñ:¨8Ð:°rÑ:ˆŒÜŸ™ IÓ.ˆŒÜ—I‘I˜c 3Ñ-¨"Ñ-ˆŒ	ÜŸ™ IÓ.ˆ�r9   c                 ó  — |j                   \  }}}| j                  |d d …dd…df   «      j                  |d| j                  || j                  z  «      j	                  dddd«      }| j                  |«      j                  ||| j                  || j                  z  «      j	                  dddd«      }| j                  |«      j                  ||| j                  || j                  z  «      j	                  dddd«      }||j                  dd«      z  | j                  z  }|j                  d¬«      }| j                  |«      }||z  j                  dd«      j                  |d|«      }| j                  |«      }| j                  |«      }|S )	Nr   r   .r+   r&   éþÿÿÿéÿÿÿÿ©rQ   )r<   rZ   ÚreshaperR   Úpermuter[   r\   r>   rX   ÚsoftmaxrT   r4   rU   )	r5   r?   r@   ÚNrA   ÚqÚkÚvÚattns	            r8   rD   zCrossAttention.forwardu   sa  € Ø—'‘'‰ˆˆ1ˆaà�G‰G�A’a˜˜1˜˜c�k‘NÓ#×+Ñ+¨A¨q°$·.±.À!ÀtÇ~Á~ÑBUÓV×^Ñ^Ð_`ÐbcÐefÐhiÓjˆà�G‰G�A‹J×Ñ˜q ! T§^¡^°Q¸$¿.¹.Ñ5HÓI×QÑQÐRSÐUVÐXYÐ[\Ó]ˆà�G‰G�A‹J×Ñ˜q ! T§^¡^°Q¸$¿.¹.Ñ5HÓI×QÑQÐRSÐUVÐXYÐ[\Ó]ˆà�A—K‘K  BÓ'Ñ'¨4¯:©:Ñ5ˆØ�|‰| ˆ|Ó#ˆØ�~‰~˜dÓ#ˆà�A‰X× Ñ   AÓ&×.Ñ.¨q°!°QÓ7ˆØ�I‰I�a‹LˆØ�N‰N˜1ÓˆØˆr9   )é   Fç        rl   NN)	rG   rH   rI   rK   rL   Úfloatr.   rD   rM   rN   s   @r8   rP   rP   \   sQ   ø„ ð Ø"Ø!Ø!ØØñ/àð/ð ð/ð ð	/ð
 ð/ð õ/ö0r9   rP   c                   óº   ‡ — e Zd Zdddddej                  ej
                  ddf	dedededed	ed
edede	ej                     de	ej                     fˆ fd„Zd„ Zˆ xZS )ÚCrossAttentionBlockç      @Frl   NrQ   rR   Ú	mlp_ratiorS   rU   rT   Ú	drop_pathÚ	act_layerÚ
norm_layerc                 óÒ   •— |
|dœ}t         ‰| �  «         |	|fi |¤Ž| _        t        |f||||dœ|¤Ž| _        |dkD  rt        |«      | _        y t        j                  «       | _        y )Nr    )rR   rS   rT   rU   rl   )	r-   r.   Únorm1rP   rj   r   r0   ÚIdentityrr   )r5   rQ   rR   rq   rS   rU   rT   rr   rs   rt   r!   r"   r6   r7   s                €r8   r.   zCrossAttentionBlock.__init__Š   su   ø€ ð ¨Ñ/ˆÜ‰ÑÔÙ Ñ* rÑ*ˆŒ
Ü"Øð
àØØØñ
ð ñ
ˆŒ	ð 1:¸B²œ )Ó,ˆ�ÄBÇKÁKÃMˆ�r9   c                 ó~   — |d d …dd…df   | j                  | j                  | j                  |«      «      «      z   }|S )Nr   r   .)rr   rj   rv   )r5   r?   s     r8   rD   zCrossAttentionBlock.forward¦   s9   € ØŠa��1��cˆk‰N˜TŸ^™^¨D¯I©I°d·j±jÀ³mÓ,DÓEÑEˆØˆr9   )rG   rH   rI   r0   ÚGELUÚ	LayerNormrK   rm   rL   r   ÚModuler.   rD   rM   rN   s   @r8   ro   ro   ˆ   s¤   ø„ ð  "Ø"Ø!Ø!Ø!Ø)+¯©Ø*,¯,©,ØØñRàðRð ðRð ð	Rð
 ðRð ðRð ðRð ðRð ˜BŸI™I‘ðRð ˜RŸY™Y™õRö8r9   ro   c                   ó@  ‡ — e Zd Zddddej                  ej
                  ddfdeedf   deedf   deedf   deedf   d	eedf   d
e	dedede
ee   ef   deej                     deej                     fˆ fd„Zdeej                      deej                      fd„Zˆ xZS )ÚMultiScaleBlockFrl   NrQ   .ÚpatchesÚdepthrR   rq   rS   rU   rT   rr   rs   rt   c                 ó  •— ||dœ}t         ‰| �  «        t        |«      }|| _        t	        j
                  «       | _        t        |«      D ]€  }g }t        ||   «      D ]1  }|j                  t        d||   ||   ||   ||||	|   |dœ|¤Ž«       Œ3 t        |«      dk7  sŒV| j                  j                  t	        j                  |Ž «       Œ‚ t        | j                  «      dk(  rd | _        t	        j
                  «       | _        t        |«      D ]t  }||   ||dz   |z     k(  r	  |||   fi |¤Ž |
«       t	        j                  ||   ||dz   |z     fi |¤Žg}| j                  j                  t	        j                  |Ž «       Œv t	        j
                  «       | _        t        |«      D ]º  }|dz   |z  }||   }|d   dk(  r7| j                  j                  t        d||   |||   ||||	d   |dœ|¤Ž«       ŒOg }t        |d   «      D ].  }|j                  t        d||   |||   ||||	d   |dœ|¤Ž«       Œ0 | j                  j                  t	        j                  |Ž «       Œ¼ t	        j
                  «       | _        t        |«      D ]z  }||dz   |z     ||   k(  r	  |||dz   |z     fi |¤Ž |
«       t	        j                  ||dz   |z     ||   fi |¤Žg}| j                   j                  t	        j                  |Ž «       Œ| y )Nr    )rQ   rR   rq   rS   rU   rT   rr   rt   r   r   ra   © )r-   r.   ÚlenÚnum_branchesr0   Ú
ModuleListÚblocksÚrangeÚappendr   r1   Úprojsrw   rY   Úfusionro   Úrevert_projs)r5   rQ   r~   r   rR   rq   rS   rU   rT   rr   rs   rt   r!   r"   r6   rƒ   ÚdÚtmpÚiÚd_ÚnhÚ_r7   s                         €r8   r.   zMultiScaleBlock.__init__­   sS  ø€ ð  ¨Ñ/ˆÜ‰ÑÔÜ˜3“xˆØ(ˆÔä—m‘m“oˆŒÜ�|Ó$ò 	8ˆAØˆCÜ˜5 ™8“_ò �Ø—
‘
œ5ð 
Ø˜A™Ø'¨™lØ'¨™lØ%Ø'Ø'Ø'¨™lØ)ñ
ð ñ
õ 
ðô �3‹x˜1‹}Ø—‘×"Ñ"¤2§=¡=°#Ð#6Õ7ð	8ô" ˆt�{‰{Ó˜qÒ ØˆDŒKä—]‘]“_ˆŒ
Ü�|Ó$ò 	3ˆAØ�1‰v˜˜a !™e |Ñ3Ñ4Ò4¸ñ " # a¡&Ñ/¨BÑ/±³¼b¿i¹iÈÈAÉÐPSÐUVÐYZÑUZÐ^jÑTjÑPkÑ>rÐoqÑ>rÐs�Ø�J‰J×ÑœbŸm™m¨SÐ1Õ2ð	3ô —m‘m“oˆŒÜ�|Ó$ò 	8ˆAØ�a‘%˜<Ñ'ˆBØ˜2‘ˆBØ�R‰y˜AŠ~Ø—‘×"Ñ"Ü'ð 
Ø ™GØ"$Ø"+¨A¡,Ø!)Ø"+Ø"+Ø"+¨B¡-Ø#-ñ
ð ñ
õð �Ü˜u R™yÓ)ò �AØ—J‘JÔ2ð 
 Ø ™GØ"$Ø"+¨A¡,Ø!)Ø"+Ø"+Ø"+¨B¡-Ø#-ñ
 ð ñ
 õ 
ðð —‘×"Ñ"¤2§=¡=°#Ð#6Õ7ð=	8ô@ ŸM™M›OˆÔÜ�|Ó$ò 	:ˆAØ�A˜‘E˜\Ñ)Ñ*¨c°!©fÒ4¸ñ " # q¨1¡u°Ñ&<Ñ"=ÑDÀÑDÁiÃkÜ—y‘y  a¨!¡e¨|Ñ%;Ñ!<¸cÀ!¹fÑKÈÑKðM�à×Ñ×$Ñ$¤R§]¡]°CÐ%8Õ9ñ	:r9   r?   Úreturnc           	      óä  — g }t        | j                  «      D ]  \  }}|j                   |||   «      «       Œ! t        j                  j                  t        t        j                     g «      }t        | j                  «      D ])  \  }}|j                   |||   d d …dd…df   «      «       Œ+ g }t        t        | j                  | j                  «      «      D ]’  \  }\  }}	t        j                  ||   ||dz   | j                  z     d d …dd …df   fd¬«      }
 ||
«      }
 |	|
d d …dd…df   «      }t        j                  |||   d d …dd …df   fd¬«      }
|j                  |
«       Œ” |S )Nr   r   .rb   )Ú	enumerater…   r‡   ÚtorchÚjitÚannotater   ÚTensorrˆ   Úzipr‰   rŠ   Úcatrƒ   )r5   r?   Úouts_br�   ÚblockÚproj_cls_tokenr4   Úoutsr‰   Úrevert_projrŒ   Úreverted_proj_cls_tokens               r8   rD   zMultiScaleBlock.forward	  sq  € àˆÜ! $§+¡+Ó.ò 	'‰HˆAˆuØ�M‰M™%  !¡›+Õ&ð	'ô Ÿ™×+Ñ+¬D´·±Ñ,>ÀÓCˆÜ  §¡Ó,ò 	@‰GˆAˆtØ×!Ñ!¡$ v¨a¡y²°A°a°C¸°Ñ'=Ó">Õ?ð	@ð ˆÜ(1´#°d·k±kÀ4×CTÑCTÓ2UÓ(Vò 	Ñ$ˆAÑ$�˜Ü—)‘)˜^¨AÑ.°¸¸A¹À×ARÑARÑ7RÑ0SÒTUÐWXÑWYÐ[^ÐT^Ñ0_Ð`ÐfgÔhˆCÙ˜“+ˆCÙ&1°#²a¸¸1¸¸c°kÑ2BÓ&CÐ#Ü—)‘)Ð4°f¸Q±iÂÀ1Á2ÀsÀ
Ñ6KÐLÐRSÔTˆCØ�K‰K˜Õð	ð ˆr9   )rG   rH   rI   r0   ry   rz   r   rK   rm   rL   r   r   r   r{   r.   r”   r—   rD   rM   rN   s   @r8   r}   r}   «   s  ø„ ð #Ø!Ø!Ø35Ø)+¯©Ø*,¯,©,ØØñZ:à�s˜C�x‘ðZ:ð ˜3 ˜8‘_ðZ:ð ˜˜c˜‘?ð	Z:ð
 ˜S #˜X‘ðZ:ð ˜U C˜ZÑ(ðZ:ð ðZ:ð ðZ:ð ðZ:ð ˜T %™[¨%Ð/Ñ0ðZ:ð ˜BŸI™I‘ðZ:ð ˜RŸY™Y™õZ:ðx˜˜eŸl™lÑ+ð °°U·\±\Ñ0B÷ r9   r}   c                 ój   — t        | |«      D ��cg c]  \  }}|d   |z  |d   z  |z  ‘Œ c}}S c c}}w )Nr   r   )r˜   )r   r~   r�   Úps       r8   Ú_compute_num_patchesr¢     s7   € Ü.1°(¸GÓ.D×E¡d a¨ˆAˆa‰D�A‰I˜˜!™Ñ Ó!ÓEÐEùÓEs   �/ÚssÚ
crop_scalec                 óz  — | j                   dd \  }}||d   k7  s||d   k7  r˜|ri|d   |k  ra|d   |k  rYt        t        ||d   z
  dz  «      «      t        t        ||d   z
  dz  «      «      }}| dd…dd…|||d   z   …|||d   z   …f   } | S t        j                  j
                  j                  | |dd¬«      } | S )	a~  
    Pulled out of CrossViT.forward_features to bury conditional logic in a leaf node for FX tracing.
    Args:
        x (Tensor): input image
        ss (tuple[int, int]): height and width to scale to
        crop_scale (bool): whether to crop instead of interpolate to achieve the desired scale. Defaults to False
    Returns:
        Tensor: the "scaled" image batch tensor
    r`   Nr   r   ç       @ÚbicubicF)ÚsizeÚmodeÚalign_corners)r<   rK   Úroundr”   r0   Ú
functionalÚinterpolate)r?   r£   r¤   rB   rC   ÚcuÚcls          r8   Úscale_imager°   #  s×   € ð �7‰7�2�3ˆ<�D€A€qØˆBˆq‰E‚z�Q˜"˜Q™%’ZÙ˜"˜Q™% 1š*¨¨A©°!ªÜœ  B q¡E¡	¨RÑ/Ó0Ó1´3´u¸aÀ"ÀQÁ%¹iÈ2Ñ=MÓ7NÓ3O�ˆBØ’!’Q˜˜2  1¡™:˜ r¨"¨r°!©u©* }Ð4Ñ5ˆAð €Hô —‘×#Ñ#×/Ñ/°¸ÀÐZ_Ð/Ó`ˆAØ€Hr9   c            '       ó¬  ‡ — e Zd ZdZdddddddd	d
dddddddd eej                  d¬«      dddfdedee	df   deedf   dededeedf   deeedf   df   deedf   dee	df   de
de
de
de	d e	d!e	d"e	d#e	d$eej                     d%ef&ˆ fd&„Zd'„ Zej"                  j$                  d(„ «       Zej"                  j$                  d3d)„«       Zej"                  j$                  d4d*„«       Zej"                  j$                  d+ej                  fd,„«       Zd5ded%ee   fd-„Zd+eej4                     fd.„Zd3d/eej4                     d0e
d+ej4                  fd1„Zd2„ Zˆ xZS )6r   zI Vision Transformer with support for patch or hybrid CNN input stage
    rE   )ç      ð?r²   )rk   r,   r&   éè  )éÀ   é€  )©r   r&   r   r¶   r¶   )é   r#   )r¦   r¦   rp   FTrl   g�íµ ÷Æ°>)ÚepsÚtokenNr   Ú	img_scale.r   r   Únum_classesr   r   rR   rq   r   r¤   rS   Ú	drop_rateÚpos_drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_ratert   Úglobal_poolc                 óà  •— t         ‰'| �  «        ||dœ}|dv sJ ‚|| _        || _        || _        t        |«      | _        t        |«      }|D ��cg c]1  }t        | j                  D �cg c]  }t        ||z  «      ‘Œ c}«      ‘Œ3 c}}| _	        || _
        t        | j                  |«      }t        |«      | _        || _        t        |«      x| _        | _        t%        j&                  «       | _        t+        | j                  «      D ]€  }t-        | d|› �t%        j.                  t1        j2                  dd||   z   ||   fi |¤Ž«      «       t-        | d|› �t%        j.                  t1        j2                  dd||   fi |¤Ž«      «       Œ‚ t5        | j                  ||«      D ]0  \  }}}| j(                  j7                  t9        d|||||
dœ|¤Ž«       Œ2 t%        j:                  |¬«      | _        t        |D �cg c]  }t        |dd  «      ‘Œ c}«      }t?        ||«      } d	}!t%        j&                  «       | _         tC        |«      D ]W  \  }"}#tE        |#d d
 «      |#d
   z   }$| |!|!|$z    }%tG        |||#f||	||||%|dœ|¤Ž}&|!|$z  }!| j@                  j7                  |&«       ŒY t%        j&                  t+        | j                  «      D �cg c]  } |||   fi |¤Ž‘Œ c}«      | _$        t%        j:                  |«      | _%        t%        j&                  t+        | j                  «      D �cg c]5  }|d	kD  rt%        jL                  ||   |fi |¤Žnt%        jN                  «       ‘Œ7 c}«      | _(        t+        | j                  «      D ]6  }tS        tU        | d|› �«      d¬«       tS        tU        | d|› �«      d¬«       Œ8 | jW                  | jX                  «       y c c}w c c}}w c c}w c c}w c c}w )Nr    ©r¹   ÚavgÚ
pos_embed_r   Ú
cls_token_)r   r   r   r   r   )r¡   r`   r   ra   )rR   rq   rS   rU   rT   rr   rt   ç{®Gáz”?©Ústdr�   )-r-   r.   r»   r   rÁ   r   r   ÚtuplerK   Úimg_size_scaledr¤   r¢   r‚   rƒ   r   ÚsumÚnum_featuresÚhead_hidden_sizer0   r„   Úpatch_embedr†   ÚsetattrÚ	Parameterr”   Úzerosr˜   r‡   r   r]   Úpos_dropr   r…   r“   Úmaxr}   ÚnormÚ	head_droprY   rw   Úheadr   ÚgetattrÚapplyÚ_init_weights)(r5   r   rº   r   r   r»   r   r   rR   rq   r   r¤   rS   r¼   r½   r¾   r¿   rÀ   rt   rÁ   r!   r"   r6   ÚsiÚsjr/   r�   Úim_sr¡   r‹   r?   Útotal_depthÚdprÚdpr_ptrÚidxÚ	block_cfgÚ
curr_depthÚdpr_Úblkr7   s(                                          €r8   r.   zCrossVit.__init__<  sÈ  ø€ ô0 	‰ÑÔØ¨Ñ/ˆØÐ.Ñ.Ð.Ð.à&ˆÔØ ˆŒØ&ˆÔÜ! (Ó+ˆŒÜ˜iÓ(ˆ	ØXa×bÐRT¤¸d¿m¹mÖ&L¸¤s¨2°©7¥|Ò&LÕ MÓbˆÔØ$ˆŒÜ*¨4×+?Ñ+?ÀÓLˆÜ 
›OˆÔØ"ˆŒÜ47¸	³NÐBˆÔ˜DÔ1ÜŸ=™=›?ˆÔô �t×(Ñ(Ó)ò 	aˆAÜ�D˜J q cÐ*¬B¯L©L¼¿¹ÀQÈÈKÐXYÉNÑHZÐ\eÐfgÑ\hÑ9oÐlnÑ9oÓ,pÔqÜ�D˜J q cÐ*¬B¯L©L¼¿¹ÀQÈÈ9ÐUVÉ<Ñ9^Ð[]Ñ9^Ó,_Õ`ð	aô ˜d×2Ñ2°JÀ	ÓJò 		‰JˆD�!�QØ×Ñ×#Ñ#Üð Ø!Ø Ø%ØØ)ñð ñõð		ô Ÿ
™
 ]Ô3ˆŒä°Ö6¨1œ3˜q  ˜v�;Ò6Ó7ˆÜ'¨¸ÓDˆØˆÜ—m‘m“oˆŒÜ'¨Ó.ò 	$‰NˆC�Ü˜Y s¨˜^Ó,¨y¸©}Ñ<ˆJØ�w˜w¨Ñ3Ð4ˆDÜ!ØØØðð $Ø#Ø!Ø(Ø(ØØ%ñð ñˆCð �zÑ!ˆGØ�K‰K×Ñ˜sÕ#ð#	$ô& —M‘MÌ5ÐQU×QbÑQbÓKcÖ"dÀa¡:¨i¸©lÑ#A¸bÓ#AÒ"dÓeˆŒ	ÜŸ™ IÓ.ˆŒÜ—M‘Mä˜4×,Ñ,Ó-ö#/àð ;FÈº/ŒB�I‰I�i ‘l KÑ6°2Ò6ÌrÏ{É{Ë}Ñ\ò#/ó 0ˆŒ	ô �t×(Ñ(Ó)ò 	DˆAÜœ' $¨*°Q°CÐ(8Ó9¸sÕCÜœ' $¨*°Q°CÐ(8Ó9¸sÖCð	Dð 	�
‰
�4×%Ñ%Õ&ùòw 'MùÓbùò4 7ùò. #eùò#/s*   ÁOÁ(OÁ=	OÇ9O!ËO&Ì&:O+ÏOc                 óê  — t        |t        j                  «      rjt        |j                  d¬«       t        |t        j                  «      r8|j
                  �+t        j                  j                  |j
                  d«       y y y t        |t        j                  «      rUt        j                  j                  |j
                  d«       t        j                  j                  |j                  d«       y y )NrÇ   rÈ   r   r²   )	Ú
isinstancer0   rY   r   ÚweightrW   ÚinitÚ	constant_rz   )r5   Úms     r8   rÚ   zCrossVit._init_weightsš  s˜   € Ü�aœŸ™Ô#Ü˜!Ÿ(™(¨Õ,Ü˜!œRŸY™YÔ'¨A¯F©FÐ,>Ü—‘×!Ñ! !§&¡&¨!Õ,ð -?Ð'ä˜œ2Ÿ<™<Ô(Ü�G‰G×Ñ˜aŸf™f aÔ(Ü�G‰G×Ñ˜aŸh™h¨Õ,ð )r9   c                 óÞ   — t        «       }t        | j                  «      D ]J  }|j                  d|› �«       t	        | d|› �d «      }|€Œ*|j
                  sŒ7|j                  d|› �«       ŒL |S )NrÆ   rÅ   )Úsetr†   rƒ   ÚaddrØ   Úrequires_grad)r5   Úoutr�   Úpes       r8   Úno_weight_decayzCrossVit.no_weight_decay£  sr   € ä‹eˆÜ�t×(Ñ(Ó)ò 	*ˆAØ�G‰G�j  Ð$Ô%Ü˜ ¨A¨3Ð/°Ó6ˆBØ‰~ "×"2Ó"2Ø—‘˜* Q CÐ(Õ)ð		*ð
 ˆ
r9   c                 ó    — t        dddg¬«      S )Nz ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)iŸ† )Ústemr…   )Údict)r5   Úcoarses     r8   Úgroup_matcherzCrossVit.group_matcher­  s   € äØ4Ø-Ð/CÐDô
ð 	
r9   c                 ó   — |rJ d«       ‚y )Nz$gradient checkpointing not supportedr�   )r5   Úenables     r8   Úset_grad_checkpointingzCrossVit.set_grad_checkpointing´  s   € áÐAÐAÓAˆz�6r9   r‘   c                 ó   — | j                   S ©N)r×   )r5   s    r8   Úget_classifierzCrossVit.get_classifier¸  s   € à�y‰yÐr9   c           
      ó(  — || _         |�|dv sJ ‚|| _        t        | j                  d   d«      r#| j                  d   j                  j
                  nd }t        | j                  d   d«      r#| j                  d   j                  j                  nd }||dœ}t        j                  t        | j                  «      D �cg c]?  }|dkD  r$t        j                  | j                  |   |fi |¤Žnt        j                  «       ‘ŒA c}«      | _        y c c}w )NrÃ   r   rè   r    )r»   rÁ   Úhasattrr×   rè   r!   r"   r0   r„   r†   rƒ   rY   r   rw   )r5   r»   rÁ   r!   r"   r6   r�   s          r8   Úreset_classifierzCrossVit.reset_classifier¼  sñ   € Ø&ˆÔØÐ"ØÐ"2Ñ2Ð2Ð2Ø*ˆDÔÜ/6°t·y±yÀ±|ÀXÔ/N�—‘˜1‘×$Ñ$×+Ò+ÐTXˆÜ-4°T·Y±Y¸q±\À8Ô-L�—	‘	˜!‘×#Ñ#×)Ò)ÐRVˆØ¨Ñ/ˆÜ—M‘Mä˜4×,Ñ,Ó-ö#
àð @KÈQºŒB�I‰I�d—n‘n QÑ'¨Ñ;¸Ò;ÔTV×T_ÑT_ÓTaÑaò#
ó ˆ�	ùò #
s   Â>ADc                 ó’  — |j                   d   }g }t        | j                  «      D ]Â  \  }}|}| j                  |   }t	        ||| j
                  «      } ||«      }|dk(  r| j                  n| j                  }|j                  |dd«      }t        j                  ||fd¬«      }|dk(  r| j                  n| j                  }	||	z   }| j                  |«      }|j                  |«       ŒÄ t        | j                  «      D ]  \  }}
 |
|«      }Œ t        | j                   «      D ��cg c]  \  }} |||   «      ‘Œ }}}|S c c}}w )Nr   ra   r   rb   )r<   r“   rÏ   rË   r°   r¤   Úcls_token_0Úcls_token_1Úexpandr”   r™   Úpos_embed_0Úpos_embed_1rÓ   r‡   r…   rÕ   )r5   r?   r@   Úxsr�   rÏ   Úx_r£   Ú
cls_tokensÚ	pos_embedrå   rÕ   s               r8   Úforward_featureszCrossVit.forward_featuresÉ  s:  € Ø�G‰G�A‰JˆØˆÜ'¨×(8Ñ(8Ó9ò 	‰NˆAˆ{ØˆBØ×%Ñ% aÑ(ˆBÜ˜R  T§_¡_Ó5ˆBÙ˜R“ˆBØ-.°!ªV˜×)Ò)¸×9IÑ9IˆJØ#×*Ñ*¨1¨b°"Ó5ˆJÜ—‘˜J¨Ð+°Ô3ˆBØ,-°ªF˜×(Ò(¸×8HÑ8HˆIØ�i‘ˆBØ—‘˜rÓ"ˆBØ�I‰I�b�Mð	ô   §¡Ó,ò 	‰FˆAˆsÙ�R“‰Bð	ô *3°4·9±9Ó)=×>™g˜a ‰d�2�a‘5�kÐ>ˆÑ>Øˆ	ùó ?s   Ä)Er  Ú
pre_logitsc                 ób  — | j                   dk(  r)|D �cg c]  }|d d …dd …f   j                  d¬«      ‘Œ c}n|D �cg c]  }|d d …df   ‘Œ c}}|D �cg c]  }| j                  |«      ‘Œ }}|s't        | j                  d   t
        j                  «      r%t        j                  |D �cg c]  }|‘Œ c}d¬«      S t        j                  t        j                  t        | j                  «      D ��cg c]  \  }} |||   «      ‘Œ c}}d¬«      d¬«      S c c}w c c}w c c}w c c}w c c}}w )NrÄ   r   rb   r   )rÁ   ÚmeanrÖ   rç   r×   r0   rw   r”   r™   Ústackr“   )r5   r  r  r?   r�   r×   s         r8   Úforward_headzCrossVit.forward_headà  sí   € Ø37×3CÑ3CÀuÒ3L¨BÖ/ qˆa’�1‘2�‰h�m‰m ˆmÕ"Ó/ÐdfÖRgÐ_`ÐSTÒUVÐXYÐUYÓSZÒRgˆØ)+Ö, Aˆd�n‰n˜QÕÐ,ˆÐ,Ùœ D§I¡I¨a¡L´"·+±+Ô>Ü—9‘9¨ž_ Ašaš_°!Ô4Ð4Ü�z‰zœ%Ÿ+™+Ä)ÈDÏIÉIÓBV×&W±w°q¸$¡t¨B¨q©E¥{Ó&WÐ]^Ô_ÐefÔgÐgùò	 0ùÒRgùÚ,ùâ,ùÛ&Ws   ”"D½DÁD!Â*	D&Ã2D+c                 óJ   — | j                  |«      }| j                  |«      }|S rü   )r  r  )r5   r?   r  s      r8   rD   zCrossVit.forwardç  s'   € Ø×"Ñ" 1Ó%ˆØ×Ñ˜bÓ!ˆØˆr9   ©F)Trü   )rG   rH   rI   rJ   r   r0   rz   rK   r   rm   rL   r   r{   Ústrr.   rÚ   r”   r•   Úignorerò   r÷   rú   rý   r   r   r   r—   r  r  rD   rM   rN   s   @r8   r   r   8  sW  ø„ ñð
  Ø+5Ø*1ØØ#Ø)3Ø1RØ)0Ø+7Ø$Ø$Ø!Ø!Ø#%Ø$&Ø$&Ø$&Ù*1°"·,±,ÀDÔ*IØ&ØØñ-\'àð\'ð ˜U C˜ZÑ(ð\'ð ˜c 3˜h™ð	\'ð
 ð\'ð ð\'ð ˜S #˜X‘ð\'ð ˜˜s C˜x™¨#Ð-Ñ.ð\'ð ˜S #˜X‘ð\'ð ˜U C˜ZÑ(ð\'ð ð\'ð ð\'ð ð\'ð ð\'ð !ð\'ð  "ð!\'ð" "ð#\'ð$ "ð%\'ð& ˜RŸY™Y™ð'\'ð( õ)\'ò|-ð ‡Y�Y×Ññó ðð ‡Y�Y×Ñò
ó ð
ð ‡Y�Y×ÑòBó ðBð ‡Y�Y×Ñð §	¡	ò ó ðñ¨Cð ¸hÀs¹mó ð T¨%¯,©,Ñ%7ó ñ.h˜t E§L¡LÑ1ð h¸tð hÐPU×P\ÑP\ó hör9   c                 ój   — |j                  dd «      rt        d«      ‚d„ }t        t        | |fd|i|¤ŽS )NÚfeatures_onlyz<features_only not implemented for Vision Transformer models.c                 ó~   — i }| j                  «       D ]'  }d|v sd|v r|j                  dd«      }n|}| |   ||<   Œ) |S )Nr
  Ú	cls_tokenú.r�   )ÚkeysÚreplace)Ú
state_dictÚnew_state_dictÚkeyÚnew_keys       r8   Úpretrained_filter_fnz._create_crossvit.<locals>.pretrained_filter_fnñ  sW   € ØˆØ—?‘?Ó$ò 	6ˆCØ˜cÑ! [°CÑ%7ØŸ+™+ c¨3Ó/‘à�Ø&0°¡oˆN˜7Ò#ð	6ð Ðr9   r   )ÚgetÚRuntimeErrorr   r   )ÚvariantÚ
pretrainedÚkwargsr   s       r8   Ú_create_crossvitr&  í  sM   € Ø‡z�z�/ 4Ô(ÜÐYÓZÐZòô  ÜØØñð 2ð	ð
 ñð r9   c                 ó2   — | ddd dt         t        dddddœ|¥S )	Nr³   )r&   éð   r(  g      ì?T)zpatch_embed.0.projzpatch_embed.1.proj)zhead.0zhead.1z
apache-2.0)Úurlr»   Ú
input_sizeÚ	pool_sizeÚcrop_pctr  rÉ   Úfixed_input_sizeÚ
first_convÚ
classifierÚlicenser	   )r)  r%  s     r8   Ú_cfgr1    s5   € àØ¨=ÀtÐY^Ü%Ô.BÐX\ØBØ*Øñð ðð r9   ztimm/)Ú	hf_hub_id)zpatch_embed.0.proj.0zpatch_embed.1.proj.0)r2  r.  )r&   é˜  r3  r²   )r2  r*  r.  r,  )zcrossvit_15_240.in1kzcrossvit_15_dagger_240.in1kzcrossvit_15_dagger_408.in1kzcrossvit_18_240.in1kzcrossvit_18_dagger_240.in1kzcrossvit_18_dagger_408.in1kzcrossvit_9_240.in1kzcrossvit_9_dagger_240.in1kzcrossvit_base_240.in1kzcrossvit_small_240.in1kzcrossvit_tiny_240.in1kr‘   c           	      óz   — t        dddgddgg d¢g d¢g d¢gddgg d¢¬	«      }t        dd
| dœt        |fi |¤Ž¤Ž}|S )N©r²   gÞÝÝÝÝÝí?r#   r,   é`   r´   ©r   r$   r   r&   ©r$   r$   r   ©rº   r   r   r   rR   rq   Úcrossvit_tiny_240©r#  r$  r�   ©rõ   r&  ©r$  r%  Ú
model_argsÚmodels       r8   r:  r:  .  sY   € äØ ¨b°"¨XÀ"ÀcÀÒS\Ò^gÒirÐRsØ�a�&¢Iô/€Jô ÐnÐ%8ÀZÑnÔSWÐXbÑSmÐflÑSmÑn€EØ€Lr9   c           	      óz   — t        dddgddgg d¢g d¢g d¢gddgg d¢¬	«      }t        dd
| dœt        |fi |¤Ž¤Ž}|S )Nr5  r#   r,   r´   rµ   r7  r·   r8  r9  Úcrossvit_small_240r;  r�   r<  r=  s       r8   rA  rA  7  sY   € äØ ¨b°"¨XÀ#ÀsÀÒT]Ò_hÒjsÐStØ�a�&¢Iô/€Jô ÐoÐ%9ÀjÑoÔTXÐYcÑTnÐgmÑTnÑo€EØ€Lr9   c           	      óz   — t        dddgddgg d¢g d¢g d¢gddgg d¢¬«      }t        dd	| d
œt        |fi |¤Ž¤Ž}|S )Nr5  r#   r,   rµ   rF   r7  r8  r9  Úcrossvit_base_240r;  r�   r<  r=  s       r8   rC  rC  @  sY   € äØ ¨b°"¨XÀ#ÀsÀÒT]Ò_hÒjsÐStØ�r�(¢iô1€Jô ÐnÐ%8ÀZÑnÔSWÐXbÑSmÐflÑSmÑn€EØ€Lr9   c           	      óz   — t        dddgddgg d¢g d¢g d¢gddgg d¢¬	«      }t        dd
| dœt        |fi |¤Ž¤Ž}|S )Nr5  r#   r,   é€   é   ©r   r&   r   r$   ©r&   r&   r   r9  Úcrossvit_9_240r;  r�   r<  r=  s       r8   rI  rI  I  sY   € äØ ¨b°"¨XÀ#ÀsÀÒT]Ò_hÒjsÐStØ�a�&¢Iô/€Jô ÐkÐ%5À*ÑkÔPTÐU_ÑPjÐciÑPjÑk€EØ€Lr9   c           	      óz   — t        dddgddgg d¢g d¢g d¢gddgg d¢¬	«      }t        dd
| dœt        |fi |¤Ž¤Ž}|S )Nr5  r#   r,   r´   rµ   ©r   é   r   r·   rH  r9  Úcrossvit_15_240r;  r�   r<  r=  s       r8   rM  rM  R  sY   € äØ ¨b°"¨XÀ#ÀsÀÒT]Ò_hÒjsÐStØ�a�&¢Iô/€Jô ÐlÐ%6À:ÑlÔQUÐV`ÑQkÐdjÑQkÑl€EØ€Lr9   c           
      ó|   — t        ddddgddgg d¢g d¢g d¢gddgg d¢d	œ|¤Ž}t        dd
| dœt        |fi |¤Ž¤Ž}|S )Nr5  r#   r,   rE   éÀ  ©r   r·   r   r%   rH  r9  Úcrossvit_18_240r;  r�   r<  r=  s       r8   rQ  rQ  [  sf   € äð 9Ø"°°B¨xÀCÈÀ:ÒV_ÒajÒluÐUvØ�a�&¢Iñ9à17ñ9€Jô ÐlÐ%6À:ÑlÔQUÐV`ÑQkÐdjÑQkÑl€EØ€Lr9   c           	      ó|   — t        dddgddgg d¢g d¢g d¢gddgg d¢d	¬
«      }t        dd| dœt        |fi |¤Ž¤Ž}|S )Nr5  r#   r,   rE  rF  rG  r$   rH  T©rº   r   r   r   rR   rq   r   Úcrossvit_9_dagger_240r;  r�   r<  r=  s       r8   rT  rT  d  s\   € äØ"°°B¨xÀCÈÀ:ÒV_ÒajÒluÐUvØ�a�&¢I¸$ô@€Jô ÐrÐ%<ÈÑrÔW[Ð\fÑWqÐjpÑWqÑr€EØ€Lr9   c           	      ó|   — t        dddgddgg d¢g d¢g d¢gddgg d¢d	¬
«      }t        dd| dœt        |fi |¤Ž¤Ž}|S )Nr5  r#   r,   r´   rµ   rK  r·   rH  TrS  Úcrossvit_15_dagger_240r;  r�   r<  r=  s       r8   rV  rV  m  ó\   € äØ ¨b°"¨XÀ#ÀsÀÒT]Ò_hÒjsÐStØ�a�&¢I¸$ô@€Jô ÐsÐ%=È*ÑsÔX\Ð]gÑXrÐkqÑXrÑs€EØ€Lr9   c           	      ó|   — t        dddgddgg d¢g d¢g d¢gddgg d¢d	¬
«      }t        dd| dœt        |fi |¤Ž¤Ž}|S )N©r²   gî?r#   r,   r´   rµ   rK  r·   rH  TrS  Úcrossvit_15_dagger_408r;  r�   r<  r=  s       r8   rZ  rZ  v  rW  r9   c           	      ó|   — t        dddgddgg d¢g d¢g d¢gddgg d¢d	¬
«      }t        dd| dœt        |fi |¤Ž¤Ž}|S )Nr5  r#   r,   rE   rO  rP  r%   rH  TrS  Úcrossvit_18_dagger_240r;  r�   r<  r=  s       r8   r\  r\    rW  r9   c           	      ó|   — t        dddgddgg d¢g d¢g d¢gddgg d¢d	¬
«      }t        dd| dœt        |fi |¤Ž¤Ž}|S )NrY  r#   r,   rE   rO  rP  r%   rH  TrS  Úcrossvit_18_dagger_408r;  r�   r<  r=  s       r8   r^  r^  ˆ  rW  r9   r  )Ú )7rJ   Ú	functoolsr   Útypingr   r   r   r   r   r”   Útorch.nnr0   Ú	timm.datar
   r   Útimm.layersr   r   r   r   r   Ú_builderr   Ú_features_fxr   Ú	_registryr   r   Úvision_transformerr   Ú__all__r{   r   rP   ro   r}   r¢   rK   rL   r°   r   r&  r1  Údefault_cfgsr:  rA  rC  rI  rM  rQ  rT  rV  rZ  r\  r^  r�   r9   r8   ú<module>rk     s�  ðñõ, ß 5Õ 5ã Ý ç Aß ^Õ ^Ý *Ý 3ß <Ý %àˆ,€ô2�—‘ô 2ôj)�R—Y‘Yô )ôX ˜"Ÿ)™)ô  ôFq�b—i‘iô qòhFð ñ�u˜S #˜X‘ð °Dò ó ðô(rˆr�y‰yô rójó.	ñ %Ù ¨7Ô3Ù#'ØØCô$ñ $(ØØ Ð-]Ðhkô$ñ !¨7Ô3Ù#'ØØCô$ñ $(ØØ Ð-]Ðhkô$ñ  ¨'Ô2Ù"&ØØCô#ñ #¨WÔ5Ù#¨gÔ6Ù"¨WÔ5ñ5&ó €ð< ñ°Xò ó ðð ñ°hò ó ðð ñ°Xò ó ðð ñ°(ò ó ðð ñ°8ò ó ðð ñ°8ò ó ðð ñ¸ò ó ðð ñ¸(ò ó ðð ñ¸(ò ó ðð ñ¸(ò ó ðð ñ¸(ò ó ñr9   