Ë
    éÓ^jîX  ã                   óú  — d Z ddlZddlmZmZmZmZmZmZ ddl	Z	ddl
mZ ddlmZmZ ddlmZmZmZmZmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZ dgZ  G d„ dejB                  «      Z" G d„ dejB                  «      Z# G d„ dejB                  «      Z$ G d„ dejB                  «      Z%dd„Z& e e&d¬«       e&d¬«       e&d¬«      dœ«      Z'd„ Z(dd„Z)edde%fd„«       Z*edde%fd„«       Z+edde%fd„«       Z,y)a¥   Transformer in Transformer (TNT) in PyTorch

A PyTorch implement of TNT as described in
'Transformer in Transformer' - https://arxiv.org/abs/2103.00112

The official mindspore code is released and available at
https://gitee.com/mindspore/mindspore/tree/master/model_zoo/research/cv/TNT

The official pytorch code is released and available at
https://github.com/huawei-noah/Efficient-AI-Backbones/tree/master/tnt_pytorch
é    N)ÚListÚOptionalÚTupleÚUnionÚTypeÚAny©ÚIMAGENET_INCEPTION_MEANÚIMAGENET_INCEPTION_STD)ÚMlpÚDropPathÚcalculate_drop_path_ratesÚtrunc_normal_Ú_assertÚ	to_2tupleÚresample_abs_pos_embedé   )Úbuild_model_with_cfg)Úfeature_take_indices)Ú
checkpoint)Úgenerate_default_cfgsÚregister_modelÚTNTc                   óP   ‡ — e Zd ZdZ	 	 	 	 	 	 d
dedededededefˆ fd„Zd	„ Zˆ xZ	S )Ú	Attentionz Multi-Head Attention
    ÚdimÚ
hidden_dimÚ	num_headsÚqkv_biasÚ	attn_dropÚ	proj_dropc	                 óš  •— ||dœ}	t         ‰| �  «        || _        || _        ||z  }
|
| _        |
dz  | _        t        j                  ||dz  fd|i|	¤Ž| _        t        j                  ||fd|i|	¤Ž| _	        t        j                  |d¬«      | _        t        j                  ||fi |	¤Ž| _        t        j                  |d¬«      | _        y )N©ÚdeviceÚdtypeg      à¿é   ÚbiasT)Úinplace)ÚsuperÚ__init__r   r   Úhead_dimÚscaleÚnnÚLinearÚqkÚvÚDropoutr    Úprojr!   )Úselfr   r   r   r   r    r!   r$   r%   Úddr+   Ú	__class__s              €úZ/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/tnt.pyr*   zAttention.__init__    s¼   ø€ ð ¨Ñ/ˆÜ‰ÑÔØ$ˆŒØ"ˆŒØ Ñ*ˆØ ˆŒØ Ñ%ˆŒ
ä—)‘)˜C ¨a¡ÑE°hÐEÀ"ÑEˆŒÜ—‘˜3 Ñ9¨(Ð9°bÑ9ˆŒÜŸ™ I°tÔ<ˆŒÜ—I‘I˜c 3Ñ-¨"Ñ-ˆŒ	ÜŸ™ I°tÔ<ˆ�ó    c                 ó|  — |j                   \  }}}| j                  |«      j                  ||d| j                  | j                  «      j                  ddddd«      }|j                  d«      \  }}| j                  |«      j                  ||| j                  d«      j                  dddd«      }||j                  dd«      z  | j                  z  }	|	j                  d¬«      }	| j                  |	«      }	|	|z  j                  dd«      j                  ||d«      }| j                  |«      }| j                  |«      }|S )	Nr&   r   é   r   é   éÿÿÿÿéþÿÿÿ©r   )Úshaper/   Úreshaper   r+   ÚpermuteÚunbindr0   Ú	transposer,   Úsoftmaxr    r2   r!   )
r3   ÚxÚBÚNÚCr/   ÚqÚkr0   Úattns
             r6   ÚforwardzAttention.forward9   s  € Ø—'‘'‰ˆˆ1ˆaØ�W‰W�Q‹Z×Ñ  1 a¨¯©¸¿¹ÓG×OÑOÐPQÐSTÐVWÐYZÐ\]Ó^ˆØ�y‰y˜‹|‰ˆˆ1Ø�F‰F�1‹I×Ñ˜a  D§N¡N°BÓ7×?Ñ?ÀÀ1ÀaÈÓKˆà�A—K‘K  BÓ'Ñ'¨4¯:©:Ñ5ˆØ�|‰| ˆ|Ó#ˆØ�~‰~˜dÓ#ˆà�A‰X× Ñ   AÓ&×.Ñ.¨q°!°RÓ8ˆØ�I‰I�a‹LˆØ�N‰N˜1ÓˆØˆr7   )é   Fç        rM   NN)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚintÚboolÚfloatr*   rK   Ú__classcell__©r5   s   @r6   r   r      s`   ø„ ñð Ø"Ø!Ø!ØØñ=àð=ð ð=ð ð	=ð
 ð=ð ð=ð õ=ö2r7   r   c                   óÔ   ‡ — e Zd ZdZdddddddej
                  ej                  dddfded	ed
edededede	dededede
ej                     de
ej                     de	fˆ fd„Zd„ Zˆ xZS )ÚBlockz TNT Block
    r:   é   ç      @FrM   Nr   Údim_outÚ	num_pixelÚnum_heads_inÚnum_heads_outÚ	mlp_ratior   r!   r    Ú	drop_pathÚ	act_layerÚ
norm_layerÚlegacyc           	      óæ  •— ||dœ}t         ‰| �  «         ||fi |¤Ž| _        t        ||f|||	|dœ|¤Ž| _         ||fi |¤Ž| _        t        d	|t        |dz  «      |||dœ|¤Ž| _        || _	        | j                  r7 ||fi |¤Ž| _
        t        j                  ||z  |fddi|¤Ž| _        d | _        n@ |||z  fi |¤Ž| _
        t        j                  ||z  |fddi|¤Ž| _         ||fi |¤Ž| _         ||fi |¤Ž| _        t        ||f|||	|dœ|¤Ž| _        |
dkD  rt#        |
«      nt        j$                  «       | _         ||fi |¤Ž| _        t        d	|t        ||z  «      |||dœ|¤Ž| _        y )
Nr#   )r   r   r    r!   r:   )Úin_featuresÚhidden_featuresÚout_featuresra   Údropr'   TFrM   © )r)   r*   Únorm_inr   Úattn_inÚnorm_mlp_inr   rR   Úmlp_inrc   Ú
norm1_projr-   r.   r2   Ú
norm2_projÚnorm_outÚattn_outr   ÚIdentityr`   Únorm_mlpÚmlp)r3   r   r[   r\   r]   r^   r_   r   r!   r    r`   ra   rb   rc   r$   r%   r4   r5   s                    €r6   r*   zBlock.__init__M   sÑ  ø€ ð$ ¨Ñ/ˆÜ‰ÑÔá! #Ñ,¨Ñ,ˆŒÜ ØØð
ð #ØØØñ
ð ñ
ˆŒñ & cÑ0¨RÑ0ˆÔÜð 
ØÜ  a¡›LØØØñ
ð ñ
ˆŒð ˆŒØ�;Š;Ù(¨Ñ3°Ñ3ˆDŒOÜŸ	™	 #¨	¡/°7ÑLÀÐLÈÑLˆDŒIØ"ˆD�Oá(¨¨y©Ñ?¸BÑ?ˆDŒOÜŸ	™	 #¨	¡/°7ÑMÀÐMÈ"ÑMˆDŒIÙ(¨Ñ7°BÑ7ˆDŒOñ # 7Ñ1¨bÑ1ˆŒÜ!ØØð
ð $ØØØñ
ð ñ
ˆŒð 1:¸B²œ )Ô,ÄBÇKÁKÃMˆŒá" 7Ñ1¨bÑ1ˆŒÜð 
ØÜ ¨)Ñ 3Ó4Ø ØØñ
ð ñ
ˆ�r7   c                 óz  — || j                  | j                  | j                  |«      «      «      z   }|| j                  | j                  | j	                  |«      «      «      z   }|j                  «       \  }}}| j                  €at        j                  |d d …dd…f   |d d …dd …f   | j                  | j                  |«      j                  ||dz
  d«      «      z   gd¬«      }not        j                  |d d …dd…f   |d d …dd …f   | j                  | j                  | j                  |j                  ||dz
  d«      «      «      «      z   gd¬«      }|| j                  | j                  | j                  |«      «      «      z   }|| j                  | j                  | j                  |«      «      «      z   }||fS )Nr   r   r;   r=   )r`   rk   rj   rm   rl   Úsizero   ÚtorchÚcatr2   rn   r?   rq   rp   rt   rs   )r3   Úpixel_embedÚpatch_embedrE   rF   rG   s         r6   rK   zBlock.forward—   s™  € à! D§N¡N°4·<±<ÀÇÁÈ[Ó@YÓ3ZÓ$[Ñ[ˆØ! D§N¡N°4·;±;¸t×?OÑ?OÐP[Ó?\Ó3]Ó$^Ñ^ˆà×"Ñ"Ó$‰ˆˆ1ˆaØ�?‰?Ð"ÜŸ)™)ØšA˜q ˜s˜FÑ#ØšA˜q™r˜EÑ" T§Y¡Y¨t¯©¸{Ó/K×/SÑ/SÐTUÐWXÐ[\ÑW\Ð^`Ó/aÓ%bÑbð%ð ô‰Kô
  Ÿ)™)ØšA˜q ˜s˜FÑ#ØšA˜q™r˜EÑ" T§_¡_°T·Y±Y¸t¿¹È{×ObÑObÐcdÐfgÐjkÑfkÐmoÓOpÓ?qÓ5rÓ%sÑsð%ð ôˆKð " D§N¡N°4·=±=ÀÇÁÈ{ÓA[Ó3\Ó$]Ñ]ˆØ! D§N¡N°4·8±8¸D¿M¹MÈ+Ó<VÓ3WÓ$XÑXˆØ˜KÐ'Ð'r7   )rN   rO   rP   rQ   r-   ÚGELUÚ	LayerNormrR   rT   rS   r   ÚModuler*   rK   rU   rV   s   @r6   rX   rX   I   sß   ø„ ñð !"Ø!#Ø!Ø"Ø!Ø!Ø!Ø)+¯©Ø*,¯,©,Ø ØØñ!H
àðH
ð ðH
ð ð	H
ð
 ðH
ð ðH
ð ðH
ð ðH
ð ðH
ð ðH
ð ðH
ð ˜BŸI™I‘ðH
ð ˜RŸY™Y™ðH
ð õH
öT(r7   rX   c                   ó  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 ddeeeeef   f   deeeeef   f   dedededefˆ fd„Zdd	eeeef   ef   fd
„Z	deeef   d	eeef   fd„Z
dej                  dej                  d	ej                  fd„Zˆ xZS )Ú
PixelEmbedz Image to Pixel Embedding
    Úimg_sizeÚ
patch_sizeÚin_chansÚin_dimÚstriderc   c	                 óR  •— ||dœ}	t         ‰| �  «        t        |«      }t        |«      }|d   |d   z  |d   |d   z  f| _        | j                  d   | j                  d   z  }
|| _        || _        || _        |
| _        || _        |D �cg c]  }t        j                  ||z  «      ‘Œ }}|| _        t        j                  || j                  fdd|dœ|	¤Ž| _        | j                  rt        j                  ||¬«      | _        y t        j                  ||¬«      | _        y c c}w )Nr#   r   r   é   r9   )Úkernel_sizeÚpaddingr„   )r‡   r„   )r)   r*   r   Ú	grid_sizer€   r�   rc   Únum_patchesrƒ   ÚmathÚceilÚnew_patch_sizer-   ÚConv2dr2   ÚUnfoldÚunfold)r3   r€   r�   r‚   rƒ   r„   rc   r$   r%   r4   rŠ   Úpsr�   r5   s                €r6   r*   zPixelEmbed.__init__°   s  ø€ ð ¨Ñ/ˆÜ‰ÑÔÜ˜XÓ&ˆÜ˜zÓ*ˆ
à" 1™+¨°A©Ñ6¸À¹ÀzÐRSÁ}Ñ8TÐUˆŒØ—~‘~ aÑ(¨T¯^©^¸AÑ->Ñ?ˆØ ˆŒØ$ˆŒØˆŒØ&ˆÔØˆŒØ;EÖF°Rœ$Ÿ)™) B¨¡KÕ0ÐFˆÐFØ,ˆÔä—I‘I˜h¨¯©ÐcÀÈAÐV\ÑcÐ`bÑcˆŒ	Ø�;Š;ÜŸ)™)°À~ÔVˆD�KäŸ)™)°
À:ÔNˆD�Kùò Gs   ÂD$Úreturnc                 óH   — |rt        | j                  «      S | j                  S ©N)Úmaxr�   )r3   Ú	as_scalars     r6   Ú
feat_ratiozPixelEmbed.feat_ratioÐ   s   € ÙÜ�t—‘Ó'Ð'à—?‘?Ð"r7   c                 óV   — |d   | j                   d   z  |d   | j                   d   z  fS )Nr   r   )r�   )r3   r€   s     r6   Údynamic_feat_sizezPixelEmbed.dynamic_feat_sizeÖ   s2   € Ø˜‰{˜dŸo™o¨aÑ0Ñ0°(¸1±+ÀÇÁÐQRÑASÑ2SÐSÐSr7   rD   Ú	pixel_posc                 ó”  — |j                   \  }}}}t        || j                  d   k(  d|› d|› d| j                  d   › d| j                  d   › d�	«       t        || j                  d   k(  d|› d|› d| j                  d   › d| j                  d   › d�	«       | j                  rx| j	                  |«      }| j                  |«      }|j                  dd«      j                  || j                  z  | j                  | j                  d   | j                  d   «      }nm| j                  |«      }|j                  dd«      j                  || j                  z  || j                  d   | j                  d   «      }| j	                  |«      }||z   }|j                  || j                  z  | j                  d«      j                  dd«      }|S )	Nr   zInput image size (Ú*z) doesn't match model (r   z).r&   r;   )r>   r   r€   rc   r2   r�   rB   r?   rŠ   rƒ   r�   r�   )r3   rD   rš   rE   rG   ÚHÚWs          r6   rK   zPixelEmbed.forwardÙ   sÄ  € Ø—W‘W‰
ˆˆ1ˆa�ÜØ�—‘˜qÑ!Ñ!Ø    1 Q CÐ'>¸t¿}¹}ÈQÑ?OÐ>PÐPQÐRV×R_ÑR_Ð`aÑRbÐQcÐceÐfô	hô 	Ø�—‘˜qÑ!Ñ!Ø    1 Q CÐ'>¸t¿}¹}ÈQÑ?OÐ>PÐPQÐRV×R_ÑR_Ð`aÑRbÐQcÐceÐfô	hð �;Š;Ø—	‘	˜!“ˆAØ—‘˜A“ˆAØ—‘˜A˜qÓ!×)Ñ)Ø�D×$Ñ$Ñ$ d§k¡k°4×3FÑ3FÀqÑ3IÈ4×K^ÑK^Ð_`ÑKaóc‰Að —‘˜A“ˆAØ—‘˜A˜qÓ!×)Ñ)¨!¨d×.>Ñ.>Ñ*>ÀÀ4Ç?Á?ÐSTÑCUÐW[×WfÑWfÐghÑWiÓjˆAØ—	‘	˜!“ˆAØ�	‰MˆØ�I‰I�a˜$×*Ñ*Ñ*¨D¯K©K¸Ó<×FÑFÀqÈ!ÓLˆØˆr7   )éà   é   r9   é0   r:   FNN©T)rN   rO   rP   rQ   r   rR   r   rS   r*   r—   r™   rw   ÚTensorrK   rU   rV   s   @r6   r   r   ¬   s÷   ø„ ñð
 58Ø68ØØØØ ØØñOà˜C  s¨C x¡Ð0Ñ1ðOð ˜c 5¨¨c¨¡?Ð2Ñ3ðOð ð	Oð
 ðOð ðOð õOñ@#¨E°%¸¸S¸±/À3Ð2FÑ,Gó #ðT¨%°°S°©/ð T¸eÀCÈÀH¹oó Tð˜Ÿ™ð °%·,±,ð À5Ç<Á<÷ r7   r   c            )       ó  ‡ — e Zd ZdZdddddddd	d
d	dddddddej
                  d
dddfdeeeeef   f   deeeeef   f   dedede	dededededede
dede
de
de
de
de
d eej                     d!ed"ef(ˆ fd#„Zd$„ Zej"                  j$                  d%„ «       Zej"                  j$                  d:d&„«       Zej"                  j$                  d;d'„«       Zej"                  j$                  d(ej                  fd)„«       Zd<dedee	   fd*„Z	 	 	 	 	 	 d=d+ej2                  d,eeeee   f      d-ed.ed/ed0e	d1ed(eeej2                     eej2                  eej2                     f   f   fd2„Z	 	 	 d>d,eeee   f   d3ed4efd5„Zd6„ Zd:d7efd8„Zd9„ Zˆ xZ S )?r   zC Transformer in Transformer - https://arxiv.org/abs/2103.00112
    rŸ   r    r9   éè  Útokeni   r¡   rY   r:   rZ   FrM   Nr€   r�   r‚   Únum_classesÚglobal_poolÚ	embed_dimÚ	inner_dimÚdepthÚnum_heads_innerÚnum_heads_outerr_   r   Ú	drop_rateÚpos_drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_raterb   Úfirst_striderc   c                 ó>  •— t         ‰| �  «        ||dœ}|dv sJ ‚|| _        || _        || _        |x| _        x| _        | _        d| _        d| _	        t        d||||||dœ|¤Ž| _        | j                  j                  }t        | j                  d«      r| j                  j                  «       n|}|| _        | j                  j                  }|d   |d   z  } |||z  fi |¤Ž| _        t#        j$                  ||z  |fi |¤Ž| _         ||fi |¤Ž| _        t#        j*                  t-        j.                  dd|fi |¤Ž«      | _        t#        j*                  t-        j.                  d|dz   |fi |¤Ž«      | _        t#        j*                  t-        j.                  d||d   |d   fi |¤Ž«      | _        t#        j6                  |¬«      | _        t;        ||«      }g }t=        |«      D ],  }|j?                  tA        d||||	|
||||||   ||d	œ|¤Ž«       Œ. t#        jB                  |«      | _"        t=        |«      D �cg c]  }tG        d
|› �||¬«      ‘Œ c}| _$         ||fi |¤Ž| _%        t#        j6                  |«      | _&        |dkD  rt#        j$                  ||fi |¤Žnt#        jN                  «       | _(        tS        | j0                  d¬«       tS        | j2                  d¬«       tS        | j4                  d¬«       | jU                  | jV                  «       y c c}w )Nr#   ©Ú r¦   Úavgr   F)r€   r�   r‚   rƒ   r„   rc   r—   r   )Úp)r   r[   r\   r]   r^   r_   r   r!   r    r`   rb   rc   zblocks.)ÚmoduleÚnum_chsÚ	reductionç{®Gáz”?©Ústdri   ),r)   r*   r§   r‚   r¨   Únum_featuresÚhead_hidden_sizer©   Únum_prefix_tokensÚgrad_checkpointingr   ry   rŠ   Úhasattrr—   r�   rn   r-   r.   r2   ro   Ú	Parameterrw   ÚzerosÚ	cls_tokenÚ	patch_posrš   r1   Úpos_dropr   ÚrangeÚappendrX   Ú
ModuleListÚblocksÚdictÚfeature_infoÚnormÚ	head_droprr   Úheadr   ÚapplyÚ_init_weights) r3   r€   r�   r‚   r§   r¨   r©   rª   r«   r¬   r­   r_   r   r®   r¯   r°   r±   r²   rb   r³   rc   r$   r%   r4   rŠ   Úrr�   r\   ÚdprrÌ   Úir5   s                                   €r6   r*   zTNT.__init__ó   s  ø€ ô2 	‰ÑÔØ¨Ñ/ˆØÐ2Ñ2Ð2Ð2Ø&ˆÔØ ˆŒØ&ˆÔØENÐNˆÔÐN˜DÔ1°D´NØ!"ˆÔØ"'ˆÔä%ð 
ØØ!ØØØØñ
ð ñ
ˆÔð ×&Ñ&×2Ñ2ˆÜ-4°T×5EÑ5EÀ|Ô-TˆD×Ñ×'Ñ'Ô)ÐZdˆØ&ˆÔØ×)Ñ)×8Ñ8ˆØ" 1Ñ%¨°qÑ(9Ñ9ˆ	á$ Y°Ñ%:ÑA¸bÑAˆŒÜ—I‘I˜i¨)Ñ3°YÑEÀ"ÑEˆŒ	Ù$ YÑ5°"Ñ5ˆŒäŸ™¤e§k¡k°!°Q¸	Ñ&HÀRÑ&HÓIˆŒÜŸ™¤e§k¡k°!°[À1±_ÀiÑ&VÐSUÑ&VÓWˆŒÜŸ™¤e§k¡k°!°YÀÈqÑ@QÐSaÐbcÑSdÑ&kÐhjÑ&kÓlˆŒÜŸ
™
 ]Ô3ˆŒä'¨¸Ó>ˆØˆÜ�u“ò 	ˆAØ�M‰Mœ%ð ØØ!Ø#Ø,Ø-Ø#Ø!Ø(Ø(Ø˜a™&Ø%Øñð ñõ ð	ô  —m‘m FÓ+ˆŒäPUÐV[ÓP\ö^ØKLŒD˜' ! ˜¨yÀAÖFò^ˆÔñ ˜yÑ/¨BÑ/ˆŒ	ÜŸ™ IÓ.ˆŒØ?JÈQº”B—I‘I˜i¨Ñ;¸Ò;ÔTV×T_ÑT_ÓTaˆŒ	ä�d—n‘n¨#Õ.Ü�d—n‘n¨#Õ.Ü�d—n‘n¨#Õ.Ø�
‰
�4×%Ñ%Õ&ùò^s   È<Lc                 óê  — t        |t        j                  «      rjt        |j                  d¬«       t        |t        j                  «      r8|j
                  �+t        j                  j                  |j
                  d«       y y y t        |t        j                  «      rUt        j                  j                  |j
                  d«       t        j                  j                  |j                  d«       y y )Nr¼   r½   r   g      ð?)	Ú
isinstancer-   r.   r   Úweightr'   ÚinitÚ	constant_r|   )r3   Úms     r6   rÓ   zTNT._init_weightsM  s˜   € Ü�aœŸ™Ô#Ü˜!Ÿ(™(¨Õ,Ü˜!œRŸY™YÔ'¨A¯F©FÐ,>Ü—‘×!Ñ! !§&¡&¨!Õ,ð -?Ð'ä˜œ2Ÿ<™<Ô(Ü�G‰G×Ñ˜aŸf™f aÔ(Ü�G‰G×Ñ˜aŸh™h¨Õ,ð )r7   c                 ó
   — h d£S )N>   rÆ   rÇ   rš   ri   ©r3   s    r6   Úno_weight_decayzTNT.no_weight_decayV  s   € â6Ð6r7   c                 ó$   — t        dddg¬«      }|S )Nz=^cls_token|patch_pos|pixel_pos|pixel_embed|norm[12]_proj|proj)z^blocks\.(\d+)N)z^norm)iŸ† )ÚstemrÌ   )rÍ   )r3   ÚcoarseÚmatchers      r6   Úgroup_matcherzTNT.group_matcherZ  s!   € äØQà)Ø$ðô
ˆð ˆr7   c                 ó   — || _         y r”   )rÂ   )r3   Úenables     r6   Úset_grad_checkpointingzTNT.set_grad_checkpointinge  s
   € à"(ˆÕr7   r’   c                 ó   — | j                   S r”   )rÑ   rÞ   s    r6   Úget_classifierzTNT.get_classifieri  s   € à�y‰yÐr7   c                 óž  — || _         |�|dv sJ ‚|| _        t        | j                  d«      r | j                  j                  j
                  nd }t        | j                  d«      r | j                  j                  j                  nd }|dkD  r)t        j                  | j                  |||¬«      | _        y t        j                  «       | _        y )Nrµ   rÙ   r   r#   )r§   r¨   rÃ   rÑ   rÙ   r$   r%   r-   r.   r©   rr   )r3   r§   r¨   r$   r%   s        r6   Úreset_classifierzTNT.reset_classifierm  s¢   € Ø&ˆÔØÐ"ØÐ"6Ñ6Ð6Ð6Ø*ˆDÔÜ,3°D·I±I¸xÔ,H�—‘×!Ñ!×(Ò(ÈdˆÜ*1°$·)±)¸XÔ*F�—	‘	× Ñ ×&Ò&ÈDˆØZeÐhiÒZi”B—I‘I˜dŸn™n¨kÀ&ÐPUÔVˆ�	Ôoq×ozÑozÓo|ˆ�	r7   rD   ÚindicesÚreturn_prefix_tokensrÏ   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 ó¤  — |dv sJ d«       ‚|dk(  }g }	t        t        | j                  «      |«      \  }
}|j                  \  }}}}| j	                  || j
                  «      }| j                  | j                  | j                  |j                  || j                  d«      «      «      «      }t        j                  | j                  j                  |dd«      |fd¬«      }|| j                  z   }| j!                  |«      }t        j"                  j%                  «       s|s| j                  }n| j                  d|dz    }t'        |«      D ]u  \  }}| j(                  r/t        j"                  j%                  «       st+        |||«      \  }}n |||«      \  }}||
v sŒR|	j-                  |r| j/                  |«      n|«       Œw | j0                  rD|	D �cg c]  }|dd…d| j0                  …f   ‘Œ }}|	D �cg c]  }|dd…| j0                  d…f   ‘Œ }	}|ra| j                  j3                  ||f«      \  }}|	D �cg c]6  }|j                  |||d«      j5                  dd	dd
«      j7                  «       ‘Œ8 }	}t        j"                  j%                  «       s|rt9        t;        |	«      «      }	|r|	S | j/                  |«      }||	fS c c}w c c}w c c}w )a<   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if an int, if is a sequence, select by matching indices
            return_prefix_tokens: Return both prefix and spatial intermediate tokens
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )ÚNCHWÚNLCz)Output format must be one of NCHW or NLC.rò   r;   r   r=   Nr   r9   r&   )r   ÚlenrÌ   r>   ry   rš   ro   r2   rn   r?   rŠ   rw   rx   rÆ   ÚexpandrÇ   rÈ   ÚjitÚis_scriptingÚ	enumeraterÂ   r   rÊ   rÏ   rÁ   r™   r@   Ú
contiguousÚlistÚzip)r3   rD   rì   rí   rÏ   rî   rï   rð   r?   ÚintermediatesÚtake_indicesÚ	max_indexrE   Ú_ÚheightÚwidthry   rz   rÌ   rÖ   ÚblkÚyÚprefix_tokensr�   rž   s                            r6   Úforward_intermediateszTNT.forward_intermediatesv  sœ  € ð. ˜_Ñ,ÐYÐ.YÓYÐ,Ø Ñ&ˆØˆÜ"6´s¸4¿;¹;Ó7GÈÓ"QÑˆ�ið  Ÿg™gÑˆˆ1ˆf�eà×&Ñ& q¨$¯.©.Ó9ˆà—o‘o d§i¡i°·±À×@SÑ@SÐTUÐW[×WgÑWgÐikÓ@lÓ0mÓ&nÓoˆÜ—i‘i §¡×!6Ñ!6°q¸"¸bÓ!AÀ;Ð OÐUVÔWˆØ! D§N¡NÑ2ˆØ—m‘m KÓ0ˆä�9‰9×!Ñ!Ô#©:Ø—[‘[‰Fà—[‘[  )¨a¡-Ð0ˆFä Ó'ò 	V‰FˆAˆsØ×&Ò&¬u¯y©y×/EÑ/EÔ/GÜ+5°c¸;ÈÓ+TÑ(�™[á+.¨{¸KÓ+HÑ(�˜[Ø�LÒ à×$Ñ$¹t T§Y¡Y¨{Ô%;ÈÕUð	Vð ×!Ò!àERÖSÀ˜Qšq ! D×$:Ñ$:Ð":Ð:Ó;ÐSˆMÐSØDQÖR¸q˜Qšq $×"8Ñ"8Ñ"9Ð9Ó:ÐRˆMÐRáà×#Ñ#×5Ñ5°v¸u°oÓF‰DˆAˆqØ^kÖlÐYZ˜QŸY™Y q¨!¨Q°Ó3×;Ñ;¸A¸qÀ!ÀQÓG×RÑRÕTÐlˆMÐlÜ�y‰y×%Ñ%Ô'Ñ,@ä ¤ ]°MÓ!BÓCˆMáØ Ð à—i‘i Ó,ˆà˜MÐ)Ð)ùò! TùÚRùò
 ms   ÇKÇ4KÈ8;KÚ
prune_normÚ
prune_headc                 óÖ   — t        t        | j                  «      |«      \  }}| j                  d|dz    | _        |rt        j                  «       | _        |r| j                  dd«       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r¶   )r   rô   rÌ   r-   rr   rÏ   rë   )r3   rì   r  r  rý   rþ   s         r6   Úprune_intermediate_layerszTNT.prune_intermediate_layers¿  s]   € ô #7´s¸4¿;¹;Ó7GÈÓ"QÑˆ�iØ—k‘k . 9¨q¡=Ð1ˆŒÙÜŸ™›ˆDŒIÙØ×!Ñ! ! RÔ(ØÐr7   c                 ój  — |j                   d   }| j                  || j                  «      }| j                  | j	                  | j                  |j                  || j                  d«      «      «      «      }t        j                  | j                  j                  |dd«      |fd¬«      }|| j                  z   }| j                  |«      }| j                  D ]I  }| j                  r/t        j                   j#                  «       st%        |||«      \  }}Œ> |||«      \  }}ŒK | j'                  |«      }|S )Nr   r;   r   r=   )r>   ry   rš   ro   r2   rn   r?   rŠ   rw   rx   rÆ   rõ   rÇ   rÈ   rÌ   rÂ   rö   r÷   r   rÏ   )r3   rD   rE   ry   rz   r  s         r6   Úforward_featureszTNT.forward_featuresÏ  s	  € Ø�G‰G�A‰JˆØ×&Ñ& q¨$¯.©.Ó9ˆà—o‘o d§i¡i°·±À×@SÑ@SÐTUÐW[×WgÑWgÐikÓ@lÓ0mÓ&nÓoˆÜ—i‘i §¡×!6Ñ!6°q¸"¸bÓ!AÀ;Ð OÐUVÔWˆØ! D§N¡NÑ2ˆØ—m‘m KÓ0ˆà—;‘;ò 	IˆCØ×&Ò&¬u¯y©y×/EÑ/EÔ/GÜ+5°c¸;ÈÓ+TÑ(�™[á+.¨{¸KÓ+HÑ(�™[ð		Ið —i‘i Ó,ˆØÐr7   Ú
pre_logitsc                 óà   — | j                   r=| j                   dk(  r%|d d …| j                  d …f   j                  d¬«      n|d d …df   }| j                  |«      }|r|S | j	                  |«      S )Nr·   r   r=   r   )r¨   rÁ   ÚmeanrÐ   rÑ   )r3   rD   r  s      r6   Úforward_headzTNT.forward_headá  sq   € Ø×ÒØ=A×=MÑ=MÐQVÒ=V�’!�T×+Ñ+Ñ,Ð,Ñ-×2Ñ2°qÐ2Ô9Ð\]Ò^_ÐabÐ^bÑ\cˆAØ�N‰N˜1ÓˆÙˆqÐ0 D§I¡I¨a£LÐ0r7   c                 óJ   — | j                  |«      }| j                  |«      }|S r”   )r  r  )r3   rD   s     r6   rK   zTNT.forwardç  s'   € Ø×!Ñ! !Ó$ˆØ×Ñ˜aÓ ˆØˆr7   ©Fr¢   r”   )NFFFrò   F)r   FT)!rN   rO   rP   rQ   r-   r|   r   rR   r   ÚstrrT   rS   r   r}   r*   rÓ   rw   rö   Úignorerß   rä   rç   ré   r   rë   r£   r   r  r	  r  r  rK   rU   rV   s   @r6   r   r   ï   sÿ  ø„ ñð
 58Ø68ØØ#Ø&Ø ØØØ#$Ø#%Ø!Ø"Ø!Ø#%Ø$&Ø$&Ø$&Ø*,¯,©,Ø !Ø ØØñ/X'à˜C  s¨C x¡Ð0Ñ1ðX'ð ˜c 5¨¨c¨¡?Ð2Ñ3ðX'ð ð	X'ð
 ðX'ð ðX'ð ðX'ð ðX'ð ðX'ð !ðX'ð !ðX'ð ðX'ð ðX'ð ðX'ð !ðX'ð  "ð!X'ð" "ð#X'ð$ "ð%X'ð& ˜RŸY™Y™ð'X'ð( ð)X'ð* õ+X'òt-ð ‡Y�Y×Ññ7ó ð7ð ‡Y�Y×Ñòó ðð ‡Y�Y×Ñò)ó ð)ð ‡Y�Y×Ñð §	¡	ò ó ðñ}¨Cð }¸hÀs¹mó }ð 8<Ø).ØØ$Ø$Ø',ñG*à�|‰|ðG*ð ˜e C¨¨c© NÑ3Ñ4ðG*ð #'ð	G*ð
 ðG*ð ðG*ð ðG*ð !%ðG*ð 
ˆt�E—L‘LÑ! 5¨¯©°t¸E¿L¹LÑ7IÐ)IÑ#JÐJÑ	KóG*ðV ./Ø$Ø#ñ	à˜3  S¡	˜>Ñ*ðð ðð ó	ò ñ$1¨$ó 1ör7   c                 ó:   — | ddd dddt         t        dddd	d
ddœ|¥S )Nr¥   )r9   rŸ   rŸ   gÍÌÌÌÌÌì?ÚbicubicTzpixel_embed.projrÑ   zarXiv:2103.00112zTransformer in TransformerzMhttps://github.com/huawei-noah/Efficient-AI-Backbones/tree/master/tnt_pytorchz
apache-2.0)Úurlr§   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚfixed_input_sizer  r¾   Ú
first_convÚ
classifierÚ	paper_idsÚ
paper_nameÚ
origin_urlÚlicenser	   )r  Úkwargss     r6   Ú_cfgr#  í  s>   € àØ¨=ÀtØ¨ÈÜ'Ô0FØ(¸Ø'Ø2ØeØñð ðð r7   ztimm/)Ú	hf_hub_id)ztnt_s_legacy_patch16_224.in1kztnt_s_patch16_224.in1kztnt_b_patch16_224.in1kc                 óÈ  — | j                  dd «       d| v r| }�nxi }| j                  «       D �]b  \  }}|j                  dd«      }|j                  dd«      }|j                  dd«      }|j                  dd	«      }|j                  d
d«      }|j                  dd«      }|j                  dd«      }|j                  dd«      }|j                  dd«      }|j                  dd«      }|j                  dd«      }|j                  dd«      }|j                  dd«      }|dk(  rh|j                  j                  dk(  rO|j
                  \  }}}t        |dz  «      x}}	||	z  |k(  sJ ‚|j                  ddd «      j                  ||||	«      }|||<   �Œe 	 |d   j
                  |j                  j
                  k7  r(t        |d   |j                  j                  d ¬!«      |d<   |S )"NÚouter_tokensrÇ   Ú	outer_posÚ	inner_posrš   rz   ry   Ú
proj_norm1rn   Ú
proj_norm2ro   Úinner_norm1rj   Ú
inner_attnrk   Úinner_norm2rl   Ú	inner_mlprm   Úouter_norm1rp   Ú
outer_attnrq   Úouter_norm2rs   Ú	outer_mlprt   Fg      à?r   r&   r   )Únew_sizerÁ   )ÚpopÚitemsÚreplacery   rc   r>   rR   r@   r?   rÇ   r   r‰   )
Ú
state_dictÚmodelÚout_dictrI   r0   rE   rF   rG   r�   rž   s
             r6   Úcheckpoint_filter_fnr:    sÙ  € Ø‡N�N�> 4Ô(Ø�jÑ ØŠàˆØ×$Ñ$Ó&ó 	‰DˆAˆqØ—	‘	˜+ {Ó3ˆAØ—	‘	˜+ {Ó3ˆAØ—	‘	˜-¨Ó7ˆAØ—	‘	˜,¨Ó5ˆAØ—	‘	˜,¨Ó5ˆAØ—	‘	˜-¨Ó3ˆAØ—	‘	˜,¨	Ó2ˆAØ—	‘	˜-¨Ó7ˆAØ—	‘	˜+ xÓ0ˆAØ—	‘	˜-¨Ó4ˆAØ—	‘	˜,¨
Ó3ˆAØ—	‘	˜-¨Ó4ˆAØ—	‘	˜+ uÓ-ˆAØ�KÒ E×$5Ñ$5×$<Ñ$<ÀÒ$EØŸ'™'‘��1�aÜ˜A ™H›Ð%��AØ˜1‘u ’zÐ!�zØ—I‘I˜a  AÓ&×.Ñ.¨q°!°Q¸Ó:�ØˆH�Q‹Kð'	ð* UØ�Ñ×"Ñ" e§o¡o×&;Ñ&;Ò;Ü 6Ø�[Ñ!Ø×&Ñ&×0Ñ0Øô!
ˆ�Ñð
 €Or7   c                 ór   — |j                  dd«      }t        t        | |ft        t	        |d¬«      dœ|¤Ž}|S )NÚout_indicesr9   Úgetter)r<  Úfeature_cls)Úpretrained_filter_fnÚfeature_cfg)r4  r   r   r:  rÍ   )ÚvariantÚ
pretrainedr"  r<  r8  s        r6   Ú_create_tntrC  1  sF   € Ø—*‘*˜]¨AÓ.€KÜ ÜˆW�jðä1Ü [¸hÔGñð ñ	€Eð
 €Lr7   r’   c           	      óX   — t        ddddddd¬«      }t        d
d	| it        |fi |¤Ž¤Ž}|S )Nr    é€  é   rY   é   FT)r�   r©   rª   r«   r­   r   rc   rB  )Útnt_s_legacy_patch16_224©rÍ   rC  ©rB  r"  Ú	model_cfgr8  s       r6   rH  rH  ;  sA   € äØ °¸"ÈaØ˜tô%€Iô Ñg¸zÐgÌTÐR[ÑMfÐ_eÑMfÑg€EØ€Lr7   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nr    rE  rF  rY   rG  F©r�   r©   rª   r«   r­   r   rB  )Útnt_s_patch16_224rI  rJ  s       r6   rN  rN  D  s>   € äØ °¸"ÈaØô€Iô Ñ`¸
Ð`ÄdÈ9ÑF_ÐX^ÑF_Ñ`€EØ€Lr7   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nr    i€  é(   rY   é
   FrM  rB  )Útnt_b_patch16_224rI  rJ  s       r6   rR  rR  M  s>   € äØ °¸"ÈbØô€Iô Ñ`¸
Ð`ÄdÈ9ÑF_ÐX^ÑF_Ñ`€EØ€Lr7   )r¶   r  )-rQ   r‹   Útypingr   r   r   r   r   r   rw   Útorch.nnr-   Ú	timm.datar
   r   Útimm.layersr   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_manipulater   Ú	_registryr   r   Ú__all__r}   r   rX   r   r   r#  Údefault_cfgsr:  rC  rH  rN  rR  ri   r7   r6   ú<module>r]     s  ðñ
ó ß :× :ã Ý ç Eß {× {Ñ {Ý *Ý +Ý #ß <àˆ'€ô*�—	‘	ô *ôZ`(ˆB�I‰Iô `(ôF@�—‘ô @ôF{ˆ"�)‰)ô {ó|ñ %Ù%)Øô&ñ #Øôñ #Øôñ&ó €ò "óJð ñ¸Cò ó ðð ñ°Sò ó ðð ñ°Sò ó ñr7   