Ë
    éÓ^jmj  ã                   óÊ  — d Z ddlmZ ddlmZmZmZmZmZm	Z	 ddl
Z
ddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZ ddlmZ dd	lmZ dd
lmZmZ ddl m!Z!m"Z" dgZ# G d„ dejH                  «      Z% G d„ dejH                  «      Z& G d„ dejH                  «      Z' G d„ dejH                  «      Z( G d„ dejH                  «      Z)d1d„Z*d2d„Z+ e! e+d¬«       e+d¬«       e+d¬«       e+d¬«       e+d¬«       e+d¬«       e+ddddd¬«       e+ddddd¬«       e+ddddd¬«       e+ddddd¬«       e+dd¬«       e+dd¬«      d œ«      Z,d!e)fd"„Z-d3d#„Z.e"d3d$e)fd%„«       Z/e"d3d$e)fd&„«       Z0e"d3d$e)fd'„«       Z1e"d3d$e)fd(„«       Z2e"d3d$e)fd)„«       Z3e"d3d$e)fd*„«       Z4e"d3d$e)fd+„«       Z5e"d3d$e)fd,„«       Z6e"d3d$e)fd-„«       Z7e"d3d$e)fd.„«       Z8e"d3d$e)fd/„«       Z9e"d3d$e)fd0„«       Z:y)4a[   FocalNet

As described in `Focal Modulation Networks` - https://arxiv.org/abs/2203.11926

Significant modifications and refactoring from the original impl at https://github.com/microsoft/FocalNet

This impl is/has:
* fully convolutional, NCHW tensor layout throughout, seemed to have minimal performance impact but more flexible
* re-ordered downsample / layer so that striding always at beginning of layer (stage)
* no input size constraints or input resolution/H/W tracking through the model
* torchscript fixed and a number of quirks cleaned up
* feature extraction support via `features_only=True`
é    )Úpartial)ÚCallableÚListÚOptionalÚTupleÚTypeÚUnionN©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚMlpÚDropPathÚLayerNorm2dÚLayerScale2dÚtrunc_normal_ÚClassifierHeadÚNormMlpClassifierHeadÚcalculate_drop_path_ratesé   )Úbuild_model_with_cfg)Úfeature_take_indices)Únamed_applyÚ
checkpoint)Úgenerate_default_cfgsÚregister_modelÚFocalNetc                   óv   ‡ — e Zd Zdddddeddfdededed	ed
ededededeej                     fˆ fd„Z
d„ Zˆ xZS )ÚFocalModulationé   TFç        NÚdimÚfocal_windowÚfocal_levelÚfocal_factorÚbiasÚuse_post_normÚnormalize_modulatorÚ	proj_dropÚ
norm_layerc                 óø  •— |
|dœ}t         ‰| �  «        || _        || _        || _        || _        || _        || _        ||| j                  dz   g| _        t        j                  |d|z  | j                  dz   z   fd|dœ|¤Ž| _        t        j                  ||fd|dœ|¤Ž| _        t        j                  «       | _        t        j                  ||fddi|¤Ž| _        t        j                   |«      | _        t        j$                  «       | _        g | _        t+        | j                  «      D ]—  }| j
                  |z  | j                  z   }| j&                  j-                  t        j.                  t        j                  ||f|||dz  ddœ|¤Žt        j                  «       «      «       | j(                  j-                  |«       Œ™ | j                  r |	|fi |¤Ž| _        y t        j0                  «       | _        y )N©ÚdeviceÚdtyper   r   )Úkernel_sizer%   r.   F)r.   ÚgroupsÚpaddingr%   )ÚsuperÚ__init__r!   r"   r#   r$   r&   r'   Úinput_splitÚnnÚConv2dÚfÚhÚGELUÚactÚprojÚDropoutr(   Ú
ModuleListÚfocal_layersÚkernel_sizesÚrangeÚappendÚ
SequentialÚIdentityÚnorm)Úselfr!   r"   r#   r$   r%   r&   r'   r(   r)   r,   r-   ÚddÚkr.   Ú	__class__s                  €ú_/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/focalnet.pyr2   zFocalModulation.__init__.   sÅ  ø€ ð ¨Ñ/ˆÜ‰ÑÔàˆŒØ(ˆÔØ&ˆÔØ(ˆÔØ*ˆÔØ#6ˆÔ Ø  d×&6Ñ&6¸Ñ&:Ð;ˆÔä—‘˜3  C¡¨4×+;Ñ+;¸aÑ+?Ñ @ÐaÈaÐVZÑaÐ^`ÑaˆŒÜ—‘˜3 ÐD°¸ÑDÀÑDˆŒä—7‘7“9ˆŒÜ—I‘I˜c 3Ñ<°AÐ<¸Ñ<ˆŒ	ÜŸ™ IÓ.ˆŒÜŸM™M›OˆÔàˆÔÜ�t×'Ñ'Ó(ò 	2ˆAØ×+Ñ+¨aÑ/°$×2CÑ2CÑCˆKØ×Ñ×$Ñ$¤R§]¡]Ü—	‘	˜#˜sÐt°ÀCÐQ\Ð`aÑQaÐhmÑtÐqsÑtÜ—‘“	ó&ô ð ×Ñ×$Ñ$ [Õ1ð	2ð .2×-?Ò-?‘J˜sÑ) bÑ)ˆ�	ÄRÇ[Á[Ã]ˆ�	ó    c                 ó,  — | j                  |«      }t        j                  || j                  d«      \  }}}d}t	        | j
                  «      D ]!  \  }} ||«      }|||d d …||dz   …f   z  z   }Œ# | j                  |j                  dd¬«      «      }|||d d …| j                  d …f   z  z   }| j                  r|| j                  dz   z  }|| j                  |«      z  }	| j                  |	«      }	| j                  |	«      }	| j                  |	«      }	|	S )Nr   r   )r   é   T)Úkeepdim)r6   ÚtorchÚsplitr3   Ú	enumerater=   r9   Úmeanr#   r'   r7   rC   r:   r(   )
rD   ÚxÚqÚctxÚgatesÚctx_allÚlÚfocal_layerÚ
ctx_globalÚx_outs
             rH   ÚforwardzFocalModulation.forwardY   s   € à�F‰F�1‹IˆÜŸ™ A t×'7Ñ'7¸Ó;‰ˆˆ3�ð ˆÜ'¨×(9Ñ(9Ó:ò 	8‰NˆAˆ{Ù˜cÓ"ˆCØ  eªA¨q°°Q±¨w¨JÑ&7Ñ 7Ñ7‰Gð	8ð —X‘X˜cŸh™h v°t˜hÓ<Ó=ˆ
Ø˜J¨ªq°$×2BÑ2BÑ2CÐ/CÑ)DÑDÑDˆð ×#Ò#Ø ×!1Ñ!1°AÑ!5Ñ6ˆGð �D—F‘F˜7“OÑ#ˆØ—	‘	˜%Ó ˆð —	‘	˜%Ó ˆØ—‘˜uÓ%ˆØˆrI   )Ú__name__Ú
__module__Ú__qualname__r   ÚintÚboolÚfloatr   r4   ÚModuler2   rZ   Ú__classcell__©rG   s   @rH   r   r   -   s’   ø„ ð !"ØØ"'Ø(-Ø!Ø*5ØØñ)Sàð)Sð ð)Sð ð	)Sð
 ð)Sð ð)Sð  ð)Sð "&ð)Sð ð)Sð ˜RŸY™Y™õ)SöVrI   r   c                   óÄ   ‡ — e Zd ZdZdddddddddej
                  eddfd	ed
ededede	de	de	de
e   dededeej                     deej                     fˆ fd„Zd„ Zˆ xZS )ÚFocalNetBlockz% Focal Modulation Network Block.
    ç      @r   rK   Fç-Cëâ6?r    Nr!   Ú	mlp_ratior#   r"   r&   Úuse_post_norm_in_modulationr'   Úlayerscale_valuer(   Ú	drop_pathÚ	act_layerr)   c           
      ój  •— ||dœ}t         ‰| �  «        || _        || _        || _        || _        || _        |s	 ||fi |¤Žnt        j                  «       | _	        t        |f|| j
                  |||	|dœ|¤Ž| _        |r	 ||fi |¤Žnt        j                  «       | _        |�t        ||fi |¤Žnt        j                  «       | _        |
dkD  rt        |
«      nt        j                  «       | _        |s	 ||fi |¤Žnt        j                  «       | _        t%        d|t'        ||z  «      ||	ddœ|¤Ž| _        |r	 ||fi |¤Žnt        j                  «       | _        |�t        ||fi |¤Žnt        j                  «       | _        |
dkD  rt        |
«      | _        yt        j                  «       | _        y)ap  
        Args:
            dim: Number of input channels.
            mlp_ratio: Ratio of mlp hidden dim to embedding dim.
            focal_level: Number of focal levels.
            focal_window: Focal window size at first focal level.
            use_post_norm: Whether to use layer norm after modulation.
            use_post_norm_in_modulation: Whether to use layer norm in modulation.
            layerscale_value: Initial layerscale value.
            proj_drop: Dropout rate.
            drop_path: Stochastic depth rate.
            act_layer: Activation layer.
            norm_layer: Normalization layer.
        r+   )r"   r#   r&   r'   r(   r)   Nr    T)Úin_featuresÚhidden_featuresrl   ÚdropÚuse_conv© )r1   r2   r!   rh   r"   r#   r&   r4   rB   Únorm1r   Ú
modulationÚ
norm1_postr   Úls1r   Ú
drop_path1Únorm2r   r^   ÚmlpÚ
norm2_postÚls2Ú
drop_path2)rD   r!   rh   r#   r"   r&   ri   r'   rj   r(   rk   rl   r)   r,   r-   rE   rG   s                   €rH   r2   zFocalNetBlock.__init__x   s�  ø€ ð> ¨Ñ/ˆÜ‰ÑÔØˆŒØ"ˆŒà(ˆÔØ&ˆÔØ*ˆÔá2?‘Z Ñ* rÒ*ÄRÇ[Á[Ã]ˆŒ
Ü)Øð	
à%Ø×(Ñ(Ø5Ø 3ØØ!ñ	
ð ñ	
ˆŒñ 4A™* SÑ/¨BÒ/ÄbÇkÁkÃmˆŒØ@PÐ@\”< Ð%5Ñ<¸Ò<Ôbd×bmÑbmÓboˆŒØ1:¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒá2?‘Z Ñ* rÒ*ÄRÇ[Á[Ã]ˆŒ
Üð 
ØÜ  i¡Ó0ØØØñ
ð ñ
ˆŒñ 4A™* SÑ/¨BÒ/ÄbÇkÁkÃmˆŒØ@PÐ@\”< Ð%5Ñ<¸Ò<Ôbd×bmÑbmÓboˆŒØ1:¸R²œ( 9Ó-ˆ�ÄRÇ[Á[Ã]ˆ�rI   c                 óV  — |}| j                  |«      }| j                  |«      }| j                  |«      }|| j                  | j	                  |«      «      z   }|| j                  | j                  | j                  | j                  | j                  |«      «      «      «      «      z   }|S ©N)
rs   rt   ru   rw   rv   r|   r{   rz   ry   rx   )rD   rQ   Úshortcuts      rH   rZ   zFocalNetBlock.forward¼   sŠ   € Øˆð �J‰J�q‹MˆØ�O‰O˜AÓˆØ�O‰O˜AÓˆØ�t—‘ t§x¡x°£{Ó3Ñ3ˆð �—‘ §¡¨¯©¸¿¹À$Ç*Á*ÈQÃ-Ó9PÓ)QÓ RÓSÑSˆàˆrI   )r[   r\   r]   Ú__doc__r4   r8   r   r^   r`   r_   r   r   ra   r2   rZ   rb   rc   s   @rH   re   re   t   sä   ø„ ñð  "Ø Ø !Ø"'Ø05Ø(-Ø04Ø!Ø!Ø)+¯©Ø*5ØØñBSàðBSð ðBSð ð	BSð
 ðBSð  ðBSð *.ðBSð "&ðBSð ' u™oðBSð ðBSð ðBSð ˜BŸI™I‘ðBSð ˜RŸY™Y™õBSöHrI   re   c                    óî   ‡ — e Zd ZdZdddddddddddeddfd	ed
ededededededededededee   dede	ee
e   f   deej                     fˆ fd„Zej                   j"                  dd„«       Zd„ Zˆ xZS )ÚFocalNetStagez4 A basic Focal Transformer layer for one stage.
    rf   Tr   Frg   r    Nr!   Úout_dimÚdepthrh   Ú
downsampler#   r"   Úuse_overlap_downr&   ri   r'   rj   r(   rk   r)   c                 óz  •— ||dœ}t         ‰| �  «        || _        || _        d| _        |rt        d||d||dœ|¤Ž| _        nt        j                  «       | _        t        j                  t        |«      D �cg c].  }t        d|||||	|
|||t        |t        «      r||   n||dœ|¤Ž‘Œ0 c}«      | _        yc c}w )a8  
        Args:
            dim: Number of input channels.
            out_dim: Number of output channels.
            depth: Number of blocks.
            mlp_ratio: Ratio of mlp hidden dim to embedding dim.
            downsample: Downsample layer at start of the layer.
            focal_level: Number of focal levels
            focal_window: Focal window size at first focal level
            use_overlap_down: User overlapped convolution in downsample layer.
            use_post_norm: Whether to use layer norm after modulation.
            use_post_norm_in_modulation: Whether to use layer norm in modulation.
            layerscale_value: Initial layerscale value
            proj_drop: Dropout rate for projections.
            drop_path: Stochastic depth rate.
            norm_layer: Normalization layer.
        r+   Fr   )Úin_chsÚout_chsÚstrideÚoverlapr)   )r!   rh   r#   r"   r&   ri   r'   rj   r(   rk   r)   Nrr   )r1   r2   r!   r„   Úgrad_checkpointingÚ
Downsampler…   r4   rB   r<   r?   re   Ú
isinstanceÚlistÚblocks)rD   r!   rƒ   r„   rh   r…   r#   r"   r†   r&   ri   r'   rj   r(   rk   r)   r,   r-   rE   ÚirG   s                       €rH   r2   zFocalNetStage.__init__Ï   sæ   ø€ ðJ ¨Ñ/ˆÜ‰ÑÔØˆŒØˆŒ
Ø"'ˆÔáÜ(ð ØØØØ(Ø%ñð ñˆD�Oô !Ÿk™k›mˆDŒOô —m‘mô ˜5“\ö%#ð ô ð ØØ#Ø'Ø)Ø+Ø,GØ$7Ø!1Ø#Ü*4°YÄÔ*E˜) Aš,È9Ø%ñð óò%#ó $ˆ�ùò %#s   Á83B8c                 ó   — || _         y r~   )rŒ   )rD   Úenables     rH   Úset_grad_checkpointingz$FocalNetStage.set_grad_checkpointing  s
   € à"(ˆÕrI   c                 óÈ   — | j                  |«      }| j                  D ]A  }| j                  r+t        j                  j                  «       st        ||«      }Œ: ||«      }ŒC |S r~   )r…   r�   rŒ   rM   ÚjitÚis_scriptingr   )rD   rQ   Úblks      rH   rZ   zFocalNetStage.forward  sW   € Ø�O‰O˜AÓˆØ—;‘;ò 	ˆCØ×&Ò&¬u¯y©y×/EÑ/EÔ/GÜ˜s AÓ&‘á˜“F‘ð		ð
 ˆrI   ©T)r[   r\   r]   r€   r   r^   r`   r_   r   r	   r   r   r4   ra   r2   rM   r–   Úignorer”   rZ   rb   rc   s   @rH   r‚   r‚   Ë   s  ø„ ñð  "Ø#Ø Ø !Ø%*Ø"'Ø05Ø(-Ø04Ø!Ø35Ø*5ØØñ%G$àðG$ð ðG$ð ð	G$ð
 ðG$ð ðG$ð ðG$ð ðG$ð #ðG$ð  ðG$ð *.ðG$ð "&ðG$ð ' u™oðG$ð ðG$ð ˜U D¨¡KÐ/Ñ0ðG$ð  ˜RŸY™Y™õ!G$ðR ‡Y�Y×Ñò)ó ð)örI   r‚   c                   óf   ‡ — e Zd Z	 	 	 	 	 ddededededeeej                        f
ˆ fd„Z	d„ Z
ˆ xZS )	r�   rˆ   r‰   rŠ   r‹   r)   c                 ó  •— ||dœ}t         ‰| �  «        || _        d}	|}
|r|dv sJ ‚|dk(  rd\  }
}	n
|dk(  rd\  }
}	t        j                  ||f|
||	dœ|¤Ž| _        |� ||fi |¤Ž| _        y	t        j                  «       | _        y	)
a  

        Args:
            in_chs: Number of input image channels.
            out_chs: Number of linear projection output channels.
            stride: Downsample stride.
            overlap: Use overlapping convolutions if True.
            norm_layer: Normalization layer.
        r+   r   )r   é   r�   )é   r   r   )rK   r   )r.   rŠ   r0   N)r1   r2   rŠ   r4   r5   r:   rB   rC   )rD   rˆ   r‰   rŠ   r‹   r)   r,   r-   rE   r0   r.   rG   s              €rH   r2   zDownsample.__init__(  s¢   ø€ ð& ¨Ñ/ˆÜ‰ÑÔØˆŒØˆØˆÙØ˜VÑ#Ð#Ð#Ø˜Š{Ø'+Ñ$�™WØ˜1’Ø'+Ñ$�˜WÜ—I‘I˜f gÐm¸;ÈvÐ_fÑmÐjlÑmˆŒ	Ø1;Ð1G‘J˜wÑ-¨"Ñ-ˆ�	ÌRÏ[É[Ë]ˆ�	rI   c                 óJ   — | j                  |«      }| j                  |«      }|S r~   )r:   rC   ©rD   rQ   s     rH   rZ   zDownsample.forwardI  s!   € Ø�I‰I�a‹LˆØ�I‰I�a‹LˆØˆrI   )r�   FNNN)r[   r\   r]   r^   r_   r   r   r4   ra   r2   rZ   rb   rc   s   @rH   r�   r�   &  sb   ø„ ð Ø!Ø48ØØñ[àð[ð ð[ð ð	[ð
 ð[ð !  b§i¡i¡Ñ1õ[öBrI   r�   c            (       ó  ‡ — e Zd ZdZdddddddd	d
d
d
d
dddddd eed¬«      ddfdededededeedf   de	deedf   deedf   de
de
de
de
dee   de	d ee	   d!e	d"e	d#e	d$eej                     f&ˆ fd%„Zej"                  j$                  d&„ «       Zej"                  j$                  d:d'„«       Zej"                  j$                  d;d(„«       Zej"                  j$                  d)ej                  fd*„«       Zd<dedee   fd+„Z	 	 	 	 	 d=d,ej0                  d-eeeee   f      d.e
d/e
d0ed1e
d)eeej0                     eej0                  eej0                     f   f   fd2„Z	 	 	 d>d-eeee   f   d3e
d4e
fd5„Zd6„ Zd:d7e
fd8„Zd9„ Zˆ xZ S )?r   z," Focal Modulation Networks (FocalNets)
    rK   éè  Úavgé`   ©r   r   é   r   rf   )r   r   r   r   ©rK   rK   rK   rK   FNç      ð?r    gš™™™™™¹?gñhãˆµøä>)ÚepsÚin_chansÚnum_classesÚglobal_poolÚ	embed_dimÚdepths.rh   Úfocal_levelsÚfocal_windowsr†   r&   ri   r'   Úhead_hidden_sizeÚhead_init_scalerj   Ú	drop_rateÚproj_drop_rateÚdrop_path_rater)   c                 óð  •— t         ‰| �  «        ||dœ}t        |«      | _        t	        | j                  «      D �cg c]
  }|d|z  z  ‘Œ }}|| _        || _        || _        |d   x| _        | _	        g | _
        t        d||d   |	|dœ|¤Ž| _        |d   }t        |t        |«      «      }g }t	        | j                  «      D ]  }||   }t        d||||   ||dkD  ||   ||   |	|
|||||t        |d| «      t        |d|dz    «       |dœ|¤Ž}|}||gz  }| xj                  t!        |d	d|z  z  d
|› �¬«      gz  c_
        Œ� t#        j$                  |Ž | _        |rBt#        j(                  «       | _        || _	        t-        | j                  |f||||dœ|¤Ž| _        n7 || j                  fi |¤Ž| _        t1        | j                  |f||dœ|¤Ž| _        t3        t5        t6        |¬«      | «       yc c}w )aa  
        Args:
            in_chans: Number of input image channels.
            num_classes: Number of classes for classification head.
            embed_dim: Patch embedding dimension.
            depths: Depth of each Focal Transformer layer.
            mlp_ratio: Ratio of mlp hidden dim to embedding dim.
            focal_levels: How many focal levels at all stages. Note that this excludes the finest-grain level.
            focal_windows: The focal window size at all stages.
            use_overlap_down: Whether to use convolutional embedding.
            use_post_norm: Whether to use layernorm after modulation (it helps stabilize training of large models)
            layerscale_value: Value for layer scale.
            drop_rate: Dropout rate.
            drop_path_rate: Stochastic depth rate.
            norm_layer: Normalization layer.
        r+   r   éÿÿÿÿr   )rˆ   r‰   r‹   r)   Nr   )r!   rƒ   r„   rh   r…   r#   r"   r†   r&   ri   r'   rj   r(   rk   r)   r�   úlayers.)Únum_chsÚ	reductionÚmodule)Úhidden_sizeÚ	pool_typer³   r)   )r½   r³   )r²   rr   )r1   r2   ÚlenÚ
num_layersr?   r«   rª   r­   Únum_featuresr±   Úfeature_infor�   Ústemr   Úsumr‚   Údictr4   rA   ÚlayersrB   rC   r   Úheadr   r   r   Ú_init_weights)rD   rª   r«   r¬   r­   r®   rh   r¯   r°   r†   r&   ri   r'   r±   r²   rj   r³   r´   rµ   r)   r,   r-   rE   r‘   Úin_dimÚdprrÅ   Úi_layerrƒ   ÚlayerrG   s                                 €rH   r2   zFocalNet.__init__S  sk  ø€ ôP 	‰ÑÔØ¨Ñ/ˆÜ˜f›+ˆŒÜ38¸¿¹Ó3IÖJ¨a�Y ! q¡&Ó)ÐJˆ	ÐJà&ˆÔØ ˆŒØ"ˆŒØ4=¸b±MÐAˆÔ˜DÔ1ØˆÔäð 
ØØ˜a‘LØ$Ø!ñ	
ð
 ñ
ˆŒ	ð ˜1‘ˆä'¨¼¸F»ÓDˆØˆÜ˜TŸ_™_Ó-ò 	qˆGØ Ñ(ˆGÜ!ð ØØØ˜W‘oØ#Ø" Q™;Ø(¨Ñ1Ø*¨7Ñ3Ø!1Ø+Ø,GØ$7Ø!1Ø(Øœc &¨¨'Ð"2Ó3´C¸¸|ÀÈ!ÁÐ8LÓ4MÐNØ%ñð  ñ!ˆEð$ ˆFØ�u�gÑˆFØ×Ò¤$¨wÀ!ÀaÈ7ÁlÑBRÐ]dÐelÐdmÐ[nÔ"oÐ!pÑpÖð-	qô0 —m‘m VÐ,ˆŒáÜŸ™›ˆDŒIØ$4ˆDÔ!Ü-Ø×!Ñ!Øðð -Ø%Ø#Ø%ñð ñˆD�Iñ # 4×#4Ñ#4Ñ;¸Ñ;ˆDŒIÜ&Ø×!Ñ!Øðð &Ø#ñ	ð
 ñˆDŒIô 	”GœM¸?ÔKÈTÕRùòG Ks   ¼G3c                 ó   — dhS )NÚ rr   ©rD   s    rH   Úno_weight_decayzFocalNet.no_weight_decayÃ  s	   € àˆtˆrI   c                 ó6   — t        d|r	ddg¬«      S g d¢¬«      S )Nz^stem)z^layers\.(\d+)N©z^norm)iŸ† ))z^layers\.(\d+).downsample)r   )z^layers\.(\d+)\.\w+\.(\d+)NrÑ   )rÂ   r�   )rÄ   )rD   Úcoarses     rH   Úgroup_matcherzFocalNet.group_matcherÇ  s3   € äØñ ð *Ø$ðô

ð 
	
ò
ô

ð 
	
rI   c                 óX   — || _         | j                  D ]  }|j                  |¬«       Œ y )N)r“   )rŒ   rÅ   r”   )rD   r“   rV   s      rH   r”   zFocalNet.set_grad_checkpointingÕ  s.   € à"(ˆÔØ—‘ò 	4ˆAØ×$Ñ$¨FÐ$Õ3ñ	4rI   Úreturnc                 ó.   — | j                   j                  S r~   )rÆ   ÚfcrÎ   s    rH   Úget_classifierzFocalNet.get_classifierÛ  s   € à�y‰y�|‰|ÐrI   c                 óL   — || _         | j                  j                  ||¬«       y )N)r½   )r«   rÆ   Úreset)rD   r«   r¬   s      rH   Úreset_classifierzFocalNet.reset_classifierß  s   € Ø&ˆÔØ�	‰	�‰˜¨{ˆÕ;rI   rQ   ÚindicesrC   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 ó  — |dv sJ d«       ‚g }t        t        | j                  «      |«      \  }}	| j                  |«      }t        j
                  j                  «       s|s| j                  }
n| j                  d|	dz    }
t        | j                  «      dz
  }t        |
«      D ]>  \  }} ||«      }||v sŒ|r||k(  r| j                  |«      }n|}|j                  |«       Œ@ |r|S |k(  r| j                  |«      }||fS )aï   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )ÚNCHWzOutput shape must be NCHW.Nr   )
r   r¾   rÅ   rÂ   rM   r–   r—   rO   rC   r@   )rD   rQ   rÜ   rC   rÝ   rÞ   rß   ÚintermediatesÚtake_indicesÚ	max_indexÚstagesÚlast_idxÚfeat_idxÚstageÚx_inters                  rH   Úforward_intermediateszFocalNet.forward_intermediatesã  s  € ð* ˜YÑ&ÐDÐ(DÓDÐ&ØˆÜ"6´s¸4¿;¹;Ó7GÈÓ"QÑˆ�ið �I‰I�a‹LˆÜ�9‰9×!Ñ!Ô#©:Ø—[‘[‰Fà—[‘[  )¨a¡-Ð0ˆFä�t—{‘{Ó# aÑ'ˆÜ(¨Ó0ò 	.‰OˆH�eÙ�a“ˆAØ˜<Ò'Ù˜H¨Ò0Ø"Ÿi™i¨›l‘Gà�GØ×$Ñ$ WÕ-ð	.ñ Ø Ð à�xÒØ—	‘	˜!“ˆAà�-ÐÐrI   Ú
prune_normÚ
prune_headc                 óÖ   — t        t        | j                  «      |«      \  }}| j                  d|dz    | _        |rt        j                  «       | _        |r| j                  dd«       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   rÍ   )r   r¾   rÅ   r4   rB   rC   rÛ   )rD   rÜ   rë   rì   rã   rä   s         rH   Úprune_intermediate_layersz"FocalNet.prune_intermediate_layers  s]   € ô #7´s¸4¿;¹;Ó7GÈÓ"QÑˆ�iØ—k‘k . 9¨q¡=Ð1ˆŒÙÜŸ™›ˆDŒIÙØ×!Ñ! ! RÔ(ØÐrI   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r~   )rÂ   rÅ   rC   r    s     rH   Úforward_featureszFocalNet.forward_features%  s.   € Ø�I‰I�a‹LˆØ�K‰K˜‹NˆØ�I‰I�a‹LˆØˆrI   Ú
pre_logitsc                 óN   — |r| j                  ||¬«      S | j                  |«      S )N)rñ   )rÆ   )rD   rQ   rñ   s      rH   Úforward_headzFocalNet.forward_head+  s%   € Ù6@ˆt�y‰y˜ zˆyÓ2ÐRÀdÇiÁiÐPQÃlÐRrI   c                 óJ   — | j                  |«      }| j                  |«      }|S r~   )rð   ró   r    s     rH   rZ   zFocalNet.forward.  s'   € Ø×!Ñ! !Ó$ˆØ×Ñ˜aÓ ˆØˆrI   ©Fr™   r~   )NFFrá   F)r   FT)!r[   r\   r]   r€   r   r   r^   Ústrr   r`   r_   r   r   r4   ra   r2   rM   r–   rš   rÏ   rÓ   r”   rØ   rÛ   ÚTensorr	   r   rê   rî   rð   ró   rZ   rb   rc   s   @rH   r   r   O  sè  ø„ ñð
 Ø#Ø$ØØ&2Ø!Ø,8Ø-9Ø%*Ø"'Ø05Ø(-Ø.2Ø%(Ø04Ø!Ø$&Ø$'Ù*1°+À4Ô*HØØñ-nSàðnSð ðnSð ð	nSð
 ðnSð ˜#˜s˜(‘OðnSð ðnSð    S ™/ðnSð !  c ™?ðnSð #ðnSð  ðnSð *.ðnSð "&ðnSð ' s™mðnSð #ðnSð  ' u™oð!nSð" ð#nSð$ "ð%nSð& "ð'nSð( ˜RŸY™Y™õ)nSð` ‡Y�Y×Ññó ðð ‡Y�Y×Ñò
ó ð
ð ‡Y�Y×Ñò4ó ð4ð
 ‡Y�Y×Ñð §	¡	ò ó ðñ<¨Cð <¸hÀs¹mó <ð 8<ØØ$Ø$Ø',ñ0 à�|‰|ð0 ð ˜e C¨¨c© NÑ3Ñ4ð0 ð ð	0 ð
 ð0 ð ð0 ð !%ð0 ð 
ˆt�E—L‘LÑ! 5¨¯©°t¸E¿L¹LÑ7IÐ)IÑ#JÐJÑ	Kó0 ðh ./Ø$Ø#ñ	à˜3  S¡	˜>Ñ*ðð ðð ó	ò ñS¨$ó SörI   r¨   c                 óF  — t        | t        j                  «      rNt        | j                  d¬«       | j
                  �*t        j                  j                  | j
                  «       y y t        | t        j                  «      rŸt        | j                  d¬«       | j
                  �)t        j                  j                  | j
                  «       |rPd|v rK| j                  j                  j                  |«       | j
                  j                  j                  |«       y y y y )Ng{®Gáz”?)Ústdúhead.fc)rŽ   r4   r5   r   Úweightr%   ÚinitÚzeros_ÚLinearÚdataÚmul_)r»   Únamer²   s      rH   rÇ   rÇ   4  sÁ   € Ü�&œ"Ÿ)™)Ô$Ü�f—m‘m¨Õ-Ø�;‰;Ð"Ü�G‰G�N‰N˜6Ÿ;™;Õ'ð #ä	�FœBŸI™IÔ	&Ü�f—m‘m¨Õ-Ø�;‰;Ð"Ü�G‰G�N‰N˜6Ÿ;™;Ô'Ù�I Ñ%Ø�M‰M×Ñ×#Ñ# OÔ4Ø�K‰K×Ñ×!Ñ! /Õ2ð &ˆ4ð	 
'rI   c                 ó2   — | dddddt         t        dddd	œ|¥S )
Nr¢   )rK   éà   r  )rž   rž   gÍÌÌÌÌÌì?Úbicubicz	stem.projrú   Úmit)Úurlr«   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationrP   rù   Ú
first_convÚ
classifierÚlicenser
   )r  Úkwargss     rH   Ú_cfgr  B  s3   € àØ¨=ÀvØ¨Ü%Ô.BØ!°Øñð #ðð rI   ztimm/)Ú	hf_hub_id)rK   é€  r  )é   r  iRU  )r  r  r  r	  r«   )r  r«   )zfocalnet_tiny_srf.ms_in1kzfocalnet_small_srf.ms_in1kzfocalnet_base_srf.ms_in1kzfocalnet_tiny_lrf.ms_in1kzfocalnet_small_lrf.ms_in1kzfocalnet_base_lrf.ms_in1kzfocalnet_large_fl3.ms_in22kzfocalnet_large_fl4.ms_in22kzfocalnet_xlarge_fl3.ms_in22kzfocalnet_xlarge_fl4.ms_in22kzfocalnet_huge_fl3.ms_in22kzfocalnet_huge_fl4.ms_in22kÚmodelc                 óP  — | j                  d| «      } d| v r| S dd l}i }|j                  «       }| j                  «       D ]ä  \  }}|j	                  dd|«      }|j                  dd«      }|j	                  dd	„ |«      }d
|v r||vr|j	                  dd|«      }|j                  dd«      }|j                  dd«      }||v r^||   j                  «       |j                  «       k(  r:||   j                  |j                  k7  r|j                  ||   j                  «      }|||<   Œæ |S )Nr  zstem.proj.weightr   zgamma_([0-9])z
ls\1.gammaÚpatch_embedrÂ   zlayers.(\d+).downsamplec                 óD   — dt        | j                  d«      «      dz   › d�S )Nr¸   r   z.downsample)r^   Úgroup)rQ   s    rH   ú<lambda>z&checkpoint_filter_fn.<locals>.<lambda>z  s#   € ¸7Ä3ÀqÇwÁwÈqÃzÃ?ÐUVÑCVÐBWÐWbÐ9c€ rI   rC   znorm([0-9])znorm\1_postzln.znorm.rÆ   rú   )	ÚgetÚreÚ
state_dictÚitemsÚsubÚreplaceÚnumelÚshapeÚreshape)r  r  r  Úout_dictÚ	dest_dictrF   Úvs          rH   Úcheckpoint_filter_fnr%  p  s+  € Ø—‘ ¨Ó4€JØ˜ZÑ'ØÐÛØ€HØ× Ñ Ó"€IØ× Ñ Ó"ò 
‰ˆˆ1Ø�F‰FÐ# ]°AÓ6ˆØ�I‰I�m VÓ,ˆØ�F‰FÐ-Ñ/cÐefÓgˆØ�Q‰;˜1 IÑ-Ø—‘�~ ~°qÓ9ˆAØ�I‰I�e˜WÓ%ˆØ�I‰I�f˜iÓ(ˆØ�	‰>˜i¨™l×0Ñ0Ó2°a·g±g³iÒ?ÀIÈaÁL×DVÑDVÐZ[×ZaÑZaÒDaØ—	‘	˜) A™,×,Ñ,Ó-ˆAØˆ�Šð
ð €OrI   c           	      óÈ   — t        d„ t        |j                  dd«      «      D «       «      }|j                  d|«      }t	        t
        | |ft        t        d|¬«      dœ|¤Ž}|S )Nc              3   ó&   K  — | ]	  \  }}|–— Œ y ­wr~   rr   )Ú.0r‘   Ú_s      rH   ú	<genexpr>z#_create_focalnet.<locals>.<genexpr>†  s   è ø€ Ò\¡d a¨¤Ñ\ùs   ‚r®   )r   r   rK   r   Úout_indicesT)Úflatten_sequentialr+  )Úpretrained_filter_fnÚfeature_cfg)ÚtuplerO   r  Úpopr   r   r%  rÄ   )ÚvariantÚ
pretrainedr  Údefault_out_indicesr+  r  s         rH   Ú_create_focalnetr4  …  sh   € ÜÑ\¬i¸¿
¹
À8È\Ó8ZÓ.[Ô\Ó\ÐØ—*‘*˜]Ð,?Ó@€Kä Ü�'˜:ðä1Ü¨D¸kÔJñð ñ	€Eð
 €LrI   rÕ   c                 ó<   — t        dg d¢ddœ|¤Ž}t        dd| i|¤ŽS )Nr¥   r¤   ©r®   r­   r2  rr   )Úfocalnet_tiny_srf©rÄ   r4  ©r2  r  Úmodel_kwargss      rH   r7  r7  ‘  s)   € äÐDš|°rÑD¸VÑD€LÜÑW¸JÐWÈ,ÑWÐWrI   c                 ó<   — t        dg d¢ddœ|¤Ž}t        dd| i|¤ŽS )N©r   r   é   r   r¤   r6  r2  rr   )Úfocalnet_small_srfr8  r9  s      rH   r>  r>  —  s)   € äÐEš}¸ÑE¸fÑE€LÜÑX¸ZÐXÈ<ÑXÐXrI   c                 ó<   — t        dg d¢ddœ|¤Ž}t        dd| i|¤ŽS )Nr<  é€   r6  r2  rr   )Úfocalnet_base_srfr8  r9  s      rH   rA  rA  �  s)   € äÐFš}¸ÑF¸vÑF€LÜÑW¸JÐWÈ,ÑWÐWrI   c                 óB   — t        dg d¢dg d¢dœ|¤Ž}t        dd| i|¤ŽS )Nr¥   r¤   r§   ©r®   r­   r¯   r2  rr   )Úfocalnet_tiny_lrfr8  r9  s      rH   rD  rD  £  s,   € äÐ_š|°rÊÑ_ÐX^Ñ_€LÜÑW¸JÐWÈ,ÑWÐWrI   c                 óB   — t        dg d¢dg d¢dœ|¤Ž}t        dd| i|¤ŽS )Nr<  r¤   r§   rC  r2  rr   )Úfocalnet_small_lrfr8  r9  s      rH   rF  rF  ©  s,   € äÐ`š}¸ÊÑ`ÐY_Ñ`€LÜÑX¸ZÐXÈ<ÑXÐXrI   c                 óB   — t        dg d¢dg d¢dœ|¤Ž}t        dd| i|¤ŽS )Nr<  r@  r§   rC  r2  rr   )Úfocalnet_base_lrfr8  r9  s      rH   rH  rH  ¯  s,   € äÐaš}¸Ê,ÑaÐZ`Ña€LÜÑW¸JÐWÈ,ÑWÐWrI   c                 óR   — t        d
g d¢dg d¢dgdz  ddddœ|¤Ž}t        dd	| i|¤ŽS )Nr<  éÀ   r§   é   r�   Trg   ©r®   r­   r¯   r°   r&   r†   rj   r2  rr   )Úfocalnet_large_fl3r8  r9  s      rH   rM  rM  ¶  sK   € äð TÚ¨º,ÐWXÐVYÐ\]ÑV]Ø¨TÀDñTàLRñT€Lô ÑX¸ZÐXÈ<ÑXÐXrI   c           
      óH   — t        dg d¢dg d¢ddddœ|¤Ž}t        d	d| i|¤ŽS )
Nr<  rJ  ©r�   r�   r�   r�   Trg   ©r®   r­   r¯   r&   r†   rj   r2  rr   )Úfocalnet_large_fl4r8  r9  s      rH   rQ  rQ  ¾  s?   € äð TÚ¨º,Ø¨TÀDñTàLRñT€Lô ÑX¸ZÐXÈ<ÑXÐXrI   c                 óR   — t        d
g d¢dg d¢dgdz  ddddœ|¤Ž}t        dd	| i|¤ŽS )Nr<  é   r§   rK  r�   Trg   rL  r2  rr   )Úfocalnet_xlarge_fl3r8  r9  s      rH   rT  rT  Æ  sK   € äð TÚ¨º,ÐWXÐVYÐ\]ÑV]Ø¨TÀDñTàLRñT€Lô ÑY¸jÐYÈLÑYÐYrI   c           
      óH   — t        dg d¢dg d¢ddddœ|¤Ž}t        d	d| i|¤ŽS )
Nr<  rS  rO  Trg   rP  r2  rr   )Úfocalnet_xlarge_fl4r8  r9  s      rH   rV  rV  Î  s?   € äð TÚ¨º,Ø¨TÀDñTàLRñT€Lô ÑY¸jÐYÈLÑYÐYrI   c                 óT   — t        d
g d¢dg d¢dgdz  dddddœ|¤Ž}t        dd	| i|¤ŽS )Nr<  é`  r§   rK   r�   Trg   )r®   r­   r¯   r°   r&   ri   r†   rj   r2  rr   )Úfocalnet_huge_fl3r8  r9  s      rH   rY  rY  Ö  sN   € äð vÚ¨º,ÐWXÐVYÐ\]ÑV]Ø¸ÈtÐfjñvàntñv€Lô ÑW¸JÐWÈ,ÑWÐWrI   c                 óJ   — t        dg d¢dg d¢dddddœ|¤Ž}t        d	d| i|¤ŽS )
Nr<  rX  rO  Trg   )r®   r­   r¯   r&   ri   r†   rj   r2  rr   )Úfocalnet_huge_fl4r8  r9  s      rH   r[  r[  Þ  sB   € äð vÚ¨º,Ø¸ÈtÐfjñvàntñv€Lô ÑW¸JÐWÈ,ÑWÐWrI   )Nr¨   )rÍ   rõ   );r€   Ú	functoolsr   Útypingr   r   r   r   r   r	   rM   Útorch.nnr4   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_manipulater   r   Ú	_registryr   r   Ú__all__ra   r   re   r‚   r�   r   rÇ   r  Údefault_cfgsr%  r4  r7  r>  rA  rD  rF  rH  rM  rQ  rT  rV  rY  r[  rr   rI   rH   ú<module>rg     sö  ðñõ& ß ?× ?ã Ý ç A÷	÷ 	ó 	õ +Ý +ß 0ß <àˆ,€ôD�b—i‘iô DôNT�B—I‘Iô TônX�B—I‘Iô Xôv&�—‘ô &ôRbˆr�y‰yô bóJ3óñ %Ù!%Øô"á"&Øô#á!%Øô"á!%Øô"á"&Øô#á!%Øô"ñ $(ØØ ¨H¸sÐPUô$Wñ $(ØØ ¨H¸sÐPUô$Wñ %)ØØ ¨H¸sÐPUô%Wñ %)ØØ ¨H¸sÐPUô%Wñ #'ØØô#ñ #'ØØô#ñ; &ó  €ðF¨Hó ó*	ð ñX°Xò Xó ðXð
 ñY°hò Yó ðYð
 ñX°Xò Xó ðXð
 ñX°Xò Xó ðXð
 ñY°hò Yó ðYð
 ñX°Xò Xó ðXð ñY°hò Yó ðYð ñY°hò Yó ðYð ñZ°xò Zó ðZð ñZ°xò Zó ðZð ñX°Xò Xó ðXð ñX°Xò Xó ñXrI   