Ë
    éÓ^j€–  ã            
       ó  — d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmc mZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZmZmZ ddl m!Z!m"Z" dd	l#m$Z$ dd
l%m&Z& ddl'm(Z( ddl)m*Z*m+Z+ dgZ,de-de	ej\                     fd„Z/e(dee-   dej`                  dej`                  fd„«       Z1dej`                  dee-   dee-   dej`                  fd„Z2 G d„ dej\                  «      Z3 G d„ dej\                  «      Z4 G d„ dej\                  «      Z5 G d„ dej\                  «      Z6 G d „ d!ej\                  «      Z7 G d"„ dej\                  «      Z8dKd#„Z9 e!i d$ e9d%d&¬'«      “d( e9d%d&d¬)«      “d* e9d%d&¬'«      “d+ e9d%d&d¬)«      “d, e9d%d&¬'«      “d- e9d%d&d¬)«      “d. e9d%d&¬'«      “d/ e9d%d&d¬)«      “d0 e9d%d&¬'«      “d1 e9d%d&d¬)«      “d2 e9d%d&¬'«      “d3 e9d%d&d¬)«      “d4 e9d%d5d6¬7«      “d8 e9d%d5d6¬7«      “d9 e9d%d:d5d6¬;«      “d< e9d%d:d5d6¬;«      “d= e9d5d6¬>«      “«      Z:dLd?„Z;dMd@e<dAe=de8fdB„Z>e"dMdC„«       Z?e"dMdD„«       Z@e"dMdE„«       ZAe"dMdF„«       ZBe"dMdG„«       ZCe"dMdH„«       ZDe"dMdI„«       ZEe"dMdJ„«       ZFy)Nzr An PyTorch implementation of Hiera

Adapted for timm from originals at https://github.com/facebookresearch/hiera
é    N)Úpartial)ÚDictÚListÚOptionalÚTupleÚTypeÚUnion©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚDropPathÚcalculate_drop_path_ratesÚMlpÚ
LayerScaleÚClNormMlpClassifierHeadÚuse_fused_attnÚ_assertÚget_norm_layerÚ	to_2tupleÚinit_weight_vitÚinit_weight_jaxé   )Úgenerate_default_cfgsÚregister_model)Úbuild_model_with_cfg)Úfeature_take_indices)Úregister_notrace_function)Únamed_applyÚ
checkpointÚHieraÚnÚreturnc                 ó„   — t         j                  t         j                  t         j                  t         j                  g|    S )z¤
    Returns a conv with nd (e.g., Conv2d for n=2). Work up to n=3.
    If you wanted a 4d Hiera, you could probably just implement this for n=4. (no promises)
    )ÚnnÚIdentityÚConv1dÚConv2dÚConv3d)r!   s    ú\/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/hiera.pyÚconv_ndr*   :   s(   € ô
 �K‰KœŸ™¤B§I¡I¬r¯y©yÐ9¸!Ñ<Ð<ó    Útarget_sizeÚmaskc                 óØ   — |€|S t        t        |j                  dd  «      t        | «      k(  d«       |j                  dd  | k7  r%t        j                  |j                  «       | ¬«      S |S )Né   z.mask spatial shape and target_size must match.)Úsize)r   ÚlenÚshapeÚFÚinterpolateÚfloat©r,   r-   s     r)   Úget_resized_maskr7   B   s`   € ð €|ØˆäŒC�—
‘
˜1˜2�Ó¤3 {Ó#3Ñ3Ð5eÔfØ‡z�z�!�"€~˜Ò$Ü�}‰}˜TŸZ™Z›\°Ô<Ð<Ø€Kr+   Úxr2   Úmu_shapec                 óü  — t        |«      }| j                  d   | j                  d   }}t        ||«      D ��cg c]
  \  }}||z  ‘Œ }}} | j                  |g|¢|¢|‘­Ž } dgt	        t        t        dd|z   «      t        d|z   dd|z  z   «      «      D �	cg c]  }	t        |	«      ‘Œ c}	g «      z   t        | j                  «      dz
  gz   }
 | j                  |
«      j                  |g|¢|‘­Ž } | S c c}}w c c}	w )a¾  
    Restore spatial organization by undoing windowed organization of mask units.

    Args:
        x: organized by mask units windows, e.g. in 2d [B, #MUy*#MUx, MUy, MUx, C]
        shape: current spatial shape, if it were not organized into mask unit
            windows, e.g. in 2d [B, #MUy*MUy, #MUx*MUx, C].
        mu_shape: current mask unit shape, e.g. in 2d [MUy, MUx]
    Returns:
        x: e.g. in 2d, [B, #MUy*MUy, #MUx*MUx, C]
    r   éÿÿÿÿr   r/   )	r1   r2   ÚzipÚviewÚsumÚrangeÚlistÚpermuteÚreshape)r8   r2   r9   ÚDÚBÚCÚsÚmuÚnum_MUsÚprA   s              r)   Úundo_windowingrJ   O   s  € ô  	ˆE‹
€AØ�7‰7�1‰:�q—w‘w˜r‘{€q€Aä$'¨¨xÓ$8×9™5˜1˜bˆq�B‹wÐ9€GÑ9Øˆ�‰ˆqÐ)�7Ð)˜XÐ) qÒ)€Að 
ˆÜ
¤¤E¨!¨Q°©U£O´U¸1¸q¹5À!ÀaÈ!ÁeÁ)Ó5LÓ MÖN˜1Œt�A�wÒNÐPRÓ
Sñ	Täˆq�w‰w‹<˜!ÑÐ
ñ	ð ð
 	#ˆ�	‰	�'Ó×"Ñ" 1Ð0 uÐ0¨aÒ0€Aà€Hùó :ùò Os   ¹C3ÂC9
c            	       óŒ   ‡ — e Zd ZdZdeedf   deedf   deeedf      fˆ fd„Zdej                  dej                  fd	„Z
ˆ xZS )
ÚUnrolla>  
    Reorders the tokens such that patches are contiguous in memory.
    E.g., given [B, (H, W), C] and stride of (Sy, Sx), this will re-order the tokens as
                           [B, (Sy, Sx, H // Sy, W // Sx), C]

    This allows operations like Max2d to be computed as x.view(B, Sx*Sy, -1, C).max(dim=1).
    Not only is this faster, but it also makes it easy to support inputs of arbitrary
    dimensions in addition to patch-wise sparsity.

    Performing this operation multiple times in sequence puts entire windows as contiguous
    in memory. For instance, if you applied the stride (2, 2) 3 times, entire windows of
    size 8x8 would be contiguous in memory, allowing operations like mask unit attention
    computed easily and efficiently, while also allowing max to be applied sequentially.

    Note: This means that intermediate values of the model are not in HxW order, so they
    need to be re-rolled if you want to use the intermediate values as a HxW feature map.
    The last block of the network is fine though, since by then the strides are all consumed.
    Ú
input_size.Úpatch_strideÚunroll_schedulec                 óŠ   •— t         ‰| �  «        t        ||«      D ��cg c]
  \  }}||z  ‘Œ c}}| _        || _        y c c}}w ©N)ÚsuperÚ__init__r<   r0   Úschedule)ÚselfrM   rN   rO   ÚirF   Ú	__class__s         €r)   rS   zUnroll.__init__„   s>   ø€ ô 	‰ÑÔÜ(+¨J¸Ó(E×F¡  1�Q˜!“VÓFˆŒ	Ø'ˆ�ùó Gs   Ÿ?r8   r"   c           
      óê  — |j                   \  }}}| j                  } |j                  |g|z   |gz   Ž }| j                  D ]ô  }t	        ||«      D ��cg c]
  \  }}||z  ‘Œ }}}|gt        t	        ||«      D ��cg c]	  \  }}||g‘Œ c}}g «      z   |gz   }	|j                  |	«      }t        |	«      }
dgt        t        d|
dz
  d«      «      z   t        t        d|
dz
  d«      «      z   |
dz
  gz   }|j                  |«      }|j                  dt        |«      «      }|t        j                  |«      z  }Œö |j                  dt        j                  | j                  «      |«      }|S c c}}w c c}}w )zª
        Input: Flattened patch embeddings [B, N, C]
        Output: Patch embeddings [B, N, C] permuted such that [B, 4, N//4, C].max(1) etc. performs MaxPoolNd
        r   r/   r   r;   )r2   r0   r=   rT   r<   r>   r1   r@   r?   rA   ÚflattenÚmathÚprodrB   )rU   r8   rD   Ú_rE   Úcur_sizeÚstridesrV   rF   Ú	new_shapeÚLrA   s               r)   ÚforwardzUnroll.forwardŽ   si  € ð
 —'‘'‰ˆˆ1ˆaØ—9‘9ˆØˆA�F‰F�a�S˜8‘^ q cÑ)Ð+ˆà—}‘}ò 	$ˆGô
 ,/¨x¸Ó+A×B¡4 1 a˜˜Q›ÐBˆHÑBØ˜œc´c¸(ÀGÓ6L×"M©d¨a° A q¢6Ó"MÈrÓRÑRÐVWÐUXÑXˆIØ—‘�yÓ!ˆAô �I“ˆAØ�cœD¤ q¨!¨a©%°Ó!3Ó4Ñ4´t¼EÀ!ÀQÈÁUÈAÓ<NÓ7OÑOÐSTÐWXÑSXÐRYÑYˆGØ—	‘	˜'Ó"ˆAð —	‘	˜!œS ›\Ó*ˆAØ”—‘˜7Ó#Ñ#‰Að#	$ð& �I‰I�bœ$Ÿ)™) D§I¡IÓ.°Ó2ˆØˆùó CùÛ"Ms   ÁE)Á:E/©Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úintr   rS   ÚtorchÚTensorra   Ú__classcell__©rW   s   @r)   rL   rL   p   sa   ø„ ñð&(à˜c 3˜h™ð(ð    S ™/ð(ð " %¨¨S¨¡/Ñ2õ	(ð˜Ÿ™ð ¨%¯,©,÷ r+   rL   c            
       óº   ‡ — e Zd ZdZdeedf   deedf   deeedf      dee   def
ˆ fd„Z	 dd	ej                  d
edej                  dej                  fd„Z
ˆ xZS )ÚRerollzQ
    Undos the "unroll" operation so that you can use intermediate features.
    rM   .rN   rO   Ú
stage_endsÚq_poolc                 óz  •— t         ‰
| �  «        t        ||«      D ��cg c]
  \  }}||z  ‘Œ c}}| _        i | _        | j                  }t        |d   dz   «      D ]R  }||f| j                  |<   ||d | v sŒt        |«      dkD  r$t        ||d   «      D �	�cg c]
  \  }	}|	|z  ‘Œ }}	}|dd  }ŒT y c c}}w c c}}	w )Nr;   r   r   )rR   rS   r<   r0   rT   r?   r1   )rU   rM   rN   rO   rn   ro   rV   rF   r0   r!   rW   s             €r)   rS   zReroll.__init__³   sÎ   ø€ ô 	‰ÑÔÜ(+¨J¸Ó(E×F¡  1�Q˜!“VÓFˆŒ	ð ˆŒØ�y‰yˆÜ�z "‘~¨Ñ)Ó*ò 	6ˆAØ.°Ð4ˆD�M‰M˜!Ñà�J˜w Ð'Ò'Ü�Ó'¨!Ò+Ü/2°4¸ÈÑ9KÓ/L×M¡t q¨!˜A ›FÐM�DÑMØ"1°!°"Ð"5‘ñ	6ùó Gùó Ns   ŸB1ÂB7r8   Ú	block_idxr-   r"   c                 ó¾  — | j                   |   \  }}|j                  \  }}}t        |«      }	dg|	z  }
|D ]ø  } |j                  |g|¢|t	        j
                  |«      z  ‘|
¢|‘­Ž }t        |j                  «      }dd|	z   gt        t        t        dd|	z   «      t        d|	z   dz   |dz
  «      «      D �cg c]  }t        |«      ‘Œ c}g «      z   |dz
  gz   }|j                  |«      }t        |	«      D ]  }|
|xx   ||   z  cc<   Œ  |j                  |dg|
¢|‘­Ž }|j                  d   }Œú  |j                  ||g|
¢|‘­Ž }|�|S t        |||
«      }|S c c}w )a&  
        Roll the given tensor back up to spatial order assuming it's from the given block.

        If no mask is provided:
            - Returns [B, H, W, C] for 2d, [B, T, H, W, C] for 3d, etc.
        If a mask is provided:
            - Returns [B, #MUs, MUy, MUx, C] for 2d, etc.
        r   r   r;   )rT   r2   r1   r=   rZ   r[   r>   r<   r?   r@   rA   rB   rJ   )rU   r8   rq   r-   rT   r0   rD   ÚNrE   rC   Úcur_mu_shaper^   r`   rI   rA   rV   s                   r)   ra   zReroll.forwardÊ   s’  € ð Ÿ™ yÑ1‰ˆ�$Ø—'‘'‰ˆˆ1ˆaä�‹IˆØ�s˜Q‘wˆàò 	ˆGà�—‘�qÐN˜7ÐN A¬¯©°7Ó);Ñ$;ÐN¸lÐNÈAÒNˆAô �A—G‘G“ˆAà�A˜‘E�
Ü¬¬E°!°Q¸±U«O¼UÀ1ÀqÁ5È1Á9ÈaÐRSÉeÓ=TÓ(UÖV 1”t˜A•wÒVÐXZÓ[ñ\à�q‘5�'ñð ð
 —	‘	˜'Ó"ˆAô ˜1“Xò .�Ø˜Q“ 7¨1¡:Ñ-”ð.à�—	‘	˜!˜RÐ2 ,Ð2°Ò2ˆAØ—‘˜‘
‰Að%	ð* ˆA�F‰F�1�aÐ*˜,Ð*¨Ò*ˆð ÐØˆHô ˜1˜d LÓ1ˆàˆùò+ Ws   Â6ErQ   rb   rk   s   @r)   rm   rm   ®   sš   ø„ ñð6à˜c 3˜h™ð6ð    S ™/ð6ð " %¨¨S¨¡/Ñ2ð	6ð
 ˜S™	ð6ð õ6ð6 "&ñ	2à�|‰|ð2ð ð2ð —,‘,ð	2ð
 
�‰÷2r+   rm   c                   óº   ‡ — e Zd ZU dZej
                  j                  e   ed<   	 	 	 	 	 dde	de	de	de	de	defˆ fd	„Z
d
ej                  dej                  fd„Zˆ xZS )ÚMaskUnitAttentionzÛ
    Computes either Mask Unit or Global Attention. Also is able to perform q pooling.

    Note: this assumes the tokens have already been flattened and unrolled into mask units.
    See `Unroll` for more details.
    Ú
fused_attnÚdimÚdim_outÚheadsÚq_strideÚwindow_sizeÚuse_mask_unit_attnc	                 óP  •— ||dœ}	t         ‰
| �  «        || _        || _        || _        || _        ||z  | _        | j                  dz  | _        t        «       | _	        t        j                  |d|z  fi |	¤Ž| _        t        j                  ||fi |	¤Ž| _        || _        || _        y)a–  
        Args:
        - dim, dim_out: The input and output feature dimensions.
        - heads: The number of attention heads.
        - q_stride: If greater than 1, pool q with this stride. The stride should be flattened (e.g., 2x2 = 4).
        - window_size: The current (flattened) size of a mask unit *after* pooling (if any).
        - use_mask_unit_attn: Use Mask Unit or Global Attention.
        ©ÚdeviceÚdtypeg      à¿é   N)rR   rS   rx   ry   rz   r{   Úhead_dimÚscaler   rw   r$   ÚLinearÚqkvÚprojr|   r}   )rU   rx   ry   rz   r{   r|   r}   r€   r�   ÚddrW   s             €r)   rS   zMaskUnitAttention.__init__  sž   ø€ ð& ¨Ñ/ˆÜ‰ÑÔØˆŒØˆŒØˆŒ
Ø ˆŒØ 5Ñ(ˆŒØ—]‘] dÑ*ˆŒ
Ü(Ó*ˆŒä—9‘9˜S ! g¡+Ñ4°Ñ4ˆŒÜ—I‘I˜g wÑ5°"Ñ5ˆŒ	à&ˆÔØ"4ˆÕr+   r8   r"   c                 óR  — |j                   \  }}}| j                  rÓ|| j                  | j                  z  z  }| j	                  |«      j                  |d|d| j                  | j                  «      j                  dddddd«      }|j                  d«      \  }}}	| j                  dkD  �r'|j                  || j                  || j                  d| j                  «      j                  d¬«      }nâ| j	                  |«      j                  ||d| j                  | j                  «      j                  ddddd«      }|j                  d«      \  }}}	| j                  dkD  rC|j                  || j                  | j                  d| j                  «      j                  d¬«      }|j                  «       |j                  «       |	j                  «       }	}}| j                  rt        j                  |||	«      }n9|| j                   z  |j#                  dd	«      z  }
|
j%                  d¬«      }
|
|	z  }| j                  r.|j#                  dd«      j                  |d| j&                  «      }n-|j#                  dd«      j                  |d| j&                  «      }| j)                  |«      }|S )
z5 Input should be of shape [batch, tokens, channels]. r;   r‚   r   é   r/   r   é   ©rx   éþÿÿÿ)r2   r}   r{   r|   r†   rB   rz   rƒ   rA   Úunbindr=   ÚamaxÚ
contiguousrw   r3   Úscaled_dot_product_attentionr„   Ú	transposeÚsoftmaxry   r‡   )rU   r8   rD   rs   r\   Únum_windowsr†   ÚqÚkÚvÚattns              r)   ra   zMaskUnitAttention.forward+  s2  € à—'‘'‰ˆˆ1ˆaà×"Ò"à §¡°×0@Ñ0@Ñ @ÑAˆKØ—(‘(˜1“+×%Ñ%Ø�2�{ A t§z¡z°4·=±=óç‰g�a˜˜A˜q ! QÓ'ð ð —j‘j “m‰GˆAˆq�!à�}‰}˜qÓ à—F‘F˜1˜dŸj™j¨+°t·}±}ÀbÈ$Ï-É-ÓX×]Ñ]ÐbcÐ]Ód‘ð —(‘(˜1“+×%Ñ% a¨¨A¨t¯z©z¸4¿=¹=ÓI×QÑQÐRSÐUVÐXYÐ[\Ð^_Ó`ˆCØ—j‘j “m‰GˆAˆq�!à�}‰}˜qÒ à—F‘F˜1˜dŸj™j¨$¯-©-¸¸T¿]¹]ÓK×PÑPÐUVÐPÓW�ð —l‘l“n a§l¡l£n°a·l±l³n�!ˆqˆAà�?Š?ä×.Ñ.¨q°!°QÓ7‰Aà˜Ÿ
™
‘N a§k¡k°"°bÓ&9Ñ9ˆDØ—<‘< B�<Ó'ˆDØ�q‘ˆAð ×"Ò"Ø—‘˜A˜qÓ!×)Ñ)¨!¨R°·±Ó>‰Aà—‘˜A˜qÓ!×)Ñ)¨!¨R°·±Ó>ˆAà�I‰I�a‹LˆØˆr+   )r   r   FNN)rc   rd   re   rf   rh   ÚjitÚFinalÚboolÚ__annotations__rg   rS   ri   ra   rj   rk   s   @r)   rv   rv   ÿ   s‰   ø… ñð —	‘	—‘ Ñ%Ó%ð Ø Ø',ØØñ!5àð!5ð ð!5ð ð	!5ð
 ð!5ð ð!5ð !%õ!5ðF,˜Ÿ™ð ,¨%¯,©,÷ ,r+   rv   c                   ó  ‡ — e Zd Zdddej                  ej
                  ddddddfded	ed
edededee   de	ej                     de	ej                     dedededefˆ fd„Zdej                  dej                  fd„Zˆ xZS )Ú
HieraBlockç      @ç        Nr   r   TFrx   ry   rz   Ú	mlp_ratioÚ	drop_pathÚinit_valuesÚ
norm_layerÚ	act_layerr{   r|   Úuse_expand_projr}   c                 óÖ  •— ||dœ}t         ‰| �  «        || _        || _         ||fi |¤Ž| _        ||k7  r8d| _        |rt        j                  ||fi |¤Ž| _        n ||dz  k(  sJ ‚d | _        nd| _        d | _        t        ||||	|
|fi |¤Ž| _
        |�t        |fd|i|¤Žnt        j                  «       | _        |dkD  rt        |«      nt        j                  «       | _         ||fi |¤Ž| _        t#        |t%        ||z  «      fd|i|¤Ž| _        |�t        |fd|i|¤Žnt        j                  «       | _        |dkD  rt        |«      | _        y t        j                  «       | _        y )Nr   Tr/   Fr£   r   r¥   )rR   rS   rx   ry   Únorm1Ú	do_expandr$   r…   r‡   rv   r˜   r   r%   Úls1r   Ú
drop_path1Únorm2r   rg   ÚmlpÚls2Ú
drop_path2)rU   rx   ry   rz   r¡   r¢   r£   r¤   r¥   r{   r|   r¦   r}   r€   r�   rˆ   rW   s                   €r)   rS   zHieraBlock.__init__[  sn  ø€ ð" ¨Ñ/ˆÜ‰ÑÔØˆŒØˆŒá Ñ* rÑ*ˆŒ
Ø�'Š>Ø!ˆDŒNÙÜŸI™I c¨7Ñ9°bÑ9�•	à #¨¡'Ò)Ð)Ð)Ø �•	à"ˆDŒNØˆDŒIÜ%ØØØØØØñ
ð ñ
ˆŒ	ð JUÐI`”:˜gÑE°;ÐEÀ"ÒEÔfh×fqÑfqÓfsˆŒØ1:¸Q²œ( 9Ô-ÄBÇKÁKÃMˆŒá Ñ.¨2Ñ.ˆŒ
Ü�w¤ G¨iÑ$7Ó 8ÑTÀIÐTÐQSÑTˆŒØITÐI`”:˜gÑE°;ÐEÀ"ÒEÔfh×fqÑfqÓfsˆŒØ1:¸Q²œ( 9Ó-ˆ�ÄBÇKÁKÃMˆ�r+   r8   r"   c           
      óz  — | j                  |«      }| j                  �r)| j                  �d| j                  |«      }|j                  |j                  d   | j
                  j                  d|j                  d   «      j                  d¬«      }n¹t        j                  |j                  |j                  d   | j
                  j                  d|j                  d   «      j                  d¬«      |j                  |j                  d   | j
                  j                  d|j                  d   «      j                  d¬«      gd¬«      }|| j                  | j                  | j                  |«      «      «      z   }|| j                  | j                  | j                  | j!                  |«      «      «      «      z   }|S )Nr   r;   r   rŒ   )r¨   r©   r‡   r=   r2   r˜   r{   r�   rh   ÚcatÚmeanr«   rª   r¯   r®   r­   r¬   )rU   r8   Úx_norms      r)   ra   zHieraBlock.forward�  sT  € à—‘˜A“ˆØ�>‹>Ø�y‰yÐ$Ø—I‘I˜fÓ%�Ø—F‘F˜1Ÿ7™7 1™: t§y¡y×'9Ñ'9¸2¸q¿w¹wÀr¹{ÓK×PÑPÐUVÐPÓW‘ä—I‘IØ—F‘F˜1Ÿ7™7 1™: t§y¡y×'9Ñ'9¸2¸q¿w¹wÀr¹{ÓK×PÑPÐUVÐPÓWØ—F‘F˜1Ÿ7™7 1™: t§y¡y×'9Ñ'9¸2¸q¿w¹wÀr¹{ÓK×PÑPÐUVÐPÓWðð ô	�ð �—‘ §¡¨¯©°6Ó):Ó ;Ó<Ñ<ˆð �—‘ §¡¨¯©°$·*±*¸Q³-Ó)@Ó AÓBÑBˆØˆr+   )rc   rd   re   r$   Ú	LayerNormÚGELUrg   r5   r   r   ÚModuler›   rS   rh   ri   ra   rj   rk   s   @r)   rž   rž   Z  sæ   ø„ ð  #Ø"Ø+/Ø*,¯,©,Ø)+¯©ØØ Ø$(Ø',ØØñ0Ràð0Rð ð0Rð ð	0Rð
 ð0Rð ð0Rð " %™ð0Rð ˜RŸY™Y™ð0Rð ˜BŸI™I‘ð0Rð ð0Rð ð0Rð "ð0Rð !%õ0Rðd˜Ÿ™ð ¨%¯,©,÷ r+   rž   c                   ó¼   ‡ — e Zd ZdZ	 	 	 ddededeedf   deedf   deedf   defˆ fd	„Z	 dd
ej                  de
ej                     dej                  fd„Zˆ xZS )Ú
PatchEmbedzHPatch embed that supports any number of spatial dimensions (1d, 2d, 3d).Údim_inry   Úkernel.ÚstrideÚpaddingrB   c	                 ó¦   •— ||dœ}	t         ‰
| �  «        t        |«      | _        || _         t        | j                  «      ||f|||dœ|	¤Ž| _        y )Nr   )Úkernel_sizer»   r¼   )rR   rS   r1   Úspatial_dimsrB   r*   r‡   )rU   r¹   ry   rº   r»   r¼   rB   r€   r�   rˆ   rW   s             €r)   rS   zPatchEmbed.__init__¥  sf   ø€ ð ¨Ñ/ˆÜ‰ÑÔä ›KˆÔØˆŒØ.”G˜D×-Ñ-Ó.ØØð
ð ØØñ
ð ñ
ˆ�	r+   r8   r-   r"   c                 óV  — |�Lt        |j                  dd  |¬«      }| j                  ||j                  t        j
                  «      z  «      }n| j                  |«      }| j                  r=|j                  |j                  d   |j                  d   d«      j                  dd«      }|S )Nr/   r6   r   r   r;   )r7   r2   r‡   Útorh   r›   rB   r’   ©rU   r8   r-   s      r)   ra   zPatchEmbed.forward¾  sˆ   € ð
 ÐÜ#°·±¸¸°À$ÔGˆDØ—	‘	˜!˜dŸg™g¤e§j¡jÓ1Ñ1Ó2‰Aà—	‘	˜!“ˆAØ�<Š<Ø—	‘	˜!Ÿ'™' !™* a§g¡g¨a¡j°"Ó5×?Ñ?ÀÀ1ÓEˆAØˆr+   )TNNrQ   )rc   rd   re   rf   rg   r   r›   rS   rh   ri   r   ra   rj   rk   s   @r)   r¸   r¸   ¢  sŸ   ø„ ÙRð !ØØñ
àð
ð ð
ð ˜#˜s˜(‘Oð	
ð
 ˜#˜s˜(‘Oð
ð ˜3 ˜8‘_ð
ð õ
ð8 ,0ñà�|‰|ðð ˜5Ÿ<™<Ñ(ðð 
�‰÷	r+   r¸   c            =       ó¼  ‡ — e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d>deedf   dededededed	eedf   d
edeedf   deedf   deedf   dedededeedf   deedf   deedf   dededee   dedede	ee
ej                     f   dedededededeeef   f:ˆ fd „Zd!„ Zej                   j"                  d"„ «       Zej                   j"                  d?d#ed$efd%„«       Zej                   j"                  d@d&ed$dfd'„«       Zej                   j"                  d(„ «       ZdAdedee   d)efd*„Zd+ej0                  d,ed$ej0                  fd-„Zd$ej0                  fd.„Z	 	 	 	 	 	 	 dBd+ej0                  d/eej0                     d0ee	eee   f      d1ed2ed3ed4ed#ed$e	eej0                     eej0                  eej0                     f   f   fd5„Z	 	 	 	 dCd0e	eee   f   d6ed7ed#efd8„Z	 	 dAd+ej0                  d/eej0                     d9ed$ej0                  fd:„Zd?d;ed$ej0                  fd<„Z	 dDd+ej0                  d/eej0                     d$ej0                  fd=„Z ˆ xZ!S )Er    NÚimg_size.Úin_chansÚ	embed_dimÚ	num_headsÚnum_classesÚglobal_poolÚstagesro   r{   Úmask_unit_sizeÚmask_unit_attnr¦   Údim_mulÚhead_mulÚpatch_kernelrN   Úpatch_paddingr¡   Údrop_path_rater£   Úfix_initÚweight_initr¤   Ú	drop_rateÚpatch_drop_rateÚhead_init_scaleÚsep_pos_embedÚabs_win_pos_embedÚglobal_pos_sizec                  óx  •— t         ‰/| �  «        ||dœ} || _        || _        d| _        t        |«      }t        |t        «      rt        |«      }|| _	        t        ||«      D �!�"cg c]
  \  }!}"|!|"z  ‘Œ c}"}!| _        t        j                  | j                  «      }#t        j                  |
«      }$t        j                  |	«      }%|t        |«      k  sJ ‚||	c| _        | _        |$|
c| _        | _        t        | j                  | j$                  «      D �!�"cg c]
  \  }!}"|!|"z  ‘Œ c}"}!| _        t)        dt        |«      dz   «      D �!cg c]  }!t+        |d |! «      dz
  ‘Œ c}!| _        || _        t1        |||||fi | ¤Ž| _        d | _        d | _        d | _        d | _        |r‹t=        j>                  tA        jB                  d| j                  d   | j                  d   z  |fi | ¤Ž«      | _        t=        j>                  tA        jB                  d| j                  d   |fi | ¤Ž«      | _        n—|ret=        j>                  tA        jB                  d|g|¢­i | ¤Ž«      | _        t=        j>                  tA        jB                  d|g|
¢­i | ¤Ž«      | _        n0t=        j>                  tA        jB                  d|#|fi | ¤Ž«      | _        tE        |||	gt        | j,                  d d «      z  «      | _#        tI        |||	gt        | j,                  d d «      z  | j,                  |«      | _%        | j,                  d | D �&cg c]  }&|&dz   ‘Œ	 }'}&d}(t+        |«      })tM        ||)«      }*t=        jN                  «       | _(        g | _)        t)        |)«      D ]Ç  }!|}+||(   },|!dz
  | j,                  v r*t        ||z  «      }+t        ||z  «      }|(dz  }(|!|'v r|$|%z  }$tU        d||+|||*|!   |||!|'v r|%nd|$||,dœ| ¤Ž}-|+}|!| j,                  v r8| xjR                  tW        |+d|(dz   z  d| j,                  |(   › �¬	«      gz  c_)        | jP                  jY                  |-«       ŒÉ |x| _-        | _.        t_        ||f|||d
dœ| ¤Ž| _0        |rWt<        jb                  je                  | j8                  d¬«       t<        jb                  je                  | j:                  d¬«       nn| j4                  �+t<        jb                  je                  | j4                  d¬«       | j6                  �+t<        jb                  je                  | j6                  d¬«       |dk7  r*|dk(  rtf        nth        }.tk        |.d¬«      }.tm        |.| «       |r| jo                  «        t        | j`                  jp                  t<        jr                  «      rs| j`                  jp                  jt                  jv                  jy                  |«       | j`                  jp                  jz                  jv                  jy                  |«       y y c c}"}!w c c}"}!w c c}!w c c}&w )Nr   Fr   r/   r   r;   )rx   ry   rz   r¡   r¢   r£   r¤   r{   r|   r¦   r}   zblocks.)Únum_chsÚ	reductionÚmoduleÚNLC)Ú	pool_typerÔ   r¤   Ú	input_fmtg{®Gáz”?)ÚstdÚskipÚjaxúhead.fc)Úclassifier_name© )>rR   rS   rÈ   rÅ   Úgrad_checkpointingr   Ú
isinstancerg   r   rN   r<   Útokens_spatial_shaperZ   r[   r1   ro   r{   Úmu_sizerË   Úmask_spatial_shaper?   r>   rn   rÕ   r¸   Úpatch_embedÚ	pos_embedÚpos_embed_winÚpos_embed_spatialÚpos_embed_temporalr$   Ú	Parameterrh   ÚzerosrL   Úunrollrm   Úrerollr   Ú
ModuleListÚblocksÚfeature_inforž   ÚdictÚappendÚnum_featuresÚhead_hidden_sizer   ÚheadÚinitÚtrunc_normal_r   r   r   r   Úfix_init_weightÚfcr…   ÚweightÚdataÚmul_Úbias)0rU   rÄ   rÅ   rÆ   rÇ   rÈ   rÉ   rÊ   ro   r{   rË   rÌ   r¦   rÍ   rÎ   rÏ   rN   rÐ   r¡   rÑ   r£   rÒ   rÓ   r¤   rÔ   rÕ   rÖ   r×   rØ   rÙ   r€   r�   rˆ   rV   rF   Ú
num_tokensÚflat_mu_sizeÚflat_q_strider8   Úq_pool_blocksÚ	cur_stageÚdepthÚdprry   r}   ÚblockÚinit_fnrW   s0                                                  €r)   rS   zHiera.__init__Ï  sÒ  ø€ ôF 	‰ÑÔØ¨Ñ/ˆØ&ˆÔØ ˆŒØ"'ˆÔÜ# JÓ/ˆ
Ü�h¤Ô$Ü  Ó*ˆHà(ˆÔÜ8;¸HÀlÓ8S×$T±°°1 Q¨!£VÓ$TˆÔ!Ü—Y‘Y˜t×8Ñ8Ó9ˆ
Ü—y‘y Ó0ˆÜŸ	™	 (Ó+ˆØœ˜F›Ò#Ð#Ð#Ø%+¨XÐ"ˆŒ�T”]Ø,8¸.Ð)ˆŒ�dÔ)Ü69¸$×:SÑ:SÐUY×UhÑUhÓ6i×"j©d¨a° 1¨£6Ó"jˆÔÜ8=¸aÄÀVÃÈqÁÓ8QÖR°1œ3˜v b q˜z›?¨QÓ.ÒRˆŒØ.ˆÔä%ØØØØØñ
ð ñ
ˆÔð 26ˆŒØ59ˆÔØ9=ˆÔØ:>ˆÔÙÜ%'§\¡\Ü—‘˜A˜t×8Ñ8¸Ñ;¸d×>WÑ>WÐXYÑ>ZÑZÐ\eÑlÐikÑló&ˆDÔ"ô ')§l¡lÜ—‘˜A˜t×8Ñ8¸Ñ;¸YÑMÈ"ÑMó'ˆDÕ#ñ !ä!#§¡¬e¯k©k¸!¸YÐ._ÈÒ._Ð\^Ñ._Ó!`�”Ü%'§\¡\´%·+±+¸aÀÐ2bÈ^Ò2bÐ_aÑ2bÓ%c�Õ"ä!#§¡¬e¯k©k¸!¸ZÈÑ.YÐVXÑ.YÓ!Z�”ô ØØØˆJœ˜TŸ_™_¨S¨bÐ1Ó2Ñ2ó
ˆŒô
 ØØØˆJœ˜TŸ_™_¨S¨bÐ1Ó2Ñ2Ø�O‰OØó
ˆŒð )-¯©¸¸Ð(@ÖA 1˜˜Q›ÐAˆÐAð ˆ	Ü�F“ˆÜ'¨¸Ó>ˆÜ—m‘m“oˆŒØˆÔÜ�u“ò  	&ˆAØˆGð "0°	Ñ!:Ðà�1‰u˜Ÿ™Ñ'Ü˜i¨'Ñ1Ó2�Ü 	¨HÑ 4Ó5�	Ø˜Q‘�	Ø˜Ñ%Ø  ]Ñ2�Läð ØØØØ#Ø˜a™&Ø'Ø%Ø+,°Ñ+=™-À1Ø(Ø /Ø#5ñð ñˆEð  ˆIØ�D—O‘OÑ#Ø×!Ò!Ü °A¸	À!¹Ñ4DÈwÐW[×WfÑWfÐgpÑWqÐVrÐMsÔtð&vñ vÕ!à�K‰K×Ñ˜uÕ%ðA 	&ðD 5>Ð=ˆÔ˜DÔ1Ü+ØØð
ð "ØØ!Øñ
ð ñ
ˆŒ	ñ Ü�G‰G×!Ñ! $×"8Ñ"8¸dÐ!ÔCÜ�G‰G×!Ñ! $×"9Ñ"9¸tÐ!ÕDà�~‰~Ð)Ü—‘×%Ñ% d§n¡n¸$Ð%Ô?Ø×!Ñ!Ð-Ü—‘×%Ñ% d×&8Ñ&8¸dÐ%ÔCà˜&Ò Ø)4¸Ò)=•oÄ?ˆGÜ˜g°yÔAˆGÜ˜ Ô&ÙØ× Ñ Ô"Ü�d—i‘i—l‘l¤B§I¡IÔ.Ø�I‰I�L‰L×Ñ×$Ñ$×)Ñ)¨/Ô:Ø�I‰I�L‰L×Ñ×"Ñ"×'Ñ'¨Õ8ð /ùóu %Uùó #kùÚRùòZ Bs   Á&X&ÄX,Å
X2Ì;X7c                 ó  — d„ }t        | j                  «      D ]m  \  }} ||j                  j                  j                  j
                  |dz   «        ||j                  j                  j                  j
                  |dz   «       Œo y )Nc                 óR   — | j                  t        j                  d|z  «      «       y )Nç       @)Údiv_rZ   Úsqrt)ÚparamÚ	_layer_ids     r)   Úrescalez&Hiera.fix_init_weight.<locals>.rescale{  s   € Ø�J‰J”t—y‘y  y¡Ó1Õ2r+   r   )Ú	enumeraterö   r˜   r‡   r  r  r­   Úfc2)rU   r  Úlayer_idÚlayers       r)   rÿ   zHiera.fix_init_weightz  si   € ò	3ô  )¨¯©Ó5ò 	=‰OˆH�eÙ�E—J‘J—O‘O×*Ñ*×/Ñ/°¸A±Ô>Ù�E—I‘I—M‘M×(Ñ(×-Ñ-¨x¸!©|Õ<ñ	=r+   c                 óH   — | j                   �dgS | j                  �ddgS ddgS )Nrí   Úpos_embed_absrî   rï   rð   )rí   r  ©rU   s    r)   Úno_weight_decayzHiera.no_weight_decay‚  s7   € à�>‰>Ð%Ø�=Ð Ø×ÑÐ+Ø# _Ð5Ð5à'Ð)=Ð>Ð>r+   Úcoarser"   c                 ó    — t        dddg¬«      S )NzW^pos_embed|pos_embed_spatial|pos_embed_temporal|pos_embed_abs|pos_embed_win|patch_embed)z^blocks\.(\d+)N)z^norm)iŸ† )Ústemrö   )rø   )rU   r  s     r)   Úgroup_matcherzHiera.group_matcher‹  s   € äØkØ-Ð/CÐDô
ð 	
r+   Úenablec                 ó   — || _         y rQ   )rç   )rU   r"  s     r)   Úset_grad_checkpointingzHiera.set_grad_checkpointing’  s
   € à"(ˆÕr+   c                 ó.   — | j                   j                  S rQ   )rü   r   r  s    r)   Úget_classifierzHiera.get_classifier–  s   € à�y‰y�|‰|Ðr+   Úreset_otherc                 óN   — || _         | j                  j                  |||¬«       y )N©r'  )rÈ   rü   Úreset)rU   rÈ   rÉ   r'  s       r)   Úreset_classifierzHiera.reset_classifierš  s    € Ø&ˆÔØ�	‰	�‰˜ [¸kˆÕJr+   r8   Ú
mask_ratioc                 óÌ  — |j                   d   }t        j                  | j                  «      }t	        |d|z
  z  «      }t        j                  |||j                  ¬«      }t        j                  |d¬«      }t        j                  |d¬«      }t        j                  ||g|j                  ¬«      }	d|	dd…d|…f<   t        j                  |	d|¬«      }	|	j                  «       S )zŠ
        Generates a random mask, mask_ratio fraction are dropped.
        1 is *keep*, 0 is *remove*. Useful for MAE, FLIP, etc.
        r   r   )r€   rŒ   N)rx   Úindex)r2   rZ   r[   rë   rg   rh   Úrandr€   Úargsortrò   Úgatherr›   )
rU   r8   r,  rD   r”   Úlen_keepÚnoiseÚids_shuffleÚids_restorer-   s
             r)   Úget_random_maskzHiera.get_random_maskž  s»   € ð
 �G‰G�A‰Jˆä—i‘i × 7Ñ 7Ó8ˆÜ�{ a¨*¡nÑ5Ó6ˆÜ—
‘
˜1˜k°!·(±(Ô;ˆô —m‘m E¨qÔ1ˆÜ—m‘m K°QÔ7ˆô �{‰{˜A˜{Ð+°A·H±HÔ=ˆØˆŠQ�	��	ˆ\Ñä�|‰|˜D a¨{Ô;ˆà�y‰y‹{Ðr+   c                 ó&  — | j                   �|| j                   j                  | j                  «      }t        j                  | j
                  |j                  dd  dd¬«      }||z   }|j                  d«      j                  dd«      }nƒ| j
                  �| j
                  }nj| j                  j                  d| j                  d   d«      t        j                  | j                  | j                  d   | j                  d   z  d¬«      z   }||z   }|S )	Nr�   ÚbicubicT)r0   ÚmodeÚ	antialiasr/   r   r   rŒ   )rî   Útilerë   r3   r4   rí   r2   rY   r’   rï   Úrepeatré   rh   Úrepeat_interleaverð   )rU   r8   rî   rí   s       r)   Ú
_pos_embedzHiera._pos_embed¶  s
  € Ø×ÑÐ)ð !×.Ñ.×3Ñ3°D×4KÑ4KÓLˆMÜŸ™Ø—‘Ø"×(Ñ(¨¨Ð-ØØô	ˆIð " MÑ1ˆIØ!×)Ñ)¨!Ó,×6Ñ6°q¸!Ó<‰IØ�^‰^Ð'ØŸ™‰Ið ×&Ñ&×-Ñ-¨a°×1JÑ1JÈ1Ñ1MÈqÓQä×'Ñ'Ø×+Ñ+Ø×-Ñ-¨aÑ0°4×3LÑ3LÈQÑ3OÑOØôñð ð �	‰MˆØˆr+   r-   ÚindicesÚnormÚ
stop_earlyÚ
output_fmtÚintermediates_onlyc	           	      óX  — |rJ d«       ‚|dv sJ d«       ‚|rNt        t        | j                  «      |«      \  }	}
|	D �cg c]  }| j                  |   ‘Œ }	}| j                  |
   }
n"t        t        | j                  «      |«      \  }	}
|�, |j                  |j
                  d   dg| j                  ¢­Ž }nd}| j                  ||¬«      }| j                  |«      }| j                  |«      }|�[||d   j                  d| j                  |j
                  d	   «         j	                  |j
                  d   d
|j
                  d
   «      }g }t        j                  j                  «       s|s| j                  }n| j                  d|
dz    }t        |«      D ]‡  \  }}| j                   r+t        j                  j                  «       st#        ||«      }n ||«      }||	v sŒJ| j%                  |||¬«      }|j'                  |dk(  r|j)                  dddd	«      n|«       Œ‰ |r|S ||fS c c}w )aè   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        z'normalization of features not supported)ÚNCHWÚNHWCz(Output format must be one of NCHW, NHWC.Nr   r   ©r-   ©.Nr/   r;   rE  r‚   )r   r1   rn   rö   r=   r2   rë   rì   r>  ró   r;  rê   rh   r™   Úis_scriptingr  rç   r   rô   rù   rA   )rU   r8   r-   r?  r@  rA  rB  rC  r  Útake_indicesÚ	max_indexrV   Ú
patch_maskÚintermediatesrö   ÚblkÚx_ints                    r)   Úforward_intermediateszHiera.forward_intermediatesÒ  s  € ñ. ÐBÐBÓBˆxØÐ-Ñ-ÐYÐ/YÓYÐ-ÙÜ&:¼3¸t¿¹Ó;OÐQXÓ&YÑ#ˆL˜)Ø8DÖE°1˜DŸO™O¨AÓ.ÐEˆLÐEØŸ™¨	Ñ2‰Iä&:¼3¸t¿{¹{Ó;KÈWÓ&UÑ#ˆL˜)àÐØ"˜Ÿ™ 1§7¡7¨1¡:¨qÐK°4×3JÑ3JÒK‰JàˆJØ×Ñ˜Q ZÐÓ0ˆØ�O‰O˜AÓˆØ�K‰K˜‹Nˆð ÐØ�$�y‘/×&Ñ& q¨$¯,©,¸¿¹À¹
ÓCÑD×IÑIÈ!Ï'É'ÐRSÉ*ÐVXÐZ[×ZaÑZaÐbdÑZeÓfˆAàˆÜ�9‰9×!Ñ!Ô#©:Ø—[‘[‰Fà—[‘[  )¨a¡-Ð0ˆFÜ Ó'ò 	c‰FˆAˆsØ×&Ò&¬u¯y©y×/EÑ/EÔ/GÜ˜s AÓ&‘á˜“F�Ø�LÒ ØŸ™ A q¨t˜Ó4�Ø×$Ñ$À*ÐPVÒBV U§]¡]°1°a¸¸AÔ%>Ð\aÕbð	cñ Ø Ð à�-ÐÐùòE Fs   ½H'Ú
prune_normÚ
prune_headc                 ó  — |r2t        t        | j                  «      |«      \  }}| j                  |   }n"t        t        | j                  «      |«      \  }}| j                  d|dz    | _        |r| j                  j                  dd¬«       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   Tr)  )r   r1   rn   rö   rü   r*  )rU   r?  rQ  rR  r  rJ  rK  s          r)   Úprune_intermediate_layerszHiera.prune_intermediate_layers  s}   € ñ Ü&:¼3¸t¿¹Ó;OÐQXÓ&YÑ#ˆL˜)ØŸ™¨	Ñ2‰Iä&:¼3¸t¿{¹{Ó;KÈWÓ&UÑ#ˆL˜)Ø—k‘k . 9¨q¡=Ð1ˆŒÙØ�I‰I�O‰O˜A¨4ˆOÔ0ØÐr+   Úreturn_intermediatesc                 ó4  — | j                   r0| j                  dkD  r!|�J ‚| j                  || j                  ¬«      }|�, |j                  |j                  d   dg| j
                  ¢­Ž }nd}| j                  ||¬«      }| j                  |«      }| j                  |«      }|�[||d   j                  d| j                  |j                  d   «         j                  |j                  d   d|j                  d   «      }g }t        | j                  «      D ]y  \  }}| j                  r+t        j                  j!                  «       st#        ||«      }n ||«      }|sŒH|| j$                  v sŒW|j'                  | j)                  |||¬«      «       Œ{ |r||fS |S )	zê
        mask should be a boolean tensor of shape [B, #MUt*#MUy*#MUx] where #MU are the number of mask units in that dim.
        Note: 1 in mask is *keep*, 0 is *remove*; mask.sum(dim=-1) should be the same across the batch.
        r   N)r,  r   rG  rH  r/   r;   )ÚtrainingrÕ   r6  r=   r2   rë   rì   r>  ró   r;  rê   r  rö   rç   rh   r™   rI  r   rn   rù   rô   )rU   r8   r-   rU  rL  rM  rV   rN  s           r)   Úforward_featureszHiera.forward_features$  s„  € ð �=Š=˜T×1Ñ1°AÒ5à�<Ð�<Ø×'Ñ'¨°d×6JÑ6JÐ'ÓKˆDàÐØ"˜Ÿ™ 1§7¡7¨1¡:¨qÐK°4×3JÑ3JÒK‰JàˆJØ×Ñ˜Q ZÐÓ0ˆØ�O‰O˜AÓˆØ�K‰K˜‹Nˆð ÐØ�$�y‘/×&Ñ& q¨$¯,©,¸¿¹À¹
ÓCÑD×IÑIÈ!Ï'É'ÐRSÉ*ÐVXÐZ[×ZaÑZaÐbdÑZeÓfˆAàˆÜ §¡Ó,ò 	C‰FˆAˆsØ×&Ò&¬u¯y©y×/EÑ/EÔ/GÜ˜s AÓ&‘á˜“F�Ú#¨¨T¯_©_Ò(<Ø×$Ñ$ T§[¡[°°A¸D [Ó%AÕBð	Cñ  Ø�mÐ#Ð#àˆr+   Ú
pre_logitsc                 óV   — |r| j                  ||¬«      }|S | j                  |«      }|S )N)rY  )rü   )rU   r8   rY  s      r)   Úforward_headzHiera.forward_headQ  s1   € Ù3=ˆD�I‰I�a JˆIÓ/ˆØˆð DHÇ9Á9ÈQÃ<ˆØˆr+   c                 óR   — | j                  ||¬«      }|€| j                  |«      }|S )NrG  )rX  r[  rÂ   s      r)   ra   zHiera.forwardU  s3   € ð
 ×!Ñ! !¨$Ð!Ó/ˆØˆ<Ø×!Ñ! !Ó$ˆAØˆr+   ))éà   r]  r‚   é`   r   éè  Úavg©r/   r‚   é   r‚   r‚   )r/   r/   )é   rc  )TTFFTr  r  )é   rd  )rŠ   rŠ   )r‚   r‚   rŸ   r    NTÚ r´   r    r    gü©ñÒMbP?FF)é   rf  NN©F)T)NF)NNFTrE  FT)r   FTTrQ   )"rc   rd   re   r   rg   Ústrr›   r5   r   r	   r   r$   r¶   rS   rÿ   rh   r™   Úignorer  r   r!  r$  r&  r+  ri   r6  r>  r   rP  rT  rX  r[  ra   rj   rk   s   @r)   r    r    Í  s‹  ø„ ð )3ØØØØ#Ø$Ø&3ØØ(.Ø.4à/IØ$(Ø Ø!Ø,2Ø,2Ø-3Ø"Ø$'Ø+/Ø!Ø!Ø6AØ"Ø%(Ø%*Ø"'Ø&+Ø/7ØØñCi9à˜C ˜H‘oði9ð ði9ð ð	i9ð
 ði9ð ði9ð ði9ð ˜#˜s˜(‘Oði9ð ði9ð ˜C ˜H‘oði9ð " # s (™Oði9ð " $¨ )Ñ,ði9ð "ði9ð ði9ð  ð!i9ð"    S ™/ð#i9ð$    S ™/ð%i9ð& !  c ™?ð'i9ð( ð)i9ð* "ð+i9ð, " %™ð-i9ð. ð/i9ð0 ð1i9ð2 ˜c 4¨¯	©	¡?Ð2Ñ3ð3i9ð4 ð5i9ð6 #ð7i9ð8 #ð9i9ð:  ð;i9ð<  $ð=i9ð> # 3¨ 8™_õ?i9òV=ð ‡Y�Y×Ññ?ó ð?ð ‡Y�Y×Ññ
 Dð 
°Tò 
ó ð
ð ‡Y�Y×Ññ)¨Tð )¸Tò )ó ð)ð ‡Y�Y×Ññó ðñK¨Cð K¸hÀs¹mð KÐaeó Kð §¡ð ¸5ð ÀUÇ\Á\ó ð0˜uŸ|™|ó ð> ,0Ø7;ØØ#Ø$Ø',Øñ= à�|‰|ð= ð ˜5Ÿ<™<Ñ(ð= ð ˜e C¨¨c© NÑ3Ñ4ð	= ð
 ð= ð ð= ð ð= ð !%ð= ð ð= ð 
ˆt�E—L‘LÑ! 5¨¯©°t¸E¿L¹LÑ7IÐ)IÑ#JÐJÑ	Kó= ðB ./Ø$Ø#Øñà˜3  S¡	˜>Ñ*ðð ðð ð	ð
 óð, ,0Ø).ñ	+à�|‰|ð+ð ˜5Ÿ<™<Ñ(ð+ð #'ð	+ð
 
�‰ó+ñZ¨$ð ¸5¿<¹<ó ð ,0ñà�|‰|ðð ˜5Ÿ<™<Ñ(ðð 
�‰÷	r+   c                 ó4   — | ddd dddt         t        dddd	œ|¥S )
Nr_  )r‚   r]  r]  gÍÌÌÌÌÌì?r8  Tzpatch_embed.projrä   z
apache-2.0)ÚurlrÈ   rM   Ú	pool_sizeÚcrop_pctÚinterpolationÚfixed_input_sizer²   rá   Ú
first_convÚ
classifierÚlicenser
   )rk  Úkwargss     r)   Ú_cfgrt  `  s5   € àØ¨=ÀtØ¨ÈÜ%Ô.BØ(¸	Øñð ðð r+   zhiera_tiny_224.mae_in1k_ft_in1kztimm/zcc-by-nc-4.0)Ú	hf_hub_idrr  zhiera_tiny_224.mae)ru  rr  rÈ   z hiera_small_224.mae_in1k_ft_in1kzhiera_small_224.maezhiera_base_224.mae_in1k_ft_in1kzhiera_base_224.maez$hiera_base_plus_224.mae_in1k_ft_in1kzhiera_base_plus_224.maez hiera_large_224.mae_in1k_ft_in1kzhiera_large_224.maezhiera_huge_224.mae_in1k_ft_in1kzhiera_huge_224.maez.hiera_small_abswin_256.sbb2_e200_in12k_ft_in1k)r‚   é   rv  gffffffî?)ru  rM   rm  z1hiera_small_abswin_256.sbb2_pd_e200_in12k_ft_in1kz&hiera_small_abswin_256.sbb2_e200_in12ki-.  )ru  rÈ   rM   rm  z)hiera_small_abswin_256.sbb2_pd_e200_in12kzhiera_base_abswin_256.untrained)rM   rm  c                 ó0  — | j                  d| «      } i }| j                  «       D ]n  \  }}d|v r|j                  dd«      }|j                  d«      r|j                  dd«      }n#|j                  d«      r|j                  dd«      }|dk(  rd}|||<   Œp |S )	NÚmodel_statezhead.projection.zhead.fc.zencoder_norm.z
head.norm.znorm.r  rí   )ÚgetÚitemsÚreplaceÚ
startswith)Ú
state_dictÚmodelÚoutputr–   r—   s        r)   Úcheckpoint_filter_fnr€  Â  s£   € Ø—‘ ¨zÓ:€JØ€FØ× Ñ Ó"ò ‰ˆˆ1ð  Ñ"Ø—	‘	Ð,¨jÓ9ˆAØ�<‰<˜Ô(Ø—	‘	˜/¨<Ó8‰AØ�\‰\˜'Ô"Ø—	‘	˜' <Ó0ˆAØ�ÒØˆAØˆˆqŠ	ð%ð& €Mr+   ÚvariantÚ
pretrainedc                 ón   — |j                  dd«      }t        t        | |ft        t	        |d¬«      dœ|¤ŽS )NÚout_indicesrŠ   Úgetter)r„  Úfeature_cls)Úpretrained_filter_fnÚfeature_cfg)Úpopr   r    r€  rø   )r�  r‚  rs  r„  s       r)   Ú_create_hierarŠ  Û  sF   € Ø—*‘*˜]¨AÓ.€KäÜØØðô 2Ü [¸hÔGñð ñð r+   c           	      óL   — t        ddd¬«      }t        dd| it        |fi |¤Ž¤ŽS )Nr^  r   )r   r/   rd  r/   ©rÆ   rÇ   rÊ   r‚  )Úhiera_tiny_224©rø   rŠ  ©r‚  rs  Ú
model_argss      r)   r�  r�  è  s.   € ä ¨a¸ÔE€JÜÑ_°jÐ_ÄDÈÑD^ÐW]ÑD^Ñ_Ð_r+   c           	      óL   — t        ddd¬«      }t        dd| it        |fi |¤Ž¤ŽS )Nr^  r   ©r   r/   é   r/   rŒ  r‚  )Úhiera_small_224rŽ  r�  s      r)   r”  r”  î  s.   € ä ¨a¸ÔF€JÜÑ`°zÐ`ÄTÈ*ÑE_ÐX^ÑE_Ñ`Ð`r+   c           	      óL   — t        ddd¬«      }t        dd| it        |fi |¤Ž¤ŽS )Nr^  r   ra  rŒ  r‚  )Úhiera_base_224rŽ  r�  s      r)   r–  r–  ô  s.   € ä ¨a¸ÔF€JÜÑ_°jÐ_ÄDÈÑD^ÐW]ÑD^Ñ_Ð_r+   c           	      óL   — t        ddd¬«      }t        dd| it        |fi |¤Ž¤ŽS )Nép   r/   ra  rŒ  r‚  )Úhiera_base_plus_224rŽ  r�  s      r)   r™  r™  ú  s.   € ä ¨q¸ÔG€JÜÑd¸:ÐdÌÈjÑIcÐ\bÑIcÑdÐdr+   c           	      óL   — t        ddd¬«      }t        dd| it        |fi |¤Ž¤ŽS )Né�   r/   ©r/   é   é$   rŠ   rŒ  r‚  )Úhiera_large_224rŽ  r�  s      r)   rŸ  rŸ     s.   € ä ¨q¸ÔG€JÜÑ`°zÐ`ÄTÈ*ÑE_ÐX^ÑE_Ñ`Ð`r+   c           	      óL   — t        ddd¬«      }t        dd| it        |fi |¤Ž¤ŽS )Nrv  rŠ   rœ  rŒ  r‚  )Úhiera_huge_224rŽ  r�  s      r)   r¡  r¡    s.   € ä ¨q¸ÔG€JÜÑ_°jÐ_ÄDÈÑD^ÐW]ÑD^Ñ_Ð_r+   c           
      óV   — t        dddddddd¬	«      }t        dd
| it        |fi |¤Ž¤ŽS )Nr^  r   r’  T)rb  rb  çñhãˆµøä>rã   F)rÆ   rÇ   rÊ   rØ   rÙ   r£   rÓ   r¦   r‚  )Úhiera_small_abswin_256rŽ  r�  s      r)   r¤  r¤    s@   € äØ ¨-È4ÐaiØ e¸Uô€Jô Ñg¸jÐgÌDÐQ[ÑLfÐ_eÑLfÑgÐgr+   c           	      óR   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤ŽS )
Nr^  r   ra  Tr£  rã   )rÆ   rÇ   rÊ   rØ   r£   rÓ   r‚  )Úhiera_base_abswin_256rŽ  r�  s      r)   r¦  r¦    s;   € äØ ¨-È4Ð]aÐotôv€JäÑf¸ZÐfÌ4ÐPZÑKeÐ^dÑKeÑfÐfr+   )re  rQ   rg  )Grf   rZ   Ú	functoolsr   Útypingr   r   r   r   r   r	   rh   Útorch.nnr$   Útorch.nn.functionalÚ
functionalr3   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   r   r   r   Ú	_registryr   r   Ú_builderr   Ú	_featuresr   Ú_features_fxr   Ú_manipulater   r   Ú__all__rg   r¶   r*   ri   r7   rJ   rL   rm   rv   rž   r¸   r    rt  Údefault_cfgsr€  rh  r›   rŠ  r�  r”  r–  r™  rŸ  r¡  r¤  r¦  ræ   r+   r)   ú<module>rµ     sþ  ðñó0 Ý ß ;× ;ã Ý ß Ð ç A÷÷ ÷ ñ ÷ =Ý *Ý +Ý 3ß 0ð ˆ)€ð=ˆsð =�t˜BŸI™I‘ó =ð ð	 $ s¡)ð 	°5·<±<ð 	ÀEÇLÁLò 	ó ð	ðØ�<‰<ðà�C‰yðð �s‘)ðð ‡\�\ó	ôB;ˆR�Y‰Yô ;ô|NˆR�Y‰Yô NôbX˜Ÿ	™	ô XôvE�—‘ô EôP(�—‘ô (ôVPˆB�I‰Iô Póf	ñ %ð S&Ø%¡tØØô(ðS&ð
 ™$ØØØôðS&ð '©ØØô)ðS&ð ™4ØØØôðS&ð* &¡tØØô(ð+S&ð2 ™$ØØØôð3S&ð> +©DØØô-ð?S&ðF ™tØØØô ðGS&ðR '©ØØô)ðSS&ðZ ™4ØØØôð[S&ðf &¡tØØô(ðgS&ðn ™$ØØØôðoS&ðz 5±dØØ ¨4ô7ð{S&ðB 8¹ØØ ¨4ô:ðCS&ðJ -©dØØØ ¨4ô/ðKS&ðT 0±ØØØ ¨4ô2ðUS&ð^ &¡tà ¨4ô(ð_S&ó S€ólñ2
˜3ð 
¨Dð 
Àuó 
ð ò`ó ð`ð
 òaó ðað
 ò`ó ð`ð
 òeó ðeð
 òaó ðað
 ò`ó ð`ð
 òhó ðhð ògó ñgr+   