
    ^j              
          d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmc mZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZmZmZ ddl m!Z!m"Z" dd	l#m$Z$ dd
l%m&Z& ddl'm(Z( ddl)m*Z*m+Z+ dgZ,de-de	ej\                     fdZ/e(dee-   dej`                  dej`                  fd       Z1dej`                  dee-   dee-   dej`                  fdZ2 G d dej\                        Z3 G d dej\                        Z4 G d dej\                        Z5 G d dej\                        Z6 G d  d!ej\                        Z7 G d" dej\                        Z8dKd#Z9 e!i d$ e9d%d&'      d( e9d%d&d)      d* e9d%d&'      d+ e9d%d&d)      d, e9d%d&'      d- e9d%d&d)      d. e9d%d&'      d/ e9d%d&d)      d0 e9d%d&'      d1 e9d%d&d)      d2 e9d%d&'      d3 e9d%d&d)      d4 e9d%d5d67      d8 e9d%d5d67      d9 e9d%d:d5d6;      d< e9d%d:d5d6;      d= e9d5d6>            Z:dLd?Z;dMd@e<dAe=de8fdBZ>e"dMdC       Z?e"dMdD       Z@e"dMdE       ZAe"dMdF       ZBe"dMdG       ZCe"dMdH       ZDe"dMdI       ZEe"dMdJ       ZFy)Nzr An PyTorch implementation of Hiera

Adapted for timm from originals at https://github.com/facebookresearch/hiera
    N)partial)DictListOptionalTupleTypeUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPathcalculate_drop_path_ratesMlp
LayerScaleClNormMlpClassifierHeaduse_fused_attn_assertget_norm_layer	to_2tupleinit_weight_vitinit_weight_jax   )generate_default_cfgsregister_model)build_model_with_cfg)feature_take_indices)register_notrace_function)named_apply
checkpointHieranreturnc                     t         j                  t         j                  t         j                  t         j                  g|    S )z
    Returns a conv with nd (e.g., Conv2d for n=2). Work up to n=3.
    If you wanted a 4d Hiera, you could probably just implement this for n=4. (no promises)
    )nnIdentityConv1dConv2dConv3d)r!   s    \/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/hiera.pyconv_ndr*   :   s(    
 KKBIIryy9!<<    target_sizemaskc                     ||S t        t        |j                  dd        t        |       k(  d       |j                  dd  | k7  r%t        j                  |j                         |       S |S )N   z.mask spatial shape and target_size must match.)size)r   lenshapeFinterpolatefloatr,   r-   s     r)   get_resized_maskr7   B   s`     |C

123{#335efzz!"~$}}TZZ\<<Kr+   xr2   mu_shapec                    t        |      }| j                  d   | j                  d   }}t        ||      D cg c]
  \  }}||z   }}} | j                  |g||| } dgt	        t        t        dd|z         t        d|z   dd|z  z               D 	cg c]  }	t        |	       c}	g       z   t        | j                        dz
  gz   }
 | j                  |
      j                  |g|| } | S c c}}w c c}	w )a  
    Restore spatial organization by undoing windowed organization of mask units.

    Args:
        x: organized by mask units windows, e.g. in 2d [B, #MUy*#MUx, MUy, MUx, C]
        shape: current spatial shape, if it were not organized into mask unit
            windows, e.g. in 2d [B, #MUy*MUy, #MUx*MUx, C].
        mu_shape: current mask unit shape, e.g. in 2d [MUy, MUx]
    Returns:
        x: e.g. in 2d, [B, #MUy*MUy, #MUx*MUx, C]
    r   r   r/   )	r1   r2   zipviewsumrangelistpermutereshape)r8   r2   r9   DBCsmunum_MUsprA   s              r)   undo_windowingrJ   O   s     	E
A771:qwwr{qA$'x$8951bqBw9G9q)7)X)q)A 

E!QUOU1q5!a!e)5L MN1tAwNPR
S	Tqww<!
	 
 	#		'""10u0a0AH : Os   C3C9
c            	            e Zd ZdZdeedf   deedf   deeedf      f fdZdej                  dej                  fd	Z
 xZS )
Unrolla>  
    Reorders the tokens such that patches are contiguous in memory.
    E.g., given [B, (H, W), C] and stride of (Sy, Sx), this will re-order the tokens as
                           [B, (Sy, Sx, H // Sy, W // Sx), C]

    This allows operations like Max2d to be computed as x.view(B, Sx*Sy, -1, C).max(dim=1).
    Not only is this faster, but it also makes it easy to support inputs of arbitrary
    dimensions in addition to patch-wise sparsity.

    Performing this operation multiple times in sequence puts entire windows as contiguous
    in memory. For instance, if you applied the stride (2, 2) 3 times, entire windows of
    size 8x8 would be contiguous in memory, allowing operations like mask unit attention
    computed easily and efficiently, while also allowing max to be applied sequentially.

    Note: This means that intermediate values of the model are not in HxW order, so they
    need to be re-rolled if you want to use the intermediate values as a HxW feature map.
    The last block of the network is fine though, since by then the strides are all consumed.
    
input_size.patch_strideunroll_schedulec                     t         |           t        ||      D cg c]
  \  }}||z   c}}| _        || _        y c c}}w N)super__init__r<   r0   schedule)selfrM   rN   rO   irF   	__class__s         r)   rS   zUnroll.__init__   s>     	(+J(EF1Q!VF	' Gs   ?r8   r"   c           
         |j                   \  }}}| j                  } |j                  |g|z   |gz    }| j                  D ]  }t	        ||      D cg c]
  \  }}||z   }}}|gt        t	        ||      D cg c]	  \  }}||g c}}g       z   |gz   }	|j                  |	      }t        |	      }
dgt        t        d|
dz
  d            z   t        t        d|
dz
  d            z   |
dz
  gz   }|j                  |      }|j                  dt        |            }|t        j                  |      z  } |j                  dt        j                  | j                        |      }|S c c}}w c c}}w )z
        Input: Flattened patch embeddings [B, N, C]
        Output: Patch embeddings [B, N, C] permuted such that [B, 4, N//4, C].max(1) etc. performs MaxPoolNd
        r   r/   r   r;   )r2   r0   r=   rT   r<   r>   r1   r@   r?   rA   flattenmathprodrB   )rU   r8   rD   _rE   cur_sizestridesrV   rF   	new_shapeLrA   s               r)   forwardzUnroll.forward   si   
 ''1a99AFFaS8^qc)+}} 	$G
 ,/x+AB41aQBHBcc(G6L"MdaAq6"MrRRVWUXXIy!A IAcDq!a%!344tE!QUA<N7OOSTWXSXRYYG		'"A 		!S\*A7##A#	$& IIb$))DII.2 C"Ms   E):E/__name__
__module____qualname____doc__r   intr   rS   torchTensorra   __classcell__rW   s   @r)   rL   rL   p   sa    &(c3h(  S/( "%S/2	( %,, r+   rL   c            
            e Zd ZdZdeedf   deedf   deeedf      dee   def
 fdZ	 dd	ej                  d
edej                  dej                  fdZ
 xZS )RerollzQ
    Undos the "unroll" operation so that you can use intermediate features.
    rM   .rN   rO   
stage_endsq_poolc                 z   t         
|           t        ||      D cg c]
  \  }}||z   c}}| _        i | _        | j                  }t        |d   dz         D ]R  }||f| j                  |<   ||d | v st        |      dkD  r$t        ||d         D 	cg c]
  \  }	}|	|z   }}	}|dd  }T y c c}}w c c}}	w )Nr;   r   r   )rR   rS   r<   r0   rT   r?   r1   )rU   rM   rN   rO   rn   ro   rV   rF   r0   r!   rW   s             r)   rS   zReroll.__init__   s     	(+J(EF1Q!VF	 yyz"~)* 	6A.4DMM!Jw'''!+/249K/LMtq!AFMDM"1!""5	6 G Ns   B1B7r8   	block_idxr-   r"   c                    | j                   |   \  }}|j                  \  }}}t        |      }	dg|	z  }
|D ]  } |j                  |g||t	        j
                  |      z  |
| }t        |j                        }dd|	z   gt        t        t        dd|	z         t        d|	z   dz   |dz
              D cg c]  }t        |       c}g       z   |dz
  gz   }|j                  |      }t        |	      D ]  }|
|xx   ||   z  cc<     |j                  |dg|
| }|j                  d   }  |j                  ||g|
| }||S t        |||
      }|S c c}w )a&  
        Roll the given tensor back up to spatial order assuming it's from the given block.

        If no mask is provided:
            - Returns [B, H, W, C] for 2d, [B, T, H, W, C] for 3d, etc.
        If a mask is provided:
            - Returns [B, #MUs, MUy, MUx, C] for 2d, etc.
        r   r   r;   )rT   r2   r1   r=   rZ   r[   r>   r<   r?   r@   rA   rB   rJ   )rU   r8   rq   r-   rT   r0   rD   NrE   rC   cur_mu_shaper^   r`   rI   rA   rV   s                   r)   ra   zReroll.forward   s    y1$''1aIsQw 	GqN7NA7);$;NlNANA AGGAAE
E!QUOU1q519aRSe=T(UV1tAwVXZ[\q5' 
 		'"A 1X .Q71:-.		!R2,22A
A%	* AFF1a*,** H 1dL1+ Ws   6ErQ   rb   rk   s   @r)   rm   rm      s    6c3h6  S/6 "%S/2	6
 S	6 66 "&	2||2 2 ,,	2
 
2r+   rm   c                        e Zd ZU dZej
                  j                  e   ed<   	 	 	 	 	 dde	de	de	de	de	def fd	Z
d
ej                  dej                  fdZ xZS )MaskUnitAttentionz
    Computes either Mask Unit or Global Attention. Also is able to perform q pooling.

    Note: this assumes the tokens have already been flattened and unrolled into mask units.
    See `Unroll` for more details.
    
fused_attndimdim_outheadsq_stridewindow_sizeuse_mask_unit_attnc	                 P   ||d}	t         
|           || _        || _        || _        || _        ||z  | _        | j                  dz  | _        t               | _	        t        j                  |d|z  fi |	| _        t        j                  ||fi |	| _        || _        || _        y)a  
        Args:
        - dim, dim_out: The input and output feature dimensions.
        - heads: The number of attention heads.
        - q_stride: If greater than 1, pool q with this stride. The stride should be flattened (e.g., 2x2 = 4).
        - window_size: The current (flattened) size of a mask unit *after* pooling (if any).
        - use_mask_unit_attn: Use Mask Unit or Global Attention.
        devicedtypeg         N)rR   rS   rx   ry   rz   r{   head_dimscaler   rw   r$   Linearqkvprojr|   r}   )rU   rx   ry   rz   r{   r|   r}   r   r   ddrW   s             r)   rS   zMaskUnitAttention.__init__  s    & /
 5(]]d*
(*99S!g+44IIgw5"5	&"4r+   r8   r"   c                 R   |j                   \  }}}| j                  r|| j                  | j                  z  z  }| j	                  |      j                  |d|d| j                  | j                        j                  dddddd      }|j                  d      \  }}}	| j                  dkD  r'|j                  || j                  || j                  d| j                        j                  d      }n| j	                  |      j                  ||d| j                  | j                        j                  ddddd      }|j                  d      \  }}}	| j                  dkD  rC|j                  || j                  | j                  d| j                        j                  d      }|j                         |j                         |	j                         }	}}| j                  rt        j                  |||	      }n9|| j                   z  |j#                  dd	      z  }
|
j%                  d      }
|
|	z  }| j                  r.|j#                  dd      j                  |d| j&                        }n-|j#                  dd      j                  |d| j&                        }| j)                  |      }|S )
z5 Input should be of shape [batch, tokens, channels]. r;   r   r      r/   r      rx   )r2   r}   r{   r|   r   rB   rz   r   rA   unbindr=   amax
contiguousrw   r3   scaled_dot_product_attentionr   	transposesoftmaxry   r   )rU   r8   rD   rs   r\   num_windowsr   qkvattns              r)   ra   zMaskUnitAttention.forward+  s2   ''1a""0@0@ @AK((1+%%2{Atzz4==gaAq!Q'  jjmGAq!}}q FF1djj+t}}b$--X]]bc]d ((1+%%aAtzz4==IQQRSUVXY[\^_`CjjmGAq!}}q FF1djj$--T]]KPPUVPW llnallnalln!qA??..q!Q7A

Nakk"b&99D<<B<'DqA ""Aq!))!R>AAq!))!R>AIIaLr+   )r   r   FNN)rc   rd   re   rf   rh   jitFinalbool__annotations__rg   rS   ri   ra   rj   rk   s   @r)   rv   rv      s     		%%  ',!5!5 !5 	!5
 !5 !5 !%!5F, ,%,, ,r+   rv   c                       e Zd Zdddej                  ej
                  ddddddfded	ed
edededee   de	ej                     de	ej                     dedededef fdZdej                  dej                  fdZ xZS )
HieraBlock      @        Nr   r   TFrx   ry   rz   	mlp_ratio	drop_pathinit_values
norm_layer	act_layerr{   r|   use_expand_projr}   c                    ||d}t         |           || _        || _         ||fi || _        ||k7  r8d| _        |rt        j                  ||fi || _        n ||dz  k(  sJ d | _        nd| _        d | _        t        ||||	|
|fi || _
        |t        |fd|i|nt        j                         | _        |dkD  rt        |      nt        j                         | _         ||fi || _        t#        |t%        ||z        fd|i|| _        |t        |fd|i|nt        j                         | _        |dkD  rt        |      | _        y t        j                         | _        y )Nr   Tr/   Fr   r   r   )rR   rS   rx   ry   norm1	do_expandr$   r   r   rv   r   r   r%   ls1r   
drop_path1norm2r   rg   mlpls2
drop_path2)rU   rx   ry   rz   r   r   r   r   r   r{   r|   r   r}   r   r   r   rW   s                   r)   rS   zHieraBlock.__init__[  sn   " /*r*
'>!DNIIc79b9	#'))) 	"DNDI%
 
	 JUI`:gE;E"Efhfqfqfs1:Q(9-BKKM.2.
wGi$7 8TITQSTITI`:gE;E"Efhfqfqfs1:Q(9-BKKMr+   r8   r"   c           
      z   | j                  |      }| j                  r)| j                  d| j                  |      }|j                  |j                  d   | j
                  j                  d|j                  d         j                  d      }nt        j                  |j                  |j                  d   | j
                  j                  d|j                  d         j                  d      |j                  |j                  d   | j
                  j                  d|j                  d         j                  d      gd      }|| j                  | j                  | j                  |                  z   }|| j                  | j                  | j                  | j!                  |                        z   }|S )Nr   r;   r   r   )r   r   r   r=   r2   r   r{   r   rh   catmeanr   r   r   r   r   r   )rU   r8   x_norms      r)   ra   zHieraBlock.forward  sT   A>>yy$IIf%FF1771:tyy'9'92qwwr{KPPUVPWIIFF1771:tyy'9'92qwwr{KPPUVPWFF1771:tyy'9'92qwwr{KPPUVPW 	 6): ;<< $**Q-)@ ABBr+   )rc   rd   re   r$   	LayerNormGELUrg   r5   r   r   Moduler   rS   rh   ri   ra   rj   rk   s   @r)   r   r   Z  s      #"+/*,,,)+ $(',0R0R 0R 	0R
 0R 0R "%0R RYY0R BII0R 0R 0R "0R !%0Rd %,, r+   r   c                        e Zd ZdZ	 	 	 ddededeedf   deedf   deedf   def fd	Z	 dd
ej                  de
ej                     dej                  fdZ xZS )
PatchEmbedzHPatch embed that supports any number of spatial dimensions (1d, 2d, 3d).dim_inry   kernel.stridepaddingrB   c	                     ||d}	t         
|           t        |      | _        || _         t        | j                        ||f|||d|	| _        y )Nr   )kernel_sizer   r   )rR   rS   r1   spatial_dimsrB   r*   r   )rU   r   ry   r   r   r   rB   r   r   r   rW   s             r)   rS   zPatchEmbed.__init__  sf     /K.GD--.
 
 
	r+   r8   r-   r"   c                 V   |Lt        |j                  dd  |      }| j                  ||j                  t        j
                        z        }n| j                  |      }| j                  r=|j                  |j                  d   |j                  d   d      j                  dd      }|S )Nr/   r6   r   r   r;   )r7   r2   r   torh   r   rB   r   rU   r8   r-   s      r)   ra   zPatchEmbed.forward  s    
 #$GD		!dggejj112A		!A<<		!''!*aggaj"5??1EAr+   )TNNrQ   )rc   rd   re   rf   rg   r   r   rS   rh   ri   r   ra   rj   rk   s   @r)   r   r     s    R !

 
 #s(O	

 #s(O
 38_
 
8 ,0|| 5<<( 
	r+   r   c            =           e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d>deedf   dededededed	eedf   d
edeedf   deedf   deedf   dedededeedf   deedf   deedf   dededee   dedede	ee
ej                     f   dedededededeeef   f: fd Zd! Zej                   j"                  d"        Zej                   j"                  d?d#ed$efd%       Zej                   j"                  d@d&ed$dfd'       Zej                   j"                  d(        ZdAdedee   d)efd*Zd+ej0                  d,ed$ej0                  fd-Zd$ej0                  fd.Z	 	 	 	 	 	 	 dBd+ej0                  d/eej0                     d0ee	eee   f      d1ed2ed3ed4ed#ed$e	eej0                     eej0                  eej0                     f   f   fd5Z	 	 	 	 dCd0e	eee   f   d6ed7ed#efd8Z	 	 dAd+ej0                  d/eej0                     d9ed$ej0                  fd:Zd?d;ed$ej0                  fd<Z	 dDd+ej0                  d/eej0                     d$ej0                  fd=Z  xZ!S )Er    Nimg_size.in_chans	embed_dim	num_headsnum_classesglobal_poolstagesro   r{   mask_unit_sizemask_unit_attnr   dim_mulhead_mulpatch_kernelrN   patch_paddingr   drop_path_rater   fix_initweight_initr   	drop_ratepatch_drop_ratehead_init_scalesep_pos_embedabs_win_pos_embedglobal_pos_sizec                  x   t         /|           ||d} || _        || _        d| _        t        |      }t        |t              rt        |      }|| _	        t        ||      D !"cg c]
  \  }!}"|!|"z   c}"}!| _        t        j                  | j                        }#t        j                  |
      }$t        j                  |	      }%|t        |      k  sJ ||	c| _        | _        |$|
c| _        | _        t        | j                  | j$                        D !"cg c]
  \  }!}"|!|"z   c}"}!| _        t)        dt        |      dz         D !cg c]  }!t+        |d |!       dz
   c}!| _        || _        t1        |||||fi | | _        d | _        d | _        d | _        d | _        |rt=        j>                  tA        jB                  d| j                  d   | j                  d   z  |fi |       | _        t=        j>                  tA        jB                  d| j                  d   |fi |       | _        n|ret=        j>                  tA        jB                  d|g|i |       | _        t=        j>                  tA        jB                  d|g|
i |       | _        n0t=        j>                  tA        jB                  d|#|fi |       | _        tE        |||	gt        | j,                  d d       z        | _#        tI        |||	gt        | j,                  d d       z  | j,                  |      | _%        | j,                  d | D &cg c]  }&|&dz   	 }'}&d}(t+        |      })tM        ||)      }*t=        jN                         | _(        g | _)        t)        |)      D ]  }!|}+||(   },|!dz
  | j,                  v r*t        ||z        }+t        ||z        }|(dz  }(|!|'v r|$|%z  }$tU        d||+|||*|!   |||!|'v r|%nd|$||,d| }-|+}|!| j,                  v r8| xjR                  tW        |+d|(dz   z  d| j,                  |(    	      gz  c_)        | jP                  jY                  |-        |x| _-        | _.        t_        ||f|||d
d| | _0        |rWt<        jb                  je                  | j8                  d       t<        jb                  je                  | j:                  d       nn| j4                  +t<        jb                  je                  | j4                  d       | j6                  +t<        jb                  je                  | j6                  d       |dk7  r*|dk(  rtf        nth        }.tk        |.d      }.tm        |.|        |r| jo                          t        | j`                  jp                  t<        jr                        rs| j`                  jp                  jt                  jv                  jy                  |       | j`                  jp                  jz                  jv                  jy                  |       y y c c}"}!w c c}"}!w c c}!w c c}&w )Nr   Fr   r/   r   r;   )rx   ry   rz   r   r   r   r   r{   r|   r   r}   zblocks.)num_chs	reductionmoduleNLC)	pool_typer   r   	input_fmtg{Gz?)stdskipjaxhead.fc)classifier_name )>rR   rS   r   r   grad_checkpointingr   
isinstancerg   r   rN   r<   tokens_spatial_shaperZ   r[   r1   ro   r{   mu_sizer   mask_spatial_shaper?   r>   rn   r   r   patch_embed	pos_embedpos_embed_winpos_embed_spatialpos_embed_temporalr$   	Parameterrh   zerosrL   unrollrm   rerollr   
ModuleListblocksfeature_infor   dictappendnum_featureshead_hidden_sizer   headinittrunc_normal_r   r   r   r   fix_init_weightfcr   weightdatamul_bias)0rU   r   r   r   r   r   r   r   ro   r{   r   r   r   r   r   r   rN   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rV   rF   
num_tokensflat_mu_sizeflat_q_strider8   q_pool_blocks	cur_stagedepthdprry   r}   blockinit_fnrW   s0                                                  r)   rS   zHiera.__init__  s   F 	/& "'#J/
h$ *H(8;Hl8S$T1Q!V$T!YYt889
yy0		(+F###%+X"T],8.)d)69$:S:SUYUhUh6i"jda16"j8=aVq8QR13vbqz?Q.R.%
 
 26599=:>%'\\At88;d>W>WXY>ZZ\elikl&D" ')llAt88;YM"M'D# !!#ekk!Y._._\^._!`%'\\%++a2b^2b_a2b%c"!#ekk!Z.YVX.Y!Z JT__Sb122

 JT__Sb122OO
 )-(@A1QAA 	F'>mmou  	&AG "0	!:1u'i'12	H 45	Q	% ]2L #a&'%+,+=-1( /#5 E  IDOO#!!A	!4DwW[WfWfgpWqVrMst&v v!KKu%A 	&D 5>=D1+
 "!
 
	 GG!!$"8"8d!CGG!!$"9"9t!D~~)%%dnn$%?!!-%%d&8&8d%C& )4)=o?GgyAG&  "diillBII.IILL$$))/:IILL""''8 /u %U #kRZ Bs   &X&X,
X2;X7c                    d }t        | j                        D ]m  \  }} ||j                  j                  j                  j
                  |dz           ||j                  j                  j                  j
                  |dz          o y )Nc                 R    | j                  t        j                  d|z               y )N       @)div_rZ   sqrt)param	_layer_ids     r)   rescalez&Hiera.fix_init_weight.<locals>.rescale{  s    JJtyyy12r+   r   )	enumerater   r   r   r  r  r   fc2)rU   r  layer_idlayers       r)   r   zHiera.fix_init_weightz  si    	3  )5 	=OHeEJJOO**//A>EIIMM((--x!|<	=r+   c                 H    | j                   dgS | j                  ddgS ddgS )Nr   pos_embed_absr   r   r   )r   r  rU   s    r)   no_weight_decayzHiera.no_weight_decay  s7    >>%= +#_55')=>>r+   coarser"   c                      t        dddg      S )NzW^pos_embed|pos_embed_spatial|pos_embed_temporal|pos_embed_abs|pos_embed_win|patch_embed)z^blocks\.(\d+)N)z^norm)i )stemr   )r   )rU   r  s     r)   group_matcherzHiera.group_matcher  s    k-/CD
 	
r+   enablec                     || _         y rQ   )r   )rU   r"  s     r)   set_grad_checkpointingzHiera.set_grad_checkpointing  s
    "(r+   c                 .    | j                   j                  S rQ   )r   r   r  s    r)   get_classifierzHiera.get_classifier  s    yy||r+   reset_otherc                 N    || _         | j                  j                  |||       y )Nr'  )r   r   reset)rU   r   r   r'  s       r)   reset_classifierzHiera.reset_classifier  s     &		[kJr+   r8   
mask_ratioc                    |j                   d   }t        j                  | j                        }t	        |d|z
  z        }t        j                  |||j                        }t        j                  |d      }t        j                  |d      }t        j                  ||g|j                        }	d|	ddd|f<   t        j                  |	d|      }	|	j                         S )z
        Generates a random mask, mask_ratio fraction are dropped.
        1 is *keep*, 0 is *remove*. Useful for MAE, FLIP, etc.
        r   r   )r   r   N)rx   index)r2   rZ   r[   r   rg   rh   randr   argsortr   gatherr   )
rU   r8   r,  rD   r   len_keepnoiseids_shuffleids_restorer-   s
             r)   get_random_maskzHiera.get_random_mask  s    
 GGAJii 7 78{a*n56

1k!((; mmEq1mmKQ7 {{A{+AHH=Q		\||Da{;yy{r+   c                 &   | j                   || j                   j                  | j                        }t        j                  | j
                  |j                  dd  dd      }||z   }|j                  d      j                  dd      }n| j
                  | j
                  }nj| j                  j                  d| j                  d   d      t        j                  | j                  | j                  d   | j                  d   z  d      z   }||z   }|S )	Nr   bicubicT)r0   mode	antialiasr/   r   r   r   )r   tiler   r3   r4   r   r2   rY   r   r   repeatr   rh   repeat_interleaver   )rU   r8   r   r   s       r)   
_pos_embedzHiera._pos_embed  s
   ) !..33D4K4KLM"((-	I "M1I!))!,66q!<I^^'I &&--a1J1J11MqQ''++--a043L3LQ3OO  	Mr+   r-   indicesnorm
stop_early
output_fmtintermediates_onlyc	           	      X   |rJ d       |dv sJ d       |rNt        t        | j                        |      \  }	}
|	D cg c]  }| j                  |    }	}| j                  |
   }
n"t        t        | j                        |      \  }	}
|, |j                  |j
                  d   dg| j                   }nd}| j                  ||      }| j                  |      }| j                  |      }|[||d   j                  d| j                  |j
                  d	            j	                  |j
                  d   d
|j
                  d
         }g }t        j                  j                         s|s| j                  }n| j                  d|
dz    }t        |      D ]  \  }}| j                   r+t        j                  j                         st#        ||      }n ||      }||	v sJ| j%                  |||      }|j'                  |dk(  r|j)                  dddd	      n|        |r|S ||fS c c}w )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        z'normalization of features not supported)NCHWNHWCz(Output format must be one of NCHW, NHWC.Nr   r   r-   .Nr/   r;   rE  r   )r   r1   rn   r   r=   r2   r   r   r>  r   r;  r   rh   r   is_scriptingr  r   r   r   r   rA   )rU   r8   r-   r?  r@  rA  rB  rC  r  take_indices	max_indexrV   
patch_maskintermediatesr   blkx_ints                    r)   forward_intermediateszHiera.forward_intermediates  s   . BBBx--Y/YY-&:3t;OQX&Y#L)8DE1DOOA.ELE	2I&:3t{{;KW&U#L)"1771:qK43J3JKJJQZ0OOAKKN $y/&&q$,,
CDII!''RS*VXZ[ZaZabdZefA99!!#:[[F[[)a-0F' 	cFAs&&uyy/E/E/GsA&FL Aqt4$$*PVBVU]]1aA%>\ab	c   -E Fs   H'
prune_norm
prune_headc                    |r2t        t        | j                        |      \  }}| j                  |   }n"t        t        | j                        |      \  }}| j                  d|dz    | _        |r| j                  j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   Tr)  )r   r1   rn   r   r   r*  )rU   r?  rQ  rR  r  rJ  rK  s          r)   prune_intermediate_layerszHiera.prune_intermediate_layers  s}     &:3t;OQX&Y#L)	2I&:3t{{;KW&U#L)kk.9q=1IIOOA4O0r+   return_intermediatesc                 4   | j                   r0| j                  dkD  r!|J | j                  || j                        }|, |j                  |j                  d   dg| j
                   }nd}| j                  ||      }| j                  |      }| j                  |      }|[||d   j                  d| j                  |j                  d            j                  |j                  d   d|j                  d         }g }t        | j                        D ]y  \  }}| j                  r+t        j                  j!                         st#        ||      }n ||      }|sH|| j$                  v sW|j'                  | j)                  |||             { |r||fS |S )	z
        mask should be a boolean tensor of shape [B, #MUt*#MUy*#MUx] where #MU are the number of mask units in that dim.
        Note: 1 in mask is *keep*, 0 is *remove*; mask.sum(dim=-1) should be the same across the batch.
        r   N)r,  r   rG  rH  r/   r;   )trainingr   r6  r=   r2   r   r   r>  r   r;  r   r  r   r   rh   r   rI  r   rn   r   r   )rU   r8   r-   rU  rL  rM  rV   rN  s           r)   forward_featureszHiera.forward_features$  s    ==T11A5<<''d6J6J'KD"1771:qK43J3JKJJQZ0OOAKKN $y/&&q$,,
CDII!''RS*VXZ[ZaZabdZefA, 	CFAs&&uyy/E/E/GsA&F#T__(<$$T[[AD[%AB	C  m##r+   
pre_logitsc                 V    |r| j                  ||      }|S | j                  |      }|S )N)rY  )r   )rU   r8   rY  s      r)   forward_headzHiera.forward_headQ  s1    3=DIIaJI/ DH99Q<r+   c                 R    | j                  ||      }|| j                  |      }|S )NrG  )rX  r[  r   s      r)   ra   zHiera.forwardU  s3    
 !!!$!/<!!!$Ar+   ))   r]  r   `   r     avgr/   r      r   r   )r/   r/   )   rc  )TTFFTr  r  )   rd  )r   r   )r   r   r   r   NT r   r   r   gMbP?FF)   rf  NNF)T)NF)NNFTrE  FT)r   FTTrQ   )"rc   rd   re   r   rg   strr   r5   r   r	   r   r$   r   rS   r   rh   r   ignorer  r   r!  r$  r&  r+  ri   r6  r>  r   rP  rT  rX  r[  ra   rj   rk   s   @r)   r    r      s    )3#$&3(..4/I$( !,2,2-3"$'+/!!6A"%(%*"'&+/7Ci9CHoi9 i9 	i9
 i9 i9 i9 #s(Oi9 i9 CHoi9 "#s(Oi9 "$),i9 "i9 i9  !i9"  S/#i9$  S/%i9& !c?'i9( )i9* "+i9, "%-i9. /i90 1i92 c4		?233i94 5i96 #7i98 #9i9:  ;i9<  $=i9> #38_?i9V= YY? ? YY
D 
T 
 
 YY)T )T ) ) YY KC Khsm Kae K 5 U\\ 0u|| > ,07;#$',= ||=  5<<(=  eCcN34	= 
 =  =  =  !%=  =  
tELL!5tELL7I)I#JJ	K= B ./$#3S	>*  	
 , ,0).	+||+ 5<<(+ #'	+
 
+Z$ 5<<  ,0|| 5<<( 
	r+   c                 4    | ddd dddt         t        dddd	|S )
Nr_  )r   r]  r]  g?r8  Tzpatch_embed.projr   z
apache-2.0)urlr   rM   	pool_sizecrop_pctinterpolationfixed_input_sizer   r   
first_conv
classifierlicenser
   )rk  kwargss     r)   _cfgrt  `  s5    =t%.B(	  r+   zhiera_tiny_224.mae_in1k_ft_in1kztimm/zcc-by-nc-4.0)	hf_hub_idrr  zhiera_tiny_224.mae)ru  rr  r   z hiera_small_224.mae_in1k_ft_in1kzhiera_small_224.maezhiera_base_224.mae_in1k_ft_in1kzhiera_base_224.maez$hiera_base_plus_224.mae_in1k_ft_in1kzhiera_base_plus_224.maez hiera_large_224.mae_in1k_ft_in1kzhiera_large_224.maezhiera_huge_224.mae_in1k_ft_in1kzhiera_huge_224.maez.hiera_small_abswin_256.sbb2_e200_in12k_ft_in1k)r      rv  gffffff?)ru  rM   rm  z1hiera_small_abswin_256.sbb2_pd_e200_in12k_ft_in1kz&hiera_small_abswin_256.sbb2_e200_in12ki-.  )ru  r   rM   rm  z)hiera_small_abswin_256.sbb2_pd_e200_in12kzhiera_base_abswin_256.untrained)rM   rm  c                 0   | j                  d|       } i }| j                         D ]n  \  }}d|v r|j                  dd      }|j                  d      r|j                  dd      }n#|j                  d      r|j                  dd      }|dk(  rd}|||<   p |S )	Nmodel_statezhead.projection.zhead.fc.zencoder_norm.z
head.norm.znorm.r  r   )getitemsreplace
startswith)
state_dictmodeloutputr   r   s        r)   checkpoint_filter_fnr    s    z:JF  " 1 "		,j9A<<(		/<8A\\'"		'<0AAq	%& Mr+   variant
pretrainedc                 n    |j                  dd      }t        t        | |ft        t	        |d      d|S )Nout_indicesr   getter)r  feature_cls)pretrained_filter_fnfeature_cfg)popr   r    r  r   )r  r  rs  r  s       r)   _create_hierar    sF    **]A.K 2[hG  r+   c           	      L    t        ddd      }t        dd| it        |fi |S )Nr^  r   )r   r/   rd  r/   r   r   r   r  )hiera_tiny_224r   r  r  rs  
model_argss      r)   r  r    s.    aEJ_j_DD^W]D^__r+   c           	      L    t        ddd      }t        dd| it        |fi |S )Nr^  r   r   r/      r/   r  r  )hiera_small_224r  r  s      r)   r  r    s.    aFJ`z`T*E_X^E_``r+   c           	      L    t        ddd      }t        dd| it        |fi |S )Nr^  r   ra  r  r  )hiera_base_224r  r  s      r)   r  r    s.    aFJ_j_DD^W]D^__r+   c           	      L    t        ddd      }t        dd| it        |fi |S )Np   r/   ra  r  r  )hiera_base_plus_224r  r  s      r)   r  r    s.    qGJd:djIc\bIcddr+   c           	      L    t        ddd      }t        dd| it        |fi |S )N   r/   r/      $   r   r  r  )hiera_large_224r  r  s      r)   r  r     s.    qGJ`z`T*E_X^E_``r+   c           	      L    t        ddd      }t        dd| it        |fi |S )Nrv  r   r  r  r  )hiera_huge_224r  r  s      r)   r  r    s.    qGJ_j_DD^W]D^__r+   c           
      V    t        dddddddd	      }t        dd
| it        |fi |S )Nr^  r   r  T)rb  rb  h㈵>r   F)r   r   r   r   r   r   r   r   r  )hiera_small_abswin_256r  r  s      r)   r  r    s@    -4aieUJ gjgDQ[Lf_eLfggr+   c           	      R    t        dddddd      }t        d	d| it        |fi |S )
Nr^  r   ra  Tr  r   )r   r   r   r   r   r   r  )hiera_base_abswin_256r  r  s      r)   r  r    s;    -4]aotvJfZf4PZKe^dKeffr+   )re  rQ   rg  )Grf   rZ   	functoolsr   typingr   r   r   r   r   r	   rh   torch.nnr$   torch.nn.functional
functionalr3   	timm.datar   r   timm.layersr   r   r   r   r   r   r   r   r   r   r   	_registryr   r   _builderr   	_featuresr   _features_fxr   _manipulater   r   __all__rg   r   r*   ri   r7   rJ   rL   rm   rv   r   r   r    rt  default_cfgsr  rh  r   r  r  r  r  r  r  r  r  r  r   r+   r)   <module>r     s  0   ; ;     A    = * + 3 0 )=s =tBII = 	$s) 	5<< 	ELL 	 	<<Cy s) \\	B;RYY ;|NRYY NbX		 XvE EP( (VPBII Pf	 % S&%t(S&
 $S& ')S& 4S&* &t(+S&2 $3S&> +D-?S&F t GS&R ')SS&Z 4[S&f &t(gS&n $oS&z 5d 47{S&B 8 4:CS&J -d 4/KS&T 0 42US&^ &t 4(_S& Sl2
3 
D 
u 
 ` `
 a a
 ` `
 e e
 a a
 ` `
 h h g gr+   