Ë
    éÓ^jY  ã            
       ó–  — d Z ddlZddlZddlmZmZ ddlZddlmZ ddl	mc m
Z ddlmZ ddlmZ ddlmZ ddlmZ  eej*                  j-                  d	d«      «      dkD  Z	 	 	 d(d
eeef   deeeef      dedej2                  fd„Zdeeef   deedf   fd„Z	 	 d)dedefd„Zdeeef   deedf   fd„Z G d„ dej>                  «      Z 	 	 	 	 d*deeef   deeef   fd„Z! G d„ dej>                  «      Z"	 	 	 d+dedee   fd „Z#d!ed"ed#ej2                  d$ej2                  dej2                  f
d%„Z$ G d&„ d'ej>                  «      Z%y),zf Relative position embedding modules and functions

Hacked together by / Copyright 2022 Ross Wightman
é    N)ÚOptionalÚTupleé   )Úndgrid)ÚRegularGridInterpolator)ÚMlp)Útrunc_normal_ÚTIMM_USE_SCIPY_INTERPÚq_sizeÚk_sizeÚclass_tokenÚreturnc           	      ó¨  — |�J d«       ‚t        j                  t        t        j                  | d   |¬«      t        j                  | d   |¬«      «      «      j	                  d«      }|d d …d d …d f   |d d …d d d …f   z
  }|j                  ddd«      }|d d …d d …dfxx   | d   dz
  z  cc<   |d d …d d …dfxx   | d   dz
  z  cc<   |d d …d d …dfxx   d| d   z  dz
  z  cc<   d| d   z  dz
  d| d   z  dz
  z  }|j                  d«      }|r5t        j                  |g d¢«      }||ddd …f<   |dz   |dd …df<   |dz   |d<   |j                  «       S )	Nz-Different q & k sizes not currently supportedr   ©Údevicer   é   éÿÿÿÿ)r   r   r   r   ©r   r   )
ÚtorchÚstackr   ÚarangeÚflattenÚpermuteÚsumÚFÚpadÚ
contiguous)r   r   r   r   ÚcoordsÚrelative_coordsÚnum_relative_distanceÚrelative_position_indexs           úd/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/pos_embed_rel.pyÚgen_relative_position_indexr#      s…  € ð ˆ>ÐJÐJÓJˆ>ä�[‰[œÜ�‰�V˜A‘Y vÔ.Ü�‰�V˜A‘Y vÔ.óó ÷ �wˆqƒzð ð šQ¢ 4˜ZÑ(¨6²!°Tº1°*Ñ+=Ñ=€OØ%×-Ñ-¨a°°AÓ6€OØ’A’q˜!�GÓ  q¡	¨A¡Ñ-ÓØ’A’q˜!�GÓ  q¡	¨A¡Ñ-ÓØ’A’q˜!�GÓ  F¨1¡I¡°Ñ 1Ñ1ÓØ ¨¡™]¨QÑ.°1°v¸a±y±=À1Ñ3DÑEÐð0 .×1Ñ1°"Ó5Ðáô #$§%¡%Ð(?ÂÓ"NÐØ)>Ð  1¡2 Ñ&Ø)>ÀÑ)BÐ ¡ A Ñ&Ø(=ÀÑ(AÐ Ñ%à"×-Ñ-Ó/Ð/ó    Únew_window_sizeÚnew_bias_shape.c           	      ól  — |d   dz  dz
  |d   dz  dz
  f}| j                   dk(  r|\  }}}| j                  \  }}}	||d   k(  r||d   k(  sJ ‚||k7  s|	|k7  rKt        j                  j                  j                  | j                  d«      |dd¬«      j                  d«      } | S | j                   dk(  sJ ‚|\  }
}| j                  \  }}|
|d   |d   z  z
  }t        ||z
  dz  «      }||f}|d   |d   k7  s|d   |d   k7  r±|r| | d …d d …f   }| d | …d d …f   } nd }t        j                  j                  j                  | j                  dd«      j                  dd	|d   |d   f«      |dd¬«      j                  d	|
|z
  «      j                  dd«      } |�t        j                  | |fd¬
«      } | S )Nr   r   r   é   ÚbicubicF)ÚsizeÚmodeÚalign_cornersç      à?r   ©Údim)ÚndimÚshaper   ÚnnÚ
functionalÚinterpolateÚ	unsqueezeÚsqueezeÚintÚ	transposeÚreshapeÚviewÚcat)Úrel_pos_biasr%   r&   Údst_sizeÚ_Údst_hÚdst_wÚnum_attn_headsÚsrc_hÚsrc_wÚdst_num_posÚsrc_num_posÚnum_extra_tokensÚsrc_sizeÚextra_tokenss                  r"   Ú resize_rel_pos_bias_table_simplerI   M   s#  € ð
   Ñ" QÑ&¨Ñ*¨O¸AÑ,>ÀÑ,BÀQÑ,FÐG€HØ×Ñ˜AÒà(‰ˆˆ5�%Ø'3×'9Ñ'9Ñ$ˆ˜˜uØ˜ ™Ò#¨°¸!±Ò(<Ð<Ð<Ø�EŠ>˜U eš^Ü Ÿ8™8×.Ñ.×:Ñ:Ø×&Ñ& qÓ)ØØØ#ð	 ;ó ÷
 ‰g�a‹jð ð@ Ðð3 × Ñ  AÒ%Ð%Ð%à'‰ˆ�QØ&2×&8Ñ&8Ñ#ˆ�^Ø&¨(°1©+¸À¹Ñ*CÑDÐÜ˜Ð&6Ñ6¸3Ñ>Ó?ˆØ˜hÐ'ˆà�A‰;˜( 1™+Ò%¨°!©¸À¹Ò)CÙØ+Ð-=Ð,=Ñ,>ÂÐ,AÑB�Ø+Ð,>Ð.>Ð->Ð,>ÂÐ,AÑB‘à#�ä Ÿ8™8×.Ñ.×:Ñ:Ø×&Ñ& q¨!Ó,×4Ñ4°a¸¸XÀa¹[È(ÐSTÉ+Ð5VÓWØØØ#ð	 ;ó ÷
 ‰d�2�{Ð%5Ñ5Ó6·y±yÀÀA³ð ð Ð'Ü$Ÿy™y¨,¸Ð)EÈ1ÔM�àÐr$   ÚinterpolationÚ	antialiasc                 ó¢  — | j                  «       \  }}|\  }}||k(  sJ ‚||k7  rª| j                  }| j                  «       } t        |dz  «      }	t        |dz  «      }
t	        j
                  | j                  dd«      j                  d||	|	«      |
|
f||¬«      }|j                  ||«      j                  dd«      }|j                  |«       |S | S )z’
    Resample relative position bias table suggested in LeVit
    Adapted from: https://github.com/microsoft/Cream/blob/main/TinyViT/utils.py
    r-   r   r   )r*   r+   rK   )	r*   ÚdtypeÚfloatr7   r   r4   r   r:   Úto)Úposition_bias_tableÚnew_sizerJ   rK   ÚL1ÚnH1ÚL2ÚnH2Ú
orig_dtypeÚS1ÚS2Ú$relative_position_bias_table_resizeds               r"   Úresize_rel_pos_bias_table_levitrZ   |   sä   € ð "×&Ñ&Ó(�G€BˆØ�G€BˆØ�#Š:Ðˆ:Ø	ˆR‚xØ(×.Ñ.ˆ
Ø1×7Ñ7Ó9Ðä��s‘‹^ˆÜ��s‘‹^ˆÜ/0¯}©}Ø×'Ñ'¨¨1Ó-×2Ñ2°1°c¸2¸rÓBØ�b�ØØô	0
Ð,ð 0T×/XÑ/XÐY\Ð^`Ó/a×/iÑ/iÐjkÐmnÓ/oÐ,Ø,×/Ñ/°
Ô;Ø3Ð3à"Ð"r$   c                 ó  ‡— t         rddlm} |d   dz  dz
  |d   dz  dz
  f}| j                  dk(  r1d}|\  }}}||d   k(  r||d   k(  sJ ‚| j                  \  }	}
}|
|f}d}nJ| j                  dk(  sJ ‚|\  }}| j                  \  }}	||d   |d   z  z
  }t        ||z
  dz  «      }||f}d}|d   |d   k7  s|d   |d   k7  �r:|r| | d	…d	d	…f   }| d	| …d	d	…f   } nd	}d
„ Šˆfd„} ||d   |d   «      } ||d   |d   «      }t        j                  |«      t        j                  |«      g}|d   dz  }|d   dz  }t        j                  | |dz   d«      }t        j                  | |dz   d«      }t        ||«      }g }t        |	«      D �]  }|r.| d	d	…|f   j                  |d   |d   «      j                  «       }n| |d	d	…d	d	…f   j                  «       }t         rgj                  |||j                  «       d¬«      }t        j                   |||«      «      j!                  «       j#                  | j$                  «      }n;t'        ||«      } ||«      j!                  «       j#                  | j$                  «      }|r|j                  dd«      }|j)                  |«       �Œ |rt        j*                  |d¬«      } nt        j*                  |d¬«      } |�|sJ ‚t        j*                  | |fd¬«      } | S )a�   Resize relative position bias table using more advanced interpolation.

    Modified from code in Microsoft Unilm (https://github.com/microsoft/unilm) repo (BeiT, BeiT-v2, etc).

    https://github.com/microsoft/unilm/blob/5255d52de86dad642810f5849dd357769346c1d7/beit/run_class_finetuning.py#L351

    Args:
        rel_pos_bias:
        new_window_size:
        new_bias_shape:

    Returns:

    r   )r4   r   r   r(   Fr-   TNc                 ó$   — | d||z  z
  z  d|z
  z  S )Nç      ð?© )ÚaÚrÚns      r"   Úgeometric_progressionz8resize_rel_pos_bias_table.<locals>.geometric_progressionÍ   s   € Ø˜˜a 1™f™Ñ%¨¨q©Ñ1Ð1r$   c                 ó(  •— d\  }}||z
  dkD  r+||z   dz  } ‰d|| dz  «      }||dz  kD  r|}n|}||z
  dkD  rŒ+g }d}t        | dz  «      D ]  }|j                  |«       ||dz   z  z  }Œ  t        |«      D �	cg c]  }	|	 ‘Œ }
}	|
dgz   |z   S c c}	w )N)g)\�Âõ(ð?g      ø?g�íµ ÷Æ°>ç       @r   r   r   )ÚrangeÚappendÚreversed)ÚsrcÚdstÚleftÚrightÚqÚgpÚdisÚcurÚir>   Úr_idsrb   s              €r"   Ú_calcz(resize_rel_pos_bias_table.<locals>._calcÐ   sÌ   ø€ Ø#‰KˆD�%Ø˜$‘, Ò%Ø˜E‘\ SÑ(�Ù*¨1¨a°¸±Ó:�Ø˜˜q™’=Ø‘Eà�Dð ˜$‘, Ó%ð ˆCØˆCÜ˜3 !™8“_ò $�Ø—
‘
˜3”Ø�q˜Q ™U‘|Ñ#‘ð$ô "*¨#£Ö/˜A�a’RÐ/ˆEÐ/Ø˜A˜3‘; Ñ$Ð$ùò 0s   Á:
Brd   gš™™™™™¹?r]   Úcubic)Úkindr   r.   )Ú
_USE_SCIPYÚscipyr4   r0   r1   r7   r   Útensorr   r   re   r:   rN   Úinterp2dÚnumpyÚTensorr   rO   r   r   rf   r;   ) r<   r%   r&   r4   r=   rF   r>   r?   r@   rA   rB   rC   rG   Úhas_flat_shaperD   rE   rH   rr   ÚyÚxÚyxÚtyÚtxÚdyÚdxÚdyxÚall_rel_pos_biasrp   ÚzÚfr`   rb   s                                   @r"   Úresize_rel_pos_bias_tabler‡   œ   sJ  ø€ õ& Ý%à Ñ" QÑ&¨Ñ*¨O¸AÑ,>ÀÑ,BÀQÑ,FÐG€HØ×Ñ˜AÒàÐØ(‰ˆˆ5�%Ø˜ ™Ò#¨°¸!±Ò(<Ð<Ð<Ø'3×'9Ñ'9Ñ$ˆ˜˜uØ˜5�>ˆØ‰à× Ñ  AÒ%Ð%Ð%à'‰ˆ�QØ&2×&8Ñ&8Ñ#ˆ�^Ø&¨(°1©+¸À¹Ñ*CÑDÐÜ˜Ð&6Ñ6¸3Ñ>Ó?ˆØ˜hÐ'ˆØˆà��{�h˜q‘kÒ! X¨a¡[°H¸Q±KÓ%?áØ'Ð)9Ð(9Ñ(:ºAÐ(=Ñ>ˆLØ'Ð(:Ð*:Ð):Ð(:ºAÐ(=Ñ>‰LàˆLò	2ô	%ñ$ �(˜1‘+˜x¨™{Ó+ˆÙ�(˜1‘+˜x¨™{Ó+ˆÜ�l‰l˜1‹oœuŸ|™|¨A›Ð/ˆð �a‰[˜CÑˆØ�a‰[˜CÑˆÜ�\‰\˜2˜#˜r C™x¨Ó-ˆÜ�\‰\˜2˜#˜r C™x¨Ó-ˆÜ�R˜‹nˆð ÐÜ�~Ó&ó 	'ˆAÙØ ¢ A Ñ&×+Ñ+¨H°Q©K¸À!¹ÓE×KÑKÓM‘à  ¢A¢q Ñ)×/Ñ/Ó1�åà×(Ñ(¨¨A¨q¯w©w«y¸wÐ(ÓG�Ü—L‘L¡ 2 r£Ó+×6Ñ6Ó8×;Ñ;¸L×<OÑ<OÓP‘ô
 ,¨B°Ó2�Ù�c“F×%Ñ%Ó'×*Ñ*¨<×+>Ñ+>Ó?�áØ—F‘F˜2˜q“M�Ø×#Ñ# AÖ&ð'	'ñ* Ü Ÿ9™9Ð%5¸2Ô>‰Lä Ÿ9™9Ð%5¸1Ô=ˆLàÐ#Ù!Ð!�>Ü Ÿ9™9 l°LÐ%AÀqÔIˆLàÐr$   c                   ó¢   ‡ — e Zd ZdZ	 	 	 ddeeef   dedefˆ fd„Zdd„Zdd„Zde	j                  fd	„Zdd
ee	j                     fd„Zdd„Zˆ xZS )Ú
RelPosBiasz_ Relative Position Bias
    Adapted from Swin-V1 relative position bias impl, modularized.
    Úwindow_sizeÚ	num_headsÚprefix_tokensc                 óú  •— ||dœ}t         ‰	| �  «        |dk  sJ ‚|| _        |d   |d   z  | _        || _        | j                  |z   fdz  |fz   | _        d|d   z  dz
  d|d   z  dz
  z  d|z  z   }t        j                  t        j                  ||fi |¤Ž«      | _
        | j                  |z   dz  }| j                  dt        j                  ||t        j                  ¬«      d¬«       | j                  «        y )	N©r   rM   r   r   r   r(   r!   F©Ú
persistent)ÚsuperÚ__init__rŠ   Úwindow_arearŒ   Ú
bias_shaper2   Ú	Parameterr   ÚemptyÚrelative_position_bias_tableÚregister_bufferÚlongÚreset_parameters)
ÚselfrŠ   r‹   rŒ   r   rM   Úddr    Ú
index_sizeÚ	__class__s
            €r"   r’   zRelPosBias.__init__  s  ø€ ð ¨Ñ/ˆÜ‰ÑÔØ Ò!Ð!Ð!Ø&ˆÔØ& q™>¨K¸©NÑ:ˆÔØ*ˆÔØ×+Ñ+¨mÑ;Ð=ÀÑAÀYÀLÑPˆŒà!" [°¡^Ñ!3°aÑ!7¸AÀÈAÁÑ<NÐQRÑ<RÑ SÐVWÐZgÑVgÑ gÐÜ,.¯L©L¼¿¹ÐEZÐ\eÑ9lÐikÑ9lÓ,mˆÔ)Ø×&Ñ&¨Ñ6¸1Ñ<ˆ
Ø×ÑØ%Ü�K‰K˜
¨6¼¿¹ÔDØð 	ô 	
ð 	×ÑÕr$   r   c                 óR   — t        | j                  d¬«       | j                  «        y©ú"Initialize parameters and buffers.g{®Gáz”?)ÚstdN)r	   r—   Ú_init_buffers©r›   s    r"   rš   zRelPosBias.reset_parameters1  s   € ä�d×7Ñ7¸SÕAØ×ÑÕr$   c                 óÆ   — | j                   j                  t        | j                  | j                  dkD  | j                   j
                  ¬«      j                  d«      «       y)ú.Compute and fill non-persistent buffer values.r   )r   r   r   N)r!   Úcopy_r#   rŠ   rŒ   r   r:   r¤   s    r"   r£   zRelPosBias._init_buffers6  sO   € à×$Ñ$×*Ñ*Ü'Ø× Ñ Ø ×.Ñ.°Ñ2Ø×3Ñ3×:Ñ:ô÷ ‰d�2‹hõ	
r$   c                 óÊ   — | j                   | j                     }|j                  | j                  «      j	                  ddd«      }|j                  d«      j                  «       S )Nr   r   r   )r—   r!   r:   r”   r   r5   r   ©r›   Úrelative_position_biass     r"   Úget_biaszRelPosBias.get_bias@  sZ   € Ø!%×!BÑ!BÀ4×C_ÑC_Ñ!`Ðà!7×!<Ñ!<¸T¿_¹_Ó!M×!UÑ!UÐVWÐYZÐ\]Ó!^ÐØ%×/Ñ/°Ó2×=Ñ=Ó?Ð?r$   Úshared_rel_posc                 ó(   — || j                  «       z   S ©N©r«   ©r›   Úattnr¬   s      r"   ÚforwardzRelPosBias.forwardF  ó   € Ø�d—m‘m“oÑ%Ð%r$   c                 ó$   — | j                  «        y©z"Initialize non-persistent buffers.N©r£   r¤   s    r"   Úinit_non_persistent_buffersz&RelPosBias.init_non_persistent_buffersI  ó   € à×ÑÕr$   ©r   NN©r   Nr®   ©Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r7   r’   rš   r£   r   rz   r«   r   r²   r·   Ú__classcell__©rž   s   @r"   r‰   r‰     sr   ø„ ñð "#ØØñ à˜s C˜x™ð ð ð ð õ	 ó8ó

ð@˜%Ÿ,™,ó @ñ&¨H°U·\±\Ñ,Bó &÷r$   r‰   Úwin_sizeÚpretrained_win_sizec                 óÀ  — |dv sJ ‚t        j                  | d   dz
   | d   |¬«      j                  t         j                  «      }t        j                  | d   dz
   | d   |¬«      j                  t         j                  «      }t        j                  t        ||«      «      }|j                  ddd«      j                  «       }|dk(  rÉ|d   dkD  r5|d d …d d …dfxx   |d   dz
  z  cc<   |d d …d d …dfxx   |d   dz
  z  cc<   n4|d d …d d …dfxx   | d   dz
  z  cc<   |d d …d d …dfxx   | d   dz
  z  cc<   |dz  }t        j                  |«      t        j                  d|j                  «       z   «      z  t        j                  d«      z  }n<t        j                  |«      t        j                  d|j                  «       z   «      z  }|j                  |«      S )	N)ÚswinÚcrr   r   r   r   rÅ   é   r]   )r   r   rO   Úfloat32r   r   r   r   ÚsignÚlog2ÚabsÚmathÚlog)rÂ   rÃ   r+   r   rM   Úrelative_coords_hÚrelative_coords_wÚrelative_coords_tables           r"   Úgen_relative_log_coordsrÑ   N  sØ  € ð �>Ñ!Ð!Ð!äŸ™ x°¡{°Q¡Ð%7¸À!¹ÈVÔT×WÑWÔX]×XeÑXeÓfÐÜŸ™ x°¡{°Q¡Ð%7¸À!¹ÈVÔT×WÑWÔX]×XeÑXeÓfÐÜ!ŸK™K¬Ð/@ÐBSÓ(TÓUÐØ1×9Ñ9¸!¸QÀÓB×MÑMÓOÐØˆv‚~Ø˜qÑ! AÒ%Ø!¢!¢Q¨ 'Ó*Ð/BÀ1Ñ/EÈÑ/IÑJÓ*Ø!¢!¢Q¨ 'Ó*Ð/BÀ1Ñ/EÈÑ/IÑJÔ*à!¢!¢Q¨ 'Ó*¨x¸©{¸Q©Ñ?Ó*Ø!¢!¢Q¨ 'Ó*¨x¸©{¸Q©Ñ?Ó*Ø Ñ"ÐÜ %§
¡
Ð+@Ó AÄEÇJÁJØÐ'×+Ñ+Ó-Ñ-óE/ñ !/Ü15·±¸1³ñ!>Ñô !&§
¡
Ð+@Ó AÄEÇIÁIØÐ'×+Ñ+Ó-Ñ-óE/ñ !/Ðð !×#Ñ# EÓ*Ð*r$   c                   óÀ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 ddeeef   dededededeeef   fˆ fd„Zd	ej                  fd
„Z
ddeej                     fd„Zdd„Zdd„Zdd„Zˆ xZS )Ú	RelPosMlpzß Log-Coordinate Relative Position MLP
    Based on ideas presented in Swin-V2 paper (https://arxiv.org/abs/2111.09883)

    This impl covers the 'swin' implementation as well as two timm specific modes ('cr', and 'rw')
    rŠ   r‹   Ú
hidden_dimrŒ   r+   Úpretrained_window_sizec	           	      óö  •— ||dœ}	t         ‰| �  «        || _        | j                  d   | j                  d   z  | _        || _        || _        | j                  fdz  |fz   | _        || _        || _        |dk(  r#t        j                  «       | _        d| _        d}
n"t        j                  «       | _        d | _        d}
t        	 d||t        j                  |
d	d
œ|	¤Ž| _        | j                  dz  }d|d   z  dz
  d|d   z  dz
  df}| j#                  dt%        j&                  ||t$        j(                  ¬«      d¬«       | j#                  dt%        j&                  |fi |	¤Žd¬«       | j+                  «        y )NrŽ   r   r   r   rÅ   é   )TFT)g      À?g        )Úhidden_featuresÚout_featuresÚ	act_layerÚbiasÚdropr!   Fr�   Úrel_coords_log)r   )r‘   r’   rŠ   r“   rŒ   r‹   r”   r+   rÕ   r2   ÚSigmoidÚbias_actÚ	bias_gainÚIdentityr   ÚReLUÚmlpr˜   r   r–   r™   rš   )r›   rŠ   r‹   rÔ   rŒ   r+   rÕ   r   rM   rœ   Úmlp_biasr�   Úrel_coords_shaperž   s                €r"   r’   zRelPosMlp.__init__s  sŒ  ø€ ð ¨Ñ/ˆÜ‰ÑÔØ&ˆÔØ×+Ñ+¨AÑ.°×1AÑ1AÀ!Ñ1DÑDˆÔØ*ˆÔØ"ˆŒØ×+Ñ+Ð-°Ñ1°Y°LÑ@ˆŒØˆŒ	Ø&<ˆÔ#Ø�6Š>ÜŸJ™J›LˆDŒMØˆDŒNØ$‰HäŸK™K›MˆDŒMØ!ˆDŒNØˆHäØð
à&Ø"Ü—g‘gØØñ
ð ñ
ˆŒð ×%Ñ%¨Ñ*ˆ
Ø ¨A¡Ñ.°Ñ2°A¸ÀA¹Ñ4FÈÑ4JÈAÐNÐØ×ÑØ%Ü�K‰K˜
¨6¼¿¹ÔDØð 	ô 	
ð
 	×ÑØÜ�K‰KÐ(Ñ/¨BÑ/Øð 	ô 	
ð 	×ÑÕr$   r   c                 ó  — | j                  | j                  «      }| j                  �D|j                  d| j                  «      | j                     }|j                  | j
                  «      }|j                  ddd«      }| j                  |«      }| j                  �| j                  |z  }| j                  r.t        j                  || j                  d| j                  dg«      }|j                  d«      j                  «       S )Nr   r   r   r   )rã   rÝ   r!   r:   r‹   r”   r   rß   rà   rŒ   r   r   r5   r   r©   s     r"   r«   zRelPosMlp.get_biasª  sç   € Ø!%§¡¨$×*=Ñ*=Ó!>ÐØ×'Ñ'Ð3Ø%;×%@Ñ%@ÀÀTÇ^Á^Ó%TÐUY×UqÑUqÑ%rÐ"Ø%;×%@Ñ%@ÀÇÁÓ%QÐ"Ø!7×!?Ñ!?ÀÀ1ÀaÓ!HÐØ!%§¡Ð/EÓ!FÐØ�>‰>Ð%Ø%)§^¡^Ð6LÑ%LÐ"Ø×ÒÜ%&§U¡UÐ+AÀD×DVÑDVÐXYÐ[_×[mÑ[mÐopÐCqÓ%rÐ"Ø%×/Ñ/°Ó2×=Ñ=Ó?Ð?r$   r¬   c                 ó(   — || j                  «       z   S r®   r¯   r°   s      r"   r²   zRelPosMlp.forward·  r³   r$   c                 ó$   — | j                  «        y)r¡   Nr¶   r¤   s    r"   rš   zRelPosMlp.reset_parametersº  r¸   r$   c           	      óh  — | j                   j                  }| j                  j                  }| j                   j	                  t        | j                  |¬«      j                  d«      «       | j                  j	                  t        | j                  | j                  | j                  ||¬«      «       y)r¦   r   r   )r+   r   rM   N)r!   r   rÝ   rM   r§   r#   rŠ   r:   rÑ   rÕ   r+   ©r›   r   rM   s      r"   r£   zRelPosMlp._init_buffers¾  s�   € à×-Ñ-×4Ñ4ˆØ×#Ñ#×)Ñ)ˆØ×$Ñ$×*Ñ*Ü'¨×(8Ñ(8ÀÔH×MÑMÈbÓQô	
ð 	×Ñ×!Ñ!Ü#Ø× Ñ Ø×+Ñ+Ø—Y‘YØØôõ	
r$   c                 ó$   — | j                  «        yrµ   r¶   r¤   s    r"   r·   z%RelPosMlp.init_non_persistent_buffersÏ  r¸   r$   )rÇ   é€   r   rÆ   r   NNr®   rº   )r¼   r½   r¾   r¿   r   r7   Ústrr’   r   rz   r«   r   r²   rš   r£   r·   rÀ   rÁ   s   @r"   rÓ   rÓ   m  s¥   ø„ ñð Ø!Ø!"ØØ6<ØØñ5 à˜s C˜x™ð5 ð ð5 ð ð	5 ð
 ð5 ð ð5 ð %*¨#¨s¨(¡Oõ5 ðn@˜%Ÿ,™,ó @ñ&¨H°U·\±\Ñ,Bó &óó
÷"r$   rÓ   ÚlengthÚmax_relative_positionc                 óÜ   — |€| dz
  }d|z  dz   }t        j                  | | |||¬«      }t        | «      D ]4  }t        | «      D ]$  }||z
  |z   }t        ||z
  «      |kD  rŒd||||f<   Œ& Œ6 |S )aÏ  Generate a one_hot lookup tensor to reindex embeddings along one dimension.

    Args:
        length: the length to reindex to.
        max_relative_position: the maximum relative position to consider.
            Relative position embeddings for distances above this threshold
            are zeroed out.
    Returns:
        a lookup Tensor of size [length, length, vocab_size] that satisfies
            ret[n,m,v] = 1{m - n + max_relative_position = v}.
    r   r   rŽ   )r   Úzerosre   rË   )	rî   rï   r   rM   Ú
vocab_sizeÚretrp   r}   Úvs	            r"   Úgenerate_lookup_tensorrõ   Ô  s™   € ð" Ð$Ø &¨¡
ÐàÐ*Ñ*¨QÑ.€JÜ
�+‰+�f˜f j¸ÀuÔ
M€CÜ�6‹]ò ˆÜ�v“ò 	ˆAØ�A‘Ð-Ñ-ˆAÜ�1�q‘5‹zÐ1Ò1ØØˆC��1�a�ŠLñ		ðð €Jr$   ÚheightÚwidthÚheight_lookupÚwidth_lookupc                 ó¨   — t        j                  d| |«      }t        j                  d||«      }||z  }|j                  | j                  d   ||«      S )a\  Reindex 2d relative position bias with 2 independent einsum lookups.

    Adapted from:
     https://github.com/google-research/maxvit/blob/2e06a7f1f70c76e64cd3dabe5cd1b8c1a23c9fb7/maxvit/models/attention_utils.py

    Args:
        relative_position_tensor: tensor of shape
            [..., vocab_height, vocab_width, ...].
        height: height to reindex to.
        width: width to reindex to.
        height_lookup: one-hot height lookup
        width_lookup: one-hot width lookup
    Returns:
        reindexed_tensor: a Tensor of shape
            [..., height * width, height * width, ...]
    znhw,ixh->nixwznixw,jyw->nijxyr   )r   Úeinsumr9   r1   )Úrelative_position_tensorrö   r÷   rø   rù   Úreindexed_tensorÚareas          r"   Úreindex_2d_einsum_lookuprÿ   ó  sW   € ô. —|‘| OÐ5MÈ}Ó]ÐÜ—|‘|Ð$5Ð7GÈÓVÐØ�E‰>€DØ×#Ñ#Ð$<×$BÑ$BÀ1Ñ$EÀtÈTÓRÐRr$   c                   ó¢   ‡ — e Zd ZdZ	 	 	 ddeeef   dedefˆ fd„Zdd„Zdd„Zde	j                  fd	„Zdd
ee	j                     fd„Zdd„Zˆ xZS )ÚRelPosBiasTfzÜ Relative Position Bias Impl (Compatible with Tensorflow MaxViT models)
    Adapted from:
     https://github.com/google-research/maxvit/blob/2e06a7f1f70c76e64cd3dabe5cd1b8c1a23c9fb7/maxvit/models/attention_utils.py
    rŠ   r‹   rŒ   c                 ó&  •— ||dœ}t         ‰| �  «        |dk  sJ ‚|| _        |d   |d   z  | _        || _        d|d   z  dz
  }d|d   z  dz
  }| j                  ||f| _        t        j                  t        j                  | j
                  fi |¤Ž«      | _
        |d   |d   |f}	|d   |d   |f}
| j                  dt        j                  |	fi |¤Žd¬«       | j                  dt        j                  |
fi |¤Žd¬«       | j                  «        y )	NrŽ   r   r   r   rø   Fr�   rù   )r‘   r’   rŠ   r“   r‹   r”   r2   r•   r   r–   r—   r˜   rš   )r›   rŠ   r‹   rŒ   r   rM   rœ   Úvocab_heightÚvocab_widthÚheight_lookup_shapeÚwidth_lookup_shaperž   s              €r"   r’   zRelPosBiasTf.__init__  s+  ø€ ð ¨Ñ/ˆÜ‰ÑÔØ Ò!Ð!Ð!Ø&ˆÔØ& q™>¨K¸©NÑ:ˆÔØ"ˆŒà˜; q™>Ñ)¨AÑ-ˆØ˜+ a™.Ñ(¨1Ñ,ˆØŸ>™>¨<¸ÐEˆŒÜ,.¯L©L¼¿¹ÀTÇ_Á_Ñ9[ÐXZÑ9[Ó,\ˆÔ)Ø*¨1™~¨{¸1©~¸|ÐLÐØ)¨!™n¨k¸!©n¸kÐJÐØ×Ñ˜_¬e¯k©kÐ:MÑ.TÐQSÑ.TÐafÐÔgØ×Ñ˜^¬U¯[©[Ð9KÑ-RÈrÑ-RÐ_dÐÔeð 	×ÑÕr$   r   c                 óz   — t         j                  j                  | j                  d¬«       | j	                  «        yr    )r2   ÚinitÚnormal_r—   r£   r¤   s    r"   rš   zRelPosBiasTf.reset_parameters0  s(   € ä
�‰�‰˜×9Ñ9¸sˆÔCØ×ÑÕr$   c                 ó,  — | j                   j                  }| j                   j                  }| j                   j                  t	        | j
                  d   ||¬«      «       | j                  j                  t	        | j
                  d   ||¬«      «       y)r¦   r   rŽ   r   N)rø   r   rM   r§   rõ   rŠ   rù   rê   s      r"   r£   zRelPosBiasTf._init_buffers5  sz   € à×#Ñ#×*Ñ*ˆØ×"Ñ"×(Ñ(ˆØ×Ñ× Ñ Ô!7¸×8HÑ8HÈÑ8KÐTZÐbgÔ!hÔiØ×Ñ×ÑÔ 6°t×7GÑ7GÈÑ7JÐSYÐafÔ gÕhr$   c                 ó�   — t        | j                  | j                  d   | j                  d   | j                  | j                  «      S )Nr   r   )rÿ   r—   rŠ   rø   rù   r¤   s    r"   r«   zRelPosBiasTf.get_bias<  sF   € ä'Ø×-Ñ-Ø×Ñ˜QÑØ×Ñ˜QÑØ×ÑØ×Ñó
ð 	
r$   r¬   c                 ó(   — || j                  «       z   S r®   r¯   r°   s      r"   r²   zRelPosBiasTf.forwardF  r³   r$   c                 ó$   — | j                  «        yrµ   r¶   r¤   s    r"   r·   z(RelPosBiasTf.init_non_persistent_buffersI  r¸   r$   r¹   rº   r®   r»   rÁ   s   @r"   r  r    sq   ø„ ñð "#ØØñ à˜s C˜x™ð ð ð ð õ	 ó6ó
ið
˜%Ÿ,™,ó 
ñ&¨H°U·\±\Ñ,Bó &÷r$   r  )NFN)r)   T)r   rÅ   NN)NNN)&r¿   rÌ   ÚosÚtypingr   r   r   Útorch.nnr2   Útorch.nn.functionalr3   r   Úgridr   r4   r   rã   r   Úweight_initr	   r7   ÚenvironÚgetru   Úboolrz   r#   rI   rí   rZ   r‡   ÚModuler‰   rÑ   rÓ   rõ   rÿ   r  r^   r$   r"   ú<module>r     sá  ðñó Û 	ß "ã Ý ß Ð å Ý 0Ý Ý &á�—‘—‘Ð 7¸Ó;Ó<¸qÑ@€
ð
 -1Ø!Øñ	50Ø�c˜3�h‘ð50à˜˜s C˜x™Ñ)ð50ð ð50ð
 ‡\�\ó50ðp,à˜s C˜x™ð,ð ˜c 3˜h™ó,ðd 'Øñ	#ð ð#ð ó	#ð@qà˜s C˜x™ðqð ˜c 3˜h™óqôh;�—‘ô ;ð@ 06ØØØñ+Ø˜˜S˜‘/ð+à" 3¨ 8™_ó+ô>d�—	‘	ô dðR 04ØØñ	Øðà'¨™}óð>SàðSð ðSð —|‘|ð	Sð
 —l‘lðSð ‡\�\óSô:;�2—9‘9õ ;r$   