Ë
    éÓ^jÇ; ã                   ó@B  — d Z ddlZddlZddlZddlZddlmZ ddlmZ ddl	m
Z
mZmZmZmZmZmZmZmZ 	 ddl	mZ ddlZddlmZ ddlmc mZ ddlmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$ dd	l%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z= d
dl>m?Z? d
dl@mAZA d
dlBmCZCmDZDmEZEmFZF d
dlGmHZHmIZImJZJ dgZK ej˜                  eM«      ZNe&e&e'dœZO	 	 	 	 	 	 	 	 �d×de<dePdePdeQdeQdeQdeQdeRdeRdeeej¦                        dePdej¦                  fd„ZT G d„ d ej¦                  «      ZU G d!„ d"ej¦                  «      ZV G d#„ d$ej¦                  «      ZW G d%„ d&ej¦                  «      ZX G d'„ d(ej¦                  «      ZY	 	 	 �dØd)ej´                  d*e[d+ePd,eQfd-„Z\ G d.„ dej¦                  «      Z]�dÙd0ej¦                  d1e[d2eQddfd3„Z^	 	 	 �dÚd0ej¦                  d1e[d4eRd2eQddf
d5„Z_�dÙd0ej¦                  d1e[d2eQddfd6„Z`�dÙd0ej¦                  d1e[d2eQddfd7„Za�dÛd8e[d4eRd2eQdefd9„Zb	 	 	 	 �dÜd;ej´                  d<ej´                  d+ePd=eePePf   d>e[d?eQdej´                  fd@„Zc ejÈ                  «       �dÝdAe]dBe[dCe[dDeQddf
dE„«       Ze	 �dÞdFee[ej´                  f   dAe]dCe[dee[ej´                  f   fdG„ZfdFee[ej´                  f   dAe]dee[ej´                  f   fdH„ZgdFee[ej´                  f   dAe]dee[ej´                  f   fdI„ZhdFeifdJ„Zj	 	 	 �dßdFee[ej´                  f   dAe]dKeQd>e[d?eQdee[ej´                  f   fdL„Zk�dàdMe[dee[e
f   fdN„Zli dO eldP¬Q«      “dR eldP¬Q«      “dS eldTdPd¬U«      “dV eldWdPddXdY¬Z«      “d[ eld\dPd¬U«      “d] eld^dPddXdY¬Z«      “d_ eld`dPd¬U«      “da eldbdPddXdY¬Z«      “dc eldddPd¬U«      “de eldfdPddXdY¬Z«      “dg eldhdPd¬U«      “di eldjdPddXdY¬Z«      “dk eldldPd¬U«      “dm eldndPd¬U«      “do eldpdPddXdY¬Z«      “dq eldrdP¬s«      “dt eldudPdXdY¬v«      “i dw eldxdPdXdY¬v«      “dy eldzdPd¬U«      “d{ eld|dPddXdY¬Z«      “d} eld~dPd¬U«      “d eld€dPddXdY¬Z«      “d� eld‚dPd¬U«      “dƒ eld„dPddXdY¬Z«      “d… eld/¬†«      “d‡ eld/¬†«      “dˆ eld/¬†«      “d‰ eld/¬†«      “dŠ eldPd¬‹«      “dŒ eldPd¬‹«      “d� eldPd¬‹«      “dŽ eldPd¬‹«      “d� eldPd¬‹«      “d� eld‘dPdd’¬“«      “¥i d” eld•dPdd’¬“«      “d– eld—dPdd’¬“«      “d˜ eld™dPdd’¬“«      “dš eld›dPdd’¬“«      “dœ eld�dPdd’¬“«      “dž eldŸdPdd’¬“«      “d  eld¡ddP¬¢«      “d£ eld¤ddP¬¢«      “d¥ eld¦dPee d¬§«      “d¨ eld©dPee d¬§«      “dª eld«dPee d¬§«      “d¬ eld­dPee d¬§«      “d® eld¯dPd°ee dd±dY¬²«      “d³ eld´dPd°ee dd±dY¬²«      “dµ eld¶dPd°ee dd±dY¬²«      “d· eld¸dPd°ee dd±dY¬²«      “d¹ eldºdPd°ee dd±dY¬²«      “¥i d» eld¼dPd°ee dd±dY¬²«      “d½ eld¾dPd°ee dd±dY¬²«      “d¿ eldÀdPd°ee dd±dY¬²«      “dÁ eldPd°dÂdÃddÄdY¬Å«      “dÆ eldPd°dÂdÃddÄdY¬Å«      “dÇ eldPd°dÂdÃddÄdY¬Å«      “dÈ eldPd°dÂdÃddÄdY¬Å«      “dÉ eldÊdPdÂdÃdËdÌdÍ¬Î«      “dÏ eldÐdPdÂdÃdËdÌ¬Ñ«      “dÒ eldÓdP¬s«      “dÔ eldPdÕdÖd×¬Ø«      “dÙ eldPdÚdÖ¬Û«      “dÜ eldPdXdÖdÝ¬Þ«      “dß eldÚdÖ¬à«      “dá el«       “dâ eldPe#e$¬ã«      “dä eldPe#e$dYdX¬å«      “¥i dæ eldPe#e$dYdÄ¬å«      “dç eldPe#e$dÖ¬è«      “dé eldPe#e$dYdXdÝ¬ê«      “dë eldPe!e"dY¬è«      “dì eldPe!e"dYdídÝ¬ê«      “dî eldPe#e$dY¬è«      “dï eldPe#e$dYdídÝ¬ê«      “dð ele#e$¬ñ«      “dò eldPe#e$dÖdXdÝ¬ê«      “dó eldPe#e$dÖ¬è«      “dô eldPe#e$dÖdXdÝ¬ê«      “dõ eldPe#e$dY¬è«      “dö eldPe#e$dYdídÝ¬ê«      “d÷ eldPe#e$¬ã«      “dø eldPe#e$dY¬è«      “dù eldPe#e$dYdXdÝ¬ê«      “dú eldPe!e"dY¬è«      “¥i dû eldPe!e"dYdídÝ¬ê«      “dü eldPe#e$dY¬è«      “dý eld/e#e$dYdídÝ¬ê«      “dþ eldPe#e$¬ã«      “dÿ eldPe#e$¬ã«      “�d  eldPe#e$dYdXdÝ¬ê«      “�d eldPe#e$dY¬è«      “�d eldPe#e$d×�¬«      “�d eldPe!e"dYd×�¬«      “�d eldPe#e$dYd×�¬«      “�d eldPe#e$d×�¬«      “�d eldPe#e$dYd×�¬«      “�d	 eldPe#e$�d
�¬«      “�d eldPe#e$dY�d
�¬«      “�d eldPe!e"dY�d�¬«      “�d eldPe#e$dY�d�¬«      “�d eldPe#e$dY�d�¬«      “¥i �d eldPe#e$dY�d�¬«      “�d eldP�d�de#e$�d
�¬«      “�d eldP�de#e$dY�d
�¬«      “�d eldP�de#e$dÕdY�d�¬«      “�d eldP�de#e$dY�d�¬«      “�d eldPe#e$dY�d
�¬«      “�d eldPe#e$dYdÚ�d
�¬«      “�d  eldPe#e$dY�d
�¬«      “�d! eldPe#e$dY�d�¬«      “�d" eldP�d#e#e$dY�d
�¬«      “�d$ eldP�d#e#e$dY�d�¬«      “�d% eldP�d#�de#e$dY�d�¬&«      “�d' eldP�d#�de#e$dY�d�¬&«      “�d( eldPe#e$�d#�ddY�d)�d�¬*«      “�d+ eldP�d,�de#e$dY�d�¬&«      “�d- eldP�d,e#e$�d)dYdÝ�d�¬.«      “�d/ eldP�d,e#e$dY�d�¬«      “¥i �d0 eldP�d,e#e$�d)dYdÝ�d�¬.«      “�d1 eldP�d,�de#e$dY�d
�¬&«      “�d2 eldP�d,�de#e$dY�d
�¬&«      “�d3 eldP�d,�de#e$dY�d�¬&«      “�d4 eldP�d,�de#e$dY�d�¬&«      “�d5 eldP�d,e#e$dY�d�¬«      “�d6 eldP�d,�de#e$dY�d�¬&«      “�d7 eldP�d,�de#e$dY�d
�¬&«      “�d8 eldP�d,�de#e$dY�d
�¬&«      “�d9 eldP�d,�de#e$dY�d�¬&«      “�d: eldP�de#e$�d
�¬;«      “�d< eldP�de#e$�d
�¬;«      “�d= eldP�de#e$dY�d�¬>«      “�d? eldP�de#e$dYdí�d�¬@«      “�dA eldP�de#e$dY�dB�¬C«      “�dD eld/dÚdÖ�¬E«      “�dF eld/dÕdÖ�¬E«      “¥i �dG eld/¬†«      “�dH eld/¬†«      “�dI eld/¬†«      “�dJ eldP�de#e$�dKdY�¬L«      “�dM eldP�de#e$dídYdÝ�¬N«      “�dO eldP�de#e$�dKdY�¬L«      “�dP eldP�de#e$dídYdÝ�¬N«      “�dQ el�dRddPdÕdÖ�¬S«      “�dT el�dUddPdÕdÖ�¬S«      “�dV el�dWddPdÕdÖ�¬S«      “�dX el�dYddPdÕdÖ�¬S«      “�dZ el�d[ddPdÕdÖ�¬S«      “�d\ el�d]ddPdÕdÖ�¬S«      “�d^ el�d_ddPdÕdÖd’�¬`«      “�da el�dbddPdÕdÖd’�¬`«      “�dc el�ddddPdÕdÖ�¬S«      “�de el�dfddPdÕdÖ�¬S«      “¥i �dg el�dhddPdÕdÖ�¬S«      “�di el�djddPdÕdÖd’�¬`«      “�dk el�dlddPdÕdÖd’�¬`«      “�dm eld/¬†«      “�dn eld/¬†«      “�do eld/¬†«      “�dp el�dqdP�d,ee d�¬r«      “�ds el�dtdP�d,ee d�¬r«      “�du el�dvdP�d,ee d�¬r«      “�dw el�dx�d,ee d�¬y«      “�dz el�d{�d,ee d�¬y«      “�d| el�d}�d,dÄdYee d�¬~«      “�d el�d€�d,ee d�¬y«      “�d� eldPdÚd�¬‚«      “�dƒ eldPd¬‹«      “�d„ eldPd¬‹«      “�d… eldPdÚd�¬‚«      “¥i �d† eldPdÚd�¬‚«      “�d‡ eldPdÚd�¬‚«      “�dˆ eldPdXd�¬‚«      “�d‰ eldPdXd�¬‚«      “�dŠ eldP�d‹d�¬‚«      “�dŒ eldP�d‹d�¬‚«      “�d� eldPdÚd�¬‚«      “�dŽ eldPdÚd�¬‚«      “�d� eldPdXd�¬‚«      “�d� eldPdXd�¬‚«      “�d‘ eldP�d‹d�¬‚«      “�d’ eldPd¬‹«      “�d“ eldPd¬‹«      “�d” eldP�d)d�¬‚«      “�d• eldP�d)d�¬‚«      “�d– eldPdXd�¬‚«      “�d— eldPdÚd�¬‚«      “¥i �d˜ eldPdÚd�¬‚«      “�d™ eldPdXd�¬‚«      “�dš eldP�d‹d�¬‚«      “�d› eldPdÚd�¬‚«      “�dœ eldPdXd�¬‚«      “�d� eldPdÚd�¬‚«      “�dž eldPd¬‹«      “�dŸ eldPd¬‹«      “�d  eldPdÚd�¬‚«      “�d¡ eldPdÚd�¬‚«      “�d¢ eldPdÚd�¬‚«      “�d£ eldPdXd�¬‚«      “�d¤ eldPdXd�¬‚«      “�d¥ eldP�d‹d�¬‚«      “�d¦ eldP�d‹d�¬‚«      “�d§ eldPdÚd�¬‚«      “�d¨ eldPdÚd�¬‚«      “¥i �d© eldPdXd�¬‚«      “�dª eldPdXd�¬‚«      “�d« eldP�d‹d�¬‚«      “�d¬ eldPd¬‹«      “�d­ eldPd¬‹«      “�d® eldPd¬‹«      “�d¯ eldPd¬‹«      “�d° eldPd¬‹«      “�d± eldPd¬‹«      “�d² eldP�d)d�¬‚«      “�d³ eldP�d)dYd¬Ø«      “�d´ eldPdXdYd¬Ø«      “�dµ eldPdÄdYd¬Ø«      “�d¶ eldPdÄdYd¬Ø«      “�d· eldPdÄdYd¬Ø«      “�d¸ eldPdÄdYd¬Ø«      “�d¹ eldPdÄdYd¬Ø«      “¥i �dº eldPdÄdYd¬Ø«      “�d» eldPdÄdYd¬Ø«      “�d¼ eldPdÄdYd¬Ø«      “�d½ eldP�d¾dYd¬Ø«      “�d¿ eldP�d¾dYd¬Ø«      “�dÀ eldP�d¾dYd¬Ø«      “�dÁ eldP�d¾dYd¬Ø«      “�dÂ eldP�d¾dYd¬Ø«      “�dÃ eldPdÚd�¬‚«      “�dÄ eldPdÚd�¬‚«      “�dÅ eldPdXd�¬‚«      “�dÆ eldP�d‹d�¬‚«      “�dÇ eldPdÚd�¬‚«      “�dÈ eldPdXd�¬‚«      “�dÉ eldP�d)dYdÝ¬Þ«      “�dÊ eldP�d)dYdÝ¬Þ«      “�dË eldP�de#e$�d
�¬Ì«      “¥i �dÍ eldP�de#e$�d
�¬Ì«      “�dÎ eldP�de#e$�d
�¬Ì«      “�dÏ eldP�de#e$�d
�¬Ì«      “�dÐ eldPdÚdÖ¬Û«      “�dÑ eldPdÚdÖ¬Û«      “�dÒ eldPdÚdÖ¬Û«      “�dÓ eldPdÚdÖ¬Û«      “�dÔ eldPdÚdÖ¬Û«      “�dÕ eldPdÚdÖ¬Û«      “�dÖ eldPdÚdÖ¬Û«      “�d× eldPd×dÚdÖ�¬Ø«      “�dÙ eldPdÚdÖ¬Û«      “�dÚ eldPdÚdÖ¬Û«      “�dÛ eldPdÚdÖ¬Û«      “�dÜ eldPdÚdÖ¬Û«      “�dÝ eldPdÚdÖ¬Û«      “�dÞ eldPd×dÚdÖ�¬Ø«      “¥i �dß eldPdÚdÖ¬Û«      “�dà eldPdÚdÖ¬Û«      “�dá eldPd×dÚdÖ�¬Ø«      “�dâ eldPd×dÚdÖ�¬Ø«      “�dã eldPdXdY¬Û«      “�dä eldPdÚdÖ¬Û«      “�då eldPdÚdÖ¬Û«      “�dæ eldPdÚdÖ¬Û«      “�dç eldPdÚdÖ¬Û«      “�dè eldPd×dÚdÖ�¬Ø«      “�dé eldPd×dÚdÖ�¬Ø«      “�dê eldPdXdY¬Û«      “�dë eldÚ�¬ì«      “�dí eldPdÚdÖ¬Û«      “�dî eldPd×dÚdÖ�¬Ø«      “�dï eldPdXdY¬Û«      “�dð eldÚ�¬ì«      “¥i �dñ eldPdÚdY¬Û«      “�dò eldPd×dÚdY�¬Ø«      “�dó eldPdXdY¬Û«      “�dô eldPdÄdYdÝ¬Þ«      “�dõ eldP�dee dÄdYd�¬«      “�dö eldPee dÄdYd�¬÷«      “�dø eldPe#e$�d#dYd�¬ù«      “�dú eldPe#e$�d#dYd�¬ù«      “�dû eldPe#e$�d#dYd�¬ù«      “�dü eldPe#e$�d#dYd�¬ù«      “�dý eldPe#e$�d#dYd�¬ù«      “�dþ eldPe#e$�d#dídYd�¬ÿ«      “�d  eldPe#e$�d#dídYd�¬ÿ«      “�d eldPe#e$�d#dídYd�¬ÿ«      “�d eldPe#e$�d#dídYd�¬ÿ«      “�d eldPe#e$�d#dídYd�¬ÿ«      “�d eldPe#e$�d#dÄdYd�¬ÿ«      “¥i �d eldPe#e$�d#dÄdYd�¬ÿ«      “�d eldPe#e$�d#dÄdYd�¬ÿ«      “�d eldPe#e$�d#dÄdYd�¬ÿ«      “�d eldP�d	dÖ¬Û«      “�d
 eldP�d	dÖ¬Û«      “�d eldP�d	dÖ¬Û«      “�d el�d	dÖ¬à«      “�d eldPee dY¬è«      “�d eldPee dY¬è«      “�d eldPee dY¬è«      “�d eldPee dY¬è«      “�d eld/ee dY�¬«      “�d eld/díee dY�¬«      “�d eldPee dYd�¬«      “�d eldPee dYd�¬«      “�d eldPee dYd�¬«      “�d eldPee dYd�¬«      “¥ZmemjÝ                  «       D � �cg c](  \  } }|jß                  �dd:«      sŒ�d|�d   d   v sŒ'| ‘Œ* c}} ZpepD ]@  Zq ejä                  emeq   «      Zses�d   dPk(  r	dPeqz   es�d<   esemeqjé                  �d�d«      <   ŒB  eHem«      Zmejê                  jß                  �d�d«      jí                  «       �d k(  Zw	 	 �dá�d!e[�d"eQ�d#eeQ   dee]�d$f   f�d%„ZxeI�dâ�d"eQde]f�d&„«       ZyeI�dâ�d"eQde]f�d'„«       ZzeI�dâ�d"eQde]f�d(„«       Z{eI�dâ�d"eQde]f�d)„«       Z|eI�dâ�d"eQde]f�d*„«       Z}eI�dâ�d"eQde]f�d+„«       Z~eI�dâ�d"eQde]f�d,„«       ZeI�dâ�d"eQde]f�d-„«       Z€eI�dâ�d"eQde]f�d.„«       Z�eI�dâ�d"eQde]f�d/„«       Z‚eI�dâ�d"eQde]f�d0„«       ZƒeI�dâ�d"eQde]f�d1„«       Z„eI�dâ�d"eQde]f�d2„«       Z…eI�dâ�d"eQde]f�d3„«       Z†eI�dâ�d"eQde]f�d4„«       Z‡eI�dâ�d"eQde]f�d5„«       ZˆeI�dâ�d"eQde]f�d6„«       Z‰eI�dâ�d"eQde]f�d7„«       ZŠeI�dâ�d"eQde]f�d8„«       Z‹eI�dâ�d"eQde]f�d9„«       ZŒeI�dâ�d"eQde]f�d:„«       Z�eI�dâ�d"eQde]f�d;„«       ZŽeI�dâ�d"eQde]f�d<„«       Z�eI�dâ�d"eQde]f�d=„«       Z�eI�dâ�d"eQde]f�d>„«       Z‘eI�dâ�d"eQde]f�d?„«       Z’eI�dâ�d"eQde]f�d@„«       Z“eI�dâ�d"eQde]f�dA„«       Z”eI�dâ�d"eQde]f�dB„«       Z•eI�dâ�d"eQde]f�dC„«       Z–eI�dâ�d"eQde]f�dD„«       Z—eI�dâ�d"eQde]f�dE„«       Z˜eI�dâ�d"eQde]f�dF„«       Z™eI�dâ�d"eQde]f�dG„«       ZšeI�dâ�d"eQde]f�dH„«       Z›eI�dâ�d"eQde]f�dI„«       ZœeI�dâ�d"eQde]f�dJ„«       Z�eI�dâ�d"eQde]f�dK„«       ZžeI�dâ�d"eQde]f�dL„«       ZŸeI�dâ�d"eQde]f�dM„«       Z eI�dâ�d"eQde]f�dN„«       Z¡eI�dâ�d"eQde]f�dO„«       Z¢eI�dâ�d"eQde]f�dP„«       Z£eI�dâ�d"eQde]f�dQ„«       Z¤eI�dâ�d"eQde]f�dR„«       Z¥eI�dâ�d"eQde]f�dS„«       Z¦eI�dâ�d"eQde]f�dT„«       Z§eI�dâ�d"eQde]f�dU„«       Z¨eI�dâ�d"eQde]f�dV„«       Z©eI�dâ�d"eQde]f�dW„«       ZªeI�dâ�d"eQde]f�dX„«       Z«eI�dâ�d"eQde]f�dY„«       Z¬eI�dâ�d"eQde]f�dZ„«       Z­eI�dâ�d"eQde]f�d[„«       Z®eI�dâ�d"eQde]f�d\„«       Z¯eI�dâ�d"eQde]f�d]„«       Z°eI�dâ�d"eQde]f�d^„«       Z±eI�dâ�d"eQde]f�d_„«       Z²eI�dâ�d"eQde]f�d`„«       Z³eI�dâ�d"eQde]f�da„«       Z´eI�dâ�d"eQde]f�db„«       ZµeI�dâ�d"eQde]f�dc„«       Z¶eI�dâ�d"eQde]f�dd„«       Z·eI�dâ�d"eQde]f�de„«       Z¸eI�dâ�d"eQde]f�df„«       Z¹eI�dâ�d"eQde]f�dg„«       ZºeI�dâ�d"eQde]f�dh„«       Z»eI�dâ�d"eQde]f�di„«       Z¼eI�dâ�d"eQde]f�dj„«       Z½eI�dâ�d"eQde]f�dk„«       Z¾eI�dâ�d"eQde]f�dl„«       Z¿eI�dâ�d"eQde]f�dm„«       ZÀeI�dâ�d"eQde]f�dn„«       ZÁeI�dâ�d"eQde]f�do„«       ZÂeI�dâ�d"eQde]f�dp„«       ZÃeI�dâ�d"eQde]f�dq„«       ZÄeI�dâ�d"eQde]f�dr„«       ZÅeI�dâ�d"eQde]f�ds„«       ZÆeI�dâ�d"eQde]f�dt„«       ZÇeI�dâ�d"eQde]f�du„«       ZÈeI�dâ�d"eQde]f�dv„«       ZÉeI�dâ�d"eQde]f�dw„«       ZÊeI�dâ�d"eQde]f�dx„«       ZËeI�dâ�d"eQde]f�dy„«       ZÌeI�dâ�d"eQde]f�dz„«       ZÍeI�dâ�d"eQde]f�d{„«       ZÎeI�dâ�d"eQde]f�d|„«       ZÏeI�dâ�d"eQde]f�d}„«       ZÐeI�dâ�d"eQde]f�d~„«       ZÑeI�dâ�d"eQde]f�d„«       ZÒeI�dâ�d"eQde]f�d€„«       ZÓeI�dâ�d"eQde]f�d�„«       ZÔeI�dâ�d"eQde]f�d‚„«       ZÕeI�dâ�d"eQde]f�dƒ„«       ZÖeI�dâ�d"eQde]f�d„„«       Z×eI�dâ�d"eQde]f�d…„«       ZØeI�dâ�d"eQde]f�d†„«       ZÙeI�dâ�d"eQde]f�d‡„«       ZÚeI�dâ�d"eQde]f�dˆ„«       ZÛeI�dâ�d"eQde]f�d‰„«       ZÜeI�dâ�d"eQde]f�dŠ„«       ZÝeI�dâ�d"eQde]f�d‹„«       ZÞeI�dâ�d"eQde]f�dŒ„«       ZßeI�dâ�d"eQde]f�d�„«       ZàeI�dâ�d"eQde]f�dŽ„«       ZáeI�dâ�d"eQde]f�d�„«       ZâeI�dâ�d"eQde]f�d�„«       ZãeI�dâ�d"eQde]f�d‘„«       ZäeI�dâ�d"eQde]f�d’„«       ZåeI�dâ�d"eQde]f�d“„«       ZæeI�dâ�d"eQde]f�d”„«       ZçeI�dâ�d"eQde]f�d•„«       ZèeI�dâ�d"eQde]f�d–„«       ZéeI�dâ�d"eQde]f�d—„«       ZêeI�dâ�d"eQde]f�d˜„«       ZëeI�dâ�d"eQde]f�d™„«       ZìeI�dâ�d"eQde]f�dš„«       ZíeI�dâ�d"eQde]f�d›„«       ZîeI�dâ�d"eQde]f�dœ„«       ZïeI�dâ�d"eQde]f�d�„«       ZðeI�dâ�d"eQde]f�dž„«       ZñeI�dâ�d"eQde]f�dŸ„«       ZòeI�dâ�d"eQde]f�d „«       ZóeI�dâ�d"eQde]f�d¡„«       ZôeI�dâ�d"eQde]f�d¢„«       ZõeI�dâ�d"eQde]f�d£„«       ZöeI�dâ�d"eQde]f�d¤„«       Z÷eI�dâ�d"eQde]f�d¥„«       ZøeI�dâ�d"eQde]f�d¦„«       ZùeI�dâ�d"eQde]f�d§„«       ZúeI�dâ�d"eQde]f�d¨„«       ZûeI�dâ�d"eQde]f�d©„«       ZüeI�dâ�d"eQde]f�dª„«       ZýeI�dâ�d"eQde]f�d«„«       ZþeI�dâ�d"eQde]f�d¬„«       ZÿeI�dâ�d"eQde]f�d­„«       �Z eI�dâ�d"eQde]f�d®„«       �ZeI�dâ�d"eQde]f�d¯„«       �ZeI�dâ�d"eQde]f�d°„«       �ZeI�dâ�d"eQde]f�d±„«       �ZeI�dâ�d"eQde]f�d²„«       �ZeI�dâ�d"eQde]f�d³„«       �ZeI�dâ�d"eQde]f�d´„«       �ZeI�dâ�d"eQde]f�dµ„«       �ZeI�dâ�d"eQde]f�d¶„«       �Z	eI�dâ�d"eQde]f�d·„«       �Z
eI�dâ�d"eQde]f�d¸„«       �ZeI�dâ�d"eQde]f�d¹„«       �ZeI�dâ�d"eQde]f�dº„«       �ZeI�dâ�d"eQde]f�d»„«       �ZeI�dâ�d"eQde]f�d¼„«       �ZeI�dâ�d"eQde]f�d½„«       �ZeI�dâ�d"eQde]f�d¾„«       �ZeI�dâ�d"eQde]f�d¿„«       �ZeI�dâ�d"eQde]f�dÀ„«       �ZeI�dâ�d"eQde]f�dÁ„«       �ZeI�dâ�d"eQde]f�dÂ„«       �Z eJeMi �dÃd�“�dÄd”“�dÅd–“�dÆd˜“�dÇdš“�dÈdœ“�dÉd�“�dÊdž“�dËd�“�dÌ�dÍ“�dÎ�dÏ“�dÐd¥“�dÑd¨“�dÒdª“�dÓd¬“�dÔdÉ“�dÕ�d	“�d�d�d�dÖœ¥«       y# e$ r
 ddlmZ Y � ŒÚw xY wc c}} w (ã  aß   Vision Transformer (ViT) in PyTorch

A PyTorch implement of Vision Transformers as described in:

'An Image Is Worth 16 x 16 Words: Transformers for Image Recognition at Scale'
    - https://arxiv.org/abs/2010.11929

`How to train your ViT? Data, Augmentation, and Regularization in Vision Transformers`
    - https://arxiv.org/abs/2106.10270

`FlexiViT: One Model for All Patch Sizes`
    - https://arxiv.org/abs/2212.08013

The official jax code is released and available at
  * https://github.com/google-research/vision_transformer
  * https://github.com/google-research/big_vision

Acknowledgments:
  * The paper authors for releasing code and weights, thanks!
  * I fixed my class token impl based on Phil Wang's https://github.com/lucidrains/vit-pytorch
  * Simple transformer style inspired by Andrej Karpathy's https://github.com/karpathy/minGPT
  * Bert reference code checks against Huggingface Transformers and Tensorflow Bert

Hacked together by / Copyright 2020, Ross Wightman
é    N)ÚOrderedDict)Úpartial)	ÚAnyÚCallableÚDictÚOptionalÚSetÚTupleÚTypeÚUnionÚList)ÚLiteral)ÚFinal)ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STDÚIMAGENET_INCEPTION_MEANÚIMAGENET_INCEPTION_STDÚOPENAI_CLIP_MEANÚOPENAI_CLIP_STD)Ú	AttentionÚDiffAttentionÚAttentionPoolLatentÚAttentionPoolPrrÚ
PatchEmbedÚMlpÚSwiGLUPackedÚSwiGLUÚ	LayerNormÚRmsNormÚDropPathÚcalculate_drop_path_ratesÚPatchDropoutÚtrunc_normal_Úlecun_normal_Úresample_patch_embedÚresample_abs_pos_embedÚuse_fused_attnÚget_act_layerÚget_norm_layerÚmaybe_add_maskÚresolve_self_attn_maskÚ	LayerTypeÚ
LayerScaleé   )Úbuild_model_with_cfg)Úfeature_take_indices)Únamed_applyÚ
checkpointÚcheckpoint_seqÚadapt_input_conv)Úgenerate_default_cfgsÚregister_modelÚregister_model_deprecationsÚVisionTransformer)Ú ÚattnÚdiffTÚ
attn_layerÚdimÚ	num_headsÚqkv_biasÚqk_normÚ
scale_normÚ	proj_biasÚ	attn_dropÚ	proj_dropÚ
norm_layerÚdepthÚreturnc                 ó´   — t        | t        «      r"t        j                  | d «      } | €
J d| › �«       ‚t	        | t
        «      r|
|d<    | |f||||||||	dœ|¤ŽS )NzUnknown attn_layer: rF   )r>   r?   r@   rA   rB   rC   rD   rE   )Ú
isinstanceÚstrÚATTN_LAYERSÚgetÚ
issubclassr   )r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   Úkwargss               úi/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/vision_transformer.pyÚ_create_attnrP   \   sƒ   € ô �*œcÔ"Ü —_‘_ Z°Ó6ˆ
ØÐ%ÐJÐ)=¸j¸\Ð'JÓJÐ%ô �*œmÔ,Øˆˆw‰áØðàØØØØØØØñð ñð ó    c            %       óX  ‡ — e Zd ZdZddddddddddej
                  eeedddfde	d	e	d
e
dededededede
de
dee
   de
deej                     deej                     deej                     dede	ddf$ˆ fd„Z	 	 ddej$                  deej$                     dedej$                  fd„Zˆ xZS ) ÚBlockz)Transformer block with pre-normalization.ç      @FTç        Nr   r=   r>   Ú	mlp_ratior?   r@   Úscale_attn_normÚscale_mlp_normrB   rD   rC   Úinit_valuesÚ	drop_pathÚ	act_layerrE   Ú	mlp_layerr<   rF   rG   c                 ó8  •— t         ‰| �  «        ||dœ} ||fi |¤Ž| _        t        ||f||||||
|	||dœ	|¤Ž| _        |rt        |fd|i|¤Žnt        j                  «       | _        |dkD  rt        |«      nt        j                  «       | _
         ||fi |¤Ž| _         |d|t        ||z  «      ||r|nd||	dœ|¤Ž| _        |rt        |fd|i|¤Žnt        j                  «       | _        |dkD  rt        |«      | _        yt        j                  «       | _        y)af  Initialize Block.

        Args:
            dim: Number of input channels.
            num_heads: Number of attention heads.
            mlp_ratio: Ratio of mlp hidden dim to embedding dim.
            qkv_bias: If True, add a learnable bias to query, key, value.
            qk_norm: If True, apply normalization to query and key.
            proj_bias: If True, add bias to output projection.
            proj_drop: Projection dropout rate.
            attn_drop: Attention dropout rate.
            init_values: Initial values for layer scale.
            drop_path: Stochastic depth rate.
            act_layer: Activation layer.
            norm_layer: Normalization layer.
            mlp_layer: MLP layer.
            attn_layer: Attention layer type (class or string).
            depth: Block index, passed to attention layer for depth-dependent init.
        ©ÚdeviceÚdtype©	r>   r?   r@   rA   rB   rC   rD   rE   rF   rY   rU   N©Úin_featuresÚhidden_featuresr[   rE   ÚbiasÚdrop© )ÚsuperÚ__init__Únorm1rP   r:   r-   ÚnnÚIdentityÚls1r    Ú
drop_path1Únorm2ÚintÚmlpÚls2Ú
drop_path2©Úselfr=   r>   rV   r?   r@   rW   rX   rB   rD   rC   rY   rZ   r[   rE   r\   r<   rF   r_   r`   ÚddÚ	__class__s                        €rO   ri   zBlock.__init__ƒ   s=  ø€ ôR 	‰ÑÔØ¨Ñ/ˆá Ñ* rÑ*ˆŒ
Ü ØØð
ð  ØØØ&ØØØØ!Øñ
ð ñ
ˆŒ	ñ FQ”:˜cÑA¨{ÐA¸bÒAÔVX×VaÑVaÓVcˆŒØ1:¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒá Ñ* rÑ*ˆŒ
Ùð 
ØÜ  i¡Ó0ØÙ%3‘z¸ØØñ
ð ñ
ˆŒñ FQ”:˜cÑA¨{ÐA¸bÒAÔVX×VaÑVaÓVcˆŒØ1:¸R²œ( 9Ó-ˆ�ÄRÇ[Á[Ã]ˆ�rQ   ÚxÚ	attn_maskÚ	is_causalc           
      ó  — || j                  | j                  | j                  | j                  |«      ||¬«      «      «      z   }|| j	                  | j                  | j                  | j                  |«      «      «      «      z   }|S ©N©ry   rz   )rn   rm   r:   rj   rs   rr   rq   ro   ©ru   rx   ry   rz   s       rO   ÚforwardzBlock.forwardÎ   sj   € ð �—‘ §¡¨¯©°4·:±:¸a³=ÈIÐaj¨Ó)kÓ lÓmÑmˆØ�—‘ §¡¨¯©°$·*±*¸Q³-Ó)@Ó AÓBÑBˆØˆrQ   ©NF©Ú__name__Ú
__module__Ú__qualname__Ú__doc__rk   ÚGELUr   r   r   rp   ÚfloatÚboolr   r   ÚModuler,   ri   ÚtorchÚTensorr   Ú__classcell__©rw   s   @rO   rS   rS   €   s}  ø„ Ù3ð  "Ø"Ø!Ø$)Ø#(Ø"Ø!Ø!Ø+/Ø!Ø)+¯©Ø*3Ø),Ø$-ØØØñ)ISàðISð ðISð ð	ISð
 ðISð ðISð "ðISð !ðISð ðISð ðISð ðISð " %™ðISð ðISð ˜BŸI™I‘ðISð ˜RŸY™Y™ðISð  ˜BŸI™I‘ð!ISð" "ð#ISð$ ð%ISð* 
õ+ISð\ 15Ø#ñ	à�|‰|ðð   §¡Ñ-ðð ð	ð
 
�‰÷rQ   rS   c            %       ó\  ‡ — e Zd Zddddddddddej                  eeedddfdeded	e	d
e
de
de
de
de
de	de	dee	   de	deej                     deej                     deej                     dededdf$ˆ fd„Zdd„Z	 	 d dej$                  deej$                     de
dej$                  fd„Zˆ xZS )!ÚResPostBlockrT   FTrU   Nr   r=   r>   rV   r?   r@   rW   rX   rB   rD   rC   rY   rZ   r[   rE   r\   r<   rF   rG   c                 ó¶  •— t         ‰| �  «        ||dœ}|| _        t        ||f||||||
|	||dœ	|¤Ž| _         ||fi |¤Ž| _        |dkD  rt        |«      nt        j                  «       | _	         |d|t        ||z  «      ||r|nd ||	dœ|¤Ž| _         ||fi |¤Ž| _        |dkD  rt        |«      nt        j                  «       | _        | j                  «        y )Nr^   ra   rU   rb   rg   )rh   ri   rY   rP   r:   rj   r    rk   rl   rn   rp   rq   ro   rs   Úinit_weightsrt   s                        €rO   ri   zResPostBlock.__init__Ú   s   ø€ ô, 	‰ÑÔØ¨Ñ/ˆØ&ˆÔä ØØð
ð  ØØØ&ØØØØ!Øñ
ð ñ
ˆŒ	ñ   Ñ* rÑ*ˆŒ
Ø1:¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒáð 
ØÜ  i¡Ó0ØÙ%3‘z¸ØØñ
ð ñ
ˆŒñ   Ñ* rÑ*ˆŒ
Ø1:¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒà×ÑÕrQ   c                 ó  — | j                   �}t        j                  j                  | j                  j
                  | j                   «       t        j                  j                  | j                  j
                  | j                   «       y y ©N)rY   rk   ÚinitÚ	constant_rj   Úweightro   ©ru   s    rO   r‘   zResPostBlock.init_weights  s[   € à×ÑÐ'Ü�G‰G×Ñ˜dŸj™j×/Ñ/°×1AÑ1AÔBÜ�G‰G×Ñ˜dŸj™j×/Ñ/°×1AÑ1AÕBð (rQ   rx   ry   rz   c           
      óÔ   — || j                  | j                  | j                  |||¬«      «      «      z   }|| j                  | j	                  | j                  |«      «      «      z   }|S r|   )rn   rj   r:   rs   ro   rq   r~   s       rO   r   zResPostBlock.forward  sY   € ð �—‘ §
¡
¨4¯9©9°QÀ)ÐW`¨9Ó+aÓ bÓcÑcˆØ�—‘ §
¡
¨4¯8©8°A«;Ó 7Ó8Ñ8ˆØˆrQ   ©rG   Nr€   )r‚   rƒ   r„   rk   r†   r   r   r   rp   r‡   rˆ   r   r   r‰   r,   ri   r‘   rŠ   r‹   r   rŒ   r�   s   @rO   r�   r�   Ù   sZ  ø„ ð
  "Ø"Ø!Ø$)Ø#(Ø"Ø!Ø!Ø+/Ø!Ø)+¯©Ø*3Ø),Ø$-ØØØñ)7àð7ð ð7ð ð	7ð
 ð7ð ð7ð "ð7ð !ð7ð ð7ð ð7ð ð7ð " %™ð7ð ð7ð ˜BŸI™I‘ð7ð ˜RŸY™Y™ð7ð  ˜BŸI™I‘ð!7ð" "ð#7ð$ ð%7ð* 
õ+7órCð 15Ø#ñ	à�|‰|ðð   §¡Ñ-ðð ð	ð
 
�‰÷rQ   r�   c            '       ó„  ‡ — e Zd ZU dZee   ed<   ddddddddddej                  e	ddddddfd	e
d
e
dededededededededee   dedeej                     deej                     deeej                        dee   de
deddf&ˆ fd„Zd"d„Z	 	 d#dej&                  deej&                     d edej&                  fd!„Zˆ xZS )$ÚParallelScalingBlockz¤ Parallel ViT block (MLP & Attention in parallel)
    Based on:
      'Scaling Vision Transformers to 22 Billion Parameters` - https://arxiv.org/abs/2302.05442
    Ú
fused_attnrT   FTrU   Nr   r=   r>   rV   r?   r@   rW   rX   rB   rD   rC   rY   rZ   r[   rE   r\   r<   rF   Úfuse_out_projrG   c                 óÖ  •— t         ‰| �  «        ||dœ}||z  dk(  sJ d«       ‚|s|rJ d«       ‚|| _        ||z  | _        | j                  dz  | _        t        «       | _        t        ||z  «      }|d|z  z   } ||fi |¤Ž| _        t        j                  ||fd|i|¤Ž| _        |g|gdz  z   | _        |r| j                  dd «       n.t        j                  t        j                   |fi |¤Ž«      | _        |r || j                  fi |¤Žnt        j$                  «       | _        |r || j                  fi |¤Žnt        j$                  «       | _        t        j*                  |
«      | _        t        j*                  |	«      | _         |«       | _        |r0t        j                  ||z   |fd|i|¤Ž| _        d | _        d | _        nCd | _        t        j                  ||fd|i|¤Ž| _        t        j                  ||fd|i|¤Ž| _        |�t9        |fd	|i|¤Žnt        j$                  «       | _        |d
kD  rt=        |«      nt        j$                  «       | _        | jA                  «        y )Nr^   r   ú$dim should be divisible by num_headsúScale norms not supportedç      à¿é   re   Úmlp_biasrY   rU   )!rh   ri   r>   Úhead_dimÚscaler'   rœ   rp   Úin_normrk   ÚLinearÚin_projÚin_splitÚregister_parameterÚ	ParameterrŠ   Úemptyr£   rl   Úq_normÚk_normÚDropoutrC   Úmlp_dropÚmlp_actÚout_projÚattn_out_projÚmlp_out_projr-   Úlsr    rZ   Úreset_parameters)ru   r=   r>   rV   r?   r@   rW   rX   rB   rD   rC   rY   rZ   r[   rE   r\   r<   rF   r�   r_   r`   rv   Úmlp_hidden_dimÚin_proj_out_dimrw   s                           €rO   ri   zParallelScalingBlock.__init__+  s.  ø€ ô. 	‰ÑÔØ¨Ñ/ˆØ�Y‰ !Ò#ÐKÐ%KÓKÐ#Ù"©>ÐVÐ;VÓVÐ9Ø"ˆŒØ˜yÑ(ˆŒØ—]‘] dÑ*ˆŒ
Ü(Ó*ˆŒÜ˜Y¨™_Ó-ˆØ(¨1¨s©7Ñ2ˆá! #Ñ,¨Ñ,ˆŒÜ—y‘y  oÑK¸HÐKÈÑKˆŒØ'Ð(¨C¨5°1©9Ñ4ˆŒÙà×#Ñ# J°Õ5äŸL™L¬¯©°^Ñ)JÀrÑ)JÓKˆDŒMá9@‘j §¡Ñ5°"Ò5ÄbÇkÁkÃmˆŒÙ9@‘j §¡Ñ5°"Ò5ÄbÇkÁkÃmˆŒÜŸ™ IÓ.ˆŒäŸ
™
 9Ó-ˆŒÙ “{ˆŒáäŸI™I c¨NÑ&:¸CÑVÀiÐVÐSUÑVˆDŒMØ!%ˆDÔØ $ˆDÕð !ˆDŒMÜ!#§¡¨3°Ñ!J¸)Ð!JÀrÑ!JˆDÔÜ "§	¡	¨.¸#Ñ TÀIÐ TÐQSÑ TˆDÔàDOÐD[”*˜SÑ@¨kÐ@¸RÒ@Ôac×alÑalÓanˆŒØ09¸B²œ )Ô,ÄBÇKÁKÃMˆŒð 	×ÑÕrQ   c                 óp   — | j                   �*t        j                  j                  | j                   «       yy)ú"Initialize parameters and buffers.N)r£   rk   r”   Úzeros_r—   s    rO   r¶   z%ParallelScalingBlock.reset_parametersn  s%   € à�=‰=Ð$Ü�G‰G�N‰N˜4Ÿ=™=Õ)ð %rQ   rx   ry   rz   c                 ó\  — |j                   \  }}}| j                  |«      }| j                  |«      }t        j                  || j
                  d¬«      \  }}	}
}| j                  �|| j                  z   }| j                  |	j                  ||| j                  | j                  «      «      j                  dd«      }	| j                  |
j                  ||| j                  | j                  «      «      j                  dd«      }
|j                  ||| j                  | j                  «      j                  dd«      }| j                  r>t        j                  |	|
||| j                   r| j"                  j$                  nd|¬«      }ng|	| j&                  z  }	|	|
j                  dd«      z  }t)        ||||¬«      }t+        ||«      }|j-                  d¬«      }| j#                  |«      }||z  }|j                  dd«      j/                  |||«      }| j1                  |«      }| j3                  |«      }| j4                  �)| j5                  t        j6                  ||fd¬«      «      }n#| j9                  |«      | j;                  |«      z   }|| j=                  | j?                  |«      «      z   }|S )	Néÿÿÿÿ©r=   r.   é   rU   ©ry   Ú	dropout_prz   éþÿÿÿ©rz   ) Úshaper¦   r¨   rŠ   Úsplitr©   r£   r­   Úviewr>   r¤   Ú	transposer®   rœ   ÚFÚscaled_dot_product_attentionÚtrainingrC   Úpr¥   r+   r*   ÚsoftmaxÚreshaper±   r°   r²   Úcatr³   r´   rZ   rµ   )ru   rx   ry   rz   ÚBÚNÚCÚyÚx_mlpÚqÚkÚvÚx_attnr:   Ú	attn_biass                  rO   r   zParallelScalingBlock.forwards  s@  € ð —'‘'‰ˆˆ1ˆað �L‰L˜‹OˆØ�L‰L˜‹OˆÜŸ™ Q¨¯©¸2Ô>‰ˆˆq�!�QØ�=‰=Ð$Ø˜DŸM™MÑ)ˆEð �K‰K˜Ÿ™˜q ! T§^¡^°T·]±]ÓCÓD×NÑNÈqÐRSÓTˆØ�K‰K˜Ÿ™˜q ! T§^¡^°T·]±]ÓCÓD×NÑNÈqÐRSÓTˆØ�F‰F�1�a˜Ÿ™¨¯©Ó7×AÑAÀ!ÀQÓGˆØ�?Š?Ü×3Ñ3Ø�1�aØ#Ø.2¯mªm˜$Ÿ.™.×*Ò*ÀØ#ô	‰Fð �D—J‘J‘ˆAØ�q—{‘{ 2 rÓ*Ñ*ˆDÜ.¨q°$¸	ÈYÔWˆIÜ! $¨	Ó2ˆDØ—<‘< B�<Ó'ˆDØ—>‘> $Ó'ˆDØ˜A‘XˆFà×!Ñ! ! QÓ'×/Ñ/°°1°aÓ8ˆð —‘˜UÓ#ˆØ—‘˜eÓ$ˆð �=‰=Ð$Ø—‘œeŸi™i¨°¨¸RÔ@ÓA‰Aà×"Ñ" 6Ó*¨T×->Ñ->¸uÓ-EÑEˆAð �—‘˜tŸw™w q›zÓ*Ñ*ˆØˆrQ   r™   r€   )r‚   rƒ   r„   r…   r   rˆ   Ú__annotations__rk   r†   r   rp   r‡   r   r   r‰   r,   ri   r¶   rŠ   r‹   r   rŒ   r�   s   @rO   r›   r›   $  s•  ø… ñð �d‘Óð  "Ø"Ø!Ø$)Ø#(Ø"Ø!Ø!Ø+/Ø!Ø)+¯©Ø*3Ø37Ø.2ØØ"'ØØñ+A àðA ð ðA ð ð	A ð
 ðA ð ðA ð "ðA ð !ðA ð ðA ð ðA ð ðA ð " %™ðA ð ðA ð ˜BŸI™I‘ðA ð ˜RŸY™Y™ðA ð     R§Y¡Y¡Ñ0ð!A ð" ! Ñ+ð#A ð$ ð%A ð&  ð'A ð, 
õ-A óF*ð 15Ø#ñ	1à�|‰|ð1ð   §¡Ñ-ð1ð ð	1ð
 
�‰÷1rQ   r›   c            '       ó°  ‡ — e Zd ZU dZee   ed<   ddddddddddej                  e	ddddddfd	e
d
e
dededededededededee   dedeej                     deej                     deeej                        dee   de
deddf&ˆ fd„Zde
fd„Zd$d„Zdej(                  fd„Z	 	 d%d ej(                  d!eej(                     d"edej(                  fd#„Zˆ xZS )&ÚDiffParallelScalingBlockaM   Parallel ViT block with Differential Attention (MLP & Attention in parallel).

    Combines the parallel MLP+Attention structure from 'Scaling Vision Transformers to
    22 Billion Parameters' (https://arxiv.org/abs/2302.05442) with differential attention
    from 'Differential Transformer' (https://arxiv.org/abs/2410.05258).
    rœ   rT   FTrU   Nr   r=   r>   rV   r?   r@   rW   rX   rB   rD   rC   rY   rZ   r[   rE   r\   r<   rF   Údual_lambdarG   c                 óT  •— t         ‰| �  «        ||dœ}||z  dk(  sJ d«       ‚|s|rJ d«       ‚|| _        ||z  dz  | _        | j                  dz  | _        t        «       | _        t        ||z  «      }|d|z  z   } ||fi |¤Ž| _        t        j                  ||fd|i|¤Ž| _        |g|gdz  z   | _        |r| j                  d	d «       n.t        j                  t        j                   |fi |¤Ž«      | _        |r || j                  fi |¤Žnt        j$                  «       | _        |r || j                  fi |¤Žnt        j$                  «       | _        t        j*                  |
«      | _        |
| _        t1        d| j                  z  fd
di|¤Ž| _        || _        |ršt        j                  t        j                   dt        j6                  |¬«      «      | _        t        j                  t        j                   dt        j6                  |¬«      «      | _        d x| _        x| _        x| _         | _!        �n.d x| _        | _        t        j                  t        j                   | j                  t        j6                  |¬«      «      | _        t        j                  t        j                   | j                  t        j6                  |¬«      «      | _        t        j                  t        j                   | j                  t        j6                  |¬«      «      | _         t        j                  t        j                   | j                  t        j6                  |¬«      «      | _!        t        j*                  |	«      | _"         |«       | _#        t        j                  ||z   |fd|i|¤Ž| _$        |�tK        |fd|i|¤Žnt        j$                  «       | _&        |dkD  rtO        |«      nt        j$                  «       | _(        d| _)        | jU                  |«       | jW                  «        y )Nr^   r   rŸ   r    r¿   r¡   r¢   re   r£   Úepsçñhãˆµøä>rg   )r`   r_   rY   rU   çš™™™™™é?),rh   ri   r>   r¤   r¥   r'   rœ   rp   r¦   rk   r§   r¨   r©   rª   r«   rŠ   r¬   r£   rl   r­   r®   r¯   rC   Úattn_drop_pr   Úsub_normrÜ   Úfloat32Úlambda_aÚlambda_bÚ	lambda_q1Ú	lambda_k1Ú	lambda_q2Ú	lambda_k2r°   r±   r²   r-   rµ   r    rZ   Úlambda_initÚset_lambda_initr¶   )ru   r=   r>   rV   r?   r@   rW   rX   rB   rD   rC   rY   rZ   r[   rE   r\   r<   rF   rÜ   r_   r`   rv   r·   r¸   rw   s                           €rO   ri   z!DiffParallelScalingBlock.__init__°  s-  ø€ ô. 	‰ÑÔØ¨Ñ/ˆØ�Y‰ !Ò#ÐKÐ%KÓKÐ#Ù"©>ÐVÐ;VÓVÐ9Ø"ˆŒØ˜yÑ(¨AÑ-ˆŒØ—]‘] dÑ*ˆŒ
Ü(Ó*ˆŒÜ˜Y¨™_Ó-ˆØ(¨1¨s©7Ñ2ˆá! #Ñ,¨Ñ,ˆŒÜ—y‘y  oÑK¸HÐKÈÑKˆŒØ'Ð(¨C¨5°1©9Ñ4ˆŒÙà×#Ñ# J°Õ5äŸL™L¬¯©°^Ñ)JÀrÑ)JÓKˆDŒMá9@‘j §¡Ñ5°"Ò5ÄbÇkÁkÃmˆŒÙ9@‘j §¡Ñ5°"Ò5ÄbÇkÁkÃmˆŒÜŸ™ IÓ.ˆŒØ$ˆÔô    D§M¡MÑ 1ÑB°tÐB¸rÑBˆŒØ&ˆÔÙÜŸL™L¬¯©°R¼u¿}¹}ÐU[Ô)\Ó]ˆDŒMÜŸL™L¬¯©°R¼u¿}¹}ÐU[Ô)\Ó]ˆDŒMØPTÐTˆDŒNÐT˜Tœ^ÐT¨d¬n¸t¾~à,0Ð0ˆDŒM˜DœMÜŸ\™\¬%¯+©+°d·m±mÌ5Ï=É=ÐagÔ*hÓiˆDŒNÜŸ\™\¬%¯+©+°d·m±mÌ5Ï=É=ÐagÔ*hÓiˆDŒNÜŸ\™\¬%¯+©+°d·m±mÌ5Ï=É=ÐagÔ*hÓiˆDŒNÜŸ\™\¬%¯+©+°d·m±mÌ5Ï=É=ÐagÔ*hÓiˆDŒNäŸ
™
 9Ó-ˆŒÙ “{ˆŒô Ÿ	™	 #¨Ñ"6¸ÑRÀ)ÐRÈrÑRˆŒàDOÐD[”*˜SÑ@¨kÐ@¸RÒ@Ôac×alÑalÓanˆŒØ09¸B²œ )Ô,ÄBÇKÁKÃMˆŒàˆÔØ×Ñ˜UÔ#ð 	×ÑÕrQ   c                 óJ   — ddt        j                  d|z  «      z  z
  | _        y )Nrà   g333333ã?g333333Ó¿)ÚmathÚexprê   )ru   rF   s     rO   rë   z(DiffParallelScalingBlock.set_lambda_initý  s!   € Ø ¤t§x¡x°°u±Ó'=Ñ!=Ñ=ˆÕrQ   c                 óŒ  — | j                   �)t        j                  j                  | j                   «       | j                  rSt        j                  j                  | j
                  «       t        j                  j                  | j                  «       yt        j                  j                  | j                  dd¬«       t        j                  j                  | j                  dd¬«       t        j                  j                  | j                  dd¬«       t        j                  j                  | j                  dd¬«       y)rº   Nr   çš™™™™™¹?©ÚmeanÚstd)r£   rk   r”   r»   rÜ   rä   rå   Únormal_ræ   rç   rè   ré   r—   s    rO   r¶   z)DiffParallelScalingBlock.reset_parameters   s¹   € à�=‰=Ð$Ü�G‰G�N‰N˜4Ÿ=™=Ô)Ø×ÒÜ�G‰G�N‰N˜4Ÿ=™=Ô)Ü�G‰G�N‰N˜4Ÿ=™=Õ)ä�G‰G�O‰O˜DŸN™N°¸ˆOÔ<Ü�G‰G�O‰O˜DŸN™N°¸ˆOÔ<Ü�G‰G�O‰O˜DŸN™N°¸ˆOÔ<Ü�G‰G�O‰O˜DŸN™N°¸ˆOÕ<rQ   c                 óø  — | j                   �?t        j                  | j                   «      }t        j                  | j                  «      }nžt        j                  t        j                  | j
                  | j                  z  d¬«      j                  «       «      }t        j                  t        j                  | j                  | j                  z  d¬«      j                  «       «      }||z
  | j                  z   S )Nr½   r¾   )rä   rŠ   rî   rå   Úsumræ   rç   r‡   rè   ré   rê   )ru   Úlambda_1Úlambda_2s      rO   Ú_compute_lambdaz(DiffParallelScalingBlock._compute_lambda  s¦   € Ø�=‰=Ð$Ü—y‘y §¡Ó/ˆHÜ—y‘y §¡Ó/‰Hä—y‘y¤§¡¨4¯>©>¸D¿N¹NÑ+JÐPRÔ!S×!YÑ!YÓ![Ó\ˆHÜ—y‘y¤§¡¨4¯>©>¸D¿N¹NÑ+JÐPRÔ!S×!YÑ!YÓ![Ó\ˆHØ˜(Ñ" T×%5Ñ%5Ñ5Ð5rQ   rx   ry   rz   c                 ó:  — |j                   \  }}}| j                  |«      }| j                  |«      }t        j                  || j
                  d¬«      \  }}	}
}| j                  �|| j                  z   }|	j                  ||d| j                  z  | j                  «      j                  dd«      }	|
j                  ||d| j                  z  | j                  «      j                  dd«      }
|j                  ||| j                  d| j                  z  «      j                  dd«      }| j                  |	«      | j                  |
«      }
}	| j                  «       j                  |	«      }| j                  rÓ|	j                  || j                  d|| j                  «      }	|
j                  || j                  d|| j                  «      }
|	j!                  d«      \  }}|
j!                  d«      \  }}| j"                  r| j$                  nd}t'        j(                  ||||||¬«      }t'        j(                  ||||||¬«      }|||z  z
  }n¢|	| j*                  z  }	|	|
j                  dd«      z  }t-        ||||¬«      }t/        ||«      }|j1                  d¬«      }| j3                  |«      }|j5                  || j                  d||«      }|d d …d d …d	f   ||d d …d d …df   z  z
  }||z  }| j7                  |«      }|d| j8                  z
  z  }|j                  dd«      j                  |||«      }| j;                  |«      }| j=                  |«      }| j?                  t        j@                  ||fd¬«      «      }|| jC                  | jE                  |«      «      z   }|S )
Nr½   r¾   r¿   r.   rU   rÀ   rÂ   rÃ   r   )#rÄ   r¦   r¨   rŠ   rÅ   r©   r£   rÍ   r>   r¤   rÇ   r­   r®   rù   Útype_asrœ   ÚunbindrÊ   rá   rÈ   rÉ   r¥   r+   r*   rÌ   rC   rÆ   râ   rê   r±   r°   r²   rÎ   rZ   rµ   )ru   rx   ry   rz   rÏ   rÐ   rÑ   rÒ   rÓ   rÔ   rÕ   rÖ   Úlambda_fullÚq1Úq2Úk1Úk2rÁ   Úattn1Úattn2r×   r:   rØ   s                          rO   r   z DiffParallelScalingBlock.forward  s'  € ð —'‘'‰ˆˆ1ˆað �L‰L˜‹OˆØ�L‰L˜‹OˆÜŸ™ Q¨¯©¸2Ô>‰ˆˆq�!�QØ�=‰=Ð$Ø˜DŸM™MÑ)ˆEð �I‰I�a˜˜A §¡Ñ.°·±Ó>×HÑHÈÈAÓNˆØ�I‰I�a˜˜A §¡Ñ.°·±Ó>×HÑHÈÈAÓNˆØ�I‰I�a˜˜DŸN™N¨A°·±Ñ,=Ó>×HÑHÈÈAÓNˆà�{‰{˜1‹~˜tŸ{™{¨1›~ˆ1ˆà×*Ñ*Ó,×4Ñ4°QÓ7ˆà�?Š?Ø—	‘	˜!˜TŸ^™^¨Q°°4·=±=ÓAˆAØ—	‘	˜!˜TŸ^™^¨Q°°4·=±=ÓAˆAØ—X‘X˜a“[‰FˆB�Ø—X‘X˜a“[‰FˆB�à,0¯MªM˜×(Ò(¸sˆIÜ×2Ñ2°2°r¸1È	Ð]fÐr{Ô|ˆEÜ×2Ñ2°2°r¸1È	Ð]fÐr{Ô|ˆEà˜[¨5Ñ0Ñ0‰Fà�D—J‘J‘ˆAØ�q—{‘{ 2 rÓ*Ñ*ˆDÜ.¨q°$¸	ÈYÔWˆIÜ! $¨	Ó2ˆDØ—<‘< B�<Ó'ˆDØ—>‘> $Ó'ˆDà—9‘9˜Q §¡°°1°aÓ8ˆDØšš1˜a˜‘= ;°²aº¸A°g±Ñ#>Ñ>ˆDØ˜A‘XˆFà—‘˜vÓ&ˆØ˜1˜t×/Ñ/Ñ/Ñ0ˆØ×!Ñ! ! QÓ'×/Ñ/°°1°aÓ8ˆð —‘˜UÓ#ˆØ—‘˜eÓ$ˆð �M‰Mœ%Ÿ)™) V¨U O¸Ô<Ó=ˆð �—‘˜tŸw™w q›zÓ*Ñ*ˆØˆrQ   r™   r€   )r‚   rƒ   r„   r…   r   rˆ   rÙ   rk   r†   r   rp   r‡   r   r   r‰   r,   ri   rë   r¶   rŠ   r‹   rù   r   rŒ   r�   s   @rO   rÛ   rÛ   §  s±  ø… ñð �d‘Óð  "Ø"Ø!Ø$)Ø#(Ø"Ø!Ø!Ø+/Ø!Ø)+¯©Ø*3Ø37Ø.2ØØ %ØØñ+K àðK ð ðK ð ð	K ð
 ðK ð ðK ð "ðK ð !ðK ð ðK ð ðK ð ðK ð " %™ðK ð ðK ð ˜BŸI™I‘ðK ð ˜RŸY™Y™ðK ð     R§Y¡Y¡Ñ0ð!K ð" ! Ñ+ð#K ð$ ð%K ð& ð'K ð, 
õ-K ðZ> Só >ó=ð6 §¡ó 6ð 15Ø#ñ	<à�|‰|ð<ð   §¡Ñ-ð<ð ð	<ð
 
�‰÷<rQ   rÛ   c            '       ó^  ‡ — e Zd ZdZdddddddddddej
                  eeedddfd	e	d
e	de	de
dedededededee
   de
de
de
deej                     deej                     deej                     dede	ddf&ˆ fd„Z	 	 d!dej$                  deej$                     dedej$                  fd „Zˆ xZS )"ÚParallelThingsBlockz¿ Parallel ViT block (N parallel attention followed by N parallel MLP)
    Based on:
      `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795
    r¿   rT   FTNrU   r   r=   r>   Únum_parallelrV   r?   r@   rW   rX   rB   rY   rD   rC   rZ   r[   rE   r\   r<   rF   rG   c                 ó^  •— ||dœ}t         ‰| �  «        || _        t        j                  «       | _        t        j                  «       | _        t        |«      D �]Q  }| j
                  j                  t        j                  t        d ||fi |¤Žfdt        ||f|||||	||||dœ	|¤Žfd|
rt        |fd|
i|¤Žnt        j                  «       fd|dkD  rt        |«      nt        j                  «       fg«      «      «       | j                  j                  t        j                  t        d ||fi |¤Žfd	 ||ft        ||z  «      ||r|nd |	|d
œ|¤Žfd|
rt        |fd|
i|¤Žnt        j                  «       fd|dkD  rt        |«      nt        j                  «       fg«      «      «       �ŒT y )Nr^   Únormr:   ra   rµ   rY   rZ   rU   rq   )rd   r[   rE   re   rf   )rh   ri   r  rk   Ú
ModuleListÚattnsÚffnsÚrangeÚappendÚ
Sequentialr   rP   r-   rl   r    rp   )ru   r=   r>   r  rV   r?   r@   rW   rX   rB   rY   rD   rC   rZ   r[   rE   r\   r<   rF   r_   r`   rv   Ú_rw   s                          €rO   ri   zParallelThingsBlock.__init__Z  sÊ  ø€ ð. ¨Ñ/ˆÜ‰ÑÔØ(ˆÔÜ—]‘]“_ˆŒ
Ü—M‘M“OˆŒ	Ü�|Ó$ó !	ˆAØ�J‰J×ÑœbŸm™m¬KØ™ CÑ.¨2Ñ.Ð/ØœØØðð (Ø%Ø#Ø.Ø'Ø'Ø'Ø)Øñð ñð ð É”z #ÑE°;ÐEÀ"ÒEÔZ\×ZeÑZeÓZgÐhØ°YÀ²^œh yÔ1ÌÏÉËÐWð#9ó -ó ô ð& �I‰I×ÑœRŸ]™]¬;Ø™ CÑ.¨2Ñ.Ð/Ø™	Øðä$'¨¨i©Ó$8Ø'Ù-;™zÀØ"Ø"ñð ñð ð É”z #ÑE°;ÐEÀ"ÒEÔZ\×ZeÑZeÓZgÐhØ°YÀ²^œh yÔ1ÌÏÉËÐWð8ó ,ó ö ñ)!	rQ   rx   ry   rz   c           	      ób  — |€|r”g }| j                   D ]Z  }|j                  |«      }|j                  |||¬«      }|j                  |«      }|j	                  |«      }|j                  |«       Œ\ |t        j                  |«      j                  d¬«      z   }nF|t        j                  | j                   D �cg c]
  } ||«      ‘Œ c}«      j                  d¬«      z   }|t        j                  | j                  D �cg c]
  } ||«      ‘Œ c}«      j                  d¬«      z   }|S c c}w c c}w )Nr}   r   r¾   )
r
  r  r:   rµ   rZ   r  rŠ   Ústackrö   r  )ru   rx   ry   rz   Úattn_outr:   r×   Úffns           rO   r   zParallelThingsBlock.forward™  s  € ð Ð ¡IØˆHØŸ
™
ò (�ØŸ™ 1›�ØŸ™ 6°YÈ)˜ÓT�ØŸ™ ›�ØŸ™¨Ó/�Ø—‘ Õ'ð(ð ”E—K‘K Ó)×-Ñ-°!Ð-Ó4Ñ4‰Aà”E—K‘K°T·Z±ZÖ @¨T¡ a¥Ò @ÓA×EÑEÈ!ÐEÓLÑLˆAØ”—‘¨t¯y©yÖ9¨™S �VÒ9Ó:×>Ñ>À1Ð>ÓEÑEˆØˆùò !AùÚ9s   Â7D'
Ã=D,
r€   r�   r�   s   @rO   r  r  U  sf  ø„ ñð !"Ø!Ø"Ø!Ø$)Ø#(Ø"Ø+/Ø!Ø!Ø!Ø)+¯©Ø*3Ø),Ø$-ØØØñ+=àð=ð ð=ð ð	=ð
 ð=ð ð=ð ð=ð "ð=ð !ð=ð ð=ð " %™ð=ð ð=ð ð=ð ð=ð ˜BŸI™I‘ð=ð  ˜RŸY™Y™ð!=ð" ˜BŸI™I‘ð#=ð$ "ð%=ð& ð'=ð, 
õ-=ðD 15Ø#ñ	à�|‰|ðð   §¡Ñ-ðð ð	ð
 
�‰÷rQ   r  rx   Ú	pool_typeÚnum_prefix_tokensÚreduce_include_prefixc                 ó&  — |s| S |dk(  r| d d …df   } | S |r| n
| d d …|d …f   } |dk(  r| j                  d¬«      } | S |dk(  r*d| j                  d¬«      | j                  d¬«      z   z  } | S |dk(  r| j                  d¬«      } | S |r
J d	|› �«       ‚| S )
NÚtokenr   Úavgr.   r¾   Úavgmaxg      à?ÚmaxzUnknown pool type )rò   Úamax)rx   r  r  r  s       rO   Úglobal_pool_nlcr  ®  sÏ   € ñ Øˆà�GÒØŠa�ˆd‰Gˆð €Hñ '‰A¨AªaÐ1BÑ1CÐ.CÑ,DˆØ˜ÒØ—‘˜1�“ˆAð €Hð ˜(Ò"Ø�q—v‘v !�v“} q§v¡v°! v£}Ñ4Ñ5ˆAð €Hð ˜%ÒØ—‘˜1�“ˆAð €Hñ !ÐBÐ$6°y°kÐ"BÓB�=à€HrQ   c            S       ó  ‡ — e Zd ZU dZee   ed<   ddddddd	d	d
ddddddddddddddddddddddddedddee	e
ddf*deeeeef   f   deeeeef   f   dededed   dedededededededed ed!ee   d"ed#ed$ed%ed&ed'ed(ee   d)eded*ed+ed,ed-ed.ed/ed0ed1ed2   d3ed4ed5ee   d6ee   d7ee   d8eej,                     d9eej,                     d:ed;dfRˆ fd<„Zdfd=„Zdgd>ed?ed;dfd@„ZdAej,                  d;dfdB„Zej8                  j;                  «       dhdCedDed;dfdE„«       Zej8                  j:                  d;ee   fdF„«       Z ej8                  j:                  didGed;e!eeee"f   f   fdH„«       Z#ej8                  j:                  djdIed;dfdJ„«       Z$ej8                  j:                  d;ej,                  fdK„«       Z%dkdedee   d;dfdL„Z&	 	 dldeeeef      deeeef      d;dfdM„Z'dNejP                  d;ejP                  fdO„Z)	 	 	 	 	 	 	 	 	 dmdNejP                  dPeeee"e   f      dQedRedSedTedUedVedWeejP                     dXed;ee"ejP                     eejP                  e"ejP                     f   e!ee*f   f   fdY„Z+	 	 	 dndPeee"e   f   dZed[ed;e"e   fd\„Z,	 	 	 	 	 dodNejP                  d]eee"e   ee   f   d^edQedRedWeejP                     d;e"ejP                     fd_„Z-	 	 dpdNejP                  dWeejP                     dXed;ejP                  fd`„Z.dkdNejP                  daee   d;ejP                  fdb„Z/didNejP                  dced;ejP                  fdd„Z0	 	 dpdNejP                  dWeejP                     dXed;ejP                  fde„Z1ˆ xZ2S )qr8   z© Vision Transformer

    A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`
        - https://arxiv.org/abs/2010.11929
    Údynamic_img_sizeéà   é   r¢   éè  r  é   é   rT   TFNÚlearnr   rU   r9   Úimg_sizeÚ
patch_sizeÚin_chansÚnum_classesÚglobal_pool©r9   r  r  r  r  ÚmapÚprrÚ	embed_dimrF   r>   rV   r?   r@   rW   rX   rB   rY   Úclass_tokenÚ	pos_embedÚno_embed_classÚ
reg_tokensÚpre_normÚ
final_normÚfc_normÚpool_include_prefixÚdynamic_img_padÚ	drop_rateÚpos_drop_rateÚpatch_drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_rateÚweight_init)ÚskipÚresetÚjaxÚjax_nlhbÚmocor9   Úfix_initÚembed_layerÚembed_norm_layerrE   r[   Úblock_fnr\   r<   rG   c+                 óð  •— t         ‰3| �  «        |)|*dœ}+|dv sJ ‚|s|dk7  sJ ‚|dv sJ ‚|€|dv n|},t        |$«      xs t        }$t        |#«      }#t	        |%«      xs t
        j                  }%|| _        || _        || _	        |x| _
        x| _        | _        |rdnd| _        | xj                  |z  c_        || _        || _        || _        || _        || _        d	| _        i }-|r|-j)                  t+        d	d
¬«      «       |#�|#|-d<    |"d-||||| |dœ|-¤|+¤Ž| _        | j,                  j.                  }.t1        | j,                  d«      r| j,                  j3                  «       n|}/|r+t        j4                  t7        j8                  dd|fi |+¤Ž«      nd| _        |r+t        j4                  t7        j8                  d||fi |+¤Ž«      nd| _        |r|.n|.| j                  z   }0|r|dk(  rd| _        n0t        j4                  t7        j8                  d|0|fi |+¤Ž«      | _        t        j@                  |¬«      | _!        |dkD  rtE        || j                  ¬«      | _#        nt        jH                  «       | _#        |r	 |$|fi |+¤Žnt        jH                  «       | _%        tM        ||«      }1t        jN                  tQ        |«      D �2cg c]@  }2 |&d-i d|“d|“d|	“d|
“d|“d|“d|“d|“d|“d|“d|“d|1|2   “d|$“d|%“d|'“d |(“d!|2“|+¤Ž‘ŒB c}2Ž | _)        tQ        |«      D �2cg c]  }2t+        d"|2› �||/¬#«      ‘Œ c}2| _*        |r|,s	 |$|fi |+¤Žnt        jH                  «       | _+        |d$k(  r!tY        | j                  f||	|$|%d%œ|+¤Ž| _-        n7|d&k(  r+t]        | j                  f||rdnd'|$d(œ|+¤Ž| _-        d)| _        nd| _-        |r|,r	 |$|fi |+¤Žnt        jH                  «       | _/        t        j@                  |«      | _0        |dkD  r!t        jb                  | j                  |fi |+¤Žnt        jH                  «       | _2        | d*k(  rd+n| | _3        |!| _4        | d*k7  r| jk                  d	¬,«       yyc c}2w c c}2w ).aò  
        Args:
            img_size: Input image size.
            patch_size: Patch size.
            in_chans: Number of image input channels.
            num_classes: Number of classes for classification head.
            global_pool: Type of global pooling for final sequence (default: 'token').
            embed_dim: Transformer embedding dimension.
            depth: Depth of transformer.
            num_heads: Number of attention heads.
            mlp_ratio: Ratio of mlp hidden dim to embedding dim.
            qkv_bias: Enable bias for qkv projections if True.
            init_values: Layer-scale init values (layer-scale enabled if not None).
            class_token: Use class token.
            no_embed_class: Don't include position embeddings for class (or reg) tokens.
            reg_tokens: Number of register tokens.
            pre_norm: Enable norm after embeddings, before transformer blocks (standard in CLIP ViT).
            final_norm: Enable norm after transformer blocks, before head (standard in most ViT).
            fc_norm: Move final norm after pool (instead of before), if None, enabled when global_pool == 'avg'.
            drop_rate: Head dropout rate.
            pos_drop_rate: Position embedding dropout rate.
            attn_drop_rate: Attention dropout rate.
            drop_path_rate: Stochastic depth rate.
            weight_init: Weight initialization scheme.
            fix_init: Apply weight initialization fix (scaling w/ layer index).
            embed_layer: Patch embedding layer.
            embed_norm_layer: Normalization layer to use / override in patch embed module.
            norm_layer: Normalization layer.
            act_layer: MLP activation layer.
            block_fn: Transformer block layer.
        r^   r+  r  )r9   Únoner%  N)r  r  r  r.   r   FÚNHWC)Ústrict_img_sizeÚ
output_fmtrE   )r&  r'  r(  r.  re   r7  Ú
feat_ratiorI  )rË   )r  r=   r>   rV   r?   r@   rW   rX   rB   rY   rD   rC   rZ   r[   r\   r<   rF   úblocks.)ÚmoduleÚnum_chsÚ	reductionr,  )r>   rV   rE   r[   r-  r  )r>   r  rE   Tr?  r@  ©Úneeds_resetrg   )6rh   ri   r)   r   r(   rk   r†   r)  r(  r*  Únum_featuresÚhead_hidden_sizer.  r  Únum_reg_tokensÚhas_class_tokenr1  r6  r  Úgrad_checkpointingÚupdateÚdictÚpatch_embedÚnum_patchesÚhasattrrM  r«   rŠ   r¬   Ú	cls_tokenÚ	reg_tokenr0  r¯   Úpos_dropr"   Ú
patch_droprl   Únorm_prer!   r  r  ÚblocksÚfeature_infor  r   Ú	attn_poolr   r5  Ú	head_dropr§   ÚheadÚweight_init_moderD  r‘   )4ru   r&  r'  r(  r)  r*  r.  rF   r>   rV   r?   r@   rW   rX   rB   rY   r/  r0  r1  r2  r3  r4  r5  r6  r  r7  r8  r9  r:  r;  r<  r=  r>  rD  rE  rF  rE   r[   rG  r\   r<   r_   r`   rv   Úuse_fc_normÚ
embed_argsr\  rQ  Ú	embed_lenÚdprÚirw   s4                                                      €rO   ri   zVisionTransformer.__init__Ï  s÷  ø€ ôX 	‰ÑÔØ¨Ñ/ˆØÐQÑQÐQÐQÙ˜k¨WÒ4Ð4Ð4ØÐ1Ñ1Ð1Ð1ØAHÀ�kÐ%=Ñ=ÐV]ˆÜ# JÓ/Ò<´9ˆ
Ü)Ð*:Ó;ÐÜ! )Ó,Ò7´·±ˆ	à&ˆÔØ ˆŒØ&ˆÔØENÐNˆÔÐN˜DÔ1°D´NÙ&1¡°qˆÔØ×Ò *Ñ,ÕØ(ˆÔØ*ˆÔØ,ˆÔØ#6ˆÔ Ø 0ˆÔØ"'ˆÔàˆ
Ùà×Ñœd°5ÀVÔLÔMØÐ'Ø'7ˆJ�|Ñ$Ù&ð 	
ØØ!ØØØ�Ø+ñ	
ð ð	
ð ñ	
ˆÔð ×&Ñ&×2Ñ2ˆÜ5<¸T×=MÑ=MÈ|Ô5\�D×$Ñ$×/Ñ/Ô1Ðblˆ	áMXœŸ™¤e§k¡k°!°Q¸	Ñ&HÀRÑ&HÔIÐ^bˆŒÙV`œŸ™¤e§k¡k°!°ZÀÑ&QÈbÑ&QÔRÐfjˆŒÙ#1‘K°{ÀT×E[ÑE[Ñ7[ˆ	Ù˜I¨Ò/Ø!ˆD�NäŸ\™\¬%¯+©+°a¸ÀIÑ*TÐQSÑ*TÓUˆDŒNÜŸ
™
 ]Ô3ˆŒØ˜QÒÜ*ØØ"&×"8Ñ"8ôˆD�Oô
 !Ÿk™k›mˆDŒOÙ7?™
 9Ñ3°Ò3ÄRÇ[Á[Ã]ˆŒä'¨¸Ó>ˆÜ—m‘mô* ˜5“\ö+&#ð* ñ) ò Ùðá#ðñ $ðñ "ð	ñ
  ðñ !0ðñ  .ðñ $ðñ (ðñ )ðñ )ðð ˜aš&ðñ &ðñ $ðñ $ðñ  &ð!ñ" Øó%ò&#ð $ˆŒô. Y^Ð^cÓXdöfØSTŒD˜' ! ˜¨yÀIÖNòfˆÔá3=Ák‘J˜yÑ/¨BÒ/ÔWY×WbÑWbÓWdˆŒ	ð ˜%ÒÜ0Ø—‘ðà#Ø#Ø%Ø#ñð ñˆD�Nð ˜EÒ!Ü-Ø—‘ðà#Ù%0™'°eØ%ñ	ð
 ñˆDŒNð (,ˆDÕ$à!ˆDŒNÙ6@Á[‘z )Ñ2¨rÒ2ÔVX×VaÑVaÓVcˆŒÜŸ™ IÓ.ˆŒØDOÐRSÂO”B—I‘I˜dŸn™n¨kÑ@¸RÒ@ÔY[×YdÑYdÓYfˆŒ	à+6¸&Ò+@¡ÀkˆÔØ ˆŒà˜&Ò Ø×Ñ¨%ÐÕ0ð !ùòm&#ùò,fs   ËAQ.ÌQ3c                 ó„  — t        j                  «       5  t        | j                  «      D ]~  \  }}t	        j
                  d|dz   z  «      }|j                  j                  j                  j                  |«       |j                  j                  j                  j                  |«       Œ€ 	 ddd«       y# 1 sw Y   yxY w)z9Apply weight initialization fix (scaling w/ layer index).g       @r.   N)rŠ   Úno_gradÚ	enumeraterc  rí   Úsqrtr:   Úprojr–   Údiv_rq   Úfc2)ru   Úlayer_idÚlayerr¥   s       rO   Úfix_init_weightz!VisionTransformer.fix_init_weight�  s‹   € ä�]‰]‹_ñ 	1Ü#,¨T¯[©[Ó#9ò 1‘�˜%ÜŸ	™	 #¨°A©Ñ"6Ó7�Ø—
‘
—‘×&Ñ&×+Ñ+¨EÔ2Ø—	‘	—‘×$Ñ$×)Ñ)¨%Õ0ñ1÷	1÷ 	1ñ 	1ús   •BB6Â6B?ÚmoderS  c                 ó  — |xs | j                   }|dv sJ ‚d|v r t        j                  | j                  «       nd}| j                  �t        | j                  d¬«       | j                  �+t        j                  j                  | j                  d¬«       | j                  �+t        j                  j                  | j                  d¬«       t        t        |||¬«      | «       | j                  r| j                  «        yy)	a*  Initialize model weights.

        Args:
            mode: Weight initialization mode ('jax', 'jax_nlhb', 'moco', or '').
            needs_reset: If True, call reset_parameters() on modules that have it.
                Set to False when modules have already self-initialized in __init__.
        )rA  rB  rC  r@  r9   ÚnlhbrU   Nç{®Gáz”?©ró   ç�íµ ÷Æ°>rR  )rh  rí   Úlogr)  r0  r#   r^  rk   r”   rô   r_  r1   Úget_init_weights_vitrD  rw  )ru   rx  rS  Ú	head_biass       rO   r‘   zVisionTransformer.init_weights˜  sË   € ð Ò,�t×,Ñ,ˆØÐ?Ñ?Ð?Ð?Ø39¸T±>”T—X‘X˜d×.Ñ.Ó/Ñ/Àrˆ	Ø�>‰>Ð%Ü˜$Ÿ.™.¨cÕ2Ø�>‰>Ð%Ü�G‰G�O‰O˜DŸN™N°ˆOÔ5Ø�>‰>Ð%Ü�G‰G�O‰O˜DŸN™N°ˆOÔ5äÔ(¨¨yÀkÔRÐTXÔYà�=Š=Ø× Ñ Õ"ð rQ   Úmc                 ó   — t        |«       y)z>Initialize weights for a single module (compatibility method).N)Úinit_weights_vit_timm)ru   r�  s     rO   Ú_init_weightszVisionTransformer._init_weights¯  s   € ô 	˜aÕ rQ   Úcheckpoint_pathÚprefixc                 ó   — t        | ||«       y)z‘Load pretrained weights.

        Args:
            checkpoint_path: Path to checkpoint.
            prefix: Prefix for state dict keys.
        N)Ú_load_weights)ru   r…  r†  s      rO   Úload_pretrainedz!VisionTransformer.load_pretrained´  s   € ô 	�d˜O¨VÕ4rQ   c                 ó
   — h d£S )z3Set of parameters that should not use weight decay.>   r^  r0  r_  Ú
dist_tokenrg   r—   s    rO   Úno_weight_decayz!VisionTransformer.no_weight_decay¾  s   € ò EÐDrQ   Úcoarsec                 ó    — t        dddg¬«      S )z¾Create regex patterns for parameter grouping.

        Args:
            coarse: Use coarse grouping.

        Returns:
            Dictionary mapping group names to regex patterns.
        z*^cls_token|reg_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)iŸ† )Ústemrc  )rZ  )ru   r�  s     rO   Úgroup_matcherzVisionTransformer.group_matcherÃ  s   € ô Ø>Ø-Ð/CÐDô
ð 	
rQ   Úenablec                 óv   — || _         t        | j                  d«      r| j                  j                  |«       yy)zEnable or disable gradient checkpointing.

        Args:
            enable: Whether to enable gradient checkpointing.
        Úset_grad_checkpointingN)rX  r]  r[  r“  )ru   r‘  s     rO   r“  z(VisionTransformer.set_grad_checkpointingÒ  s6   € ð #)ˆÔÜ�4×#Ñ#Ð%=Ô>Ø×Ñ×3Ñ3°FÕ;ð ?rQ   c                 ó   — | j                   S )zGet the classifier head.)rg  r—   s    rO   Úget_classifierz VisionTransformer.get_classifierÝ  s   € ð �y‰yÐrQ   c                 óJ  — || _         |�V|dv sJ ‚|dv r| j                  €J d«       ‚|dvr| j                  �d| _        n|dv r| j                  |k7  rJ d«       ‚|| _        |dkD  r&t        j                  | j
                  |«      | _        yt        j                  «       | _        y)zŸReset the classifier head.

        Args:
            num_classes: Number of classes for new classifier.
            global_pool: Global pooling type.
        Nr+  )r,  r-  z=Cannot currently add attention pooling in reset_classifier().zECannot currently change attention pooling type in reset_classifier().r   )r)  re  r*  rk   r§   r.  rl   rg  )ru   r)  r*  s      rO   Úreset_classifierz"VisionTransformer.reset_classifierâ  s©   € ð 'ˆÔØÐ"ØÐ"UÑUÐUÐUØ˜nÑ,°·±Ð1GØ]Ð]Ó]�uØ NÑ2°t·~±~Ð7QØ!%�•Ø Ñ.°4×3CÑ3CÀ{Ò3RØeÐeÓe�uØ*ˆDÔØ>IÈAºo”B—I‘I˜dŸn™n¨kÓ:ˆ�	ÔSU×S^ÑS^ÓS`ˆ�	rQ   c           	      ó°  — | j                   j                  }| j                   j                  ||¬«       | j                  �—| j                  rdn| j
                  }| j                   j                  |z   }|| j                  j                  d   k7  rGt        j                  t        | j                  | j                   j                  ||d¬«      «      | _        yyy)zæUpdate the input image resolution and patch size.

        Args:
            img_size: New input resolution, if None current resolution is used.
            patch_size: New patch size, if None existing patch size is used.
        )r&  r'  Nr   r.   T)Únew_sizeÚold_sizer  Úverbose)r[  Ú	grid_sizeÚset_input_sizer0  r1  r  r\  rÄ   rk   r«   r&   )ru   r&  r'  Úprev_grid_sizer  Únum_new_tokenss         rO   r�  z VisionTransformer.set_input_sizeõ  s½   € ð ×)Ñ)×3Ñ3ˆØ×Ñ×'Ñ'°ÀjÐ'ÔQØ�>‰>Ð%Ø%)×%8Ò%8¡¸d×>TÑ>TÐØ!×-Ñ-×9Ñ9Ð<MÑMˆNØ §¡×!5Ñ!5°aÑ!8Ò8Ü!#§¡Ô.DØ—N‘NØ!×-Ñ-×7Ñ7Ø+Ø&7Ø ô/ó "�•ð 9ð &rQ   rx   c           	      óŽ  — g }| j                   �9|j                  | j                   j                  |j                  d   dd«      «       | j                  �9|j                  | j                  j                  |j                  d   dd«      «       | j
                  €Ft        j                  ||j                  |j                  d   d|j                  d   «      gz   d¬«      S | j                  rn|j                  \  }}}}| j                  j                  }t        | j
                  ||f|| j                  rdn| j                  ¬«      }|j                  |d|«      }n| j
                  }| j                  r#||z   }|r>t        j                  ||gz   d¬«      }n"|rt        j                  ||gz   d¬«      }||z   }| j                  |«      S )z$Apply positional embedding to input.r   r½   r.   r¾   )r™  rš  r  )r^  r  ÚexpandrÄ   r_  r0  rŠ   rÎ   rÆ   r  r[  rœ  r&   r1  r  r`  )	ru   rx   Úto_catrÏ   ÚHÚWrÑ   rž  r0  s	            rO   Ú
_pos_embedzVisionTransformer._pos_embed  s€  € àˆØ�>‰>Ð%Ø�M‰M˜$Ÿ.™.×/Ñ/°·±¸±
¸BÀÓCÔDØ�>‰>Ð%Ø�M‰M˜$Ÿ.™.×/Ñ/°·±¸±
¸BÀÓCÔDà�>‰>Ð!Ü—9‘9˜V q§v¡v¨a¯g©g°a©j¸"¸a¿g¹gÀb¹kÓ'JÐ&KÑKÐQRÔSÐSà× Ò ØŸ™‰JˆAˆq�!�QØ!×-Ñ-×7Ñ7ˆNÜ.Ø—‘Ø˜Q˜Ø'Ø'+×':Ò':¡!À×@VÑ@Vô	ˆIð —‘�q˜"˜aÓ ‰AàŸ™ˆIà×Òð �I‘ˆAÙÜ—I‘I˜f¨ s™l°Ô2‘ñ Ü—I‘I˜f¨ s™l°Ô2�Ø�I‘ˆAà�}‰}˜QÓÐrQ   ÚindicesÚreturn_prefix_tokensr  Ú
stop_earlyrL  Úintermediates_onlyÚoutput_dictry   rz   c           	      ó*  — |dv sJ d«       ‚|dk(  }g }t        t        | j                  «      |«      \  }}|j                  \  }}}}| j	                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }t        j                  j                  «       s|s| j                  }n| j                  d|dz    }t        |«      D ]}  \  }}|	€|
r |||	|
¬«      }n?| j                  r+t        j                  j                  «       st        ||«      }n ||«      }||v sŒZ|j                  |r| j                  |«      n|«       Œ | j                   rE|D �cg c]  }|dd…d| j                   …f   ‘Œ }}|D �cg c]  }|dd…| j                   d…f   ‘Œ }}nd}|ra| j                  j#                  ||f«      \  }}|D �cg c]6  }|j%                  |||d«      j'                  dd	dd
«      j)                  «       ‘Œ8 }}|r*i }||d<   |�|r||d<   |s| j                  |«      }||d<   |S t        j                  j                  «       s|r|�t+        t-        ||«      «      }|r|S | j                  |«      }||fS c c}w c c}w c c}w )a,   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            return_prefix_tokens: Return both prefix and spatial intermediate tokens
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
            output_dict: Return outputs as a dictionary with 'image_features' and 'image_intermediates' keys
            attn_mask: Optional attention mask for masked attention (e.g., for NaFlex)
            is_causal: If True, use causal (autoregressive) masking in attention
        Returns:
            A tuple with (final_features, intermediates), a list of intermediate features, or a dictionary containing
            'image_features' and 'image_intermediates' (and optionally 'image_intermediates_prefix')
        )ÚNCHWÚNLCz)Output format must be one of NCHW or NLC.r¬  Nr.   r}   r   r½   r¢   r¿   Úimage_intermediatesÚimage_intermediates_prefixÚimage_features)r0   Úlenrc  rÄ   r[  r¥  ra  rb  rŠ   ÚjitÚis_scriptingrp  rX  r2   r  r  r  Údynamic_feat_sizerÍ   ÚpermuteÚ
contiguousÚlistÚzip)ru   rx   r¦  r§  r  r¨  rL  r©  rª  ry   rz   rÍ   ÚintermediatesÚtake_indicesÚ	max_indexrÏ   r  ÚheightÚwidthrc  rm  ÚblkrÒ   Úprefix_tokensr£  r¤  Úresult_dictÚx_finals                               rO   Úforward_intermediatesz'VisionTransformer.forward_intermediates5  s¡  € ð< ˜_Ñ,ÐYÐ.YÓYÐ,Ø Ñ&ˆØˆÜ"6´s¸4¿;¹;Ó7GÈÓ"QÑˆ�ið  Ÿg™gÑˆˆ1ˆf�eØ×Ñ˜QÓˆØ�O‰O˜AÓˆØ�O‰O˜AÓˆØ�M‰M˜!Óˆä�9‰9×!Ñ!Ô#©:Ø—[‘[‰Fà—[‘[  )¨a¡-Ð0ˆFÜ Ó'ò 		B‰FˆAˆsØÐ$©	Ù˜ Y¸)ÔD‘Ø×(Ò(´·±×1GÑ1GÔ1IÜ˜s AÓ&‘á˜“F�Ø�LÒ à×$Ñ$±T T§Y¡Y¨q¤\¸qÕAð		Bð ×!Ò!àERÖSÀ˜Qšq ! D×$:Ñ$:Ð":Ð:Ó;ÐSˆMÐSØDQÖR¸q˜Qšq $×"8Ñ"8Ñ"9Ð9Ó:ÐRˆMÑRà ˆMáà×#Ñ#×5Ñ5°v¸u°oÓF‰DˆAˆqØ^kÖlÐYZ˜QŸY™Y q¨!¨Q°Ó3×;Ñ;¸A¸qÀ!ÀQÓG×RÑRÕTÐlˆMÐlñ ØˆKà1>ˆKÐ-Ñ.ØÐ(Ñ-AØ<I�Ð8Ñ9ñ &ØŸ)™) A›,�Ø07�Ð,Ñ-àÐô �y‰y×%Ñ%Ô'Ñ,@À]ÐE^ä ¤ ]°MÓ!BÓCˆMáØ Ð à�I‰I�a‹Lˆà�-ÐÐùòG TùÚRùò ms   Å$JÆJÇ;JÚ
prune_normÚ
prune_headc                 ó  — t        t        | j                  «      |«      \  }}| j                  d|dz    | _        |rt        j                  «       | _        |r+t        j                  «       | _        | j                  dd«       |S )aE  Prune layers not required for specified intermediates.

        Args:
            indices: Indices of intermediate layers to keep.
            prune_norm: Whether to prune normalization layer.
            prune_head: Whether to prune the classifier head.

        Returns:
            List of indices that were kept.
        Nr.   r   r9   )r0   r±  rc  rk   rl   r  r5  r—  )ru   r¦  rÃ  rÄ  rº  r»  s         rO   Úprune_intermediate_layersz+VisionTransformer.prune_intermediate_layers–  sj   € ô  #7´s¸4¿;¹;Ó7GÈÓ"QÑˆ�iØ—k‘k . 9¨q¡=Ð1ˆŒÙÜŸ™›ˆDŒIÙÜŸ;™;›=ˆDŒLØ×!Ñ! ! RÔ(ØÐrQ   ÚnrÍ   c           	      ó:   — | j                  |||||rdndd|¬«      S )aÈ  Get intermediate layer outputs (DINO interface compatibility).

        NOTE: This API is for backwards compat, favour using forward_intermediates() directly.

        Args:
            x: Input tensor.
            n: Number or indices of layers.
            reshape: Reshape to NCHW format.
            return_prefix_tokens: Return prefix tokens.
            norm: Apply normalization.

        Returns:
            List of intermediate features.
        r¬  r­  T)r§  r  rL  r©  ry   )rÂ  )ru   rx   rÇ  rÍ   r§  r  ry   s          rO   Úget_intermediate_layersz)VisionTransformer.get_intermediate_layers¯  s4   € ð. ×)Ñ)ØˆqØ!5ØÙ!(‘v¨eØ#Øð *ó 
ð 	
rQ   c                 ó–  — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|€|r| j                  D ]  } ||||¬«      }Œ nR| j
                  r5t        j                  j                  «       st        | j                  |«      }n| j	                  |«      }| j                  |«      }|S )z\Forward pass through feature layers (embeddings, transformer blocks, post-transformer norm).r}   )r[  r¥  ra  rb  rc  rX  rŠ   r²  r³  r3   r  )ru   rx   ry   rz   r¾  s        rO   Úforward_featuresz"VisionTransformer.forward_featuresÏ  s°   € ð ×Ñ˜QÓˆØ�O‰O˜AÓˆØ�O‰O˜AÓˆØ�M‰M˜!ÓˆàÐ ¡Ià—{‘{ò E�Ù˜ Y¸)ÔD‘ñEà×$Ò$¬U¯Y©Y×-CÑ-CÔ-EÜ˜tŸ{™{¨AÓ.‰Aà—‘˜A“ˆAà�I‰I�a‹LˆØˆrQ   r  c                 óì   — | j                   �4| j                  s|dd…| j                  d…f   }| j                  |«      }|S |€| j                  n|}t	        ||| j                  | j                  ¬«      }|S )z´Apply pooling to feature tokens.

        Args:
            x: Feature tensor.
            pool_type: Pooling type override.

        Returns:
            Pooled features.
        N)r  r  r  )re  r6  r  r*  r  )ru   rx   r  s      rO   ÚpoolzVisionTransformer.poolç  s}   € ð �>‰>Ð%Ø×+Ò+Ø’a˜×/Ñ/Ñ0Ð0Ñ1�Ø—‘˜qÓ!ˆAØˆHØ(1Ð(9�D×$Ò$¸yˆ	ÜØØØ"×4Ñ4Ø"&×":Ñ":ô	
ˆð ˆrQ   Ú
pre_logitsc                 ó’   — | j                  |«      }| j                  |«      }| j                  |«      }|r|S | j                  |«      S )zÊForward pass through classifier head.

        Args:
            x: Feature tensor.
            pre_logits: Return features before final classifier.

        Returns:
            Output tensor.
        )rÍ  r5  rf  rg  )ru   rx   rÎ  s      rO   Úforward_headzVisionTransformer.forward_headÿ  sA   € ð �I‰I�a‹LˆØ�L‰L˜‹OˆØ�N‰N˜1ÓˆÙˆqÐ0 D§I¡I¨a£LÐ0rQ   c                 óP   — | j                  |||¬«      }| j                  |«      }|S r|   )rË  rÐ  r~   s       rO   r   zVisionTransformer.forward  s0   € ð ×!Ñ! !¨yÀIÐ!ÓNˆØ×Ñ˜aÓ ˆØˆrQ   r™   ©r9   T©r9   ©F)Tr“   )NN)	NFFFr¬  FFNF)r.   FT)r.   FFFNr€   )3r‚   rƒ   r„   r…   r   rˆ   rÙ   r   rS   r   r   r   rp   r
   r   r‡   r   rJ   r   r,   r   rk   r‰   ri   rw  r‘   r„  rŠ   r²  Úignorer‰  r	   rŒ  r   r   r�  r“  r•  r—  r�  r‹   r¥  r   rÂ  rÆ  rÉ  rË  rÍ  rÐ  r   rŒ   r�   s   @rO   r8   r8   Ç  s  ø… ñð
 ˜D‘kÓ!ð 58Ø68ØØ#ØV]Ø ØØØ!Ø!Ø!Ø$)Ø#(Ø"Ø+/Ø $Ø$Ø#(ØØ"Ø#Ø&*Ø(-Ø%*Ø$)Ø!Ø#%Ø%'Ø$&Ø$&Ø$&ØSUØ"Ø$.Ø48Ø.2Ø-1Ø(-Ø),Ø$-ØØñW1à˜C  s¨C x¡Ð0Ñ1ð1ð ˜c 5¨¨c¨¡?Ð2Ñ3ð1ð ð	1ð
 ð1ð !Ð!RÑSð1ð ð1ð ð1ð ð1ð ð1ð ð1ð ð1ð "ð1ð !ð1ð ð1ð  " %™ð!1ð" ð#1ð$ ð%1ð& !ð'1ð( ð)1ð* ð+1ð, ð-1ð. ˜d‘^ð/1ð0 "&ð11ð2 #ð31ð4 "ð51ð6 ð71ð8 !ð91ð: #ð;1ð< "ð=1ð> "ð?1ð@ "ðA1ðB !Ð!OÑPðC1ðD ðE1ðF "ðG1ðH ' yÑ1ðI1ðJ ! Ñ+ðK1ðL   	Ñ*ðM1ðN ˜2Ÿ9™9‘oðO1ðP ˜BŸI™I‘ðQ1ðR "ðS1ðX 
õY1óB1ñ# ð #¸ð #Èó #ð.!˜rŸy™yð !¨Tó !ð
 ‡Y�Y×ÑÓñ5¨sð 5¸Cð 5Èò 5ó ð5ð ‡Y�Y×ÑðE  S¡ò Eó ðEð ‡Y�Y×Ññ
 Dð 
°T¸#¸uÀSÈ$ÀYÑ?OÐ:OÑ5Pò 
ó ð
ð ‡Y�Y×Ññ<¨Tð <¸Tò <ó ð<ð ‡Y�Y×Ñð §	¡	ò ó ðña¨Cð a¸hÀs¹mð aÐW[ó að* 37Ø48ñà˜u S¨# X™Ñ/ðð !  s¨C x¡Ñ1ðð 
ó	ð2% ˜EŸL™Lð % ¨U¯\©\ó % ðT 8<Ø).ØØ$Ø$Ø',Ø %Ø04Ø#ñ_ à�|‰|ð_ ð ˜e C¨¨c© NÑ3Ñ4ð_ ð #'ð	_ ð
 ð_ ð ð_ ð ð_ ð !%ð_ ð ð_ ð   §¡Ñ-ð_ ð ð_ ð 
ˆt�E—L‘LÑ! 5¨¯©°t¸E¿L¹LÑ7IÐ)IÑ#JÈDÐQTÐVYÐQYÉNÐZÑ	[ó_ ðF ./Ø$Ø#ñ	à˜3  S¡	˜>Ñ*ðð ðð ð	ð
 
ˆc‰óð8 45Ø!Ø).ØØ04ñ
à�|‰|ð
ð �S˜$˜s™) U¨3¡ZÐ/Ñ0ð
ð ð	
ð
 #'ð
ð ð
ð   §¡Ñ-ð
ð 
ˆe�l‰lÑ	ó
ðF 15Ø#ñ	à�|‰|ðð   §¡Ñ-ðð ð	ð
 
�‰óñ0�e—l‘lð ¨x¸©}ð ÈÏÉó ñ01˜eŸl™lð 1¸ð 1ÈÏÉó 1ð$ 15Ø#ñ	à�|‰|ðð   §¡Ñ-ðð ð	ð
 
�‰÷rQ   r9   rO  ÚnamerS  c                 óN  — t        | t        j                  «      rNt        | j                  d¬«       | j
                  �*t        j                  j                  | j
                  «       yyt        | d«      r| j                  «        y|rt        | d«      r| j                  «        yyy)zðViT weight initialization, original timm impl (for reproducibility).

    Args:
        module: Module to initialize.
        name: Module name for context.
        needs_reset: If True, call reset_parameters() on modules that have it.
    r{  r|  Nr‘   r¶   )rI   rk   r§   r#   r–   re   r”   r»   r]  r‘   r¶   ©rO  rÖ  rS  s      rO   rƒ  rƒ    sy   € ô �&œ"Ÿ)™)Ô$Ü�f—m‘m¨Õ-Ø�;‰;Ð"Ü�G‰G�N‰N˜6Ÿ;™;Õ'ð #ä	�˜Ô	(Ø×ÑÕÙ	œ Ð);Ô<Ø×ÑÕ!ð =ˆrQ   r€  c                 óf  — t        | t        j                  «      rô|j                  d«      rTt        j                  j                  | j                  «       t        j                  j                  | j                  |«       yt        j                  j                  | j                  «       | j                  �Yd|v r+t        j                  j                  | j                  d¬«      n(t        j                  j                  | j                  «       yyt        | t        j                  «      rLt        | j                  «       | j                  �*t        j                  j                  | j                  «       yyt        | d«      r| j                  «        y|rt        | d«      r| j                  «        yyy)a  ViT weight initialization, matching JAX (Flax) impl.

    Args:
        module: Module to initialize.
        name: Module name for context.
        head_bias: Bias value for head layer.
        needs_reset: If True, call reset_parameters() on modules that have it.
    rg  Nrq   r}  r|  r‘   r¶   )rI   rk   r§   Ú
startswithr”   r»   r–   r•   re   Úxavier_uniform_rô   ÚConv2dr$   r]  r‘   r¶   )rO  rÖ  r€  rS  s       rO   Úinit_weights_vit_jaxrÝ  +  s  € ô �&œ"Ÿ)™)Ô$Ø�?‰?˜6Ô"Ü�G‰G�N‰N˜6Ÿ=™=Ô)Ü�G‰G×Ñ˜fŸk™k¨9Õ5ä�G‰G×#Ñ# F§M¡MÔ2Ø�{‰{Ð&Ø:?À4¹-”—‘—‘ §¡°�Ô6ÌRÏWÉWÏ^É^Ð\b×\gÑ\gÓMhùð 'ä	�FœBŸI™IÔ	&Ü�f—m‘mÔ$Ø�;‰;Ð"Ü�G‰G�N‰N˜6Ÿ;™;Õ'ð #ä	�˜Ô	(Ø×ÑÕÙ	œ Ð);Ô<Ø×ÑÕ!ð =ˆrQ   c                 ó~  — t        | t        j                  «      ræd|v r‚t        j                  dt        | j                  j                  d   dz  | j                  j                  d   z   «      z  «      }t        j                  j                  | j                  | |«       n)t        j                  j                  | j                  «       | j                  �*t        j                  j                  | j                  «       yyt        | d«      r| j                  «        y|rt        | d«      r| j                  «        yyy)	zôViT weight initialization, matching moco-v3 impl minus fixed PatchEmbed.

    Args:
        module: Module to initialize.
        name: Module name for context.
        needs_reset: If True, call reset_parameters() on modules that have it.
    Úqkvg      @r   r¢   r.   Nr‘   r¶   )rI   rk   r§   rí   rq  r‡   r–   rÄ   r”   Úuniform_rÛ  re   r»   r]  r‘   r¶   )rO  rÖ  rS  Úvals       rO   Úinit_weights_vit_mocorâ  K  så   € ô �&œ"Ÿ)™)Ô$Ø�D‰=ä—)‘)˜B¤ v§}¡}×':Ñ':¸1Ñ'=ÀÑ'BÀVÇ]Á]×EXÑEXÐYZÑE[Ñ'[Ó!\Ñ\Ó]ˆCÜ�G‰G×Ñ˜VŸ]™]¨S¨D°#Õ6ä�G‰G×#Ñ# F§M¡MÔ2Ø�;‰;Ð"Ü�G‰G�N‰N˜6Ÿ;™;Õ'ð #ä	�˜Ô	(Ø×ÑÕÙ	œ Ð);Ô<Ø×ÑÕ!ð =ˆrQ   c                 óD   — |rt        | d«      r| j                  «        y y y )Nr¶   )r]  r¶   rØ  s      rO   Úinit_weights_reset_parametersrä  b  s"   € Ù”w˜vÐ'9Ô:Ø×ÑÕ!ð ;€{rQ   rx  c                 óÚ   — | j                  d«      rt        t        ||¬«      S | j                  d«      rt        t        |¬«      S | dk(  rt        t        |¬«      S t        t
        |¬«      S )NrA  )r€  rS  rC  rR  r@  )rÚ  r   rÝ  râ  rä  rƒ  )rx  r€  rS  s      rO   r  r  g  s]   € Ø‡��uÔÜÔ+°yÈkÔZÐZØ	�‰˜Ô	 ÜÔ,¸+ÔFÐFØ	�ŠäÔ4À+ÔNÐNô Ô,¸+ÔFÐFrQ   rg   ÚposembÚ
posemb_newÚgs_newÚinterpolationÚ	antialiasc           	      ó  — |j                   d   |z
  }| j                   d   |z
  }t        t        j                  |«      «      gdz  }t	        |«      s"t        t        j                  |«      «      gdz  }t        | |||||d¬«      S )z¨ Rescale the grid of position embeddings when loading from state_dict.
    *DEPRECATED* This function is being deprecated in favour of using resample_abs_pos_embed
    r.   r¿   T)r  ré  rê  r›  )rÄ   rp   rí   rq  r±  r&   )	ræ  rç  r  rè  ré  rê  Úntok_newÚntok_oldÚgs_olds	            rO   Úresize_pos_embedrï  t  sŠ   € ð ×Ñ Ñ"Ð%6Ñ6€HØ�|‰|˜A‰Ð!2Ñ2€HÜ”$—)‘)˜HÓ%Ó&Ð'¨!Ñ+€FÜˆvŒ;Ü”d—i‘i Ó)Ó*Ð+¨aÑ/ˆÜ!Ø�˜Ø+Ø#ØØôð rQ   Úmodelr…  r†  Úload_bfloat16c                 ó¦  ‡‡‡‡— ddl Š‰r
ddlm Š ddlŠdIˆˆˆˆfd„	}‰r‰j                  |«      }n‰j                  |«      }d}d}d}|sd|v rd}nd	|v rd
}d}nd|v rd}d}t	        | j
                  d«      �r×| j
                  j                  }	t	        |	d«       }
|
r|	n|	j                  }|j                  j                  j                  t        |j                  j                  j                  d    |||› d�   «      «      «       |j                  j                  j                   |||› d�   «      «       |j                  j                  j                   |||› d�   «      «       |
�sÔt        |	j                   «      D �]»  \  }}t        |j"                  «      D �]œ  \  }}|› d|dz   › d|dz   › d�}t%        d«      D ]¹  }t'        |d|dz   › �«      j                  j                   |||› d|dz   › d�   «      «       t'        |d|dz   › �«      j                  j                   |||› d|dz   › d�   «      «       t'        |d|dz   › �«      j                  j                   |||› d|dz   › d�   «      «       Œ» |j(                  €Œì|j(                  j                  j                  j                   |||› d�   «      «       |j(                  j                  j                  j                   |||› d�   «      «       |j(                  j                  j                  j                   |||› d�   «      «       �ŒŸ �Œ¾  |||› d �   «      }nCt        | j
                  j*                  j                  j                  d    |||› d �   «      «      }|j                  d!d | j
                  j*                  j                  j                  d!d k7  r;t-        || j
                  j*                  j                  j                  d!d ||d¬"«      }| j
                  j*                  j                  j                  |«       | j
                  j*                  j                  j                   |||› d#�   «      «       | j.                  �)| j.                  j                   |||› d$�   d¬%«      «       |r |||› d&�   d¬%«      }n |||› d'�   d¬%«      }|j                  | j0                  j                  k7  rAt'        | d(d«      rdnt'        | d)d«      }t3        || j
                  j4                  |||d¬*«      }| j0                  j                  |«       | j                  j                  j                   |||› d+�   «      «       | j                  j                  j                   |||› d,�   «      «       t7        | j8                  t:        j<                  «      r¢|› d-�|v r›| j8                  j                  j                  d   ||› d-�   j                  d.   k(  rb| j8                  j                  j                   |||› d/�   «      «       | j8                  j                  j                   |||› d-�   «      «       t7        | j>                  t@        «      �rn|› d0�}|d1z   }| j>                  jB                  j                   |||› d2�   d¬%«      «       | j>                  jD                  j                  j                  tG        jH                  d3D �cg c]-  } |||› |› d�   d¬%«      jK                  d«      jL                  ‘Œ/ c}«      «       | j>                  jD                  j                  j                  tG        jH                  d3D �cg c]#  } |||› |› d�   d¬%«      jO                  d.«      ‘Œ% c}«      «       | j>                  jP                  j                  j                   |||› d4�   d¬%«      jK                  d«      jL                  «       | j>                  jP                  j                  j                   |||› d5�   d¬%«      jO                  d.«      «       | j>                  j*                  j                  j                   |||› d6�   «      jK                  d«      «       | j>                  j*                  j                  j                   |||› d7�   «      «       | j>                  j                  j                  j                   |||› d8�   «      «       | j>                  j                  j                  j                   |||› d9�   «      «       t%        d:«      D ]ž  }t'        | j>                  jR                  d;|dz   › �«      j                  j                   |||› d<|› d�   «      «       t'        | j>                  jR                  d;|dz   › �«      j                  j                   |||› d<|› d�   «      «       Œ  |rd=nd>\  }}}t        | j"                  jU                  «       «      D �]  \  }}|› d?�|v r|› d@�}|}n
|› dA|› d�}d}|dB|› d�z   }|jV                  j                  j                   |||› d8�   |¬C«      «       |jV                  j                  j                   |||› d9�   |¬C«      «       |jX                  jZ                  j                  j                  tG        jH                  dDD �cg c].  } |||› |› d�   d|¬E«      jK                  d«      jL                  ‘Œ0 c}«      «       |jX                  jZ                  j                  j                  tG        jH                  dDD �cg c]$  } |||› |› d�   d|¬E«      jO                  d.«      ‘Œ& c}«      «       |jX                  j*                  j                  j                   |||› d6�   |¬C«      jK                  d«      «       |jX                  j*                  j                  j                   |||› d7�   |¬C«      «       |j\                  j                  j                   |||› dF|› d�   |¬C«      «       |j\                  j                  j                   |||› dF|› d�   |¬C«      «       t%        d:«      D ]”  }t'        |jR                  d;|dz   › �«      j                  j                   |||› dG|› dH|› d�   |¬C«      «       t'        |jR                  d;|dz   › �«      j                  j                   |||› dG|› dH|› d�   |¬C«      «       Œ– �Œ yc c}w c c}w c c}w c c}w )JzV Load weights from .npz checkpoints for official Google Brain Flax implementation
    r   NTc                 óV  •— |�| |   } ‰rE| j                  ‰j                  «      j                  ‰j                  «      } ‰j	                  | «      } | j
                  dk(  rK| j                  d   | j                  d   cxk(  r| j                  d   cxk(  rdk(  rn n| j                  «       } |rh| j
                  dk(  r| j                  g d¢«      } nE| j
                  dk(  r| j                  g d¢«      } n"| j
                  dk(  r| j                  ddg«      } t        j                  | «      } | S )Né   r   r.   r¿   )r¢   r¿   r   r.   r¢   )r¿   r   r.   )rÆ   Úbfloat16Úastyperã   ÚarrayÚndimrÄ   ÚflattenrÇ   rŠ   Ú
from_numpy)Ú_wÚtÚidxÚjnprñ  Ú	ml_dtypesÚnps      €€€€rO   Ú_n2pz_load_weights.<locals>._n2p–  sê   ø€ Øˆ?Ø�C‘ˆBáØ—‘˜×+Ñ+Ó,×3Ñ3°C·K±KÓ@ˆBØ—‘˜"“ˆBà�7‰7�aŠ<˜BŸH™H Q™K¨2¯8©8°A©;ÔJ¸"¿(¹(À1¹+ÔJÈÕJØ—‘“ˆBÙØ�w‰w˜!Š|Ø—\‘\¢,Ó/‘Ø—‘˜A’Ø—\‘\¢)Ó,‘Ø—‘˜A’Ø—\‘\ 1 a &Ó)�ä×Ñ˜bÓ!ˆØˆ	rQ   ÚbilinearFzopt/target/embedding/kernelzopt/target/zparams/embedding/kernelzparams/zparams/img/embedding/kernelzparams/img/Úbackboner�  r.   zconv_root/kernelzgn_root/scalezgn_root/biasÚblockz/unitú/r¢   Úconvz/kernelr  Úgnz/scalez/biaszconv_proj/kernelzgn_proj/scalezgn_proj/biaszembedding/kernelrÂ   ©ré  rê  r›  zembedding/biasÚcls)rü  Úpos_embeddingz(Transformer/posembed_input/pos_embeddingr1  r  ©r™  r  ré  rê  r›  zTransformer/encoder_norm/scalezTransformer/encoder_norm/biasz	head/biasr½   zhead/kernelz
MAPHead_0/zMultiHeadDotProductAttention_0/Úprobe)ÚkeyÚvaluezquery/kernelz
query/biasz
out/kernelzout/biaszLayerNorm_0/scalezLayerNorm_0/biasr¿   ÚfczMlpBlock_0/Dense_)r   r   r.   )r.   r¢   r¿   z*Transformer/encoderblock/LayerNorm_0/scalezTransformer/encoderblock/zTransformer/encoderblock_ÚMultiHeadDotProductAttention_)rý  )Úqueryr  r  )rü  rý  Ú
LayerNorm_Ú	MlpBlock_z/Dense_)TN)/ÚnumpyÚ	jax.numpyrÿ  Úloadr]  r[  r  r�  r  r–   Úcopy_r4   rÄ   r  re   rp  Ústagesrc  r  ÚgetattrÚ
downsamplerr  r%   r^  r0  r&   rœ  rI   rg  rk   r§   re  r   ÚlatentÚkvrŠ   rÎ   rù  ÚTrÍ   rÔ   rq   Úchildrenrj   r:   rß  ro   )rð  r…  r†  rñ  r  Úwré  rê  Ú
big_visionr  Ú	stem_onlyr�  rm  ÚstageÚjr  ÚbpÚrÚembed_conv_wÚpos_embed_wr  Úblock_prefixÚ
mha_prefixrÇ  Úmha_subÚb_subÚln1_subrý  rþ  rÿ  r   s      `                        @@@rO   rˆ  rˆ  �  sn  û€ ó ÙÝÛ÷ð ñ* Ø�H‰H�_Ó%‰à�G‰G�OÓ$ˆà€MØ€IØ€JÙØ(¨AÑ-Ø"‰FØ&¨!Ñ+ØˆFØ‰JØ*¨aÑ/Ø"ˆFØˆJäˆu× Ñ  *Õ-à×$Ñ$×-Ñ-ˆÜ ¨&Ó1Ð1ˆ	Ù$‰x¨(¯-©-ˆØ�	‰	×Ñ×ÑÔ/°·	±	×0@Ñ0@×0FÑ0FÀqÑ0IÉ4ÐPQÐU[ÐT\Ð\lÐRmÑPnÓKoÓpÔqØ�	‰	×Ñ×Ñ™t A¨¨°Ð&>Ñ$?Ó@ÔAØ�	‰	�‰×Ñ™T ! v h¨lÐ$;Ñ"<Ó=Ô>ÚÜ% h§o¡oÓ6ó 
W‘��5Ü )¨%¯,©,Ó 7ó 	W‘H�A�uØ"˜8 5¨¨Q©¨¨u°Q¸±U°G¸1Ð=�BÜ" 1›Xò b˜Ü ¨¨a°!©e¨W ~Ó6×=Ñ=×CÑCÁDÈÈbÈTÐQUÐVWÐZ[ÑV[ÐU\Ð\cÐKdÑIeÓDfÔgÜ ¨¨a°!©e¨W ~Ó6×=Ñ=×CÑCÁDÈÈbÈTÐQSÐTUÐXYÑTYÐSZÐZ`ÐKaÑIbÓDcÔdÜ ¨¨a°!©e¨W ~Ó6×;Ñ;×AÑAÁ$ÀqÈBÈ4ÈrÐRSÐVWÑRWÐQXÐX]ÐI^ÑG_ÓB`Õaðbð ×'Ñ'Ñ3Ø×(Ñ(×-Ñ-×4Ñ4×:Ñ:¹4ÀÀRÀDÐHXÐBYÑ@ZÓ;[Ô\Ø×(Ñ(×-Ñ-×4Ñ4×:Ñ:¹4ÀÀRÀDÈÐBVÑ@WÓ;XÔYØ×(Ñ(×-Ñ-×2Ñ2×8Ñ8¹¸aÀ2À$ÀlÐ@SÑ>TÓ9UÖVò	Wð
Wñ ˜A  Ð(8Ð9Ñ:Ó;‰ä'Ø×Ñ×"Ñ"×)Ñ)×/Ñ/°Ñ2±D¸¸f¸XÐEUÐ;VÑ9WÓ4XóZˆà×Ñ˜"˜#Ð %×"3Ñ"3×"8Ñ"8×"?Ñ"?×"EÑ"EÀbÀcÐ"JÒJÜ+ØØ×Ñ×"Ñ"×)Ñ)×/Ñ/°°Ð4Ø'ØØô
ˆð 
×Ñ×Ñ×!Ñ!×'Ñ'¨Ô5Ø	×Ñ×Ñ×Ñ×%Ñ%¡d¨1°¨x°~Ð-FÑ+GÓ&HÔIØ‡�Ð"Ø�‰×Ñ™d 1¨ x¨s ^Ñ#4¸Ô>Ô?ÙÙ˜1 ˜x }Ð5Ñ6¸%Ô@‰á˜1 ˜xÐ'OÐPÑQÐUZÔ[ˆØ×Ñ˜EŸO™O×1Ñ1Ò1Ü!(¨Ð0@À%Ô!H™AÌgÐV[Ð]pÐrsÓNtÐÜ,ØØ×&Ñ&×0Ñ0Ø/Ø'ØØô
ˆð 
‡O�O×Ñ˜+Ô&Ø	‡J�J×Ñ×Ñ™D  f XÐ-KÐ#LÑ!MÓNÔOØ	‡J�J‡O�O×Ñ™$˜q F 8Ð+HÐ!IÑJÓKÔLÜ�5—:‘:œrŸy™yÔ)Øˆh�iÐ  AÑ%Ø�J‰J�O‰O×!Ñ! !Ñ$¨¨V¨H°IÐ*>Ñ(?×(EÑ(EÀbÑ(IÒIØ�
‰
×Ñ×Ñ¡ Q¨&¨°Ð'=Ñ%>Ó ?Ô@Ø�
‰
�‰×Ñ™d 1¨ x¨yÐ%9Ñ#:Ó;Ô<ô
 �%—/‘/Ô#6Õ7Ø ˜ Ð,ˆØ!Ð&EÑFˆ
Ø�‰×Ñ×$Ñ$¡T¨!¨|¨n¸EÐ,BÑ*CÀuÔ%MÔNØ�‰×Ñ×!Ñ!×'Ñ'¬¯	©	ØN^ö3`ØIJ‰D��j�\ !  GÐ,Ñ-°Ô7×?Ñ?ÀÓB×DÓDò3`ó )aô 	bà�‰×Ñ×Ñ×%Ñ%¤e§i¡iØK[ö1]ØFG‰D��j�\ !  EÐ*Ñ+¨uÔ5×=Ñ=¸bÕAò1]ó '^ô 	_à�‰×Ñ× Ñ ×&Ñ&¡t¨A°°¸LÐ.IÑ,JÈeÔ'T×'\Ñ'\Ð]^Ó'_×'aÑ'aÔbØ�‰×Ñ×Ñ×$Ñ$¡T¨!¨z¨l¸*Ð,EÑ*FÈ%Ô%P×%XÑ%XÐY[Ó%\Ô]Ø�‰×Ñ×#Ñ#×)Ñ)©$¨q°J°<¸zÐ1JÑ/KÓ*L×*TÑ*TÐUVÓ*WÔXØ�‰×Ñ×!Ñ!×'Ñ'©¨Q°*°¸XÐ/FÑ-GÓ(HÔIØ�‰×Ñ×#Ñ#×)Ñ)©$¨q°L°>ÐARÐ1SÑ/TÓ*UÔVØ�‰×Ñ×!Ñ!×'Ñ'©¨Q°,°Ð?OÐ/PÑ-QÓ(RÔSÜ�q“ò 	wˆAÜ�E—O‘O×'Ñ'¨2¨a°!©e¨W¨Ó6×=Ñ=×CÑCÁDÈÈlÈ^Ð[lÐmnÐloÐovÐKwÑIxÓDyÔzÜ�E—O‘O×'Ñ'¨2¨a°!©e¨W¨Ó6×;Ñ;×AÑAÁ$ÀqÈLÈ>ÐYjÐklÐjmÐmrÐIsÑGtÓBuÕvð	wñ ,6™i¸9Ñ€GˆU�GÜ˜eŸl™l×3Ñ3Ó5Ó6ó T‰ˆˆ5ØˆXÐ?Ð@ÀAÑEØ$˜XÐ%>Ð?ˆLØ‰Cà$˜XÐ%>¸q¸cÀÐCˆLØˆCØ!Ð&CÀGÀ9ÈAÐ$NÑNˆ
Ø�‰×Ñ× Ñ ¡ a¨<¨.Ð8IÐ(JÑ&KÐQTÔ!UÔVØ�‰×Ñ×Ñ™t A¨¨Ð6FÐ&GÑ$HÈcÔRÔSØ�
‰
�‰×Ñ×#Ñ#¤E§I¡IØWpö/rØRS‰D��j�\ !  GÐ,Ñ-°¸CÔ@×HÑHÈÓK×MÓMò/ró %sô 	tà�
‰
�‰×Ñ×!Ñ!¤%§)¡)ØTmö-oØOP‰D��j�\ !  EÐ*Ñ+¨u¸#Ô>×FÑFÀrÕJò-oó #pô 	qà�
‰
�‰×Ñ×$Ñ$¡T¨!¨z¨l¸*Ð,EÑ*FÈCÔ%P×%XÑ%XÐYZÓ%[Ô\Ø�
‰
�‰×Ñ×"Ñ"¡4¨¨Z¨L¸Ð*AÑ(BÈÔ#LÔMØ�‰×Ñ× Ñ ¡ a¨<¨.¸
À7À)È6Ð(RÑ&SÐY\Ô!]Ô^Ø�‰×Ñ×Ñ™t A¨¨°jÀÀ	ÈÐ&OÑ$PÐVYÔZÔ[Ü�q“ò 	TˆAÜ�E—I‘I  A¨¡E 7˜|Ó,×3Ñ3×9Ñ9Ù�Q˜,˜ y°°°w¸q¸cÀÐIÑJÐPSÔTôVä�E—I‘I  A¨¡E 7˜|Ó,×1Ñ1×7Ñ7Ù�Q˜,˜ y°°°w¸q¸cÀÐGÑHÈcÔRõTò	Tñ%Tùò3`ùò1]ùò./rùò-os   Ü	2t?Þ(uë63u	í0)uÚ
state_dictc                 óÌ  — i }g d¢}| j                  «       D ]Ê  \  }}|j                  |«      sŒ|j                  |d«      }|D ]  }|j                  |d   |d   «      }Œ |dk(  r:d}|j                  dd«      }t	        j
                  |j                  d   «      |d<   n>|dk(  r#d	}|j                  d«      j                  d«      }n|d
k(  r|j                  d«      }|||<   ŒÌ |S )N)
)Úconv1úpatch_embed.proj)Úpositional_embeddingr0  )ztransformer.resblocks.rN  )Úln_prerb  )Úln_postr  )Úln_r  )Úin_proj_zqkv.©r²   rr  )zmlp.c_fcúmlp.fc1)z
mlp.c_projzmlp.fc2r9   r   r.   rr  úhead.weightú	head.biasÚclass_embeddingr^  r0  )ÚitemsrÚ  ÚreplacerÇ   rŠ   ÚzerosrÄ   Ú	unsqueeze)r-  rð  r†  Úout_dictÚswapsrÕ   rÖ   Úsps           rO   Ú_convert_openai_cliprB  (  sø   € ð
 €Hò€Eð × Ñ Ó"ò ‰ˆˆ1Ø�|‰|˜FÔ#ØØ�I‰I�f˜bÓ!ˆØò 	(ˆBØ—	‘	˜"˜Q™%  A¡Ó'‰Að	(ð �Š;ØˆAØ—‘˜A˜qÓ!ˆAÜ$)§K¡K°·±¸±
Ó$;ˆH�[Ò!ØÐ#Ò#ØˆAØ—‘˜A“×(Ñ(¨Ó+‰AØ�+ÒØ—‘˜A“ˆAØˆ�Šð!ð" €OrQ   c                 ó¼  — dd l }i }| j                  dd «       d| v rR| j                  d«      |d<   | j                  d«      | d   d d …df   z   |d<   | j                  d«      d d …dd …f   |d<   | j                  «       D ]Z  \  }}|j                  d|«      r|||j	                  d	d
«      <   Œ.|j                  d|«      r|||j	                  dd«      <   ŒV|||<   Œ\ |S )Nr   Ú
mask_tokenÚregister_tokensr_  r^  r0  r.   z(blocks\.(\d+)\.mlp\.w12\.(?:weight|bias)Úw12Úfc1z'blocks\.(\d+)\.mlp\.w3\.(?:weight|bias)Úw3rt  )ÚreÚpopr;  Úmatchr<  )r-  rð  rI  r?  rÕ   rÖ   s         rO   Ú_convert_dinov2rL  N  s÷   € ó Ø€HØ‡N�N�< Ô&Ø˜JÑ&à *§¡Ð/@Ó Aˆ�ÑØ *§¡¨{Ó ;¸jÈÑ>UÒVWÐYZÐVZÑ>[Ñ [ˆ�ÑØ *§¡¨{Ó ;ºA¸q¹r¸EÑ Bˆ�ÑØ× Ñ Ó"ò ‰ˆˆ1Ø�8‰8Ð?ÀÔCØ01ˆH�Q—Y‘Y˜u eÓ,Ñ-ØØ�X‰XÐ@À!ÔDØ/0ˆH�Q—Y‘Y˜t UÓ+Ñ,ØØˆ�Šðð €OrQ   c                 ód  — i }| j                  «       D ]š  \  }}|j                  dd«      }|j                  dd«      }|j                  dd«      }|j                  dd«      }|j                  d	d
«      }|j                  dd«      }|j                  dd«      }|j                  dd«      }|||<   Œœ |S )NÚnorm_1rj   Únorm_2ro   zpreprocessor.patchifier.úpatch_embed.zpreprocessor.pos_embedr0  ztrunk.r9   zpost_trunk_norm.únorm.r7  z	mlp.fc1_gzmlp.fc3z	mlp.fc1_x)r;  r<  )r-  rð  r?  rÕ   rÖ   s        rO   Ú_convert_aimv2rR  e  s¸   € ð €HØ× Ñ Ó"ò 	‰ˆˆ1Ø�I‰I�h Ó(ˆØ�I‰I�h Ó(ˆØ�I‰IÐ0°.ÓAˆØ�I‰IÐ.°Ó<ˆØ�I‰I�h Ó#ˆØ�I‰IÐ(¨'Ó2ˆØ�I‰I�i Ó-ˆØ�I‰I�i Ó-ˆØˆ�Šð	ð €OrQ   c                 ó„  — ddl }| j                  d| «      } dD ]  }| j                  |d«       Œ g d¢}i }| j                  «       D ]I  \  }}d|v rŒ|D ]  \  }}|j	                  |||«      }Œ |dk(  r|dd j                  d«      |d	<   ŒE|||<   ŒK i i }
}	|j                  d
«      }|j                  «       D ]†  \  }}|j                  |«      }|s||	|<   Œ|j                  «       \  }}}|
j                  ||fi «      }|||<   t        |«      dk(  sŒ[t        j                  |d   |d   |d   gd¬«      |	d|› d|› �<   Œˆ |	S )zP
    Turn a BEiT-3 checkpoint into a standard VisionTransformer state-dict.
    r   Nrð  )zbeit3.text_embed.weightzbeit3.vision_embed.mask_token))zbeit3\.r9   )zvision_embed\.cls_tokenr^  )zvision_embed\.rP  )zembed_positions\.z
pos_embed.)z	encoder\.r9   )zlayers\.rN  )zffn_layernorm\.rQ  )zffn\.zmlp.)zself_attn_layer_norm\.znorm1.)zself_attn\.zattn.)zfinal_layer_norm\.znorm2.)Úinner_attn_lnr  r6  )z\.A\.ú.z.B.zpos_embed.weightr¿   r0  z1blocks\.(\d+)\.attn\.(q|k|v)_proj\.(weight|bias)$r¢   rÔ   rÕ   rÖ   r¾   rN  z
.attn.qkv.)rI  rL   rJ  r;  Úsubr>  ÚcompileÚ	fullmatchÚgroupsÚ
setdefaultr±  rŠ   rÎ   )r-  rð  rI  rÕ   ÚrulesÚtmprÖ   ÚoldÚnewÚoutÚbufÚpatr�  r¾  ÚwhichÚkindÚstashs                    rO   Ú_convert_beit3re  w  s„  € ó Ø—‘ ¨Ó4€Jð Jò  ˆØ�‰�q˜$Õð ò€Eð  €CØ× Ñ Ó"ò 	‰ˆˆ1Ø�A‰:ØØò 	$‰HˆC�Ø—‘�s˜C Ó#‰Að	$àÐ"Ò"à   ˜uŸ™¨qÓ1ˆC�ÒàˆC�ŠFð	ð �2ˆ€CØ
�*‰*ÐIÓ
J€CØ—	‘	“ò ‰ˆˆ1Ø�M‰M˜!ÓˆÙØˆC�‰FØàŸ8™8›:ÑˆˆU�DØ—‘  T˜{¨BÓ/ˆØˆˆe‰Üˆu‹:˜‹?Ü38·9±9Ø�s‘˜U 3™Z¨¨s©Ð4¸!ô4ˆC�'˜#˜˜j¨¨Ð/Ò0ðð €JrQ   Úadapt_layer_scalec           	      ób  — ddl }i }| j                  d| «      } | j                  d| «      } d}d| v rt        | |«      } �n-d| v rt        | |d¬	«      } �nd
| v rt        | |«      } �nd| v rPd}| j	                  «       D �	�
ci c]&  \  }	}
|	j                  |«      sŒ|	t        |«      d |
“Œ( } }	}
t        | |«      } n³t        d„ | j                  «       D «       «      rt        | |«      } n†d| v r| d   } d}nzd| v sd| v r[d}d| v rlt        |j                  t        j                  «      rH| d   |d<   t        j                  | d   j                   d   «      |d<   nd| v rd}nd| v rt#        | |«      } |rA| j	                  «       D �	�
ci c]&  \  }	}
|	j                  |«      sŒ|	t        |«      d |
“Œ( } }	}
| j	                  «       D �]`  \  }	}
d|	v rÁ|j$                  j&                  j(                  j                   \  }}}}t        |
j                   «      dk  rC|j$                  j&                  j(                  j                   \  }}}}|
j+                  |d||«      }
|
j                   d   |k7  s|
j                   d   |k7  r£t-        |
||f||d¬«      }
n�|	dk(  rk|
j                   d    |j.                  j                   d    k7  rBt1        |d!d"«      rdnt1        |d#d «      }t3        |
|j$                  j4                  |||d¬$«      }
n |rd%|	v r|j7                  d&d'|	«      }	nd(|	v r�Œ\|
||	<   �Œc |S c c}
}	w c c}
}	w ))zJ convert patch embedding weight from manual patchify + linear proj to convr   Nrð  r-  r9   zvisual.class_embeddingzmodule.visual.class_embeddingzmodule.visual.)r†  rD  zvision_encoder.mask_tokenzvision_encoder.c              3   ó$   K  — | ]  }d |v –— Œ
 y­w)zbeit3.Nrg   )Ú.0rÕ   s     rO   ú	<genexpr>z'checkpoint_filter_fn.<locals>.<genexpr>Ì  s   è ø€ Ò6˜qˆX˜Œ]Ñ6ùs   ‚Úencoderzmodule.zvisual.trunk.pos_embedz"visual.trunk.blocks.0.norm1.weightzvisual.trunk.zvisual.head.proj.weightr8  r9  zmodule.visual.trunk.pos_embedzmodule.visual.trunk.z#preprocessor.patchifier.proj.weightzpatch_embed.proj.weightrô  r½   rÂ   Tr  r0  r.   r1  Fr  r  Úgamma_zgamma_([0-9])z
ls\1.gammarÎ  )rI  rL   rB  rL  r;  rÚ  r±  ÚanyÚkeysre  rI   rg  rk   r§   rŠ   r=  rÄ   rR  r[  rr  r–   rÍ   r%   r0  r  r&   rœ  rV  )r-  rð  rf  ré  rê  rI  r?  r†  Ú	ve_prefixrÕ   rÖ   ÚOÚIr£  r¤  r  s                   rO   Úcheckpoint_filter_fnrr  ³  sG  € ó Ø€HØ—‘ ¨Ó4€JØ—‘ ¨jÓ9€JØ€Fà :Ñ-Ü)¨*°eÓ<Š
Ø	(¨JÑ	6Ü)¨*°eÐDTÔUŠ
Ø	˜Ñ	#Ü$ Z°Ó7Š
Ø	$¨
Ñ	2à%ˆ	Ø8B×8HÑ8HÓ8J×f±°°1ÈaÏlÉlÐ[dÕNe�aœ˜I›˜Ð(¨!Ñ+Ðfˆ
ÑfÜ$ Z°Ó7‰
Ü	Ñ6 J§O¡OÓ$5Ô6Ô	6ä# J°Ó6‰
Ø	�jÑ	 à 	Ñ*ˆ
Ø‰Ø	! ZÑ	/Ð3WÐ[eÑ3eà ˆØ$¨
Ñ2´zÀ%Ç*Á*ÌbÏiÉiÔ7Xà&0Ð1JÑ&KˆH�]Ñ#Ü$)§K¡K°
Ð;TÑ0U×0[Ñ0[Ð\]Ñ0^Ó$_ˆH�[Ò!Ø	(¨JÑ	6Ø'‰Ø	.°*Ñ	<Ü# J°Ó6ˆ
áà5?×5EÑ5EÓ5G×`©T¨Q°È1Ï<É<ÐX^ÕK_�aœ˜F›˜�o qÑ(Ð`ˆ
Ñ`à× Ñ Ó"ó  ‰ˆˆ1Ø$¨Ñ)Ø×*Ñ*×/Ñ/×6Ñ6×<Ñ<‰JˆAˆq�!�QÜ�1—7‘7‹|˜aÒà"×.Ñ.×3Ñ3×:Ñ:×@Ñ@‘
��1�a˜Ø—I‘I˜a  Q¨Ó*�Ø�w‰w�r‰{˜aÒ 1§7¡7¨2¡;°!Ò#3Ü(ØØ˜�FØ"/Ø'Ø ô‘ð �+Ò !§'¡'¨!¡*°·±×0EÑ0EÀaÑ0HÒ"Hä%,¨UÐ4DÀeÔ%L¡ÔRYÐZ_ÐatÐvwÓRxÐÜ&ØØ×*Ñ*×4Ñ4Ø"3Ø+Ø#Øô‰Añ  8¨q¡=à—‘Ð'¨¸Ó:‰AØ˜QÑáØˆ�‹ðA ðB €Oùóu gùó. as   Á>L%ÂL%ÆL+ÆL+Úurlc                 ó4   — | ddd dddt         t        dddd	œ|¥S )
Nr"  )r¢   r   r   gÍÌÌÌÌÌì?ÚbicubicTr0  rg  ú
apache-2.0)rs  r)  Ú
input_sizeÚ	pool_sizeÚcrop_pctré  Úfixed_input_sizerò   ró   Ú
first_convÚ
classifierÚlicense)r   r   )rs  rN   s     rO   Ú_cfgr~    s:   € àØØ#ØØØ"Ø Ü'Ü%Ø(ØØñð ðð rQ   z*vit_base_patch16_224.augreg2_in21k_ft_in1kztimm/)Ú	hf_hub_idz)vit_base_patch8_224.augreg2_in21k_ft_in1kz)vit_tiny_patch16_224.augreg_in21k_ft_in1kz•https://storage.googleapis.com/vit_models/augreg/Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npz)rs  r  Úcustom_loadz)vit_tiny_patch16_384.augreg_in21k_ft_in1kz•https://storage.googleapis.com/vit_models/augreg/Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npz)r¢   é€  r�  ç      ð?)rs  r  r€  rw  ry  z*vit_small_patch32_224.augreg_in21k_ft_in1kz–https://storage.googleapis.com/vit_models/augreg/S_32-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npzz*vit_small_patch32_384.augreg_in21k_ft_in1kz–https://storage.googleapis.com/vit_models/augreg/S_32-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npzz*vit_small_patch16_224.augreg_in21k_ft_in1kz–https://storage.googleapis.com/vit_models/augreg/S_16-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npzz*vit_small_patch16_384.augreg_in21k_ft_in1kz–https://storage.googleapis.com/vit_models/augreg/S_16-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npzz)vit_base_patch32_224.augreg_in21k_ft_in1kz—https://storage.googleapis.com/vit_models/augreg/B_32-i21k-300ep-lr_0.001-aug_medium1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npzz)vit_base_patch32_384.augreg_in21k_ft_in1kz•https://storage.googleapis.com/vit_models/augreg/B_32-i21k-300ep-lr_0.001-aug_light1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npzz)vit_base_patch16_224.augreg_in21k_ft_in1kz–https://storage.googleapis.com/vit_models/augreg/B_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_224.npzz)vit_base_patch16_384.augreg_in21k_ft_in1kz–https://storage.googleapis.com/vit_models/augreg/B_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_384.npzz(vit_base_patch8_224.augreg_in21k_ft_in1kz•https://storage.googleapis.com/vit_models/augreg/B_8-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_224.npzz*vit_large_patch16_224.augreg_in21k_ft_in1kz–https://storage.googleapis.com/vit_models/augreg/L_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_224.npzz*vit_large_patch16_384.augreg_in21k_ft_in1kz–https://storage.googleapis.com/vit_models/augreg/L_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_384.npzz'vit_base_patch16_224.orig_in21k_ft_in1kzohttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_vit_base_p16_224-80ecf9dd.pth)rs  r  z'vit_base_patch16_384.orig_in21k_ft_in1kzohttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_vit_base_p16_384-83fb41ba.pth)rs  r  rw  ry  z(vit_large_patch32_384.orig_in21k_ft_in1kzphttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_vit_large_p32_384-9b920ba8.pthz!vit_small_patch16_224.augreg_in1kz•https://storage.googleapis.com/vit_models/augreg/S_16-i1k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_224.npzz!vit_small_patch16_384.augreg_in1kz•https://storage.googleapis.com/vit_models/augreg/S_16-i1k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_384.npzz vit_base_patch32_224.augreg_in1kz•https://storage.googleapis.com/vit_models/augreg/B_32-i1k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_224.npzz vit_base_patch32_384.augreg_in1kz•https://storage.googleapis.com/vit_models/augreg/B_32-i1k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_384.npzz vit_base_patch16_224.augreg_in1kz•https://storage.googleapis.com/vit_models/augreg/B_16-i1k-300ep-lr_0.001-aug_strong2-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_224.npzz vit_base_patch16_384.augreg_in1kz•https://storage.googleapis.com/vit_models/augreg/B_16-i1k-300ep-lr_0.001-aug_strong2-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_384.npzzvit_large_patch14_224.untrained)rs  zvit_huge_patch14_224.untrainedzvit_giant_patch14_224.untrainedz"vit_gigantic_patch14_224.untrainedzvit_base_patch32_224.orig_in21k)r  r)  zvit_base_patch16_224.orig_in21kz vit_large_patch32_224.orig_in21kz vit_large_patch16_224.orig_in21kzvit_huge_patch14_224.orig_in21kz!vit_tiny_patch16_224.augreg_in21kzmhttps://storage.googleapis.com/vit_models/augreg/Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0.npziSU  )rs  r  r€  r)  z"vit_small_patch32_224.augreg_in21kznhttps://storage.googleapis.com/vit_models/augreg/S_32-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0.npzz"vit_small_patch16_224.augreg_in21kznhttps://storage.googleapis.com/vit_models/augreg/S_16-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0.npzz!vit_base_patch32_224.augreg_in21kzohttps://storage.googleapis.com/vit_models/augreg/B_32-i21k-300ep-lr_0.001-aug_medium1-wd_0.03-do_0.0-sd_0.0.npzz!vit_base_patch16_224.augreg_in21kznhttps://storage.googleapis.com/vit_models/augreg/B_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0.npzz vit_base_patch8_224.augreg_in21kzmhttps://storage.googleapis.com/vit_models/augreg/B_8-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0.npzz"vit_large_patch16_224.augreg_in21kznhttps://storage.googleapis.com/vit_models/augreg/L_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1.npzzvit_base_patch32_224.sam_in1kz:https://storage.googleapis.com/vit_models/sam/ViT-B_32.npz)rs  r€  r  zvit_base_patch16_224.sam_in1kz:https://storage.googleapis.com/vit_models/sam/ViT-B_16.npzzvit_small_patch16_224.dinoz[https://dl.fbaipublicfiles.com/dino/dino_deitsmall16_pretrain/dino_deitsmall16_pretrain.pth)rs  r  rò   ró   r)  zvit_small_patch8_224.dinozYhttps://dl.fbaipublicfiles.com/dino/dino_deitsmall8_pretrain/dino_deitsmall8_pretrain.pthzvit_base_patch16_224.dinozWhttps://dl.fbaipublicfiles.com/dino/dino_vitbase16_pretrain/dino_vitbase16_pretrain.pthzvit_base_patch8_224.dinozUhttps://dl.fbaipublicfiles.com/dino/dino_vitbase8_pretrain/dino_vitbase8_pretrain.pthz vit_small_patch14_dinov2.lvd142mzNhttps://dl.fbaipublicfiles.com/dinov2/dinov2_vits14/dinov2_vits14_pretrain.pthrv  )r¢   é  rƒ  )rs  r  r}  rò   ró   r)  rw  ry  zvit_base_patch14_dinov2.lvd142mzNhttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitb14/dinov2_vitb14_pretrain.pthz vit_large_patch14_dinov2.lvd142mzNhttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_pretrain.pthz vit_giant_patch14_dinov2.lvd142mzNhttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitg14/dinov2_vitg14_pretrain.pthz%vit_small_patch14_reg4_dinov2.lvd142mzShttps://dl.fbaipublicfiles.com/dinov2/dinov2_vits14/dinov2_vits14_reg4_pretrain.pthz$vit_base_patch14_reg4_dinov2.lvd142mzShttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitb14/dinov2_vitb14_reg4_pretrain.pthz%vit_large_patch14_reg4_dinov2.lvd142mzShttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_reg4_pretrain.pthz%vit_giant_patch14_reg4_dinov2.lvd142mzShttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitg14/dinov2_vitg14_reg4_pretrain.pthz"vit_base_patch14_reg1_tipsv2.webli)rU   rU   rU   )r‚  r‚  r‚  )r¢   éÀ  r„  )r  r}  rò   ró   r)  rw  ry  z#vit_large_patch14_reg1_tipsv2.webliz$vit_so400m_patch14_reg1_tipsv2.webliz#vit_giant_patch14_reg1_tipsv2.weblizvit_base_patch16_224_miil.in21kz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tresnet/vit_base_patch16_224_in21k_miil-887286df.pthg      ì?r  iÕ+  )rs  r  rò   ró   ry  ré  r)  z'vit_base_patch16_224_miil.in21k_ft_in1kzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tresnet/vit_base_patch16_224_1k_miil_84_4-2deb18e3.pth)rs  r  rò   ró   ry  ré  z vit_base_patch16_rpn_224.sw_in1kz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_base_patch16_rpn_224-sw-3b07e89d.pthz#vit_medium_patch16_gap_240.sw_in12k)r¢   éð   r…  gffffffî?i-.  )r  rw  ry  r)  z+vit_medium_patch16_gap_256.sw_in12k_ft_in1k)r¢   é   r†  )r  rw  ry  z+vit_medium_patch16_gap_384.sw_in12k_ft_in1kÚsquash)r  rw  ry  Ú	crop_modez%vit_betwixt_patch16_gap_256.untrained)rw  ry  z"vit_base_patch16_gap_224.untrainedz/vit_base_patch32_clip_224.laion2b_ft_in12k_in1k)r  rò   ró   z/vit_base_patch32_clip_384.laion2b_ft_in12k_in1k)r  rò   ró   ry  rw  z/vit_base_patch32_clip_448.laion2b_ft_in12k_in1kz/vit_base_patch16_clip_224.laion2b_ft_in12k_in1k)r  rò   ró   ry  z/vit_base_patch16_clip_384.laion2b_ft_in12k_in1k)r  rò   ró   ry  rw  rˆ  z0vit_large_patch14_clip_224.laion2b_ft_in12k_in1kz0vit_large_patch14_clip_336.laion2b_ft_in12k_in1k)r¢   éP  r‰  z/vit_huge_patch14_clip_224.laion2b_ft_in12k_in1kz/vit_huge_patch14_clip_336.laion2b_ft_in12k_in1kz.vit_base_patch32_clip_224.openai_ft_in12k_in1krñ   z.vit_base_patch32_clip_384.openai_ft_in12k_in1kz.vit_base_patch16_clip_224.openai_ft_in12k_in1kz.vit_base_patch16_clip_384.openai_ft_in12k_in1kz/vit_large_patch14_clip_224.openai_ft_in12k_in1kz/vit_large_patch14_clip_336.openai_ft_in12k_in1kz)vit_base_patch32_clip_224.laion2b_ft_in1kz)vit_base_patch16_clip_224.laion2b_ft_in1kz)vit_base_patch16_clip_384.laion2b_ft_in1kz*vit_large_patch14_clip_224.laion2b_ft_in1kz*vit_large_patch14_clip_336.laion2b_ft_in1kz)vit_huge_patch14_clip_224.laion2b_ft_in1kz)vit_huge_patch14_clip_336.laion2b_ft_in1kz(vit_base_patch32_clip_224.openai_ft_in1kz(vit_base_patch16_clip_224.openai_ft_in1kz(vit_base_patch16_clip_384.openai_ft_in1kz)vit_large_patch14_clip_224.openai_ft_in1kz*vit_base_patch16_clip_224.laion2b_ft_in12k)r  rò   ró   r)  z+vit_large_patch14_clip_224.laion2b_ft_in12k)r  rò   ró   ry  r)  z*vit_huge_patch14_clip_224.laion2b_ft_in12kz)vit_base_patch16_clip_224.openai_ft_in12kz*vit_large_patch14_clip_224.openai_ft_in12kz!vit_base_patch32_clip_224.laion2bé   z!vit_base_patch16_clip_224.laion2bz"vit_large_patch14_clip_224.laion2br#  z!vit_huge_patch14_clip_224.laion2bé   z"vit_giant_patch14_clip_224.laion2bz%vit_gigantic_patch14_clip_224.laion2bé   z'vit_base_patch32_clip_224.laion400m_e32Úmit)zDnatively QuickGELU, use quickgelu model variant for original results)r  r}  Únotesrò   ró   r)  z'vit_base_patch16_clip_224.laion400m_e32)r  r}  rò   ró   ry  r)  z,vit_base_patch16_plus_clip_240.laion400m_e32é€  )r  r}  rò   ró   rw  ry  r)  z(vit_large_patch14_clip_224.laion400m_e32z$vit_base_patch32_clip_224.datacompxlz$vit_base_patch32_clip_256.datacompxl)r  rò   ró   ry  rw  r)  z$vit_base_patch16_clip_224.datacompxlz%vit_large_patch14_clip_224.datacompxlzvit_base_patch16_clip_224.dfn2bz
apple-asclz%vit_large_patch14_clip_224.dfn2b_s39bz vit_large_patch14_clip_224.dfn2b)r  r}  rŽ  rò   ró   ry  r)  zvit_huge_patch14_clip_224.dfn5bzvit_huge_patch14_clip_378.dfn5b)r¢   éz  r�  )r  rò   ró   r}  rŽ  ry  rw  r)  z-vit_huge_patch14_clip_224.metaclip2_worldwidezcc-by-nc-4.0z-vit_huge_patch14_clip_378.metaclip2_worldwide)r  r}  rò   ró   rw  ry  rˆ  r)  z1vit_gigantic_patch14_clip_224.metaclip2_worldwidez1vit_gigantic_patch14_clip_378.metaclip2_worldwidez(vit_base_patch32_clip_224.metaclip_2pt5bz(vit_base_patch16_clip_224.metaclip_2pt5bz)vit_large_patch14_clip_224.metaclip_2pt5bz(vit_huge_patch14_clip_224.metaclip_2pt5bz-vit_huge_patch14_clip_224.metaclip_altogetherz,vit_gigantic_patch14_clip_224.metaclip_2pt5bz'vit_base_patch32_clip_224.metaclip_400mz'vit_base_patch16_clip_224.metaclip_400mz(vit_large_patch14_clip_224.metaclip_400mz vit_base_patch32_clip_224.openai)r  rŽ  rò   ró   r)  z vit_base_patch16_clip_224.openaiz!vit_large_patch14_clip_224.openai)r  rŽ  rò   ró   ry  r)  z!vit_large_patch14_clip_336.openai)r  rŽ  rò   ró   ry  rw  r)  z/vit_large_patch14_clip_224.apple_mclip2_dfndr2bz
apple-amlr)r  r)  rò   ró   ry  r}  z#vit_base_patch32_plus_256.untrained)rs  rw  ry  z#vit_base_patch16_plus_240.untrainedz$vit_small_patch16_36x1_224.untrainedz$vit_small_patch16_18x2_224.untrainedz#vit_base_patch16_18x2_224.untrainedz)eva_large_patch14_196.in22k_ft_in22k_in1k)r¢   éÄ   r‘  )r  r}  rò   ró   rw  ry  z)eva_large_patch14_336.in22k_ft_in22k_in1k)r  r}  rò   ró   rw  ry  rˆ  z#eva_large_patch14_196.in22k_ft_in1kz#eva_large_patch14_336.in22k_ft_in1kzflexivit_small.1200ep_in1kzEhttps://storage.googleapis.com/big_vision/flexivit/flexivit_s_i1k.npz)rs  r€  r  rw  ry  zflexivit_small.600ep_in1kzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_s_i1k_600ep.npzzflexivit_small.300ep_in1kzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_s_i1k_300ep.npzzflexivit_base.1200ep_in1kzEhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i1k.npzzflexivit_base.600ep_in1kzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i1k_600ep.npzzflexivit_base.300ep_in1kzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i1k_300ep.npzzflexivit_base.1000ep_in21kzMhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i21k_1000ep.npz)rs  r€  r  rw  ry  r)  zflexivit_base.300ep_in21kzLhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i21k_300ep.npzzflexivit_large.1200ep_in1kzEhttps://storage.googleapis.com/big_vision/flexivit/flexivit_l_i1k.npzzflexivit_large.600ep_in1kzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_l_i1k_600ep.npzzflexivit_large.300ep_in1kzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_l_i1k_300ep.npzzflexivit_base.patch16_in21kzIhttps://storage.googleapis.com/big_vision/flexivit/vit_b16_i21k_300ep.npzzflexivit_base.patch30_in21kzIhttps://storage.googleapis.com/big_vision/flexivit/vit_b30_i21k_300ep.npzz!vit_base_patch16_xp_224.untrainedz"vit_large_patch14_xp_224.untrainedz!vit_huge_patch14_xp_224.untrainedzvit_base_patch16_224.maezEhttps://dl.fbaipublicfiles.com/mae/pretrain/mae_pretrain_vit_base.pth)rs  r  r}  rò   ró   r)  zvit_large_patch16_224.maezFhttps://dl.fbaipublicfiles.com/mae/pretrain/mae_pretrain_vit_large.pthzvit_huge_patch14_224.maezEhttps://dl.fbaipublicfiles.com/mae/pretrain/mae_pretrain_vit_huge.pthz#vit_huge_patch14_gap_224.in1k_ijepaz?https://dl.fbaipublicfiles.com/ijepa/IN1K-vit.h.14-300e.pth.tar)rs  r}  rò   ró   r)  z$vit_huge_patch14_gap_224.in22k_ijepaz@https://dl.fbaipublicfiles.com/ijepa/IN22K-vit.h.14-900e.pth.tarz#vit_huge_patch16_gap_448.in1k_ijepazEhttps://dl.fbaipublicfiles.com/ijepa/IN1K-vit.h.16-448px-300e.pth.tar)rs  r}  rw  ry  rò   ró   r)  z%vit_giant_patch16_gap_224.in22k_ijepaz@https://dl.fbaipublicfiles.com/ijepa/IN22K-vit.g.16-600e.pth.tarz$vit_base_patch32_siglip_256.v2_webli)r  rw  r)  z$vit_base_patch16_siglip_224.v2_webliz!vit_base_patch16_siglip_224.webliz$vit_base_patch16_siglip_256.v2_webliz!vit_base_patch16_siglip_256.webliz&vit_base_patch16_siglip_256.webli_i18nz$vit_base_patch16_siglip_384.v2_webliz!vit_base_patch16_siglip_384.webliz$vit_base_patch16_siglip_512.v2_webli)r¢   rŠ  rŠ  z!vit_base_patch16_siglip_512.webliz%vit_large_patch16_siglip_256.v2_webliz"vit_large_patch16_siglip_256.webliz%vit_large_patch16_siglip_384.v2_webliz"vit_large_patch16_siglip_384.webliz%vit_large_patch16_siglip_512.v2_webliz&vit_so400m_patch14_siglip_224.v2_webliz#vit_so400m_patch14_siglip_224.webliz&vit_so400m_patch14_siglip_378.v2_webliz#vit_so400m_patch14_siglip_378.webliz#vit_so400m_patch14_siglip_384.webliz&vit_so400m_patch16_siglip_256.v2_webliz(vit_so400m_patch16_siglip_256.webli_i18nz&vit_so400m_patch16_siglip_384.v2_webliz&vit_so400m_patch16_siglip_512.v2_webliz(vit_giantopt_patch16_siglip_256.v2_webliz(vit_giantopt_patch16_siglip_384.v2_webliz(vit_base_patch32_siglip_gap_256.v2_webliz(vit_base_patch16_siglip_gap_224.v2_webliz%vit_base_patch16_siglip_gap_224.webliz(vit_base_patch16_siglip_gap_256.v2_webliz%vit_base_patch16_siglip_gap_256.webliz*vit_base_patch16_siglip_gap_256.webli_i18nz(vit_base_patch16_siglip_gap_384.v2_webliz%vit_base_patch16_siglip_gap_384.webliz(vit_base_patch16_siglip_gap_512.v2_webliz%vit_base_patch16_siglip_gap_512.webliz)vit_large_patch16_siglip_gap_256.v2_webliz&vit_large_patch16_siglip_gap_256.webliz)vit_large_patch16_siglip_gap_384.v2_webliz&vit_large_patch16_siglip_gap_384.webliz)vit_large_patch16_siglip_gap_512.v2_webliz*vit_so400m_patch14_siglip_gap_224.v2_webliz'vit_so400m_patch14_siglip_gap_224.webliz*vit_so400m_patch14_siglip_gap_224.pali_mixz)vit_so400m_patch14_siglip_gap_224.pali_ptz-vit_so400m_patch14_siglip_gap_224.pali2_3b_ptz.vit_so400m_patch14_siglip_gap_224.pali2_10b_ptz*vit_so400m_patch14_siglip_gap_378.v2_webliz'vit_so400m_patch14_siglip_gap_378.webliz'vit_so400m_patch14_siglip_gap_384.webliz*vit_so400m_patch14_siglip_gap_448.pali_mixz)vit_so400m_patch14_siglip_gap_448.pali_ptz2vit_so400m_patch14_siglip_gap_448.pali_refcoco_segz-vit_so400m_patch14_siglip_gap_448.pali_ocrvqaz-vit_so400m_patch14_siglip_gap_448.pali2_3b_ptz.vit_so400m_patch14_siglip_gap_448.pali2_10b_ptz0vit_so400m_patch14_siglip_gap_448.pali2_3b_docciz1vit_so400m_patch14_siglip_gap_448.pali2_10b_docciz)vit_so400m_patch14_siglip_gap_896.pali_pt)r¢   é€  r’  z2vit_so400m_patch14_siglip_gap_896.pali_refcoco_segz-vit_so400m_patch14_siglip_gap_896.pali_ocrvqaz-vit_so400m_patch14_siglip_gap_896.pali2_3b_ptz.vit_so400m_patch14_siglip_gap_896.pali2_10b_ptz*vit_so400m_patch16_siglip_gap_256.v2_webliz,vit_so400m_patch16_siglip_gap_256.webli_i18nz*vit_so400m_patch16_siglip_gap_384.v2_webliz*vit_so400m_patch16_siglip_gap_512.v2_webliz,vit_giantopt_patch16_siglip_gap_256.v2_webliz,vit_giantopt_patch16_siglip_gap_384.v2_webliz+vit_so400m_patch14_siglip_378.webli_ft_in1kz/vit_so400m_patch14_siglip_gap_378.webli_ft_in1kz,vit_xsmall_patch16_clip_224.tinyclip_yfcc15m)r  r}  rò   ró   r)  z.vit_medium_patch32_clip_224.tinyclip_laion400mz,vit_medium_patch16_clip_224.tinyclip_yfcc15mz/vit_betwixt_patch32_clip_224.tinyclip_laion400mz%vit_wee_patch16_reg1_gap_256.sbb_in1kz/vit_dwee_patch16_reg1_gap_256.sbb_nadamuon_in1kz&vit_dwee_patch16_reg1_gap_256.sbb_in1kz&vit_pwee_patch16_reg1_gap_256.sbb_in1kz0vit_dpwee_patch16_reg1_gap_256.sbb_nadamuon_in1kz'vit_dpwee_patch16_reg1_gap_256.sbb_in1kz1vit_little_patch16_reg1_gap_256.sbb_in12k_ft_in1kz)vit_little_patch16_reg1_gap_256.sbb_in12k)r  r)  rw  ry  z(vit_little_patch16_reg4_gap_256.sbb_in1kz2vit_dlittle_patch16_reg1_gap_256.sbb_nadamuon_in1kz(vit_medium_patch16_reg1_gap_256.sbb_in1kz1vit_medium_patch16_reg4_gap_256.sbb_in12k_ft_in1kz(vit_medium_patch16_reg4_gap_256.sbb_in1kz)vit_medium_patch16_reg4_gap_256.sbb_in12kz8vit_mediumd_patch16_reg4_gap_256.sbb2_e200_in12k_ft_in1kz2vit_mediumd_patch16_reg4_gap_256.sbb_in12k_ft_in1kz0vit_mediumd_patch16_reg4_gap_256.sbb2_e200_in12kz*vit_mediumd_patch16_reg4_gap_256.sbb_in12kz8vit_mediumd_patch16_reg4_gap_384.sbb2_e200_in12k_ft_in1kz)vit_betwixt_patch16_reg1_gap_256.sbb_in1kz8vit_betwixt_patch16_reg4_gap_256.sbb2_e200_in12k_ft_in1kz2vit_betwixt_patch16_reg4_gap_256.sbb_in12k_ft_in1kz)vit_betwixt_patch16_reg4_gap_256.sbb_in1kz0vit_betwixt_patch16_reg4_gap_256.sbb2_e200_in12kz*vit_betwixt_patch16_reg4_gap_256.sbb_in12kz8vit_betwixt_patch16_reg4_gap_384.sbb2_e200_in12k_ft_in1kz'vit_base_patch16_reg4_gap_256.untrained)rw  z6vit_so150m_patch16_reg4_gap_256.sbb_e250_in12k_ft_in1kz.vit_so150m_patch16_reg4_gap_256.sbb_e250_in12kz6vit_so150m_patch16_reg4_gap_384.sbb_e250_in12k_ft_in1kz)vit_so150m_patch16_reg4_map_256.untrainedz7vit_so150m2_patch16_reg1_gap_256.sbb_e200_in12k_ft_in1kz/vit_so150m2_patch16_reg1_gap_256.sbb_e200_in12kz7vit_so150m2_patch16_reg1_gap_384.sbb_e200_in12k_ft_in1kz7vit_so150m2_patch16_reg1_gap_448.sbb_e200_in12k_ft_in1kz$vit_intern300m_patch14_448.ogvl_distz$vit_intern300m_patch14_448.ogvl_2pt5)r  rò   ró   rw  ry  r)  z aimv2_large_patch14_224.apple_pt)r  rò   ró   r}  ry  r)  z%aimv2_large_patch14_224.apple_pt_distzaimv2_huge_patch14_224.apple_ptzaimv2_1b_patch14_224.apple_ptzaimv2_3b_patch14_224.apple_ptz aimv2_large_patch14_336.apple_pt)r  rò   ró   r}  rw  ry  r)  z%aimv2_large_patch14_336.apple_pt_distzaimv2_huge_patch14_336.apple_ptzaimv2_1b_patch14_336.apple_ptzaimv2_3b_patch14_336.apple_ptz aimv2_large_patch14_448.apple_ptzaimv2_huge_patch14_448.apple_ptzaimv2_1b_patch14_448.apple_ptzaimv2_3b_patch14_448.apple_ptztest_vit.r160_in1k)r¢   é    r“  ztest_vit2.r160_in1kztest_vit3.r160_in1kztest_vit4.r160_in1kz$beit3_base_patch16_224.in22k_ft_in1kz-beit3_base_patch16_224.indomain_in22k_ft_in1kz%beit3_large_patch16_224.in22k_ft_in1kz.beit3_large_patch16_224.indomain_in22k_ft_in1kz!beit3_giant_patch14_224.untrained)rs  rò   ró   ry  z!beit3_giant_patch14_336.untrained)rs  rw  rò   ró   ry  zbeit3_base_patch16_224.ptz"beit3_base_patch16_224.indomain_ptzbeit3_large_patch16_224.ptz#beit3_large_patch16_224.indomain_ptrŽ  Ú	quickgelur  Ú_clip_Ú_clip_quickgelu_ÚTIMM_USE_NAFLEXVITÚfalseÚtrueÚvariantÚ
pretrainedÚ
use_naflexÚ	NaFlexVitc           	      ó0  — |€t         }|rddlm}  || |fi |¤ŽS |j                  dd«      }d| v rt	        t
        dd¬«      }nt
        }|j                  d	d
«      }d| v r|j                  dd «      dk7  rd}t        t        | |f||t        |d¬«      dœ|¤ŽS )Nr.   )Ú_create_naflexvit_from_classicÚout_indicesr¢   Úflexir  F)ré  rê  Úpretrained_strictTÚsiglipr*  r,  Úgetter)r   Úfeature_cls)Úpretrained_filter_fnr¢  Úfeature_cfg)
Ú_USE_NAFLEX_DEFAULTÚ	naflexvitrŸ  rJ  r   rr  rL   r/   r8   rZ  )rš  r›  rœ  rN   rŸ  r   Ú
_filter_fnÚstricts           rO   Ú_create_vision_transformerr¬  ú  s¿   € ð ÐÜ(ˆ
Ùå=Ù-¨g°zÑLÀVÑLÐLà—*‘*˜]¨AÓ.€KØ�'Ñô Ô1ÀÐW\Ô]‰
ä)ˆ
ð �Z‰ZÐ+¨TÓ2€FØ�7Ñ˜vŸz™z¨-¸Ó>À%ÒGØˆäÜØØðð (Ø Ü [¸hÔGñð ñð rQ   c           	      óR   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )z ViT-Tiny (Vit-Ti/16)
    r!  éÀ   r$  r¢   ©r'  r.  rF   r>   r›  )Úvit_tiny_patch16_224©rZ  r¬  ©r›  rN   Ú
model_argsrð  s       rO   r°  r°     ó8   € ô  ¨s¸"ÈÔJ€JÜ&ÑsÈ*ÐsÔX\Ð]gÑXrÐkqÑXrÑs€EØ€LrQ   c           	      óR   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )z% ViT-Tiny (Vit-Ti/16) @ 384x384.
    r!  r®  r$  r¢   r¯  r›  )Úvit_tiny_patch16_384r±  r²  s       rO   r¶  r¶  )  r´  rQ   c           	      óR   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )z ViT-Small (ViT-S/32)
    é    r�  r$  é   r¯  r›  )Úvit_small_patch32_224r±  r²  s       rO   rº  rº  2  ó8   € ô  ¨s¸"ÈÔJ€JÜ&ÑtÈ:ÐtÔY]Ð^hÑYsÐlrÑYsÑt€EØ€LrQ   c           	      óR   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )z& ViT-Small (ViT-S/32) at 384x384.
    r¸  r�  r$  r¹  r¯  r›  )Úvit_small_patch32_384r±  r²  s       rO   r½  r½  ;  r»  rQ   c           	      óR   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )ú ViT-Small (ViT-S/16)
    r!  r�  r$  r¹  r¯  r›  )Úvit_small_patch16_224r±  r²  s       rO   rÀ  rÀ  D  r»  rQ   c           	      óR   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )r¿  r!  r�  r$  r¹  r¯  r›  )Úvit_small_patch16_384r±  r²  s       rO   rÂ  rÂ  M  r»  rQ   c           	      óR   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )z ViT-Small (ViT-S/8)
    é   r�  r$  r¹  r¯  r›  )Úvit_small_patch8_224r±  r²  s       rO   rÅ  rÅ  V  s8   € ô  ¨c¸ÀqÔI€JÜ&ÑsÈ*ÐsÔX\Ð]gÑXrÐkqÑXrÑs€EØ€LrQ   c           	      óR   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )z¾ ViT-Base (ViT-B/32) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k, source https://github.com/google-research/vision_transformer.
    r¸  r#  r$  r¯  r›  )Úvit_base_patch32_224r±  r²  s       rO   rÇ  rÇ  _  ó8   € ô
  ¨s¸"ÈÔK€JÜ&ÑsÈ*ÐsÔX\Ð]gÑXrÐkqÑXrÑs€EØ€LrQ   c           	      óR   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )zÎ ViT-Base model (ViT-B/32) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    r¸  r#  r$  r¯  r›  )Úvit_base_patch32_384r±  r²  s       rO   rÊ  rÊ  i  rÈ  rQ   c           	      óR   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )zÈ ViT-Base (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 224x224, source https://github.com/google-research/vision_transformer.
    r!  r#  r$  r¯  r›  )Úvit_base_patch16_224r±  r²  s       rO   rÌ  rÌ  s  rÈ  rQ   c           	      óR   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )zÎ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    r!  r#  r$  r¯  r›  )Úvit_base_patch16_384r±  r²  s       rO   rÎ  rÎ  }  rÈ  rQ   c           	      óR   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )zÇ ViT-Base (ViT-B/8) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 224x224, source https://github.com/google-research/vision_transformer.
    rÄ  r#  r$  r¯  r›  )Úvit_base_patch8_224r±  r²  s       rO   rÐ  rÐ  ‡  s8   € ô
  ¨c¸ÀrÔJ€JÜ&ÑrÈÐrÔW[Ð\fÑWqÐjpÑWqÑr€EØ€LrQ   c           	      óR   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )zo ViT-Large model (ViT-L/32) from original paper (https://arxiv.org/abs/2010.11929). No pretrained weights.
    r¸  r‹  é   r!  r¯  r›  )Úvit_large_patch32_224r±  r²  s       rO   rÓ  rÓ  ‘  ó8   € ô  ¨t¸2ÈÔL€JÜ&ÑtÈ:ÐtÔY]Ð^hÑYsÐlrÑYsÑt€EØ€LrQ   c           	      óR   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )zÏ ViT-Large model (ViT-L/32) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    r¸  r‹  rÒ  r!  r¯  r›  )Úvit_large_patch32_384r±  r²  s       rO   rÖ  rÖ  š  ó8   € ô
  ¨t¸2ÈÔL€JÜ&ÑtÈ:ÐtÔY]Ð^hÑYsÐlrÑYsÑt€EØ€LrQ   c           	      óR   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )zÏ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 224x224, source https://github.com/google-research/vision_transformer.
    r!  r‹  rÒ  r¯  r›  )Úvit_large_patch16_224r±  r²  s       rO   rÙ  rÙ  ¤  r×  rQ   c           	      óR   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )zÏ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    r!  r‹  rÒ  r¯  r›  )Úvit_large_patch16_384r±  r²  s       rO   rÛ  rÛ  ®  r×  rQ   c           	      óR   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )z  ViT-Large model (ViT-L/14)
    é   r‹  rÒ  r!  r¯  r›  )Úvit_large_patch14_224r±  r²  s       rO   rÞ  rÞ  ¸  rÔ  rQ   c           	      óR   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )zW ViT-Huge model (ViT-H/14) from original paper (https://arxiv.org/abs/2010.11929).
    rÝ  rŒ  r¸  r!  r¯  r›  )Úvit_huge_patch14_224r±  r²  s       rO   rà  rà  Á  s8   € ô  ¨t¸2ÈÔL€JÜ&ÑsÈ*ÐsÔX\Ð]gÑXrÐkqÑXrÑs€EØ€LrQ   c           	      óT   — t        ddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	zq ViT-Giant (little-g) model (ViT-g/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560
    rÝ  é€  çtÑE]t@é(   r!  ©r'  r.  rV   rF   r>   r›  )Úvit_giant_patch14_224r±  r²  s       rO   ræ  ræ  Ê  s;   € ô  ¨t¸uÈBÐZ\Ô]€JÜ&ÑtÈ:ÐtÔY]Ð^hÑYsÐlrÑYsÑt€EØ€LrQ   c           	      óV   — t        ddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	zq ViT-Gigantic (big-G) model (ViT-G/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560
    rÝ  é€  ç;±;±@é0   r!  rå  r›  )Úvit_gigantic_patch14_224r±  r²  s       rO   rë  rë  Ó  sG   € ô  ¨t¸uÈBÐZ\Ô]€JÜ&Ø"ñYØ/9ðYÜ=AÀ*Ñ=WÐPVÑ=WñY€Eà€LrQ   c           	      óV   — t        ddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )z• ViT-Base (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929).
    Weights taken from: https://github.com/Alibaba-MIIL/ImageNet21K
    r!  r#  r$  F)r'  r.  rF   r>   r?   r›  )Úvit_base_patch16_224_miilr±  r²  s       rO   rí  rí  Ý  sG   € ô
  ¨s¸"ÈÐUZÔ[€JÜ&Ø#ñZØ0:ðZÜ>BÀ:Ñ>XÐQWÑ>XñZ€Eà€LrQ   c                 ó^   — t        ddddddddd¬«	      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )zB ViT-Medium (ViT-M/16) w/o class token, w/ avg-pool @ 240x240
    r!  rŠ  r$  rÄ  Fr  r}  ©	r'  r.  rF   r>   r/  r*  r?   rY   r5  r›  )Úvit_medium_patch16_gap_240r±  r²  s       rO   rð  rð  è  óU   € ô Ø ¨B¸!ÈØ E°tÀUôL€Jô 'Ø$ñ[Ø1;ð[Ü?CÀJÑ?YÐRXÑ?Yñ[€Eà€LrQ   c                 ó^   — t        ddddddddd¬«	      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )zB ViT-Medium (ViT-M/16) w/o class token, w/ avg-pool @ 256x256
    r!  rŠ  r$  rÄ  Fr  r}  rï  r›  )Úvit_medium_patch16_gap_256r±  r²  s       rO   ró  ró  ô  rñ  rQ   c                 ó^   — t        ddddddddd¬«	      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )zB ViT-Medium (ViT-M/16) w/o class token, w/ avg-pool @ 384x384
    r!  rŠ  r$  rÄ  Fr  r}  rï  r›  )Úvit_medium_patch16_gap_384r±  r²  s       rO   rõ  rõ     rñ  rQ   c                 ó^   — t        ddddddddd¬«	      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )zC ViT-Betwixt (ViT-b/16) w/o class token, w/ avg-pool @ 256x256
    r!  r�  r$  é
   Fr  r}  rï  r›  )Úvit_betwixt_patch16_gap_256r±  r²  s       rO   rø  rø    sU   € ô Ø ¨B¸"È%Ø E°tÀUôL€Jô 'Ø%ñ\Ø2<ð\Ü@DÀZÑ@ZÐSYÑ@Zñ\€Eà€LrQ   c           	      óZ   — t        ddddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	z@ ViT-Base (ViT-B/16) w/o class token, w/ avg-pool @ 224x224
    r!  r#  r$  Fr  ©r'  r.  rF   r>   r/  r*  r5  r›  )Úvit_base_patch16_gap_224r±  r²  s       rO   rû  rû    sP   € ô Ø ¨B¸"È%Ð]bÐlqôs€Jä&Ø"ñYØ/9ðYÜ=AÀ*Ñ=WÐPVÑ=WñY€Eà€LrQ   c           	      óZ   — t        ddddddd¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
z; ViT-Huge model (ViT-H/14) w/ no class token, avg pool
    rÝ  rŒ  r¸  r!  Fr  rú  r›  )Úvit_huge_patch14_gap_224r±  r²  s       rO   rý  rý  #  óP   € ô Ø ¨R¸2È5Ð^cÐmrôt€Jä&Ø"ñYØ/9ðYÜ=AÀ*Ñ=WÐPVÑ=WñY€Eà€LrQ   c           	      óZ   — t        ddddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	zE ViT-Huge model (ViT-H/16) w/ no class token, avg pool @ 448x448
    r!  rŒ  r¸  Fr  rú  r›  )Úvit_huge_patch16_gap_448r±  r²  s       rO   r   r   .  rþ  rQ   c           
      ó\   — t        dddddddd¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
zH ViT-Giant (little-gg) model (ViT-g/16) w/ no class token, avg pool
    r!  râ  rä  rã  Fr  ©r'  r.  rF   r>   rV   r/  r*  r5  r›  )Úvit_giant_patch16_gap_224r±  r²  s       rO   r  r  9  sR   € ô Ø ¨R¸2ÈØ u°eô=€Jô 'Ø#ñZØ0:ðZÜ>BÀ:Ñ>XÐQWÑ>XñZ€Eà€LrQ   c           
      ót   — t        ddddt        t        d¬«      ¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
Nr†  r÷  rô  Trß   ©rÞ   ©r.  rF   r>   r3  rE   r›  )Úvit_xsmall_patch16_clip_224©rZ  r   r   r¬  r²  s       rO   r  r  E  óP   € ô  ¨2¸ÀTÔV]Ô^gÐmqÔVrÔs€JÜ&Ø%ñ\Ø2<ð\Ü@DÀZÑ@ZÐSYÑ@Zñ\€Eà€LrQ   c                 óv   — t        dddddt        t        d¬«      ¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )Nr¸  rŠ  r$  rÄ  Trß   r  ©r'  r.  rF   r>   r3  rE   r›  )Úvit_medium_patch32_clip_224r  r²  s       rO   r  r  N  sV   € ô Ø ¨B¸!ÀdÔW^Ô_hÐnrÔWsôu€Jä&Ø%ñ\Ø2<ð\Ü@DÀZÑ@ZÐSYÑ@Zñ\€Eà€LrQ   c           
      ót   — t        ddddt        t        d¬«      ¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
NrŠ  r$  rÄ  Trß   r  r  r›  )Úvit_medium_patch16_clip_224r  r²  s       rO   r  r  X  r	  rQ   c                 óv   — t        dddddt        t        d¬«      ¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )Nr¸  r�  r$  r÷  Trß   r  r  r›  )Úvit_betwixt_patch32_clip_224r  r²  s       rO   r  r  a  sV   € ô Ø ¨B¸"ÀtÔX_Ô`iÐosÔXtôv€Jä&Ø&ñ]Ø3=ð]ÜAEÀjÑA[ÐTZÑA[ñ]€Eà€LrQ   c                 óv   — t        dddddt        t        d¬«      ¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
ú) ViT-B/32 CLIP image tower @ 224x224
    r¸  r#  r$  Trß   r  r  r›  )Úvit_base_patch32_clip_224r  r²  s       rO   r  r  k  óV   € ô Ø ¨B¸"ÀtÔX_Ô`iÐosÔXtôv€Jä&Ø#ñZØ0:ðZÜ>BÀ:Ñ>XÐQWÑ>XñZ€Eà€LrQ   c                 óv   — t        dddddt        t        d¬«      ¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
z) ViT-B/32 CLIP image tower @ 256x256
    r¸  r#  r$  Trß   r  r  r›  )Úvit_base_patch32_clip_256r  r²  s       rO   r  r  v  r  rQ   c                 óv   — t        dddddt        t        d¬«      ¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
z) ViT-B/32 CLIP image tower @ 384x384
    r¸  r#  r$  Trß   r  r  r›  )Úvit_base_patch32_clip_384r  r²  s       rO   r  r  �  r  rQ   c                 óv   — t        dddddt        t        d¬«      ¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
z) ViT-B/32 CLIP image tower @ 448x448
    r¸  r#  r$  Trß   r  r  r›  )Úvit_base_patch32_clip_448r  r²  s       rO   r  r  Œ  r  rQ   c                 óv   — t        dddddt        t        d¬«      ¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
z ViT-B/16 CLIP image tower
    r!  r#  r$  Trß   r  r  r›  )Úvit_base_patch16_clip_224r  r²  s       rO   r  r  —  r  rQ   c                 óv   — t        dddddt        t        d¬«      ¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
z) ViT-B/16 CLIP image tower @ 384x384
    r!  r#  r$  Trß   r  r  r›  )Úvit_base_patch16_clip_384r  r²  s       rO   r  r  ¢  r  rQ   c                 óv   — t        dddddt        t        d¬«      ¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )z5 ViT-Base (ViT-B/16+) CLIP image tower @ 240x240
    r!  r’  r$  rÝ  Trß   r  r  r›  )Úvit_base_patch16_plus_clip_240r  r²  s       rO   r   r   ­  sV   € ô Ø ¨B¸"ÀtÔX_Ô`iÐosÔXtôv€Jä&Ø(ñ_Ø5?ð_ÜCGÈ
ÑC]ÐV\ÑC]ñ_€Eà€LrQ   c                 óv   — t        dddddt        t        d¬«      ¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )z1 ViT-Large model (ViT-L/14) CLIP image tower
    rÝ  r‹  rÒ  r!  Trß   r  r  r›  )Úvit_large_patch14_clip_224r  r²  s       rO   r"  r"  ¸  óV   € ô Ø ¨R¸2ÈÔY`ÔajÐptÔYuôw€Jä&Ø$ñ[Ø1;ð[Ü?CÀJÑ?YÐRXÑ?Yñ[€Eà€LrQ   c                 óv   — t        dddddt        t        d¬«      ¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )z; ViT-Large model (ViT-L/14) CLIP image tower @ 336x336
    rÝ  r‹  rÒ  r!  Trß   r  r  r›  )Úvit_large_patch14_clip_336r  r²  s       rO   r%  r%  Ã  r#  rQ   c                 óv   — t        dddddt        t        d¬«      ¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )z1 ViT-Huge model (ViT-H/14) CLIP image tower.
    rÝ  rŒ  r¸  r!  Trß   r  r  r›  )Úvit_huge_patch14_clip_224r  r²  s       rO   r'  r'  Î  óV   € ô Ø ¨R¸2ÈÔY`ÔajÐptÔYuôw€Jä&Ø#ñZØ0:ðZÜ>BÀ:Ñ>XÐQWÑ>XñZ€Eà€LrQ   c                 óv   — t        dddddt        t        d¬«      ¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )z: ViT-Huge model (ViT-H/14) CLIP image tower @ 336x336
    rÝ  rŒ  r¸  r!  Trß   r  r  r›  )Úvit_huge_patch14_clip_336r  r²  s       rO   r*  r*  Ù  r(  rQ   c                 óv   — t        dddddt        t        d¬«      ¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )z: ViT-Huge model (ViT-H/14) CLIP image tower @ 378x378
    rÝ  rŒ  r¸  r!  Trß   r  r  r›  )Úvit_huge_patch14_clip_378r  r²  s       rO   r,  r,  ä  r(  rQ   c                 óx   — t        ddddddt        t        d¬«      ¬	«      }t        	 dd
| it        |fi |¤Ž¤Ž}|S )zŸ ViT-Giant (little-g) model (ViT-g/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560
    Pretrained weights from CLIP image tower.
    rÝ  râ  rã  rä  r!  Trß   r  ©r'  r.  rV   rF   r>   r3  rE   r›  )Úvit_giant_patch14_clip_224r  r²  s       rO   r/  r/  ï  sX   € ô
 Ø °¸bÈBÐY]Üœ9¨$Ô/ô€Jô 'Ø$ñ[Ø1;ð[Ü?CÀJÑ?YÐRXÑ?Yñ[€Eà€LrQ   c                 óx   — t        ddddddt        t        d¬«      ¬	«      }t        	 dd
| it        |fi |¤Ž¤Ž}|S )ú“ ViT-bigG model (ViT-G/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560
    Pretrained weights from CLIP image tower.
    rÝ  rè  ré  rê  r!  Trß   r  r.  r›  )Úvit_gigantic_patch14_clip_224r  r²  s       rO   r2  r2  ý  óX   € ô
 Ø °¸bÈBÐY]Üœ9¨$Ô/ô€Jô 'Ø'ñ^Ø4>ð^ÜBFÀzÑB\ÐU[ÑB\ñ^€Eà€LrQ   c                 óx   — t        ddddddt        t        d¬«      ¬	«      }t        	 dd
| it        |fi |¤Ž¤Ž}|S )r1  rÝ  rè  ré  rê  r!  Trß   r  r.  r›  )Úvit_gigantic_patch14_clip_378r  r²  s       rO   r5  r5    r3  rQ   c                 óx   — t        dddddt        t        d¬«      d¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )r  r¸  r#  r$  Trß   r  Ú
quick_gelu©r'  r.  rF   r>   r3  rE   r[   r›  )Ú#vit_base_patch32_clip_quickgelu_224r  r²  s       rO   r9  r9    óW   € ô Ø ¨B¸"ÀtÜœ9¨$Ô/¸<ô€Jô 'Ø-ñdØ:DðdÜHLÈZÑHbÐ[aÑHbñd€Eà€LrQ   c                 óx   — t        dddddt        t        d¬«      d¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )z0 ViT-B/16 CLIP image tower w/ QuickGELU act
    r!  r#  r$  Trß   r  r7  r8  r›  )Ú#vit_base_patch16_clip_quickgelu_224r  r²  s       rO   r<  r<  &  r:  rQ   c                 óx   — t        dddddt        t        d¬«      d¬	«      }t        	 dd
| it        |fi |¤Ž¤Ž}|S )zB ViT-Large model (ViT-L/14) CLIP image tower w/ QuickGELU act
    rÝ  r‹  rÒ  r!  Trß   r  r7  r8  r›  )Ú$vit_large_patch14_clip_quickgelu_224r  r²  s       rO   r>  r>  3  óW   € ô Ø ¨R¸2ÈÜœ9¨$Ô/¸<ô€Jô 'Ø.ñeØ;EðeÜIMÈjÑIcÐ\bÑIcñe€Eà€LrQ   c                 óx   — t        dddddt        t        d¬«      d¬	«      }t        	 dd
| it        |fi |¤Ž¤Ž}|S )zL ViT-Large model (ViT-L/14) CLIP image tower @ 336x336 w/ QuickGELU act
    rÝ  r‹  rÒ  r!  Trß   r  r7  r8  r›  )Ú$vit_large_patch14_clip_quickgelu_336r  r²  s       rO   rA  rA  @  r?  rQ   c                 óx   — t        dddddt        t        d¬«      d¬	«      }t        	 dd
| it        |fi |¤Ž¤Ž}|S )zB ViT-Huge model (ViT-H/14) CLIP image tower w/ QuickGELU act.
    rÝ  rŒ  r¸  r!  Trß   r  r7  r8  r›  )Ú#vit_huge_patch14_clip_quickgelu_224r  r²  s       rO   rC  rC  M  óW   € ô Ø ¨R¸2ÈÜœ9¨$Ô/¸<ô€Jô 'Ø-ñdØ:DðdÜHLÈZÑHbÐ[aÑHbñd€Eà€LrQ   c                 óx   — t        dddddt        t        d¬«      d¬	«      }t        	 dd
| it        |fi |¤Ž¤Ž}|S )zK ViT-Huge model (ViT-H/14) CLIP image tower @ 378x378 w/ QuickGELU act
    rÝ  rŒ  r¸  r!  Trß   r  r7  r8  r›  )Ú#vit_huge_patch14_clip_quickgelu_378r  r²  s       rO   rF  rF  Z  rD  rQ   c                 óz   — t        ddddddt        t        d¬«      d	¬
«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )z0 ViT-bigG model (ViT-G/14) w/ QuickGELU act
    rÝ  rè  ré  rê  r!  Trß   r  r7  )r'  r.  rV   rF   r>   r3  rE   r[   r›  )Ú'vit_gigantic_patch14_clip_quickgelu_224r  r²  s       rO   rH  rH  g  s[   € ô Ø °¸bÈBÐY]Üœ9¨$Ô/¸<ô€Jô 'Ø1ñhØ>HðhÜLPÐQ[ÑLfÐ_eÑLfñh€Eà€LrQ   c           	      óV   — t        ddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	z ViT-Base (ViT-B/32+)
    r¸  r’  r$  rÝ  rß   ©r'  r.  rF   r>   rY   r›  )Úvit_base_patch32_plus_256r±  r²  s       rO   rK  rK  v  óG   € ô  ¨s¸"ÈÐX\Ô]€JÜ&Ø#ñZØ0:ðZÜ>BÀ:Ñ>XÐQWÑ>XñZ€Eà€LrQ   c           	      óV   — t        ddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	z ViT-Base (ViT-B/16+)
    r!  r’  r$  rÝ  rß   rJ  r›  )Úvit_base_patch16_plus_240r±  r²  s       rO   rN  rN  €  rL  rQ   c                 óf   — t        dddddddt        d¬«	      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
z/ ViT-Base (ViT-B/16) w/ residual post-norm
    r!  r#  r$  Frß   r  )	r'  r.  rF   r>   r?   rY   r/  rG  r*  r›  )Úvit_base_patch16_rpn_224)rZ  r�   r¬  r²  s       rO   rP  rP  Š  sV   € ô Ø ¨B¸"ÀuÐZ^Ø¤L¸eôE€Jô 'Ø"ñYØ/9ðYÜ=AÀ*Ñ=WÐPVÑ=WñY€Eà€LrQ   c           	      óV   — t        ddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	a   ViT-Base w/ LayerScale + 36 x 1 (36 block serial) config. Experimental, may remove.
    Based on `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795
    Paper focuses on 24x2 + 48x1 for 'Small' width but those are extremely slow.
    r!  r�  é$   r¹  rß   rJ  r›  )Úvit_small_patch16_36x1_224r±  r²  s       rO   rS  rS  –  sG   € ô  ¨s¸"ÈÐW[Ô\€JÜ&Ø$ñ[Ø1;ð[Ü?CÀJÑ?YÐRXÑ?Yñ[€Eà€LrQ   c           	      ó`   — t        dddddt        ¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	a   ViT-Small w/ LayerScale + 18 x 2 (36 block parallel) config. Experimental, may remove.
    Based on `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795
    Paper focuses on 24x2 + 48x1 for 'Small' width but those are extremely slow.
    r!  r�  é   r¹  rß   ©r'  r.  rF   r>   rY   rG  r›  )Úvit_small_patch16_18x2_224©rZ  r  r¬  r²  s       rO   rW  rW  ¢  sM   € ô Ø ¨B¸!ÈÔXkôm€Jä&Ø$ñ[Ø1;ð[Ü?CÀJÑ?YÐRXÑ?Yñ[€Eà€LrQ   c           	      ó`   — t        dddddt        ¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	zÉ ViT-Base w/ LayerScale + 18 x 2 (36 block parallel) config. Experimental, may remove.
    Based on `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795
    r!  r#  rU  r$  rß   rV  r›  )Úvit_base_patch16_18x2_224rX  r²  s       rO   rZ  rZ  ¯  sM   € ô
 Ø ¨B¸"È$ÔYlôn€Jä&Ø#ñZØ0:ðZÜ>BÀ:Ñ>XÐQWÑ>XñZ€Eà€LrQ   c           	      óV   — t        ddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	zG EVA-large model https://arxiv.org/abs/2211.07636 /via MAE MIM pretrainrÝ  r‹  rÒ  r!  r  ©r'  r.  rF   r>   r*  r›  )Úeva_large_patch14_196r±  r²  s       rO   r]  r]  »  sF   € ô  ¨t¸2ÈÐY^Ô_€JÜ&ØñVØ,6ðVÜ:>¸zÑ:TÈVÑ:TñV€Eà€LrQ   c           	      óT   — t        ddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	zF EVA-large model https://arxiv.org/abs/2211.07636 via MAE MIM pretrainrÝ  r‹  rÒ  r!  r  r\  r›  )Úeva_large_patch14_336r±  r²  s       rO   r_  r_  Ä  s;   € ô  ¨t¸2ÈÐY^Ô_€JÜ&ÑtÈ:ÐtÔY]Ð^hÑYsÐlrÑYsÑt€EØ€LrQ   c           	      óT   — t        ddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	z FlexiViT-Small
    r!  r�  r$  r¹  T©r'  r.  rF   r>   r1  r›  )Úflexivit_smallr±  r²  s       rO   rb  rb  Ì  s;   € ô  ¨s¸"ÈÐZ^Ô_€JÜ&ÑmÀJÐmÔRVÐWaÑRlÐekÑRlÑm€EØ€LrQ   c           	      óT   — t        ddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )z FlexiViT-Base
    r!  r#  r$  Tra  r›  )Úflexivit_baser±  r²  s       rO   rd  rd  Õ  s;   € ô  ¨s¸"ÈÐ[_Ô`€JÜ&ÑlÀ:ÐlÔQUÐV`ÑQkÐdjÑQkÑl€EØ€LrQ   c           	      óT   — t        ddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )z FlexiViT-Large
    r!  r‹  rÒ  Tra  r›  )Úflexivit_larger±  r²  s       rO   rf  rf  Þ  s;   € ô  ¨t¸2ÈÐ\`Ôa€JÜ&ÑmÀJÐmÔRVÐWaÑRlÐekÑRlÑm€EØ€LrQ   c                 óp   — t        ddddddt        t        dd¬«
      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	úH ViT-Large model (ViT-L/14) w/ parallel blocks and qk norm enabled.
    r!  r#  r$  TF©
r'  r.  rF   r>   r3  r1  rE   rG  r?   r@   r›  )Úvit_base_patch16_xp_224©rZ  r   r›   r¬  r²  s       rO   rj  rj  ç  sX   € ô Ø ¨B¸"ÀtÐ\`ÜÔ%9ÀEÐSWô€Jô 'Ø!ñXØ.8ðXÜ<@ÀÑ<VÈvÑ<VñX€Eà€LrQ   c                 óp   — t        ddddddt        t        dd¬«
      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
rh  rÝ  r‹  rÒ  r!  TFri  r›  )Úvit_large_patch14_xp_224rk  r²  s       rO   rm  rm  ô  sY   € ô Ø ¨R¸2ÈÐ]aÜÔ%9ÀEÐSWô€Jô 'Ø"ñYØ/9ðYÜ=AÀ*Ñ=WÐPVÑ=WñY€Eà€LrQ   c                 óp   — t        ddddddt        t        dd¬«
      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
zG ViT-Huge model (ViT-H/14) w/ parallel blocks and qk norm enabled.
    rÝ  rŒ  r¸  r!  TFri  r›  )Úvit_huge_patch14_xp_224rk  r²  s       rO   ro  ro    sX   € ô Ø ¨R¸2ÈÐ]aÜÔ%9ÀEÐSWô€Jô 'Ø!ñXØ.8ðXÜ<@ÀÑ<VÈvÑ<VñX€Eà€LrQ   c           	      óV   — t        ddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	z ViT-S/14 for DINOv2
    rÝ  r�  r$  r¹  rß   rJ  r›  )Úvit_small_patch14_dinov2r±  r²  s       rO   rq  rq    sG   € ô  ¨s¸"ÈÐW[Ô\€JÜ&Ø"ñYØ/9ðYÜ=AÀ*Ñ=WÐPVÑ=WñY€Eà€LrQ   c           	      óV   — t        ddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )z ViT-B/14 for DINOv2
    rÝ  r#  r$  rß   rJ  r›  )Úvit_base_patch14_dinov2r±  r²  s       rO   rs  rs    sF   € ô  ¨s¸"ÈÐX\Ô]€JÜ&Ø!ñXØ.8ðXÜ<@ÀÑ<VÈvÑ<VñX€Eà€LrQ   c           	      óV   — t        ddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	z ViT-L/14 for DINOv2
    rÝ  r‹  rÒ  r!  rß   rJ  r›  )Úvit_large_patch14_dinov2r±  r²  s       rO   ru  ru  "  sG   € ô  ¨t¸2ÈÐY]Ô^€JÜ&Ø"ñYØ/9ðYÜ=AÀ*Ñ=WÐPVÑ=WñY€Eà€LrQ   c           
      ó€   — t        ddddddt        t        j                  ¬«      }t	        	 d	d| it        |fi |¤Ž¤Ž}|S )
ú ViT-G/14 for DINOv2
    rÝ  é   rä  rÒ  rß   çhË¹WU@)r'  r.  rF   r>   rY   rV   r\   r[   r›  )Úvit_giant_patch14_dinov2©rZ  r   rk   ÚSiLUr¬  r²  s       rO   rz  rz  ,  sV   € ô Ø ¨R¸2È4Ø¬ÄÇÁô€Jô 'Ø"ñYØ/9ðYÜ=AÀ*Ñ=WÐPVÑ=WñY€Eà€LrQ   c           	      óZ   — t        ddddddd¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )z( ViT-S/14 for DINOv2 w/ 4 registers
    rÝ  r�  r$  r¹  rß   rô  T©r'  r.  rF   r>   rY   r2  r1  r›  )Úvit_small_patch14_reg4_dinov2r±  r²  s       rO   r  r  =  sP   € ô Ø ¨B¸!ÈØ Tô€Jô 'Ø'ñ^Ø4>ð^ÜBFÀzÑB\ÐU[ÑB\ñ^€Eà€LrQ   c           	      óZ   — t        ddddddd¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
z( ViT-B/14 for DINOv2 w/ 4 registers
    rÝ  r#  r$  rß   rô  Tr~  r›  )Úvit_base_patch14_reg4_dinov2r±  r²  s       rO   r�  r�  J  sP   € ô Ø ¨B¸"È$Ø Tô€Jô 'Ø&ñ]Ø3=ð]ÜAEÀjÑA[ÐTZÑA[ñ]€Eà€LrQ   c           	      óZ   — t        ddddddd¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )z( ViT-L/14 for DINOv2 w/ 4 registers
    rÝ  r‹  rÒ  r!  rß   rô  Tr~  r›  )Úvit_large_patch14_reg4_dinov2r±  r²  s       rO   rƒ  rƒ  W  sP   € ô Ø ¨R¸2È4Ø Tô€Jô 'Ø'ñ^Ø4>ð^ÜBFÀzÑB\ÐU[ÑB\ñ^€Eà€LrQ   c                 ó„   — t        ddddddt        t        j                  dd¬	«
      }t	        	 dd
| it        |fi |¤Ž¤Ž}|S )rw  rÝ  rx  rä  rÒ  rß   ry  rô  T©
r'  r.  rF   r>   rY   rV   r\   r[   r2  r1  r›  )Úvit_giant_patch14_reg4_dinov2r{  r²  s       rO   r†  r†  d  s\   € ô Ø ¨R¸2È4Ð[fÜ¬"¯'©'¸aÐPTô€Jô 'Ø'ñ^Ø4>ð^ÜBFÀzÑB\ÐU[ÑB\ñ^€Eà€LrQ   c           	      óZ   — t        ddddddd¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
zR ViT-B/14 for TIPSv2 (DINOv2-style w/ 1 register token, LayerScale init=1.0).
    rÝ  r#  r$  r‚  r.   Tr~  r›  )Úvit_base_patch14_reg1_tipsv2r±  r²  s       rO   rˆ  rˆ  u  sP   € ô Ø ¨B¸"È#Ø Tô€Jô 'Ø&ñ]Ø3=ð]ÜAEÀjÑA[ÐTZÑA[ñ]€Eà€LrQ   c           	      óZ   — t        ddddddd¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )zR ViT-L/14 for TIPSv2 (DINOv2-style w/ 1 register token, LayerScale init=1.0).
    rÝ  r‹  rÒ  r!  r‚  r.   Tr~  r›  )Úvit_large_patch14_reg1_tipsv2r±  r²  s       rO   rŠ  rŠ  ‚  sP   € ô Ø ¨R¸2È3Ø Tô€Jô 'Ø'ñ^Ø4>ð^ÜBFÀzÑB\ÐU[ÑB\ñ^€Eà€LrQ   c           
      ó\   — t        dddddddd¬	«      }t        	 dd
| it        |fi |¤Ž¤Ž}|S )zW SoViT-400M/14 for TIPSv2 (DINOv2-style w/ 1 register token, LayerScale init=1.0).
    rÝ  é€  é   r!  r‚  g9Žã8Žã@r.   T)r'  r.  rF   r>   rY   rV   r2  r1  r›  )Úvit_so400m_patch14_reg1_tipsv2r±  r²  s       rO   rŽ  rŽ  �  sR   € ô Ø ¨R¸2È3Ø¨!¸Dô€Jô 'Ø(ñ_Ø5?ð_ÜCGÈ
ÑC]ÐV\ÑC]ñ_€Eà€LrQ   c                 ó„   — t        ddddddt        t        j                  dd¬	«
      }t	        	 dd
| it        |fi |¤Ž¤Ž}|S )z^ ViT-G/14 for TIPSv2 (DINOv2-style w/ SwiGLU FFN, 1 register token, LayerScale init=1.0).
    rÝ  rx  rä  rÒ  r‚  ry  r.   Tr…  r›  )Úvit_giant_patch14_reg1_tipsv2r{  r²  s       rO   r�  r�  œ  s[   € ô Ø ¨R¸2È3Ø¬ÄÇÁØ Tô€Jô
 'Ø'ñ^Ø4>ð^ÜBFÀzÑB\ÐU[ÑB\ñ^€Eà€LrQ   c           	      óZ   — t        ddddddd¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
Nr¸  r#  r$  Fr,  Ú	gelu_tanh©r'  r.  rF   r>   r/  r*  r[   r›  )Úvit_base_patch32_siglip_256r±  r²  s       rO   r”  r”  ¬  sO   € äØ ¨B¸"È%Ð]bØô€Jô 'Ø%ñ\Ø2<ð\Ü@DÀZÑ@ZÐSYÑ@Zñ\€Eà€LrQ   c           	      óX   — t        dddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	Nr!  r#  r$  Fr,  ©r'  r.  rF   r>   r/  r*  r›  )Úvit_base_patch16_siglip_224r±  r²  s       rO   r—  r—  ·  óL   € äØ ¨B¸"È%Ð]bô€Jô 'Ø%ñ\Ø2<ð\Ü@DÀZÑ@ZÐSYÑ@Zñ\€Eà€LrQ   c           	      óX   — t        dddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	Nr!  r#  r$  Fr,  r–  r›  )Úvit_base_patch16_siglip_256r±  r²  s       rO   rš  rš  Á  r˜  rQ   c           	      óX   — t        dddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	Nr!  r#  r$  Fr,  r–  r›  )Úvit_base_patch16_siglip_384r±  r²  s       rO   rœ  rœ  Ë  r˜  rQ   c           	      óX   — t        dddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	Nr!  r#  r$  Fr,  r–  r›  )Úvit_base_patch16_siglip_512r±  r²  s       rO   rž  rž  Õ  r˜  rQ   c           	      óX   — t        dddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	Nr!  r‹  rÒ  Fr,  r–  r›  )Úvit_large_patch16_siglip_256r±  r²  s       rO   r   r   ß  óL   € äØ ¨R¸2È5Ð^cô€Jô 'Ø&ñ]Ø3=ð]ÜAEÀjÑA[ÐTZÑA[ñ]€Eà€LrQ   c           	      óX   — t        dddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	Nr!  r‹  rÒ  Fr,  r–  r›  )Úvit_large_patch16_siglip_384r±  r²  s       rO   r£  r£  é  r¡  rQ   c           	      óZ   — t        ddddddd¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
Nr!  r‹  rÒ  Fr,  r’  r“  r›  )Úvit_large_patch16_siglip_512r±  r²  s       rO   r¥  r¥  ó  sO   € äØ ¨R¸2È5Ð^cØô€Jô 'Ø&ñ]Ø3=ð]ÜAEÀjÑA[ÐTZÑA[ñ]€Eà€LrQ   c           	      óZ   — t        ddddddd¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )NrÝ  rŒ  r�  r!  çˆ…ZÓ¼ã@Fr,  ©r'  r.  rF   r>   rV   r/  r*  r›  )Úvit_so400m_patch14_siglip_224r±  r²  s       rO   r©  r©  þ  óO   € äØ ¨R¸2ÈÐ]bÐpuô€Jô 'Ø'ñ^Ø4>ð^ÜBFÀzÑB\ÐU[ÑB\ñ^€Eà€LrQ   c           	      óZ   — t        ddddddd¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )NrÝ  rŒ  r�  r!  r§  Fr,  r¨  r›  )Úvit_so400m_patch14_siglip_378r±  r²  s       rO   r¬  r¬    sQ   € ô Ø ¨R¸2ÈÐ]bÐpuô€Jô 'Ø'ñ^Ø4>ð^ÜBFÀzÑB\ÐU[ÑB\ñ^€Eà€LrQ   c           	      óZ   — t        ddddddd¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )NrÝ  rŒ  r�  r!  r§  Fr,  r¨  r›  )Úvit_so400m_patch14_siglip_384r±  r²  s       rO   r®  r®    rª  rQ   c           
      ó\   — t        dddddddd¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )Nr!  rŒ  r�  r§  Fr,  r’  ©r'  r.  rF   r>   rV   r/  r*  r[   r›  )Úvit_so400m_patch16_siglip_256r±  r²  s       rO   r±  r±    óR   € äØ ¨R¸2ÈÐ]bÐpuØô€Jô 'Ø'ñ^Ø4>ð^ÜBFÀzÑB\ÐU[ÑB\ñ^€Eà€LrQ   c           
      ó\   — t        dddddddd¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )Nr!  rŒ  r�  r§  Fr,  r’  r°  r›  )Úvit_so400m_patch16_siglip_384r±  r²  s       rO   r´  r´  (  r²  rQ   c           
      ó\   — t        dddddddd¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )Nr!  rŒ  r�  r§  Fr,  r’  r°  r›  )Úvit_so400m_patch16_siglip_512r±  r²  s       rO   r¶  r¶  3  r²  rQ   c           	      óZ   — t        ddddddd¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
Nr!  rx  rä  Fr,  r’  r“  r›  )Úvit_giantopt_patch16_siglip_256r±  r²  s       rO   r¸  r¸  >  óO   € äØ ¨R¸2È5Ð^cØô€Jô 'Ø)ñ`Ø6@ð`ÜDHÈÑD^ÐW]ÑD^ñ`€Eà€LrQ   c           	      óZ   — t        ddddddd¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
Nr!  rx  rä  Fr,  r’  r“  r›  )Úvit_giantopt_patch16_siglip_384r±  r²  s       rO   r»  r»  I  r¹  rQ   c           
      ó\   — t        dddddddd¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
Nr¸  r#  r$  Fr  r’  ©r'  r.  rF   r>   r/  r*  r5  r[   r›  )Úvit_base_patch32_siglip_gap_256r±  r²  s       rO   r¾  r¾  T  sR   € äØ ¨B¸"È%Ð]bÐlqØô€Jô 'Ø)ñ`Ø6@ð`ÜDHÈÑD^ÐW]ÑD^ñ`€Eà€LrQ   c           	      óZ   — t        ddddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	ú^ A SigLIP variant of ViT with global average pooling (GAP) instead of attention pooling (MAP).r!  r#  r$  Fr  rú  r›  )Úvit_base_patch16_siglip_gap_224r±  r²  s       rO   rÁ  rÁ  _  óQ   € ô Ø ¨B¸"È%Ð]bÐlqô€Jô 'Ø)ñ`Ø6@ð`ÜDHÈÑD^ÐW]ÑD^ñ`€Eà€LrQ   c           	      óZ   — t        ddddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	rÀ  r!  r#  r$  Fr  rú  r›  )Úvit_base_patch16_siglip_gap_256r±  r²  s       rO   rÄ  rÄ  j  rÂ  rQ   c           	      óZ   — t        ddddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	rÀ  r!  r#  r$  Fr  rú  r›  )Úvit_base_patch16_siglip_gap_384r±  r²  s       rO   rÆ  rÆ  u  rÂ  rQ   c           	      óZ   — t        ddddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	rÀ  r!  r#  r$  Fr  rú  r›  )Úvit_base_patch16_siglip_gap_512r±  r²  s       rO   rÈ  rÈ  €  rÂ  rQ   c           	      óZ   — t        ddddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	rÀ  r!  r‹  rÒ  Fr  rú  r›  )Ú vit_large_patch16_siglip_gap_256r±  r²  s       rO   rÊ  rÊ  ‹  óQ   € ô Ø ¨R¸2È5Ð^cÐmrô€Jô 'Ø*ñaØ7AðaÜEIÈ*ÑE_ÐX^ÑE_ña€Eà€LrQ   c           	      óZ   — t        ddddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	rÀ  r!  r‹  rÒ  Fr  rú  r›  )Ú vit_large_patch16_siglip_gap_384r±  r²  s       rO   rÍ  rÍ  –  rË  rQ   c           
      ó\   — t        dddddddd¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
Nr!  r‹  rÒ  Fr  r’  r½  r›  )Ú vit_large_patch16_siglip_gap_512r±  r²  s       rO   rÏ  rÏ  ¡  sP   € äØ ¨R¸2È5Ø 5°Kô€Jô 'Ø*ñaØ7AðaÜEIÈ*ÑE_ÐX^ÑE_ña€Eà€LrQ   c           
      ó\   — t        dddddddd¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )rÀ  rÝ  rŒ  r�  r!  r§  Fr  r  r›  )Ú!vit_so400m_patch14_siglip_gap_224r±  r²  s       rO   rÑ  rÑ  ¬  óR   € ô Ø ¨R¸2ÈØ u°eô€Jô 'Ø+ñbØ8BðbÜFJÈ:ÑF`ÐY_ÑF`ñb€Eà€LrQ   c           
      ó\   — t        dddddddd¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )rÀ  rÝ  rŒ  r�  r!  r§  Fr  r  r›  )Ú!vit_so400m_patch14_siglip_gap_378r±  r²  s       rO   rÔ  rÔ  ¸  rÒ  rQ   c           
      ó\   — t        dddddddd¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )rÀ  rÝ  rŒ  r�  r!  r§  Fr  r  r›  )Ú!vit_so400m_patch14_siglip_gap_384r±  r²  s       rO   rÖ  rÖ  Ä  rÒ  rQ   c           
      ó\   — t        dddddddd¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )rÀ  rÝ  rŒ  r�  r!  r§  Fr  r  r›  )Ú!vit_so400m_patch14_siglip_gap_448r±  r²  s       rO   rØ  rØ  Ð  rÒ  rQ   c           
      ó\   — t        dddddddd¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )rÀ  rÝ  rŒ  r�  r!  r§  Fr  r  r›  )Ú!vit_so400m_patch14_siglip_gap_896r±  r²  s       rO   rÚ  rÚ  Ü  rÒ  rQ   c                 ó^   — t        ddddddddd¬«	      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )rÀ  r!  rŒ  r�  r§  Fr  r’  ©	r'  r.  rF   r>   rV   r/  r*  r5  r[   r›  )Ú!vit_so400m_patch16_siglip_gap_256r±  r²  s       rO   rÝ  rÝ  è  sT   € ô Ø ¨R¸2ÈØ u°eÀ{ô€Jô 'Ø+ñbØ8BðbÜFJÈ:ÑF`ÐY_ÑF`ñb€Eà€LrQ   c                 ó^   — t        ddddddddd¬«	      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )Nr!  rŒ  r�  r§  Fr  r’  rÜ  r›  )Ú!vit_so400m_patch16_siglip_gap_384r±  r²  s       rO   rß  rß  ô  óS   € äØ ¨R¸2ÈÐ]bØ 5°Kô€Jô 'Ø+ñbØ8BðbÜFJÈ:ÑF`ÐY_ÑF`ñb€Eà€LrQ   c                 ó^   — t        ddddddddd¬«	      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )Nr!  rŒ  r�  r§  Fr  r’  rÜ  r›  )Ú!vit_so400m_patch16_siglip_gap_512r±  r²  s       rO   râ  râ  ÿ  rà  rQ   c           
      ó\   — t        dddddddd¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
Nr!  rx  rä  Fr  r’  r½  r›  )Ú#vit_giantopt_patch16_siglip_gap_256r±  r²  s       rO   rä  rä  
  óP   € äØ ¨R¸2È5Ø 5°Kô€Jô 'Ø-ñdØ:DðdÜHLÈZÑHbÐ[aÑHbñd€Eà€LrQ   c           
      ó\   — t        dddddddd¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
Nr!  rx  rä  Fr  r’  r½  r›  )Ú#vit_giantopt_patch16_siglip_gap_384r±  r²  s       rO   rç  rç    rå  rQ   c                 ó`   — t        ddddddddd	d
¬«
      }t        	 dd| it        |fi |¤Ž¤Ž}|S )Nr!  r†  rÝ  rô  rß   é   FTr.   r  ©
r'  r.  rF   r>   rY   rV   r/  r1  r2  r*  r›  )Úvit_wee_patch16_reg1_gap_256r±  r²  s       rO   rë  rë     sU   € äØ ¨B¸!ÈÐYZØ¨$¸1È%ô€Jô 'Ø&ñ]Ø3=ð]ÜAEÀjÑA[ÐTZÑA[ñ]€Eà€LrQ   c                 ób   — t        ddddddddd	d
d¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )Nr!  r†  rÝ  rô  rß   ré  FTr.   r  r;   ©r'  r.  rF   r>   rY   rV   r/  r1  r2  r*  r<   r›  )Úvit_dwee_patch16_reg1_gap_256r±  r²  s       rO   rî  rî  +  sX   € äØ ¨B¸!ÈÐYZØ¨$¸1È%Ð\bô€Jô 'Ø'ñ^Ø4>ð^ÜBFÀzÑB\ÐU[ÑB\ñ^€Eà€LrQ   c                 ój   — t        dddddddddd	t        ¬
«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )Nr!  r†  rô  rß   ré  FTr.   r  ©r'  r.  rF   r>   rY   rV   r/  r1  r2  r*  rG  r›  )Úvit_pwee_patch16_reg1_gap_256)rZ  r›   r¬  r²  s       rO   rñ  rñ  6  sX   € äØ ¨B¸!ÈÐYZØ¨$¸1È%ÔZnô€Jô 'Ø'ñ^Ø4>ð^ÜBFÀzÑB\ÐU[ÑB\ñ^€Eà€LrQ   c                 ój   — t        dddddddddd	t        ¬
«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )Nr!  r†  rô  rß   ré  FTr.   r  rð  r›  )Úvit_dpwee_patch16_reg1_gap_256)rZ  rÛ   r¬  r²  s       rO   ró  ró  A  sX   € äØ ¨B¸!ÈÐYZØ¨$¸1È%ÔZrô€Jô 'Ø(ñ_Ø5?ð_ÜCGÈ
ÑC]ÐV\ÑC]ñ_€Eà€LrQ   c                 ó`   — t        ddddddddd	d
¬«
      }t        	 dd| it        |fi |¤Ž¤Ž}|S )Nr!  é@  rÝ  ré  rß   çffffff@FTr.   r  rê  r›  )Úvit_little_patch16_reg1_gap_256r±  r²  s       rO   r÷  r÷  L  óU   € äØ ¨B¸!ÈÐY\Ø¨$¸1È%ô€Jô 'Ø)ñ`Ø6@ð`ÜDHÈÑD^ÐW]ÑD^ñ`€Eà€LrQ   c                 ób   — t        ddddddddd	d
d¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )Nr!  rõ  rÝ  ré  rß   rö  FTr.   r  r;   rí  r›  )Ú vit_dlittle_patch16_reg1_gap_256r±  r²  s       rO   rú  rú  W  sX   € äØ ¨B¸!ÈÐY\Ø¨$¸1È%Ð\bô€Jô 'Ø*ñaØ7AðaÜEIÈ*ÑE_ÐX^ÑE_ña€Eà€LrQ   c                 ó`   — t        ddddddddd	d
¬«
      }t        	 dd| it        |fi |¤Ž¤Ž}|S )Nr!  rõ  rÝ  ré  rß   rö  FTrô  r  rê  r›  )Úvit_little_patch16_reg4_gap_256r±  r²  s       rO   rü  rü  b  rø  rQ   c                 ó^   — t        ddddddddd	¬
«	      }t        	 dd| it        |fi |¤Ž¤Ž}|S )Nr!  rŠ  r$  rÄ  rß   FTr.   r  ©	r'  r.  rF   r>   rY   r/  r1  r2  r*  r›  )Úvit_medium_patch16_reg1_gap_256r±  r²  s       rO   rÿ  rÿ  m  óR   € äØ ¨B¸!ÈØ¨$¸1È%ô€Jô 'Ø)ñ`Ø6@ð`ÜDHÈÑD^ÐW]ÑD^ñ`€Eà€LrQ   c                 ó^   — t        ddddddddd	¬
«	      }t        	 dd| it        |fi |¤Ž¤Ž}|S )Nr!  rŠ  r$  rÄ  rß   FTrô  r  rþ  r›  )Úvit_medium_patch16_reg4_gap_256r±  r²  s       rO   r  r  x  r   rQ   c                 ó^   — t        ddddddddd	¬
«	      }t        	 dd| it        |fi |¤Ž¤Ž}|S )Nr!  rŠ  é   rÄ  rß   FTrô  r  rþ  r›  )Ú vit_mediumd_patch16_reg4_gap_256r±  r²  s       rO   r  r  ƒ  óR   € äØ ¨B¸!ÈØ¨$¸1È%ô€Jô 'Ø*ñaØ7AðaÜEIÈ*ÑE_ÐX^ÑE_ña€Eà€LrQ   c                 ó^   — t        ddddddddd	¬
«	      }t        	 dd| it        |fi |¤Ž¤Ž}|S )Nr!  rŠ  r  rÄ  rß   FTrô  r  rþ  r›  )Ú vit_mediumd_patch16_reg4_gap_384r±  r²  s       rO   r  r  Ž  r  rQ   c                 ó^   — t        ddddddddd	¬
«	      }t        	 dd| it        |fi |¤Ž¤Ž}|S )Nr!  r�  r$  r÷  rß   FTr.   r  rþ  r›  )Ú vit_betwixt_patch16_reg1_gap_256r±  r²  s       rO   r
  r
  ™  óR   € äØ ¨B¸"È$Ø¨$¸1È%ô€Jô 'Ø*ñaØ7AðaÜEIÈ*ÑE_ÐX^ÑE_ña€Eà€LrQ   c                 ó^   — t        ddddddddd	¬
«	      }t        	 dd| it        |fi |¤Ž¤Ž}|S )Nr!  r�  r$  r÷  rß   FTrô  r  rþ  r›  )Ú vit_betwixt_patch16_reg4_gap_256r±  r²  s       rO   r  r  ¤  r  rQ   c                 ó^   — t        ddddddddd	¬
«	      }t        	 dd| it        |fi |¤Ž¤Ž}|S )Nr!  r�  r$  r÷  rß   FTrô  r  rþ  r›  )Ú vit_betwixt_patch16_reg4_gap_384r±  r²  s       rO   r  r  ¯  r  rQ   c           
      ó\   — t        dddddddd¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )Nr!  r#  r$  FTr  rô  )r'  r.  rF   r>   r/  r1  r*  r2  r›  )Úvit_base_patch16_reg4_gap_256r±  r²  s       rO   r  r  º  sP   € äØ ¨B¸"È%Ø¨¸1ô€Jô 'Ø'ñ^Ø4>ð^ÜBFÀzÑB\ÐU[ÑB\ñ^€Eà€LrQ   c           
      ó\   — t        dddddddd¬	«      }t        	 dd
| it        |fi |¤Ž¤Ž}|S )úF SO150M (shape optimized, but diff than paper def, optimized for GPU) r!  r’  rU  rÝ  çú~j¼t“@Frô  r,  )r'  r.  rF   r>   rV   r/  r2  r*  r›  )Úvit_so150m_patch16_reg4_map_256r±  r²  s       rO   r  r  Å  sR   € ô Ø ¨B¸"ÈØ a°Uô€Jô 'Ø)ñ`Ø6@ð`ÜDHÈÑD^ÐW]ÑD^ñ`€Eà€LrQ   c                 ó^   — t        ddddddddd¬	«	      }t        	 dd
| it        |fi |¤Ž¤Ž}|S )r  r!  r’  rU  rÝ  r  Frô  r  ©	r'  r.  rF   r>   rV   r/  r2  r*  r5  r›  )Úvit_so150m_patch16_reg4_gap_256r±  r²  s       rO   r  r  Ñ  óT   € ô Ø ¨B¸"ÈØ a°UÀEô€Jô 'Ø)ñ`Ø6@ð`ÜDHÈÑD^ÐW]ÑD^ñ`€Eà€LrQ   c                 ó^   — t        ddddddddd¬	«	      }t        	 dd
| it        |fi |¤Ž¤Ž}|S )r  r!  r’  rU  rÝ  r  Frô  r  r  r›  )Úvit_so150m_patch16_reg4_gap_384r±  r²  s       rO   r  r  Ý  r  rQ   c                 ó`   — t        dddddddddd	¬
«
      }t        	 dd| it        |fi |¤Ž¤Ž}|S )úI SO150M v2 (shape optimized, but diff than paper def, optimized for GPU) r!  é@  é   é   çÅNìÄNì@rß   Fr.   r  ©
r'  r.  rF   r>   rV   rY   r?   r/  r2  r*  r›  )Ú vit_so150m2_patch16_reg1_gap_256r±  r²  s       rO   r#  r#  é  óW   € ô Ø ¨B¸"ÈÐ[_Ø E°aÀUô€Jô 'Ø*ñaØ7AðaÜEIÈ*ÑE_ÐX^ÑE_ña€Eà€LrQ   c                 ó`   — t        dddddddddd	¬
«
      }t        	 dd| it        |fi |¤Ž¤Ž}|S )r  r!  r  r  r   r!  rß   Fr.   r  r"  r›  )Ú vit_so150m2_patch16_reg1_gap_384r±  r²  s       rO   r&  r&  õ  r$  rQ   c                 ó`   — t        dddddddddd	¬
«
      }t        	 dd| it        |fi |¤Ž¤Ž}|S )r  r!  r  r  r   r!  rß   Fr.   r  r"  r›  )Ú vit_so150m2_patch16_reg1_gap_448r±  r²  s       rO   r(  r(    r$  rQ   c           	      óZ   — t        ddddddd¬«      }t        	 d
d	| it        |fi |¤Ž¤Ž}|S )NrÝ  r‹  rÒ  r!  rð   FT)r'  r.  rF   r>   rY   r4  r  r›  )Úvit_intern300m_patch14_448r±  r²  s       rO   r*  r*    sN   € äØ ¨R¸2Ø E¸Dô€Jô 'Ø$ñ[Ø1;ð[Ü?CÀJÑ?YÐRXÑ?Yñ[€Eà€LrQ   c                 ó¬   — t        dddddddddddt        t        d	¬
«      t        t        d	¬
«      t        ¬«      }t	        	 dd| it        |fi |¤Ž¤Ž}|S )ú ViT Large AIM-v2 model
    rÝ  r‹  rÒ  rÄ  Fç      @r  Úsilurß   r  ©r'  r.  rF   r>   r/  r5  rV   r*  r?   rB   r[   rE   rF  r\   r›  )Úaimv2_large_patch14_224©rZ  r   r   r   r¬  r²  s       rO   r0  r0    óp   € ô Ø ¨R¸1È%ÐY^Ø E°EÀUÐV\Üœ7¨Ô-ÄÌÐUYÔ@ZÔflô€Jô
 'Ø!ñXØ.8ðXÜ<@ÀÑ<VÈvÑ<VñX€Eà€LrQ   c                 ó¬   — t        dddddddddddt        t        d	¬
«      t        t        d	¬
«      t        ¬«      }t	        	 dd| it        |fi |¤Ž¤Ž}|S )ú ViT Huge AIM-v2 model
    rÝ  rx  rÒ  r$  FçòAÏfU@r  r.  rß   r  r/  r›  )Úaimv2_huge_patch14_224r1  r²  s       rO   r6  r6  &  sp   € ô
 Ø ¨R¸2È5ÐZ_Ø e°eÀuÐX^Üœ7¨Ô-ÄÌÐUYÔ@ZÔflô€Jô
 'Ø ñWØ-7ðWÜ;?À
Ñ;UÈfÑ;UñW€Eà€LrQ   c                 ó¬   — t        dddddddddddt        t        d	¬
«      t        t        d	¬
«      t        ¬«      }t	        	 dd| it        |fi |¤Ž¤Ž}|S )ú ViT 1B AIM-v2 model
    rÝ  é   rÒ  r!  Fr-  r  r.  rß   r  r/  r›  )Úaimv2_1b_patch14_224r1  r²  s       rO   r:  r:  5  óp   € ô Ø ¨R¸2È5ÐZ_Ø E°EÀUÐV\Üœ7¨Ô-ÄÌÐUYÔ@ZÔflô€Jô
 'ØñUØ+5ðUÜ9=¸jÑ9SÈFÑ9SñU€Eà€LrQ   c                 ó¬   — t        dddddddddddt        t        d¬	«      t        t        d¬	«      t        ¬
«      }t	        	 dd| it        |fi |¤Ž¤Ž}|S )ú ViT 3B AIM-v2 model
    rÝ  é   rÒ  Fr5  r  r.  rß   r  r/  r›  )Úaimv2_3b_patch14_224r1  r²  s       rO   r?  r?  C  óp   € ô Ø ¨R¸2È5ÐZ_Ø e°eÀuÐX^Üœ7¨Ô-ÄÌÐUYÔ@ZÔflô€Jô
 'ØñUØ+5ðUÜ9=¸jÑ9SÈFÑ9SñU€Eà€LrQ   c                 ó¬   — t        dddddddddddt        t        d	¬
«      t        t        d	¬
«      t        ¬«      }t	        	 dd| it        |fi |¤Ž¤Ž}|S )r,  rÝ  r‹  rÒ  rÄ  Fr-  r  r.  rß   r  r/  r›  )Úaimv2_large_patch14_336r1  r²  s       rO   rB  rB  Q  r2  rQ   c                 ó¬   — t        dddddddddddt        t        d	¬
«      t        t        d	¬
«      t        ¬«      }t	        	 dd| it        |fi |¤Ž¤Ž}|S )r4  rÝ  rx  rÒ  r$  Fr5  r  r.  rß   r  r/  r›  )Úaimv2_huge_patch14_336r1  r²  s       rO   rD  rD  _  óp   € ô Ø ¨R¸2È5ÐZ_Ø e°eÀuÐX^Üœ7¨Ô-ÄÌÐUYÔ@ZÔflô€Jô
 'Ø ñWØ-7ðWÜ;?À
Ñ;UÈfÑ;UñW€Eà€LrQ   c                 ó¬   — t        dddddddddddt        t        d	¬
«      t        t        d	¬
«      t        ¬«      }t	        	 dd| it        |fi |¤Ž¤Ž}|S )r8  rÝ  r9  rÒ  r!  Fr-  r  r.  rß   r  r/  r›  )Úaimv2_1b_patch14_336r1  r²  s       rO   rG  rG  m  r;  rQ   c                 ó¬   — t        dddddddddddt        t        d¬	«      t        t        d¬	«      t        ¬
«      }t	        	 dd| it        |fi |¤Ž¤Ž}|S )r=  rÝ  r>  rÒ  Fr5  r  r.  rß   r  r/  r›  )Úaimv2_3b_patch14_336r1  r²  s       rO   rI  rI  {  r@  rQ   c                 ó¬   — t        dddddddddddt        t        d	¬
«      t        t        d	¬
«      t        ¬«      }t	        	 dd| it        |fi |¤Ž¤Ž}|S )r,  rÝ  r‹  rÒ  rÄ  Fr-  r  r.  rß   r  r/  r›  )Úaimv2_large_patch14_448r1  r²  s       rO   rK  rK  ‰  r2  rQ   c                 ó¬   — t        dddddddddddt        t        d	¬
«      t        t        d	¬
«      t        ¬«      }t	        	 dd| it        |fi |¤Ž¤Ž}|S )r4  rÝ  rx  rÒ  r$  Fr5  r  r.  rß   r  r/  r›  )Úaimv2_huge_patch14_448r1  r²  s       rO   rM  rM  —  rE  rQ   c                 ó¬   — t        dddddddddddt        t        d	¬
«      t        t        d	¬
«      t        ¬«      }t	        	 dd| it        |fi |¤Ž¤Ž}|S )r8  rÝ  r9  rÒ  r!  Fr-  r  r.  rß   r  r/  r›  )Úaimv2_1b_patch14_448r1  r²  s       rO   rO  rO  ¥  r;  rQ   c                 ó¬   — t        dddddddddddt        t        d¬	«      t        t        d¬	«      t        ¬
«      }t	        	 dd| it        |fi |¤Ž¤Ž}|S )r=  rÝ  r>  rÒ  Fr5  r  r.  rß   r  r/  r›  )Úaimv2_3b_patch14_448r1  r²  s       rO   rQ  rQ  ³  r@  rQ   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
ú ViT Test
    r!  é@   r¹  r¿   r¢   T)r'  r.  rF   r>   rV   r  r›  )Útest_vitr±  r²  s       rO   rU  rU  Á  s=   € ô  ¨r¸ÀaÐSTÐgkÔl€JÜ&Ñg¸jÐgÌDÐQ[ÑLfÐ_eÑLfÑg€EØ€LrQ   c                 ó^   — t        ddddddddd	d
¬«
      }t        dd| it        |fi |¤Ž¤Ž}|S )rS  r!  rT  rÄ  r¿   r¢   Fr.   r  rß   T)
r'  r.  rF   r>   rV   r/  r2  r*  rY   r  r›  )Ú	test_vit2r±  r²  s       rO   rW  rW  Ê  sK   € ô Ø ¨1¸ÀQØ a°UÈÐ_côe€Jô 'Ñh¸zÐhÌTÐR\ÑMgÐ`fÑMgÑh€EØ€LrQ   c                 ó^   — t        ddddddddd	d
¬«
      }t        dd| it        |fi |¤Ž¤Ž}|S )rS  r!  é`   é	   r¢   r¿   Fr.   r,  Trß   )
r'  r.  rF   r>   rV   r/  r2  r*  r6  rY   r›  )Ú	test_vit3r±  r²  s       rO   r[  r[  Õ  sL   € ô Ø ¨1¸ÀQØ a°UÐPTÐbfôh€Jô 'Ñh¸zÐhÌTÐR\ÑMgÐ`fÑMgÑh€EØ€LrQ   c                 ó`   — t        dddddddddd	d
¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )rS  r!  rY  rZ  r¢   Fr.   r  rß   TÚrmsnorm)r'  r.  rF   r>   rV   r/  r2  r*  rY   r  rE   r›  )Ú	test_vit4r±  r²  s       rO   r^  r^  à  sM   € ô Ø ¨1¸ÀQØ a°UÈÐ_cØô€Jô
 'Ñh¸zÐhÌTÐR\ÑMgÐ`fÑMgÑh€EØ€LrQ   c                 ó|   — t        dddddddddt        t        d¬«      ¬	«
      }t        dd
| it        |fi |¤Ž¤Ž}|S )zu BEiT3 Base model (ViT-Base size) with patch size 16x16.
    Remapped to VisionTransformer with scale_norm=True.
    r!  r#  r$  rô  Tr  rß   r  ©
r'  r.  rF   r>   rV   rW   rX   r/  r*  rE   r›  )Úbeit3_base_patch16_224r  r²  s       rO   ra  ra  í  sS   € ô
 Ø ¨B¸"ÈØ¨T¸tÐQVÜœ9¨$Ô/ô€Jô
 'ÑuÈJÐuÔZ^Ð_iÑZtÐmsÑZtÑu€EØ€LrQ   c                 ó|   — t        dddddddddt        t        d¬«      ¬	«
      }t        dd
| it        |fi |¤Ž¤Ž}|S )zw BEiT3 Large model (ViT-Large size) with patch size 16x16.
    Remapped to VisionTransformer with scale_norm=True.
    r!  r‹  rÒ  rô  Tr  rß   r  r`  r›  )Úbeit3_large_patch16_224r  r²  s       rO   rc  rc  û  sS   € ô
 Ø ¨R¸2ÈØ¨T¸tÐQVÜœ9¨$Ô/ô€Jô
 'ÑvÈZÐvÔ[_Ð`jÑ[uÐntÑ[uÑv€EØ€LrQ   c                 ó|   — t        dddddddddt        t        d¬	«      ¬
«
      }t        dd| it        |fi |¤Ž¤Ž}|S )zf BEiT3 Giant model with patch size 14x14.
    Remapped to VisionTransformer with scale_norm=True.
    rÝ  râ  rä  r!  ç†8ÖÅmt@Tr  rß   r  r`  r›  )Úbeit3_giant_patch14_224r  r²  s       rO   rf  rf  	  sS   € ô
 Ø ¨R¸2ÈØ¨T¸tÐQVÜœ9¨$Ô/ô€Jô
 'ÑvÈZÐvÔ[_Ð`jÑ[uÐntÑ[uÑv€EØ€LrQ   c                 ó~   — t        ddddddddddt        t        d	¬
«      ¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )z} BEiT3 Giant model with patch size 14x14 and image size 336x336.
    Remapped to VisionTransformer with scale_norm=True.
    r‰  rÝ  râ  rä  r!  re  Tr  rß   r  )r&  r'  r.  rF   r>   rV   rW   rX   r/  r*  rE   r›  )Úbeit3_giant_patch14_336r  r²  s       rO   rh  rh    sV   € ô
 Ø ¨t¸2ÈÐW]Ø¨T¸tÐQVÜœ9¨$Ô/ô€Jô
 'ÑvÈZÐvÔ[_Ð`jÑ[uÐntÑ[uÑv€EØ€LrQ   Úvit_tiny_patch16_224_in21kÚvit_small_patch32_224_in21kÚvit_small_patch16_224_in21kÚvit_base_patch32_224_in21kÚvit_base_patch16_224_in21kÚvit_base_patch8_224_in21kÚvit_large_patch32_224_in21kÚvit_large_patch16_224_in21kÚvit_huge_patch14_224_in21kÚvit_base_patch32_224_samzvit_base_patch32_224.samÚvit_base_patch16_224_samzvit_base_patch16_224.samÚvit_small_patch16_224_dinoÚvit_small_patch8_224_dinoÚvit_base_patch16_224_dinoÚvit_base_patch8_224_dinoÚvit_base_patch16_224_miil_in21kÚ!vit_base_patch32_224_clip_laion2b)Ú"vit_large_patch14_224_clip_laion2bÚ!vit_huge_patch14_224_clip_laion2bÚ"vit_giant_patch14_224_clip_laion2b)FFFTrU   rU   Nr   )r  r.   FrÒ  )r9   rU   T)rA  rU   T)r.   rg   ru  F)r9   F)zvisual.)Fru  TrÓ  )FNrÔ  (  r…   ÚcopyÚloggingrí   ÚosÚcollectionsr   Ú	functoolsr   Útypingr   r   r   r   r	   r
   r   r   r   r   ÚImportErrorÚtyping_extensionsrŠ   Útorch.nnrk   Útorch.nn.functionalÚ
functionalrÈ   Ú	torch.jitr   Ú	timm.datar   r   r   r   r   r   Útimm.layersr   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   Ú_builderr/   Ú	_featuresr0   Ú_manipulater1   r2   r3   r4   Ú	_registryr5   r6   r7   Ú__all__Ú	getLoggerr‚   Ú_loggerrK   rp   rˆ   r‡   r‰   rP   rS   r�   r›   rÛ   r  r‹   rJ   r  r8   rƒ  rÝ  râ  rä  r  rï  ro  rˆ  rB  rL  rR  rZ  re  rr  r~  Údefault_cfgsr;  rL   Ú_quick_gelu_cfgsrÇ  ÚdeepcopyÚcr<  ÚenvironÚlowerr¨  r¬  r°  r¶  rº  r½  rÀ  rÂ  rÅ  rÇ  rÊ  rÌ  rÎ  rÐ  rÓ  rÖ  rÙ  rÛ  rÞ  rà  ræ  rë  rí  rð  ró  rõ  rø  rû  rý  r   r  r  r  r  r  r  r  r  r  r  r  r   r"  r%  r'  r*  r,  r/  r2  r5  r9  r<  r>  rA  rC  rF  rH  rK  rN  rP  rS  rW  rZ  r]  r_  rb  rd  rf  rj  rm  ro  rq  rs  ru  rz  r  r�  rƒ  r†  rˆ  rŠ  rŽ  r�  r”  r—  rš  rœ  rž  r   r£  r¥  r©  r¬  r®  r±  r´  r¶  r¸  r»  r¾  rÁ  rÄ  rÆ  rÈ  rÊ  rÍ  rÏ  rÑ  rÔ  rÖ  rØ  rÚ  rÝ  rß  râ  rä  rç  rë  rî  rñ  ró  r÷  rú  rü  rÿ  r  r  r  r
  r  r  r  r  r  r  r#  r&  r(  r*  r0  r6  r:  r?  rB  rD  rG  rI  rK  rM  rO  rQ  rU  rW  r[  r^  ra  rc  rf  rh  )rÇ  r•  s   00rO   ú<module>r˜     s�B  ðñó2 Û Û Û 	Ý #Ý ß O× OÕ Oð*Ýó Ý ß Ð Ý ÷÷ ÷
÷ ÷ ÷ ÷ ÷ ó õ4 +Ý +ß RÓ Rß YÑ YàÐ
€ð ˆ'×
Ñ
˜HÓ
%€ð 	ØØñ€ð ØØ ØØØØ04Øò!Øð!àð!ð ð!ð ð	!ð
 ð!ð ð!ð ð!ð ð!ð ð!ð ˜T "§)¡)™_Ñ-ð!ð ð!ð ‡Y�Yó!ôHVˆB�I‰Iô VôrH�2—9‘9ô HôV@˜2Ÿ9™9ô @ôFk˜rŸy™yô kô\V˜"Ÿ)™)ô Vðv !Ø!"Ø&+ò	Ø�<‰<ðàðð ðð  $ó	ô2O	˜Ÿ	™	ô O	òd" "§)¡)ð "°3ð "È$ð "ÐZ^ó "ð( ØØ ò	"Ø—	‘	ð"àð"ð ð"ð ð	"ð
 
ó"ò@" "§)¡)ð "°3ð "È$ð "ÐZ^ó "ò."¨"¯)©)ð "¸3ð "ÐRVð "Ðbfó "ò

G˜sð 
G°uð 
GÐQUð 
GÐaió 
Gð  "#Ø"$Ø&ØòØ—‘ðà—L‘Lðð ðð �c˜3�h‘ð	ð
 ðð ðð ‡\�\óð2 €‡�ƒòWTÐ*ð WT¸Sð WTÈ#ð WTÐcgð WTÐtxò WTó ðWTðz  ò#Ø˜˜eŸl™lÐ*Ñ+ð#à ð#ð ð#ð 
ˆ#ˆu�|‰|Ð
Ñó	#ðLØ˜˜eŸl™lÐ*Ñ+ðà ðð 
ˆ#ˆu�|‰|Ð
Ñóð.Ø˜˜eŸl™lÐ*Ñ+ðà ðð 
ˆ#ˆu�|‰|Ð
Ñóð$9˜tó 9ð~ #(Ø&ØòQØ˜˜eŸl™lÐ*Ñ+ðQà ðQð  ðQð ð	Qð
 ðQð 
ˆ#ˆu�|‰|Ð
ÑóQòhˆcð  T¨#¨s¨(¡^ó ð"Sð 1±$Øô3ðSð
 0±Øô2ðSð 0±ð dØØô2ðSð 0±ð dØØ ]¸Sô2BðSð" 1±$ð eØØô3ð#Sð* 1±$ð eØØ ]¸Sô3Bð+Sð2 1±$ð eØØô3ð3Sð: 1±$ð eØØ ]¸Sô3Bð;SðB 0±ð fØØô2ðCSðJ 0±ð dØØ ]¸Sô2BðKSðR 0±ð eØØô2ðSSðZ 0±ð eØØ ]¸Sô2Bð[Sðb /±ð dØØô1ðcSðj 1±$ð eØØô3ðkSðr 1±$ð eØØ ]¸Sô3BðsSð~ .©tØ}Øô0ðSðD .©tØ}ØØ ¨3ô00ñESðL /±Ø~ØØ ¨3ô10ðMSðX (©ð dØØô*ðYSð` (©ð dØØ ]¸Sô*BðaSðh '©ð dØØô)ðiSðp '©ð dØØ ]¸Sô)BðqSðx '©ð dØØô)ðySð@ '©ð dØØ ]¸Sô)BðASðJ &¡t°¤|ðKSðL %¡d¨r¤lðMSðN &¡t°¤|ðOSðP )©$°2¬,ðQSðV &¡tàØô(ðWSð^ &¡tàØô(ð_Sðf '©àØô)ðgSðn '©àØô)ðoSðv &¡tØØô(ðwSð@ (©Ø{ØØ eô*-òASðH )©$Ø|ØØ eô+-ðISðP )©$Ø|ØØ eô+-ðQSðX (©Ø}ØØ eô*-ðYSð` (©Ø|ØØ eô*-ðaSðh '©Ø{ØØ eô)-ðiSðp )©$Ø|ØØ eô+-ðqSð| $¡TØHÐVZØô&ð}SðB $¡TØHÐVZØô&ðCSðL !¡$ØiØØ"Ð(<È!ô#MðMSðT  ¡ØgØØ"Ð(<È!ô"MðUSð\  ¡ØeØØ"Ð(<È!ô"Mð]Sðd ¡ØcØØ"Ð(<È!ô!MðeSðp '©Ø\ØØØ"Ð(<È!Ø ¨3ô)0ðqSð| &¡tØ\ØØØ"Ð(<È!Ø ¨3ô(0ð}SðH '©Ø\ØØØ"Ð(<È!Ø ¨3ô)0ðISðT '©Ø\ØØØ"Ð(<È!Ø ¨3ô)0ðUSðd ,©TØaØØØ"Ð(<È!Ø ¨3ô.0òeSðp +©DØaØØØ"Ð(<È!Ø ¨3ô-0ðqSð| ,©TØaØØØ"Ð(<È!Ø ¨3ô.0ð}SðH ,©TØaØØØ"Ð(<È!Ø ¨3ô.0ðISðZ )©$ØØØ˜|¸Ø ¨3ô	+0ð[Sðd *©4ØØØ˜|¸Ø ¨3ô	,0ðeSðn +©DØØØ˜|¸Ø ¨3ô	-0ðoSðx *©4ØØØ˜|¸Ø ¨3ô	,0ðySðF &¡tð LØØ˜|°eÈ:Ðchô(jðGSðN .©tð NØØ˜|°eÈ:ô0WðOSðZ '©ð LØô)ð[Sð` *©4ØØ ¨4¸Uô,DðaSðf 2±4ØØ ¨4ô41ðgSðl 2±4ØØ ¨4¸8ô4EðmSðr ,©TØ ¨4ô.1ðsSðv )©$«&ðwSð| 6±tØØ ?ô84ð}SðB	 6±tØØ ?¸SÈ]ô8\òC	SðH	 6±tØØ ?¸SÈ]ô8\ðI	SðN	 6±tØØ ?¸Tô8CðO	SðT	 6±tØØ ?Ø ¸(ô8DðU	Sð\	 7¹ØØ$Ð*@È3ô9Pð]	Sðb	 7¹ØØ$Ð*@Ø ¸(ô9Dðc	Sðj	 6±tØØ ?¸Sô8Bðk	Sðp	 6±tØØ ?Ø ¸(ô8Dðq	Sðz	 5±dà ?ô74ð{	Sð@
 5±dØØ ?Ø -¸8ô7EðA
SðH
 5±dØØ ?¸Tô7CðI
SðN
 5±dØØ ?Ø -¸8ô7EðO
SðV
 6±tØØ ?¸Sô8BðW
Sð\
 6±tØØ ?Ø ¸(ô8Dð]
Sðf
 0±ØØ ?ô24ðg
Sðl
 0±ØØ ?¸Sô2Bðm
Sðr
 0±ØØ ?Ø ¸(ô2Dðs
Sðz
 1±$ØØ$Ð*@È3ô3Pò{
Sð@ 1±$ØØ$Ð*@Ø ¸(ô3DðASðH 0±ØØ ?¸Sô2BðISðN 0±ØØ ?Ø ¸(ô2DðOSðX /±ØØ ?ô14ðYSð^ /±ØØ ?ô14ð_Sñd /±ØØ ?Ø ¸(ô1DðeSñl 0±ØØ ?¸Sô2BðmSñt 1±$ØØ ?Àõ3GðuSñz 2±4ØØ$Ð*@È3Ð\aõ4cð{Sñ@ 1±$ØØ ?¸SÈeõ3UðASñH 0±ØØ ?Àõ2GðISñN 1±$ØØ ?¸SÈeõ3UðOSñV (©ØØ ?Áõ*EðWSñ\ (©ØØ ?¸SÉcõ*Sð]Sñb )©$ØØ$Ð*@È3Ñ\_õ+aðcSñh (©ØØ ?¸SÉdõ*TðiSñn )©$ØØ ?¸SÉdõ+TòoSñt ,©TØØ ?¸SÉdõ.TðuSñ| .©tØÙÙWØ ?Áõ	0Eð}SñF .©tØÙØ ?¸SÉcõ0SðGSñN 3±DØÙØ ?Ø ¨3¹Cõ	5AðOSñX /±ØÙØ ?¸SÉcõ1SðYSñb +©DØØ ?¸SÉcõ-SðcSñh +©DØØ ?Ø ¹Cõ-AðiSñp +©DØØ ?¸SÉcõ-SðqSñv ,©TØØ ?¸SÉcõ.SðwSñ~ &¡tØÙØ ?¸SÉcõ(SðSñF ,©TØÙØ ?¸SÉcõ.SðGSñN '©ØÙÙWØ ?¸SÉcõ	)SðOSñX &¡tØÙÙWØ ?¸SÉdõ	(TðYSñb &¡tØØ ?ÙÙWØ¡¹Dõ(BðcSñz 4±TØÙÙWØ ?¸SÉdõ	6Tð{SñD 4±TØÙØ ?Ù ¨3¸(ÑPTõ	6VðESñN 8¹ØÙØ ?¸SÉdõ:TòOSñV 8¹ØÙØ ?Ù ¨3¸(ÑPTõ	:VðWSñb /±ØÙÙWØ ?¸SÉcõ	1SðcSñl /±ØÙÙWØ ?¸SÉcõ	1SðmSñv 0±ØÙÙWØ ?¸SÉcõ	2SðwSñ@ /±ØÙÙWØ ?¸SÉdõ	1TðASñJ 4±TØÙØ ?¸SÉdõ6TðKSñR 3±DØÙÙWØ ?¸SÉdõ	5TðSSñ\ .©tØÙÙWØ ?¸SÉcõ	0Sð]Sñf .©tØÙÙWØ ?¸SÉcõ	0SðgSñp /±ØÙÙWØ ?¸SÉcõ	1SðqSñ| '©ØÙWØ ?Áõ)Eð}SñD '©ØÙWØ ?Áõ)EðESñL (©ØÙWØ ?¸SÉcõ*SðMSñT (©ØÙWØ ?Ø ¹Cõ	*AðUSñ` 6±tØÙØ ?¸SÙõ	8ðaSñp *©4°BÀ=Ð[_Õ+`ðqSñr *©4°BÀ=Ð[_Õ+`òsSñt +©D°R¬LðuSñv +©D°R¬LðwSñx *©4°B¬<ðySñ@ 0±à¡5Ø ?Ù ¨3õ	20ðASñJ 0±à¡5Ø ?Ø ¨3¸(õ	2DðKSñT *©4à¡5Ø ?Ù ¨3õ	,0ðUSñ^ *©4à¡5Ø ?Ø ¨3¸(õ	,Dð_Sñj !¡$ÙSÐaeØØ ¨4õ#1ðkSñr  ¡ÙYÐgkØØ ¨4õ"1ðsSñz  ¡ÙYÐgkØØ ¨4õ"1ð{SñD  ¡ÙSÐaeØØ ¨4õ"1ðESñL ¡ÙYÐgkØØ ¨4õ!1ðMSñT ¡ÙYÐgkØØ ¨4õ!1ðUSñ\ !¡$Ù[ÐimØØ ¨4¸Uõ#Dð]Sñd  ¡ÙZÐhlØØ ¨4¸Uõ"DðeSñn !¡$ÙSÐaeØØ ¨4õ#1ðoSñv  ¡ÙYÐgkØØ ¨4õ"1òwSñ~  ¡ÙYÐgkØØ ¨4õ"1ðSñH "¡4ÙWÐeiØØ ¨4¸Uõ$DðISñP "¡4ÙWÐeiØØ ¨4¸Uõ$DðQSñZ (©°"¬ð[Sñ\ )©$°2¬,ð]Sñ^ (©°"¬ð_Sñb ¡ÙSØÙØ"Ð(<È!õ	!MðcSñl  ¡ÙTØÙØ"Ð(<È!õ	"MðmSñv ¡ÙSØÙØ"Ð(<È!õ	!MðwSñB *©4ÙMáØ"Ð(<È!õ	,MðCSñL +©DÙNáØ"Ð(<È!õ	-MðMSñV *©4ÙSáØ ¨3Ø"Ð(<È!õ,MðWSñb ,©TÙNáØ"Ð(<È!õ	.MðcSñn +©DØØ Øõ-ðoSñv +©DØØô-ðwSñ| (©ØØô*ð}SñB +©DØØ Øõ-òCSñJ (©ØØ Øõ*ðKSñR -©dØØ Øõ/ðSSñZ +©DØØ Øõ-ð[Sñb (©ØØ Øõ*ðcSñj +©DØÙ Øõ-ðkSñr (©ØÙ Øõ*ðsSñz ,©TØØ Øõ.ð{SñB )©$ØØ Øõ+ðCSñJ ,©TØØ Øõ.ðKSñR )©$ØØ Øõ+ðSSñZ ,©TØÙ Øõ.ð[Sñb -©dØØô/ðcSñh *©4ØØô,ðiSñn -©dØÙ Øõ/ðoSñv *©4ØÙ Øõ,ðwSñ~ *©4ØØ Øõ,ðSñF -©dØØ Øõ/òGSñN /±ØØ Øõ1ðOSñV -©dØØ Øõ/ðWSñ^ -©dØÙ Øõ/ð_Sñf /±ØØ Øõ1ðgSñn /±ØØ Øõ1ðoSñx /±ØØ Øõ1ðySñ@ /±ØØô1ðASñF ,©TØØô.ðGSñL /±ØØ Øõ1ðMSñT ,©TØØ Øõ.ðUSñ\ 1±$ØØ Øõ3ð]Sñd /±ØØ Øõ1ðeSñl ,©TØØ Øõ.ðmSñt /±ØÙ Øõ1ðuSñ| ,©TØÙ Øõ.ð}SñD 0±ØØ Øõ2ðESñL -©dØØ Øõ/òMSñT 0±ØØ Øõ2ðUSñ\ -©dØØ Øõ/ð]Sñd 0±ØÙ Øõ2ðeSñl 1±$ØØô3ðmSñr .©tØØô0ðsSñx 1±$ØØô3ðySñ~ 0±ØØô2ðSñD 4±TØØô6ðESñJ 5±dØØô7ðKSñZ 1±$ØÙ Øõ3ð[Sñb .©tØÙ ¨3Øô0ðcSñj .©tØØ ¨3Øô0ðkSñr 1±$ØØ ¨3Øô3ðsSñz 0±ØØ ¨3Øô2ð{SñB 9¹$ØØ ¨3Øô;ðCSñJ 4±TØØ ¨3Øô6ðKSñR 4±TØØ ¨3Øô6òSSñZ 5±dØØ ¨3Øô7ð[Sñn 7¹ØØ ¨3Øô9ðoSñv 8¹ØØ ¨3Øô:ðwSñ~ 0±ØÙ ¨3Øô2ðSñF 9¹$ØÙ ¨3Øô;ðGSñN 4±TØÙ ¨3Øô6ðOSñV 4±TØÙ ¨3Øô6ðWSñ^ 5±dØÙ ¨3Øô7ð_Sñr 1±$ØØ Øõ3ðsSñz 3±DØØ Øõ5ð{SñB 1±$ØØ Øõ3ðCSñJ 1±$ØÙ Øõ3ðKSñR 3±DØØ Øõ5ðSSñZ 3±DØØ Øõ5ð[Sñd 2±4ØÙ ¨3¸(ô4ðeSñl 6±tØÙ ¨3¸(ô8ðmSñv 3±DØÙØ ?Áõ5EòwSñ~ 5±dØÙØ ?Áõ7EðSñF 3±DØÙØ ?Áõ5EðGSñN 6±tØÙØ ?Áõ8EðOSñX ,©TØØ ¨4ô.1ðYSñ^ 6±tØØ ¨4ô81ð_Sñd -©dØØ ¨4ô/1ðeSñj -©dØØ ¨4ô/1ðkSñp 7¹ØØ ¨4ô91ðqSñv .©tØØ ¨4ô01ðwSñ| 8¹ØØ ¨4ô:1ð}SñB  0±ØØØ ¨4õ21ðC SñJ  /±ØØ ¨4ô11ðK SñP  9¹$ØØ ¨4ô;1ðQ SñV  /±ØØ ¨4ô11ðW Sñ\  8¹ØØ ¨4ô:1ð] Sñb  /±ØØ ¨4ô11ðc Sñh  0±ØØØ ¨4õ21òi Sñp  ?ÁØØ ¨4ôA1ðq Sñv  9¹$ØØ ¨4ô;1ðw Sñ|  7¹ØØØ ¨4õ91ð} SñD! 1±$ØØØ ¨4õ31ðE!SñL! ?ÁØØ ¨3ôA0ðM!SñR! 0±ØØ ¨4ô21ðS!SñX! ?ÁØØ ¨4ôA1ðY!Sñ^! 9¹$ØØ ¨4ô;1ð_!Sñd! 0±ØØ ¨4ô21ðe!Sñj! 7¹ØØØ ¨4õ91ðk!Sñr! 1±$ØØØ ¨4õ31ðs!Sñz! ?ÁØØ ¨3ôA0ð{!Sñ@" .©tØ õ0"ðA"SñF" =¹dØØ ¨4ô?1ðG"SñL" 5±dØØØ ¨4õ71ðM"SñT" =¹dØØ ¨3ô?0ðU"SñZ" 0±Ø õ2"ò["Sñ^" >¹tØØ ¨3ô@0ð_"Sñd" 6±tØØØ ¨3õ80ðe"Sñl" >¹tØØ ¨3ô@0ðm"Sñr" >¹tØØ ¨3¸(ô@Dðs"Sñz" +©DØÙØ"Ð(<Ø ¨3¸Aõ	-ð{"SñF# +©DØØ"Ð(<Ø ¨3¸Aõ-ðG#SñR# '©ØØ ?¹LØ !õ)%ðS#SñZ# ,©TØØ ?¹LØ !õ.%ð[#Sñb# &¡tØØ ?¹LØ !õ(%ðc#Sñj# $¡TØØ ?¹LØ !õ&%ðk#Sñr# $¡TØØ ?¹LØ !õ&%ðs#Sñz# '©ØØ ?¹LØ ¨3¸Aõ)?ð{#SñB$ ,©TØØ ?¹LØ ¨3¸Aõ.?ðC$SñJ$ &¡tØØ ?¹LØ ¨3¸Aõ(?ðK$SñR$ $¡TØØ ?¹LØ ¨3¸Aõ&?ðS$SñZ$ $¡TØØ ?¹LØ ¨3¸Aõ&?ð[$Sñb$ '©ØØ ?¹LØ ¨3¸Aõ)?òc$Sñj$ &¡tØØ ?¹LØ ¨3¸Aõ(?ðk$Sñr$ $¡TØØ ?¹LØ ¨3¸Aõ&?ðs$Sñz$ $¡TØØ ?¹LØ ¨3¸Aõ&?ð{$SñD% ™$ØÙ ¨4ô1ðE%SñJ% ™4ØÙ ¨4ô1ðK%SñP% ™4ØÙ ¨4ô1ðQ%SñV% ™4Ù ¨4ô1ðW%Sñ^% +©DØØ"Ð(<Àsô-Lð_%Sñd% 4±TØØ"Ð(<Àsô6Lðe%Sñj% ,©TØØ"Ð(<Àsô.Lðk%Sñp% 5±dØØ"Ð(<Àsô7Lðq%Sñv% (©ØÐ*Ð0DÈsõ*Tðw%Sñz% (©Ø˜=Ð/DÐJ^Ðilõ*nð{%Sñ~%  ¡ØØ"Ð(<ÀsØõ"ð%SñH& )©$ØØ"Ð(<ÀsØõ+ðI&SñR& !¡$ØØ"Ð(<ÀsØõ#ðS&Sñ\& *©4ØØ"Ð(<ÀsØõ,ñ]&S€ðj& #/×"4Ñ"4Ó"6×n™$˜!˜Q¸!¿%¹%ÁÈÕ:LÑQ\Ð`aÑbiÑ`jÐklÑ`mÒQm’AÓnÐ Ø	ò >€Aàˆ�‰�l 1‘oÓ&€AØ‰�~˜Ò Ø  1™ˆ‰+‰Ø<=€L�—‘™8Ñ%7Ó8Ò9ð>ñ % \Ó2€ð —j‘j—n‘nÑ%9¹7ÓC×IÑIÓKÉvÑUÐ ð !Ø%)ó#Øñ#àñ#ð ˜T‘Nð#ð
 Ð™kÐ)Ñ*ô#ðL ó Tð Ð@Qó ó ðð ó Tð Ð@Qó ó ðð ó dð ÐARó ó ðð ó dð ÐARó ó ðð ó dð ÐARó ó ðð ó dð ÐARó ó ðð ó Tð Ð@Qó ó ðð ó Tð Ð@Qó ó ðð ó Tð Ð@Qó ó ðð ó Tð Ð@Qó ó ðð ó Tð Ð@Qó ó ðð ó Dð Ð?Pó ó ðð ó dð ÐARó ó ðð ó dð ÐARó ó ðð ó dð ÐARó ó ðð ó dð ÐARó ó ðð ó dð ÐARó ó ðð ó Tð Ð@Qó ó ðð ó dð ÐARó ó ðð ó¨ð ÐDUó ó ðð ó¨$ð ÐEVó ó ðð ó¨4ð ÐFWó ó ðð ó¨4ð ÐFWó ó ðð ó¨4ð ÐFWó ó ðð ó¨Dð ÐGXó ó ðð ó¨ð ÐDUó ó ðð ó¨ð ÐDUó ó ðð ó¨ð ÐDUó ó ðð ó¨$ð ÐEVó ó ðð ó¨Dð ÐGXó ó ðð ó¨Dð ÐGXó ó ðð ó¨Dð ÐGXó ó ðð ó¨Tð ÐHYó ó ðð ó¨$ð ÐEVó ó ðð ó¨$ð ÐEVó ó ðð ó¨$ð ÐEVó ó ðð ó¨$ð ÐEVó ó ðð ó¨$ð ÐEVó ó ðð ó¨$ð ÐEVó ó ðð ó¨tð ÐJ[ó ó ðð ó¨4ð ÐFWó ó ðð ó¨4ð ÐFWó ó ðð ó¨$ð ÐEVó ó ðð ó¨$ð ÐEVó ó ðð ó¨$ð ÐEVó ó ðð ó
¨4ð 
ÐFWó 
ó ð
ð ó
¨dð 
ÐIZó 
ó ð
ð ó
¨dð 
ÐIZó 
ó ð
ð ó	°Dð 	ÐO`ó 	ó ð	ð ó	°Dð 	ÐO`ó 	ó ð	ð ó	°Tð 	ÐPaó 	ó ð	ð ó	°Tð 	ÐPaó 	ó ð	ð ó	°Dð 	ÐO`ó 	ó ð	ð ó	°Dð 	ÐO`ó 	ó ð	ð ó	¸ð 	ÐSdó 	ó ð	ð ó¨$ð ÐEVó ó ðð ó¨$ð ÐEVó ó ðð ó¨ð ÐDUó ó ðð ó¨4ð ÐFWó ó ðð ó	¨4ð 	ÐFWó 	ó ð	ð ó¨$ð ÐEVó ó ðð ó dð ÐARó ó ðð ó dð ÐARó ó ðð ó˜tð Ð:Kó ó ðð ó˜dð Ð9Jó ó ðð ó˜tð Ð:Kó ó ðð ó	¨ð 	ÐCTó 	ó ð	ð ó	¨ð 	ÐDUó 	ó ð	ð ó	¨ð 	ÐCTó 	ó ð	ð ó¨ð ÐDUó ó ðð ó¨ð ÐCTó ó ðð ó¨ð ÐDUó ó ðð ó¨ð ÐDUó ó ðð  ó	¨dð 	ÐIZó 	ó ð	ð ó	¨Tð 	ÐHYó 	ó ð	ð ó	¨dð 	ÐIZó 	ó ð	ð ó¨dð ÐIZó ó ðð  ó	¨Tð 	ÐHYó 	ó ð	ð ó	¨dð 	ÐIZó 	ó ð	ð ó	¨tð 	ÐJ[ó 	ó ð	ð ó¨dð ÐIZó ó ðð ó¨Dð ÐGXó ó ðð ó¨Dð ÐGXó ó ðð ó¨Dð ÐGXó ó ðð ó¨Dð ÐGXó ó ðð ó¨Dð ÐGXó ó ðð ó¨Tð ÐHYó ó ðð ó¨Tð ÐHYó ó ðð ó¨Tð ÐHYó ó ðð ó¨dð ÐIZó ó ðð ó¨dð ÐIZó ó ðð ó¨dð ÐIZó ó ðð ó¨dð ÐIZó ó ðð ó¨dð ÐIZó ó ðð ó¨dð ÐIZó ó ðð ó°ð ÐK\ó ó ðð ó°ð ÐK\ó ó ðð ó°ð ÐK\ó ó ðð ó°ð ÐK\ó ó ðð ó°ð ÐK\ó ó ðð ó°ð ÐK\ó ó ðð ó°ð ÐK\ó ó ðð ó°ð ÐL]ó ó ðð ó°ð ÐL]ó ó ðð ó°ð ÐL]ó ó ðð ó°$ð ÐM^ó ó ðð ó°$ð ÐM^ó ó ðð ó°$ð ÐM^ó ó ðð ó°$ð ÐM^ó ó ðð ó°$ð ÐM^ó ó ðð ó°$ð ÐM^ó ó ðð ó°$ð ÐM^ó ó ðð ó°$ð ÐM^ó ó ðð ó°Dð ÐO`ó ó ðð ó°Dð ÐO`ó ó ðð ó¨Tð ÐHYó ó ðð ó¨dð ÐIZó ó ðð ó¨dð ÐIZó ó ðð ó¨tð ÐJ[ó ó ðð ó°ð ÐK\ó ó ðð ó°ð ÐL]ó ó ðð ó°ð ÐK\ó ó ðð ó°ð ÐK\ó ó ðð ó°ð ÐK\ó ó ðð ó°ð ÐL]ó ó ðð ó°ð ÐL]ó ó ðð ó°ð ÐL]ó ó ðð ó°ð ÐL]ó ó ðð ó°ð ÐL]ó ó ðð ó¨dð ÐIZó ó ðð ó°ð ÐK\ó ó ðð ó°ð ÐK\ó ó ðð ó°ð ÐK\ó ó ðð ó°ð ÐL]ó ó ðð ó°ð ÐL]ó ó ðð ó°ð ÐL]ó ó ñð ó¨4ð ÐFWó ó ñð ó
¨ð 
ÐCTó 
ó ñ
ð ó tð ÐBSó ó ñð ó
 Tð 
Ð@Qó 
ó ñ
ð ó
 Tð 
Ð@Qó 
ó ñ
ð ó
¨ð 
ÐCTó 
ó ñ
ð ó
 tð 
ÐBSó 
ó ñ
ð ó
 Tð 
Ð@Qó 
ó ñ
ð ó
 Tð 
Ð@Qó 
ó ñ
ð ó
¨ð 
ÐCTó 
ó ñ
ð ó
 tð 
ÐBSó 
ó ñ
ð ó
 Tð 
Ð@Qó 
ó ñ
ð ó
 Tð 
Ð@Qó 
ó ñ
ð ó˜ð Ð4Eó ó ñð ó˜$ð Ð5Fó ó ñð ó˜$ð Ð5Fó ó ñð ó	˜$ð 	Ð5Fó 	ó ñ	ð ó
 tð 
ÐBSó 
ó ñ
ð ó
¨ð 
ÐCTó 
ó ñ
ð ó
¨ð 
ÐCTó 
ó ñ
ð ó
¨ð 
ÐCTó 
ó ñ
ñ ˜Hð 'Ù Ð"Eð'á!Ð#Gð'ñ "Ð#Gð'ñ !Ð"Eð	'ñ
 !Ð"Eð'ñ  Ð!Cð'ñ "Ð#Eð'ñ "Ð#Gð'ñ !Ð"Cð'ñ Ñ :ð'ñ Ñ :ð'ñ !Ð">ð'ñ  Ð!<ð'ñ  Ð!<ð'ñ Ð :ð'ñ  &Ð'Hð!'ñ" (Ñ)Lð#'ñ$ +OÙ)LÙ*Nó)'õ øðEX ò *ß)Ð)ð*üóT^ os.   ¶BD ÁUBDÁU;BDÁVBDÂDBDÂDBD