Ë
    éÓ^j.3  ã            '       ó4  — d Z ddlmZmZmZmZ ddlZddlmZ ddlm	Z	 ddl
mZmZmZ ddlmZ d	eed
f   dededeeeef      fd„Z G d„ de	«      Zdee   dee   deee      deee      deee      deee      dee   dededededededee   deeej.                  f   dee   d ed!ed"ee   f&d#„Zdee   dee   deee      deee      deee      deee      dee   dededededededee   deeej.                  f   dee   d ed!ed"ee   f&d$„Zy)%aÌ   Adafactor (Big Vision variant) for PyTorch

Adapted from the implementation in big vision: https://github.com/google-research/big_vision

Described in 'Scaling Vision Transformers': https://arxiv.org/abs/2106.04560

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Why Gradients Rapidly Increase Near the End of Training: https://arxiv.org/abs/2506.02285

Adaptation and PyTorch modifications by Ross Wightman
é    )ÚListÚOptionalÚTupleÚUnionN)ÚTensor)Ú	Optimizeré   )Ú
_get_valueÚ_init_scalarÚ_validate_scalar)ÚParamsTÚshape.ÚfactoredÚmin_dim_size_to_factorÚreturnc                 ó¼   — |rt        | «      dk  ryt        d„ t        | «      D «       «      }| |d   d      |k  ryt        |d   d   «      t        |d   d   «      fS )aØ  Whether to use a factored second moment estimator.

    This function returns a tuple with the two largest axes to reduce over.
    If no two dimensions have size >= min_dim_size_to_factor, return None.

    Args:
      shape: an input shape
      factored: whether to use factored second-moment estimator for > 2d vars.
      min_dim_size_to_factor: only factor accumulator if two array dimensions have at least this size.

    Returns:
      None or a tuple of ints
    é   Nc              3   ó*   K  — | ]  \  }}||f–— Œ y ­w©N© )Ú.0ÚiÚxs      úb/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/optim/adafactor_bv.pyú	<genexpr>z!_factored_dims.<locals>.<genexpr>+   s   è ø€ Ò>¡T Q¨˜1˜aœ&Ñ>ùs   ‚éþÿÿÿr	   éÿÿÿÿ)ÚlenÚsortedÚ	enumerateÚint)r   r   r   Úsorted_dimss       r   Ú_factored_dimsr#      sj   € ñ$ ”s˜5“z A’~ØÜÑ>¬Y°uÓ-=Ô>Ó?€KØˆ[˜‰_˜QÑÑ Ð#9Ò9ØÜˆ{˜2‰˜qÑ!Ó"¤C¨°B©¸Ñ(:Ó$;Ð;Ð;ó    c            !       ó  ‡ — e Zd ZdZddddddej
                  dd	dd
d
d
fd
dœdedededededede	e   de
eej                  f   de	e   dede	e   dededede	e   fˆ fd„Zˆ fd„Z ej                   «       dd„«       Zˆ xZS )ÚAdafactorBigVisionzÅ
    PyTorch implementation of BigVision's Adafactor variant with both single and multi tensor implementations.

    Adapted from https://github.com/google-research/big_vision by Ross Wightman
    ç      ð?é   gš™™™™™é?r   g+‡ÙÎ÷ï?gÍÌÌÌÌÌì?Ng        F)ÚforeachÚparamsÚlrr   Ú
decay_rateÚdecay_offsetÚ	beta2_capÚmomentumÚmomentum_dtypeÚepsÚweight_decayÚclipping_thresholdÚunscaled_wdÚcautionÚcorrected_weight_decayr)   c                ó>  •— t        |t        «      rK|dk(  rt        j                  }n5|dk(  rt        j                  }n|dk(  s
J |› d�«       ‚t        j
                  }t        d|«       t        d|
«       t        ||||||||	|
|||||¬«      }t        ‰| �%  ||«       y )NÚfloat16Úbfloat16Úfloat32z dtype not supportedzlearning rater2   )r+   r   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r)   )
Ú
isinstanceÚstrÚtorchr8   r9   r:   r   ÚdictÚsuperÚ__init__)Úselfr*   r+   r   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r)   ÚdefaultsÚ	__class__s                    €r   r@   zAdafactorBigVision.__init__8   s¯   ø€ ô& �n¤cÔ*Ø Ò*Ü!&§¡‘Ø :Ò-Ü!&§¡‘à%¨Ò2Ð[°~Ð6FÐFZÐ4[Ó[Ð2Ü!&§¡�ä˜¨"Ô-Ü˜¨Ô6äØØ#9Ø!Ø%ØØØ)ØØ%Ø1Ø#ØØ#9Øô
ˆô  	‰Ñ˜ Õ*r$   c                 óì  •— t         ‰| �  |«       | j                  D ]Õ  }|j                  dd«       |j                  dd«       |j                  dd «       |d   D ]•  }| j                  j                  |i «      }t        |«      dk7  r&d|v r"t        |d   dt        j                  ¬	«      |d<   d
|v sŒXt        j                  |d
   «      sŒq|d
   j                  | j                  d   ¬«      |d
<   Œ— Œ× y )Nr5   Fr6   r)   r*   r   ÚstepÚcpu©ÚdeviceÚdtypeÚexp_avgr0   ©rI   )r?   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetr   r   r=   Úfloat64Ú	is_tensorÚtorB   )rA   rO   ÚgroupÚpÚp_staterC   s        €r   rL   zAdafactorBigVision.__setstate__i   sò   ø€ Ü‰Ñ˜UÔ#Ø×&Ñ&ò 	fˆEØ×Ñ˜Y¨Ô.Ø×ÑÐ5°uÔ=Ø×Ñ˜Y¨Ô-Ø˜8‘_ò 	f�ØŸ*™*Ÿ.™.¨¨BÓ/�Ü�w“< 1Ò$¨°7Ñ):Ü&2°7¸6±?È5ÔX]×XeÑXeÔ&f�G˜F‘Oà Ò'¬E¯O©O¸GÀIÑ<NÕ,Oð *1°Ñ);×)>Ñ)>ÀTÇ]Á]ÐScÑEdÐ)>Ó)e�G˜IÒ&ñ	fñ		fr$   c                 ó�  — d }|�$t        j                  «       5   |«       }d d d «       | j                  D �]   }g }g }g }g }g }g }	g }
|d   D �]`  }|j                  €Œ|j                  j                  rt        d«      ‚|j                  |«       |j                  |j                  «       | j                  |   }t        |«      dk(  �rMt        dt         j                  ¬«      |d<   |j                  j                  }t        |d| j                  d   ¬	«      }|�Š|\  }}t        |j                  j                  «      }d
||<   t        |j                  j                  «      }d
||<   |j                  j                  |«      |d<   |j                  j                  |«      |d<   n2t        j                   |j                  t         j"                  ¬«      |d<   | j                  d   �1t        j                   |j                  | j                  d   ¬«      |d<   |	j                  |d   «       |j                  |j%                  dd «      «       |j                  |j%                  dd «      «       |j                  |j%                  dd «      «       |
j                  |j%                  dd «      «       �Œc |d   rt&        }nt(        } |d%i d|“d|“d|“d|“d|“d|
“d|	“d|d   “d|d   “d|d   “d|d   “d|d   “d|d   “d|d   “d|d   “d |d    “d!|d!   “d"|d"   “d#|d$   r| j                  d   nd “Ž �Œ |S # 1 sw Y   �ŒxY w)&Nr*   zSparse gradients not supportedr   rF   rG   rE   Tr   )r   r   r	   Úexp_avg_sq_rÚexp_avg_sq_c)Úmemory_formatÚ
exp_avg_sqr/   r0   rK   rJ   r)   ÚgradsÚexp_avg_sq_rsÚexp_avg_sq_csÚexp_avg_sqsÚexp_avgsÚstate_stepsÚbeta2_decayr,   r.   r1   r+   r2   r3   r4   r5   Úmax_lrr6   r   )r=   Úenable_gradrM   ÚgradÚ	is_sparseÚRuntimeErrorÚappendrO   r   r   rQ   r   r#   rB   ÚlistÚ	new_zerosÚ
zeros_likeÚpreserve_formatrP   Ú_multi_tensor_adafactorÚ_single_tensor_adafactor)rA   ÚclosureÚlossrT   Úparams_with_gradr\   r]   r^   r_   ra   r`   rU   rO   r   Úfactored_dimsÚdcÚdrÚ	row_shapeÚ	col_shapeÚfuncs                       r   rE   zAdafactorBigVision.stepz   s�  € àˆØÐÜ×"Ñ"Ó$ñ !Ù“y�÷!ð ×&Ñ&ó L	ˆEØ!ÐØˆEØˆMØˆMØˆKØˆKØˆHà˜8‘_ó (<�Ø—6‘6�>Øà—6‘6×#Ò#Ü&Ð'GÓHÐHà ×'Ñ'¨Ô*Ø—‘˜QŸV™VÔ$àŸ
™
 1™�ä�u“: “?Ü$0¸ÄUÇ]Á]Ô$S�E˜&‘MàŸF™FŸL™L�EÜ$2ØØ!%Ø/3¯}©}Ð=UÑ/Vô%�Mð %Ð0Ø!.™˜˜BÜ$(¨¯©¯©Ó$6˜	Ø()˜	 "™Ü$(¨¯©¯©Ó$6˜	Ø()˜	 "™Ø01·±×0@Ñ0@ÀÓ0K˜˜nÑ-Ø01·±×0@Ñ0@ÀÓ0K˜˜nÒ-ä.3×.>Ñ.>¸q¿v¹vÔUZ×UjÑUjÔ.k˜˜lÑ+à—}‘} ZÑ0Ð<Ü+0×+;Ñ+;¸A¿F¹FÈ$Ï-É-ÐXhÑJiÔ+j˜˜iÑ(à×"Ñ" 5¨¡=Ô1Ø×$Ñ$ U§Y¡Y¨~¸tÓ%DÔEØ×$Ñ$ U§Y¡Y¨~¸tÓ%DÔEØ×"Ñ" 5§9¡9¨\¸4Ó#@ÔAØ—‘ §	¡	¨)°TÓ :Ö;ðQ(<ðT �YÒÜ.‘ä/�áò Ù'ðáðñ ,ðñ ,ð	ñ
 (ðñ "ðñ (ðð " ,Ò/ðð   Ò,ðð (-Ð-EÒ'Fðð ˜%’Lðð ˜’;ðð # >Ò2ðð ˜zÒ*ðð  %Ð%5Ò6ðð  $)Ð)=Ò#>ð!ð" " -Ò0ð#ð$ ˜iÒ(ð%ð& /4Ð4LÒ.M�t—}‘} TÒ*ÐSWøó'ðqL	ð\ ˆ÷c!ñ !ús   ™L;Ì;Mr   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r=   r9   r   Úfloatr!   r   r   r<   rI   Úboolr@   rL   Úno_gradrE   Ú__classcell__)rC   s   @r   r&   r&   1   s  ø„ ñð Ø*,Ø #Ø !Ø$Ø(+Ø6;·n±nØ#'Ø"%Ø26Ø %Ø!Ø+0ð/+ð" ',ò#/+àð/+ð ð/+ð %(ð	/+ð
 ð/+ð ð/+ð ð/+ð ˜u‘oð/+ð " # u§{¡{Ð"2Ñ3ð/+ð ˜%‘ð/+ð  ð/+ð !)¨¡ð/+ð ð/+ð ð/+ð %)ð/+ð" ˜d‘^õ#/+ôbfð" €U‡]�]ƒ_òTó ôTr$   r&   r*   r\   r]   r^   r_   r`   ra   rb   r.   r1   r+   r2   r/   r0   r3   r4   r5   rc   c          	      óN  — t        | «      D �]–  \  }}||   }||   }||   }||   }||   }||   }|
€!|j                  t        j                  k(  rdnd}
|j	                  d«       t        |«      }t        j                  |«      r[t        j                  ||j                  |j                  ¬«      }t        j                  |dt        j                  || «      z
  «      }nt        |d|| z  z
  «      }d|z
  }t        j                  |«      |
z   }|€¬t        |j                  d|	¬«      \  } }!|j                  |j!                  |!d¬«      |«       |j                  |j!                  | d¬«      |«       | |!kD  r| dz
  n| }"|j!                  |"d¬«      }#||#z  j#                  «       }$|j#                  «       }%||$z  |%z  }&n+|€|�J ‚|j                  ||«       ||j#                  «       z  }&|�I|&j%                  d	«      |&j'                  «       d
z  |z  z  j)                  d¬«      }'|&j+                  |'«       |�Ù|�×||j                  k7  r@|j                  |&j-                  |«      d|z
  «       |j-                  |j                  «      }&n%|j                  |&d|z
  «       |j/                  «       }&|ra|&|z  dkD  j-                  |j                  «      }(|(j+                  |(j!                  «       j)                  d¬«      «       |&j1                  |(«       |&j1                  |«       |dk7  rk|r2|€|j1                  d|z
  «       nR|j1                  d||z  |z  z
  «       n7|€|j1                  d||z  z
  «       n|j1                  d|d	z  |z  |z  z
  «       |j	                  |&d¬«       �Œ™ y )NgH¯¼šò×z>g ÂëþKH´9r	   )rI   rH   r'   T)r   )ÚdimÚkeepdimr   g      à?)Úmaxr   gü©ñÒMbP?)Úming      ð¿)Úalpha)r    rI   r=   r8   Úadd_r
   rR   ÚtensorrH   ÚminimumÚpowr„   Úsquarer#   r   Úlerp_ÚmeanÚrsqrtÚnormÚnumelÚclamp_Údiv_rS   ÚcloneÚmul_))r*   r\   r]   r^   r_   r`   ra   rb   r.   r   r1   r+   r2   r/   r0   r3   r4   r5   rc   r   Úparamre   rX   rY   r[   rJ   Ústep_trE   Úbeta2_cap_tÚbeta2_tÚone_minus_beta2_tÚgrad_sqrrs   rt   Ú	reduce_dcÚrow_col_meanÚ
row_factorÚ
col_factorÚupdateÚdenomÚmasks)                                            r   rn   rn   Ò   s}  € ô, ˜fÓ%ó R'‰ˆˆ5Ø�Q‰xˆØ$ QÑ'ˆØ$ QÑ'ˆØ  ‘^ˆ
Ø˜1‘+ˆØ˜Q‘ˆØˆ;àŸ*™*¬¯©Ò5‘$¸5ˆCð 	�‰�AŒÜ˜&Ó!ˆÜ�?‰?˜4Ô ÜŸ,™, y¸¿
¹
È4Ï;É;ÔWˆKÜ—m‘m K°´u·y±yÀÈÀ|Ó7TÑ1TÓU‰Gä˜) S¨4°[°LÑ+AÑ%AÓBˆGØ ™KÐä—<‘< Ó%¨Ñ+ˆàÐä# D§J¡J°ÐMcÔd‰FˆB�Ø×Ñ˜xŸ}™}°¸T˜}ÓBÐDUÔVØ×Ñ˜xŸ}™}°¸T˜}ÓBÐDUÔVà"$ r¢'˜˜Qš¨rˆIØ'×,Ñ,°ÀDÐ,ÓIˆLØ&¨Ñ5×<Ñ<Ó>ˆJØ%×+Ñ+Ó-ˆJà˜JÑ&¨Ñ3‰Fð  Ð'¨LÐ,@Ð@Ð@Ø×Ñ˜XÐ'8Ô9Ø˜J×,Ñ,Ó.Ñ.ˆFð Ð)Ø—[‘[ “^¨¯©«¸#Ñ(=ÐASÑ'SÑT×\Ñ\ÐadÐ\ÓeˆEØ�K‰K˜Ôð Ð GÐ$7Ø §¡Ò+Ø—‘˜fŸi™i¨Ó7¸¸X¹ÔFØ Ÿ™ D§J¡JÓ/‘à—‘˜f a¨(¡lÔ3Ø Ÿ™›�áà ™¨Ñ)×-Ñ-¨d¯j©jÓ9�Ø—	‘	˜$Ÿ)™)›+×,Ñ,°Ð,Ó6Ô7Ø—‘˜DÔ!ð 	�‰�BŒð ˜1ÒÙà�>Ø—J‘J˜r LÑ0Õ1ð —J‘J˜r R¨&¡[°LÑ$@Ñ@ÕAð �>Ø—J‘J˜r B¨Ñ$5Ñ5Õ6ð —J‘J˜r R¨1¡W¨vÑ%5¸Ñ$EÑEÔFð 	�
‰
�6 ˆ
Ö&ñeR'r$   c                ó   — J d«       ‚)Nz2multi-tensor fn (foreach=True) not implemented yetr   )r*   r\   r]   r^   r_   r`   ra   rb   r.   r   r1   r+   r2   r/   r0   r3   r4   r5   rc   s                      r   rm   rm   =  s   € ð. GÐFÓFˆ5r$   )r{   Útypingr   r   r   r   r=   r   Útorch.optimr   Ú_helpersr
   r   r   Ú_typesr   r!   r}   Útupler#   r&   r|   r<   rI   rn   rm   r   r$   r   ú<module>r§      s´  ðñ÷ 0Ó /ã Ý Ý !ç @Ñ @Ý ð<Ø�S˜#�X‰ð<àð<ð !$ð<ð ˆe�C˜�H‰oÑó	<ô4^˜ô ^ðBh'Ø�V‘ðh'à�F‰|ðh'ð ˜H VÑ,Ñ-ðh'ð ˜H VÑ,Ñ-ð	h'ð
 ˜( 6Ñ*Ñ+ðh'ð �x Ñ'Ñ(ðh'ð ˜&‘\ðh'ð ðh'ð ðh'ð !$ðh'ð ðh'ð ðh'ð ðh'ð ˜5‘/ðh'ð  ˜c 5§;¡;Ð.Ñ/ð!h'ð" % U™Oð#h'ð$ ð%h'ð& ð'h'ð( ˜‘ó)h'ðVGØ�V‘ðGà�F‰|ðGð ˜H VÑ,Ñ-ðGð ˜H VÑ,Ñ-ð	Gð
 ˜( 6Ñ*Ñ+ðGð �x Ñ'Ñ(ðGð ˜&‘\ðGð ðGð ðGð !$ðGð ðGð ðGð ðGð ˜5‘/ðGð  ˜c 5§;¡;Ð.Ñ/ð!Gð" % U™Oð#Gð$ ð%Gð& ð'Gð( ˜‘ô)Gr$   