
    ^j$              	           d Z ddlZddlmc mZ ddlmZ ddlZdej                  fdZ
dej                  fdZddeded	ed
efdZ G d de      Zy)a  
AdamP Optimizer Implementation copied from https://github.com/clovaai/AdamP/blob/master/adamp/adamp.py

Paper: `Slowing Down the Weight Norm Increase in Momentum-based Optimizers` - https://arxiv.org/abs/2006.08217
Code: https://github.com/clovaai/AdamP


References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Spherical Cautious Optimizers: https://openreview.net/forum?id=OyT2CJ4fh7 
Copyright (c) 2020-present NAVER Corp.
MIT license
    N)	Optimizerreturnc                 D    | j                  | j                  d      d      S )Nr   )reshapesizexs    [/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/optim/adamp.py_channel_viewr      s    99QVVAY##    c                 &    | j                  dd      S )N   r   )r   r	   s    r   _layer_viewr      s    99Qr   deltawd_ratioepscautionc                 t   d}ddt        | j                        dz
  z  z   }t        t        fD ]  }	 |	|       }
 |	|      }t	        j
                  ||
d|      j                         }|j                         |t        j                  |
j                  d            z  k  st| |
j                  dd      j                  |      j                  |      z  }|| |	||z        j                  d      j                  |      z  z  }|r| |	||z        j                  d      j                  |      z  }||z
  }||z  d	kD  j                  |j                         }|j#                  |j%                         j'                  d
             |j)                  |       || |	||z        j                  d      j                  |      z  z  }|}||fc S  |ra||z  d	kD  j                  |j                         }|j#                  |j%                         j'                  d
             |j)                  |       ||fS )N      ?)r   )r   r   )dimr      )pr   )r   r   MbP?min)lenshaper   r   Fcosine_similarityabs_maxmathsqrtr   normadd_r   sumtodtypediv_meanclamp_mul_)r   gradperturbr   r   r   r   wdexpand_size	view_func
param_view	grad_view
cosine_simp_ngrad_radial	grad_perpmasks                    r   
projectionr:      s	   	B$#agg,"233K#[1 	q\
dO	((JA3OTTV
 >>edii
0B&CCCjooqo166s;CCKPPCsYsW}599a9@HHUUUG!IcDj$9$=$=!$=$D$L$L[$YY ;.	)+a/33DJJ?		$))+,,,67T"33=!9!=!=!!=!D!L!L[!YYYBB;+. $"&&tzz2		$))+$$$./TB;r   c                   `     e Zd Z	 	 	 	 	 	 	 	 d fd	Z ej
                         dd       Z xZS )AdamPc
           
      T    t        ||||||||	      }
t        t        |   ||
       y )N)lrbetasr   weight_decayr   r   nesterovr   )dictsuperr<   __init__)selfparamsr>   r?   r   r@   r   r   rA   r   defaults	__class__s              r   rD   zAdamP.__init__A   s:     %	
 	eT#FH5r   c                    d }|$t        j                         5   |       }d d d        | j                  D ]A  }|d   D ]5  }|j                  |j                  }|d   \  }}|d   }|j	                  dd      }	| j
                  |   }
t        |
      dk(  r5d|
d<   t        j                  |      |
d<   t        j                  |      |
d	<   |
d   |
d	   }}|
dxx   d
z  cc<   d
||
d   z  z
  }d
||
d   z  z
  }|j                  |      j                  |d
|z
         |j                  |      j                  ||d
|z
         |j                         t        j                  |      z  j                  |d         }|d   |z  }|r||z  d
|z
  |z  z   |z  }n||z  }d}t        |j                        d
kD  rt        ||||d   |d   |d   |	      \  }}nc|	ra||z  dkD  j                  |j                         }|j#                  |j%                         j'                  d             |j                  |       |d   dkD  r |j                  d|d   |d   z  |z  z
         |j                  ||        8 D |S # 1 sw Y   ]xY w)NrF   r?   rA   r   Fr   stepexp_avg
exp_avg_sqr   )alpha)valuer   r>   r   r   r   r   r   r@   )torchenable_gradparam_groupsr.   getstater   
zeros_liker-   r&   addcmul_r$   r#   r   r:   r(   r)   r*   r+   r,   )rE   closurelossgroupr   r.   beta1beta2rA   r   rS   rK   rL   bias_correction1bias_correction2denom	step_sizer/   r   r9   s                       r   rJ   z
AdamP.stepY   s   ""$ !y! && 5	2E8_ 4266>vv$W~u ,))Iu5

1 u:?$%E&M','7'7':E)$*/*:*:1*=E,' ',I&6l8Kf"#$uf'=#= #$uf'=#= U#((QY(?&//d!e)/L#*TYY7G-HHNNuUZ|\!$K*::	$w!e)t1CCuLG%oG qww<!#(24%.%
:KUSX\[b)%GX #dNQ.224::>DIIdiik00T0:;LL& (1,FF2deN.C Ch NNO wyj1i425	2n u! !s   I<<J)r   )g?g+?g:0yE>r   皙?r_   FF)N)__name__
__module____qualname__rD   rO   no_gradrJ   __classcell__)rH   s   @r   r<   r<   @   s>     60 U]]_= =r   r<   )F)__doc__rO   torch.nn.functionalnn
functionalr   torch.optim.optimizerr   r#   Tensorr   r   floatboolr:   r<    r   r   <module>rn      sl       + $ $ell      U  UY  FWI Wr   