Ë
    éÓ^jEW  ã                   óD  — d Z ddlZddlZddlZddlZddlmZmZmZm	Z	m
Z
mZ ddlZddlZ	 ddlZddlZdej"                  j                  _        dej"                  j                  _        dZ	 dej,                  j.                  _        dZdd	lmZmZ dd
lmZ  ej@                  e!«      Z"	 	 	 	 dde#de#de#de#de#dejH                  fd„Z% G d„ dejL                  jN                  «      Z(dd„Z)	 	 dd„Z*d„ Z+d„ Z,d„ Z-d„ Z.d„ Z/d„ Z0d„ Z1y# e$ r dZY Œ¥w xY w# e$ r dZY Œ”w xY w) a²   PyTorch Implementation of the Kron (PSGD) optimizer

This is a PSGD optimizer using a Kronecker-factored preconditioner.

This impl was adapted from https://github.com/evanatyourservice/kron_torch
by Evan Walters, licensed CC-BY-4.0.

Contributions to above also made by
* Lucas Nestler, added to his https://github.com/ClashLuke/HeavyBall implementation.
* Omead Pooladzandi https://github.com/opooladz

The above work drew from https://github.com/lixilinx/psgd_torch by Xi-Lin Li

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Why Gradients Rapidly Increase Near the End of Training: https://arxiv.org/abs/2506.02285

This `timm` impl
* works with a wider variety of torch versions
* fixes some checkpoint save/restore (resume issues)
* adds decoupled weight-decay option
* has some refactoring, cleanup of args, default/group items
* warning about not having opt_einsum (unusable without)

é    N)ÚAnyÚCallableÚDictÚOptionalÚTupleÚUnionTzauto-hqFi@B é   )Ú_add_scaled_Ú_validate_scalar)ÚParamsTÚnÚmax_probÚmin_probÚdecayÚ
flat_startÚreturnc                 ó¶   — 	 t        j                  | t         j                  ¬«      } |t        j                  | | |z
  z  «      z  }|j	                  ||¬«       |S )aø  Anneal preconditioner update probability during beginning of training.

    PSGD benefits from more preconditioner updates at the beginning of training,
    but once the preconditioner is learned the update probability can drop low.

    This schedule is an exponential anneal with a flat start. Default settings keep
    update probability at 1.0 for 200 steps then exponentially anneal down to
    `min_prob` by 4000 steps. Default settings work very well for most models and
    training regimes.
    ©Údtype)ÚminÚmax)ÚtorchÚtensorÚfloat32ÚexpÚclamp_)r   r   r   r   r   Úprobs         úZ/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/optim/kron.pyÚprecond_update_prob_scheduler   8   sN   € ð$ .Ü�‰�QœeŸm™mÔ,€AØ”e—i‘i  ¨!¨j©.Ñ 9Ó:Ñ:€DØ‡K�K�H (€KÔ+à€Kó    c            )       ól  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d dededededeeeef      de	d	e	d
ee
   dedededeej                     deej                     dededede	de	dedef(ˆ fd„Zˆ fd„Zˆ fd„Zdee
ef   fˆ fd„Zdee
ef   ddfˆ fd„Zˆ fd„Z ej*                  «       d!d„«       Zˆ xZS )"ÚKronaÑ  Implements PSGD Kron from https://github.com/lixilinx/psgd_torch.

    Args:
        params: Iterable of parameters to optimize or dicts defining parameter groups.
        lr: Learning rate.
        momentum: Momentum parameter.
        weight_decay: Weight decay.
        preconditioner_update_probability: Probability of updating the preconditioner.
            If None, defaults to a schedule that anneals from 1.0 to 0.03 by 4000 steps.
        max_size_triangular: Max size for dim's preconditioner to be triangular.
        min_ndim_triangular: Minimum number of dimensions a layer needs to have triangular preconditioners.
        memory_save_mode: 'one_diag', 'smart_one_diag', or 'all_diag', None is default
            to set all preconditioners to be triangular, 'one_diag' sets the largest
            or last dim to be diagonal per layer, and 'all_diag' sets all preconditioners to be diagonal.
        momentum_into_precond_update: whether to send momentum into preconditioner
            update instead of raw gradients.
        mu_dtype: Dtype of the momentum accumulator.
        precond_dtype: Dtype of the preconditioner.
        decoupled_decay: AdamW style decoupled weight decay
        corrected_weight_decay: apply corrected weight decay when using decoupled_decay (lr**2 / max_lr)
        flatten: Flatten dimensions instead of fully relying on expressions for higher rank params
        flatten_start_dim: Start of flatten range, defaults to 2. Seems good tradeoff for ConvNets.
        flatten_end_dim: End of flatten range, defaults to -1.
        stochastic_weight_decay: Enable random modulation of weight decay
        deterministic: Deterministic behaviour across save / load (resume). FIXME slow, needs work
    NÚparamsÚlrÚmomentumÚweight_decayÚ!preconditioner_update_probabilityÚmax_size_triangularÚmin_ndim_triangularÚmemory_save_modeÚmomentum_into_precond_updateÚ
precond_lrÚprecond_init_scaleÚmu_dtypeÚprecond_dtypeÚdecoupled_decayÚcorrected_weight_decayÚflattenÚflatten_start_dimÚflatten_end_dimÚstochastic_weight_decayÚdeterministicc                 óx  •— t         st        j                  d«       t        d|«       d|cxk  rdk  sn t	        d|› �«      ‚d|k  st	        d|› �«      ‚t        di d|“d|“d	|“d
|“d|“d|“d|“d|	“d|
“d|“d|“d|“d|“d|“d|“d|“d|“d|“Ž}t        t        | �#  ||«       i | _	        t        j                  t        j                  «      j                  | _        t        j                   d«      | _        || _        t&        r…t        j(                  t*        dd¬«      | _        t        j(                  t,        dd¬«      | _        t        j(                  t.        dd¬«      | _        t        j(                  t0        dd¬«      | _        y t*        | _        t,        | _        t.        | _        t0        | _        y )NzKIt is highly recommended to have 'opt_einsum' installed for this optimizer.zlearning rateç        ç      ð?zInvalid beta parameter: zInvalid weight_decay value: r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   i9  TF)Ú	fullgraphÚdynamic© )Úhas_opt_einsumÚwarningsÚwarnr   Ú
ValueErrorÚdictÚsuperr"   Ú__init__Ú_param_exprsr   ÚfinfoÚbfloat16ÚtinyÚ_tinyÚrandomÚRandomÚrngr6   Ú
has_dynamoÚcompileÚ_calc_A_and_conjBÚ_q_termsÚ_precond_gradÚ
_balance_Q)Úselfr#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   ÚdefaultsÚ	__class__s                         €r   rC   zKron.__init__n   sÝ  ø€ õ. Ü�M‰MÐgÔhä˜¨"Ô-Ø�hÔ$ Ô$ÜÐ7¸°zÐBÓCÐCØ�lÒ"ÜÐ;¸L¸>ÐJÓKÐKäò 
Ùð
áð
ñ &ð
ñ /Pð	
ñ
 !4ð
ñ !4ð
ñ .ð
ñ *Fð
ñ "ð
ñ  2ð
ñ ð
ñ (ð
ñ ,ð
ñ $:ð
ñ ð
ñ  0ð!
ñ" ,ð#
ñ$ %<ð%
ˆô( 	Œd�DÑ" 6¨8Ô4àˆÔÜ—[‘[¤§¡Ó0×5Ñ5ˆŒ
Ü—=‘= Ó&ˆŒØ*ˆÔõ Ü%*§]¡]Ô3DÐPTÐ^cÔ%dˆDÔ"Ü!ŸM™M¬(¸dÈEÔRˆDŒMÜ!&§¡¬}ÈÐV[Ô!\ˆDÔÜ#Ÿm™m¬JÀ$ÐPUÔVˆD�Oä%6ˆDÔ"Ü$ˆDŒMÜ!.ˆDÔÜ(ˆD�Or    c                 ój   •— t         ‰| �  |«       | j                  D ]  }|j                  dd«       Œ y )Nr1   F)rB   Ú__setstate__Úparam_groupsÚ
setdefault)rR   ÚstateÚgrouprT   s      €r   rV   zKron.__setstate__µ   s6   ø€ Ü‰Ñ˜UÔ#Ø×&Ñ&ò 	>ˆEØ×ÑÐ5°uÕ=ñ	>r    c                 óB   •— t         ‰| �  «       }| j                  |d<   |S )NrK   )rB   Ú__getstate__rK   )rR   Ú_dictrT   s     €r   r\   zKron.__getstate__º   s"   ø€ Ü‘Ñ$Ó&ˆØ—x‘xˆˆe‰Øˆr    r   c                 ó^   •— t         ‰| �  «       }| j                  j                  «       |d<   |S ©NÚ	rng_state)rB   Ú
state_dictrK   Úgetstate)rR   Úoptimizer_staterT   s     €r   ra   zKron.state_dict¿   s/   ø€ ä™'Ñ,Ó.ˆð (,§x¡x×'8Ñ'8Ó':ˆ˜Ñ$ØÐr    ra   c                 óÀ   •— i }d|v r|j                  d«      |d<   t        ‰| �	  |«       |j                  |«       d|v r| j                  j                  |d   «       y y r_   )ÚpoprB   Úload_state_dictÚupdaterK   Úsetstate)rR   ra   Ú
rng_statesrT   s      €r   rf   zKron.load_state_dictÇ   sf   ø€ àˆ
Ø˜*Ñ$Ø&0§n¡n°[Ó&AˆJ�{Ñ#ô 	‰Ñ 
Ô+Ø×Ñ˜*Ô%ð ˜*Ñ$Ø�H‰H×Ñ˜j¨Ñ5Õ6ð %r    c                 ó2   •— t         ‰| �  |«       i | _        y ©N)rB   rV   rD   )rR   rY   rT   s     €r   rV   zKron.__setstate__Õ   s   ø€ Ü‰Ñ˜UÔ#ØˆÕr    c                 ó8  — d }|�$t        j                  «       5   |«       }d d d «       d}d}d}d}| j                  D �]�  }|j                  d«      }|j                  dt         j                  «      }	|j                  dd«      }
|j                  dd «      }|d   D �],  }|j
                  €Œ|j
                  }| j                  |   }d}|d	   rt        ||d
   |d   «      }d}t        |«      dk(  rÒd|d<   d|d<   t        j                  ||xs |j                  ¬«      |d<   t        ||d   |d   |d   |d   |	¬«      \  |d<   }|| j                  |<   |d   j                  «       }||d   j                  «       z  dz  }||z  }||z  }t        d„ |d   D «       «      }t        d„ |d   D «       «      dz  }||z  }||z  }nK|| j                  vr.t        ||d   |d   |d   |d   |	d¬«      }|| j                  |<   n| j                  |   }|€t         }t#        |«      r ||d   «      }|dxx   dz  cc<   |d   d|z  k\  }|rd|d<   |dxx   dz  cc<   |d   }d||d   z  z
  }|d   }|j%                  |d   «      j'                  |d|d   z
  ¬«       |�!|j)                  |j+                  |¬«      «       ||z  j+                  |	¬«      }| j,                  j/                  «       dk  xr |}|j1                  «       dkD  r|r| j3                  |d   «       |�ru|\  }}}|d   }| j4                  rLt        j6                  |j8                  ¬«      }|j;                  | j,                  j=                  dd«      «       nd }t        j>                  |j@                  ||	|j8                  ¬«      } |
r|n|}!| jC                  ||!|| «      \  }"}#| jE                  ||"|#«      }$tG        ||$«      D ]¤  \  }%\  }&}'|&|'z
  }(|(|d    z  }(|%j1                  «       d!k  r3|(|%z  }(|(|&|'z   jI                  tK        d"«      «      | jL                  z   z  }(n8t        jN                  |(«      }(|(tQ        |&|'z   «      | jL                  z   z  }(|(|%z  }(|%jS                  |(«       Œ¦ | jU                  |d   ||«      j+                  |j                  ¬«      })|)j%                  t        jV                  d#|)jY                  «       j[                  «       j]                  «       d$z   z  d%¬&«      «       |r|)j_                  |j@                  «      })|d'   }*|*dk7  rx|d(   r d!| j,                  j/                  «       z  |*z  }*|d)   r;|d*   r|d+   d!z  | j`                  d+   z  }+n|d+   }+|j%                  d%|+|*z  z
  «       n|)j'                  ||*¬«       tc        ||)|d+    «       �Œ/ �Œ� |dkD  r:td        jg                  d,|› d-|d.›d/�«       td        jg                  d0|› d-|d.›d/�«       |S # 1 sw Y   �ŒðxY w)1Nr   r.   r/   r+   Tr'   r#   Fr2   r3   r4   ÚstepÚupdate_counterr   Úmomentum_bufferr-   r(   r)   r*   ÚQi   c              3   ó<   K  — | ]  }|j                  «       –— Œ y ­wrk   )Únumel©Ú.0Úqs     r   ú	<genexpr>zKron.step.<locals>.<genexpr>  s   è ø€ Ò&E°Q q§w¡w§yÑ&Eùs   ‚c              3   ó^   K  — | ]%  }|j                  «       |j                  «       z  –— Œ' y ­wrk   )rr   Úelement_sizers   s     r   rv   zKron.step.<locals>.<genexpr>  s"   è ø€ Ò$VÀa Q§W¡W£Y°·±Ó1AÕ%AÑ$Vùs   ‚+-)r   Úinit_qr	   r%   )Úalphag{®Gáz„?)Údevicel        )Ú	generatorr   r{   r,   é   Úinfgš™™™™™ñ?g:Œ0âŽyE>r9   )r   r&   r5   r0   r1   r$   zPSGD Momentum buffer size: z elements, z.2fz MBzPSGD Preconditioners size: )4r   Úenable_gradrW   Úgetr   ÚgradrY   Úsafe_flattenÚlenÚ
zeros_liker   Ú_init_Q_exprsrD   rr   rx   Úsumr   ÚcallableÚmul_Úadd_Úcopy_ÚtorK   rI   ÚdimrQ   r6   Ú	Generatorr{   Úmanual_seedÚrandintÚrandnÚshaperN   rO   ÚzipÚnormÚfloatrH   ÚtriuÚ_norm_lower_boundÚsub_rP   ÚclampÚsquareÚmeanÚsqrt_ÚviewrS   r
   Ú_loggerÚinfo),rR   ÚclosureÚlossÚtotal_momentum_sizeÚtotal_momentum_mbÚtotal_precond_sizeÚtotal_precond_mbrZ   r.   r/   r+   Úupdate_probÚpr�   rY   Ú	flattenedÚexprsÚmomentum_sizeÚmomentum_mbÚprecond_sizeÚ
precond_mbÚ	do_updateÚbetaÚbias_correctionro   Údebiased_momentumÚbalanceÚexprAÚexprGsÚ_rp   Ú	torch_rngÚVÚGÚAÚconjBÚtermsru   Úterm1Úterm2ÚtmpÚpre_gradr&   Úwd_scales,                                               r   rm   z	Kron.stepÙ   sÃ  € àˆØÐÜ×"Ñ"Ó$ñ !Ù“y�÷!ð  ÐØÐØÐØÐà×&Ñ&ó T	8ˆEØ—y‘y Ó,ˆHØ!ŸI™I o´u·}±}ÓEˆMØ+0¯9©9Ð5SÐUYÓ+ZÐ(ØŸ)™)Ð$GÈÓNˆKà˜8‘_ó N8�Ø—6‘6�>Øà—v‘v�ØŸ
™
 1™�à!�	Ø˜Ò#Ü'¨¨eÐ4GÑ.HÈ%ÐPaÑJbÓc�DØ $�Iä�u“: ’?Ø$%�E˜&‘MØ./�EÐ*Ñ+Ü/4×/?Ñ/?ÀÈHÒLbÐX\×XbÑXbÔ/c�EÐ+Ñ,ä(5ØØÐ2Ñ3ØÐ3Ñ4ØÐ3Ñ4ØÐ0Ñ1Ø+ô)Ñ%�E˜#‘J ð ,1�D×%Ñ% aÑ(ð %*Ð*;Ñ$<×$BÑ$BÓ$D�MØ"/°%Ð8IÑ2J×2WÑ2WÓ2YÑ"YÐ\aÑ"a�KØ'¨=Ñ8Ð'Ø%¨Ñ4Ð%ä#&Ñ&E¸%À¹*Ô&EÓ#E�LÜ!$Ñ$VÈ5ÐQTÉ:Ô$VÓ!VÐY^Ñ!^�JØ&¨,Ñ6Ð&Ø$¨
Ñ2Ñ$Ø˜d×/Ñ/Ñ/ä)ØØÐ2Ñ3ØÐ3Ñ4ØÐ3Ñ4ØÐ0Ñ1Ø+Ø$ô�Eð ,1�D×%Ñ% aÒ(ð !×-Ñ-¨aÑ0�Eð Ð&Ü">�KÜ˜KÔ(Ù"-¨e°F©mÓ"<�KØÐ&Ó'¨1Ñ,Ó'Ø!Ð"2Ñ3°q¸;±ÑF�	ÙØ./�EÐ*Ñ+à�f“ Ñ"“ð ˜ZÑ(�Ø"# d¨e°F©mÑ&;Ñ";�Ø"'Ð(9Ñ":�Ø×$Ñ$ U¨:Ñ%6Ó7×<Ñ<¸TÈÈUÐS]ÑM^ÑI^Ð<Ô_ð Ð'Ø#×)Ñ)¨/×*<Ñ*<À8Ð*<Ó*LÔMØ%4°Ñ%F×$JÑ$JÐQ^Ð$JÓ$_Ð!ð Ÿ(™(Ÿ/™/Ó+¨dÑ2Ò@°y�Ø—8‘8“: ’>¡gØ—O‘O E¨#¡JÔ/ò Ø',Ñ$�E˜6 1Ø˜c™
�AØ×)Ò)Ü$)§O¡OÐ;L×;SÑ;SÔ$T˜	Ø!×-Ñ-¨d¯h©h×.>Ñ.>¸qÀ'Ó.JÕKà$(˜	ÜŸ™Ø)×/Ñ/Ø"+Ø+Ø0×7Ñ7ô	�Añ .JÑ)Èt�Aà#×5Ñ5°e¸QÀÀ1ÓE‘H�A�uà ŸM™M¨&°!°UÓ;�Eä-0°°E«]ò 
$Ñ)˜™>˜E 5Ø# e™m˜Ø˜u \Ñ2Ñ2˜ØŸ5™5›7 Qš;Ø 1™H˜CØ E¨E¡M×#7Ñ#7¼¸e»Ó#EÈÏ
É
Ñ#RÑR™Cä"'§*¡*¨S£/˜CØÔ#4°U¸U±]Ó#CÀdÇjÁjÑ#PÑP˜CØ 1™H˜CØŸ™˜s�ð
$ð  ×-Ñ-Ø˜#‘JØØ%ó÷ ‘"˜1Ÿ7™7�"Ó#ð	 ð —‘œeŸk™k¨#°·±Ó1B×1GÑ1GÓ1I×1OÑ1OÓ1QÐTXÑ1XÑ*YÐ_bÔcÔdÙØ'Ÿ}™}¨Q¯W©WÓ5�Hð  % ^Ñ4�Ø 1Ò$ØÐ6Ò7Ø'(¨4¯8©8¯?©?Ó+<Ñ'<¸|Ñ'K˜àÐ.Ò/Ø Ð!9Ò:Ø',¨T¡{°aÑ'7¸$¿-¹-ÈÑ:MÑ'M™Hà',¨T¡{˜HØŸ™˜r H¨|Ñ$;Ñ;Õ<à Ÿ™ a¨|˜Ô<ô ˜Q ¨5°©;¨,Ö7ò]N8ðT	8ðl  Ò"Ü�L‰LÐ6Ð7JÐ6KÈ;ÐWhÐilÐVmÐmpÐqÔrÜ�L‰LÐ6Ð7IÐ6JÈ+ÐVfÐgjÐUkÐknÐoÔpàˆ÷E!ñ !ús   ™XØX)çü©ñÒMbP?gÍÌÌÌÌÌì?r8   Ni   r}   NTgš™™™™™¹?r9   NNFFFr}   éÿÿÿÿFFrk   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r”   r   r   r   ÚintÚstrÚboolr   r   rC   rV   r\   r   r   ra   rf   Úno_gradrm   Ú__classcell__)rT   s   @r   r"   r"   R   s¤  ø„ ñð< ØØ!ØNRØ#'Ø#$Ø*.Ø-1ØØ$'Ø*.Ø/3Ø %Ø',ØØ!"Ø!Ø(-Ø#ñ+E)àðE)ð ðE)ð ð	E)ð
 ðE)ð ,4°E¸(ÀE¸/Ñ4JÑ+KðE)ð !ðE)ð !ðE)ð # 3™-ðE)ð '+ðE)ð ðE)ð "ðE)ð ˜5Ÿ;™;Ñ'ðE)ð   §¡Ñ,ðE)ð ðE)ð  !%ð!E)ð" ð#E)ð$ ð%E)ð& ð'E)ð( "&ð)E)ð* õ+E)ôN>ô
ð
˜D  c ™Nõ ð7¨$¨s°C¨x©.ð 7¸Tõ 7ôð €U‡]�]ƒ_òeó ôer    r"   c                 óˆ   — | j                   }t        |dk\  r|n||z   |dz
  «      }||k  s||kD  r| S | j                  ||«      S )Nr   r	   )Úndimr   r2   )r   Ú	start_dimÚend_dimrÌ   s       r   r‚   r‚   ‚  sR   € Ø�;‰;€Dô ˜W¨š\‘'¨t°g©~¸tÀa¹xÓH€Gð ˆyÒ˜I¨Ò/Øˆð �>‰>˜) WÓ-Ð-r    c           	      ó–	  — t         j                  t         j                  z   }|�|n| j                  }| j                  }g }	t        |«      dk(  r4|r)|	j                  |t        j                  | |¬«      z  «       d}
dg}d}�nt        |«      dkD  r"t        dt        | j                  «      › d�«      ‚|d	t        |«      z  z  }|€|D �cg c]  }d
‘Œ }}n¸|dk(  r3t        j                  |«      ddd…   }|D �cg c]  }d
‘Œ }}d||d   <   n€|dk(  rWt        j                  |«      ddd…   }t        |«      }|D �cg c]  }d
‘Œ }}t        |«      dk\  r8|d   |d   kD  r-d||d   <   n$|dk(  r|D �cg c]  }d‘Œ }}nt        d|› d�«      ‚g dd}}}g }g g ddf\  }}}}t        t        ||«      «      D �]‡  \  }\  }}|d	k(  s||kD  st        |«      |k  s|�r|r4|	j                  |t        j                  ||| j                   ¬«      z  «       |j                  ||   «       |||   z   }|||   z   }dj#                  t%        t        |«      «      D �cg c]  }||k(  r||dz      n||   ‘Œ c}«      }|dz   |z   dz   ||dz      z   }|j                  |«       |j                  ||dz      «       |j                  ||dz      «       |||dz      z   }|||dz      z   }�Œ+|r4|	j                  |t        j&                  ||| j                   ¬«      z  «       |j                  ||   ||dz      z   «       |||dz      z   }|||   z   }dj#                  t%        t        |«      «      D �cg c]  }||k(  r||dz      n||   ‘Œ c}«      }dj#                  t%        t        |«      «      D �cg c]  }||k(  r||dz      n||   ‘Œ c}«      }|dz   |z   dz   ||dz      z   ||dz      z   }|j                  |«       ||   ||dz      ||dz      }!} }|j                  || z   «       |j                  ||!z   «       ||!z   }|| z   }�ŒŠ dj#                  |«      dz   |z   dz   |z   }
dj#                  |«      dz   dj#                  |«      z   dz   |z   dz   |z   }t)        |«      }|r|	|
||fgS |
||fS c c}w c c}w c c}w c c}w c c}w c c}w c c}w )z’For a scalar or tensor t, we initialize its preconditioner Q and
    reusable einsum expressions for updating Q and preconditioning gradient.
    Nr   r   z,->z,,->é   zGot tensor with dim z; Einstein runs out of letters!r	   FÚone_diagrÁ   TÚsmart_one_diagr}   éþÿÿÿÚall_diagzInvalid memory_save_mode: z/, must be one of [None, 'one_diag', 'all_diag']Ú )r   r{   ú,z->é   )ÚstringÚascii_lowercaseÚascii_uppercaser   r‘   rƒ   Úappendr   Ú	ones_liker@   ÚnpÚargsortÚsortedÚ	enumerater’   Úonesr{   ÚjoinÚrangeÚeyeÚtuple)"ÚtÚscaleÚmax_sizer)   r*   r   ry   Úlettersr‘   rp   r²   r³   ÚexprPr´   Údim_diagÚrev_sorted_dimsÚsorted_shapeÚpiece1AÚpiece2AÚpiece3AÚpiece1PÚpiece2PÚpiece3PÚpiece4PÚiÚsizeÚdim_dÚjÚpiece1Ú
subscriptsÚpiece2ÚaÚbÚcs"                                     r   r…   r…   �  s8  € ô ×$Ñ$¤v×'=Ñ'=Ñ=€GàÐ&‰E¨A¯G©G€EØ�G‰G€EØ
€AÜ
ˆ5ƒz�Q‚ÙØ�H‰H�UœUŸ_™_¨Q°eÔ<Ñ<Ô=ØˆØ�ˆØŠäˆu‹:˜Š?ÜÐ3´C¸¿¹³L°>ÐA`ÐaÓbÐbà˜!œc %›j™.Ñ)ˆàÐ#Ø',Ö- !šÐ-ˆHÑ-Ø Ò+Ü Ÿj™j¨Ó/±°"°Ñ5ˆOØ',Ö- !šÐ-ˆHÐ-Ø+/ˆH�_ QÑ'Ò(ØÐ!1Ò1ä Ÿj™j¨Ó/±°"°Ñ5ˆOÜ! %›=ˆLØ',Ö- !šÐ-ˆHÐ-Ü�5‹z˜QŠ <°Ñ#3°lÀ2Ñ6FÒ#FØ/3�˜¨Ñ+Ò,Ø Ò+Ø&+Ö, šÐ,ˆHÑ,äØ,Ð-=Ð,>Ð>mÐnópð pð &(¨¨R˜'�ˆØˆØ.0°"°b¸"Ð-=Ñ*ˆ�˜' 7Ü )¬#¨e°XÓ*>Ó ?ó )	&ÑˆA‰}��eà˜’	Ø˜(’?Ü�u“:Ð 3Ò3Úñ Ø—H‘H˜U¤U§Z¡Z°¸EÈ!Ï(É(Ô%SÑSÔTà—‘˜w q™zÔ*Ø! G¨A¡JÑ.�Ø! G¨A¡JÑ.�àŸ™ÔUZÔ[^Ð_dÓ[eÓUfÖ!gÐPQ°Q¸!²V '¨!¨b©&¢/ÀÈÁÑ"KÒ!gÓh�Ø# c™\¨FÑ2°TÑ9¸GÀAÈÁF¹OÑK�
Ø—‘˜jÔ)à—‘˜w q¨2¡v™Ô/Ø—‘˜w q¨2¡v™Ô/Ø! G¨A°©F¡OÑ3�Ø! G¨A°©F¡OÑ3’ñ Ø—H‘H˜U¤U§Y¡Y¨t¸5ÈÏÉÔ%RÑRÔSà—‘˜w q™z¨G°A¸±F©OÑ;Ô<Ø! G¨A°©F¡OÑ3�Ø! G¨A¡JÑ.�àŸ™ÔUZÔ[^Ð_dÓ[eÓUfÖ!gÐPQ°Q¸!²V '¨!¨b©&¢/ÀÈÁÑ"KÒ!gÓh�ØŸ™ÔUZÔ[^Ð_dÓ[eÓUfÖ!gÐPQ°Q¸!²V '¨!¨b©&¢/ÀÈÁÑ"KÒ!gÓh�Ø# c™\¨FÑ2°TÑ9¸GÀAÈÁF¹OÑKÈgÐVWÐZ\ÑV\ÉoÑ]�
Ø—‘˜jÔ)à" 1™: w¨q°2©v¡¸ÀÀBÁ¹�a�1�Ø—‘˜q 1™uÔ%Ø—‘˜q 1™uÔ%Ø! A™+�Ø! A™+’ðS)	&ðV —‘˜Ó! CÑ'¨'Ñ1°DÑ8¸7ÑBˆØ—‘˜Ó! CÑ'¨#¯(©(°7Ó*;Ñ;¸cÑAÀGÑKÈdÑRÐU\Ñ\ˆä�6‹]€FÙØ�E˜6 5Ð)Ð*Ð*à�f˜eÐ#Ð#ùòO .ùò .ùò .ùò -ùò. "hùò" "hùÚ!gs*   Ã		R(Ã9	R-Ä<	R2Å2	R7ÉR<
Í'S
Î(S
c                 ó¶  — | |z  } t        j                  | | j                  «       z  «      }t        j                  t        j                  |d¬«      d«      \  }}t        j                  t        j                  |d¬«      d«      \  }}||kD  ri| d d …|f   j                  «       | z  }|t         j
                  j                  |t         j
                  j                  |«      z  | j                  z  «      z  S | | |   j                  «       z  }|t         j
                  j                  | j                  |t         j
                  j                  |«      z  z  «      z  S )Nr   )rŒ   r	   )r   ÚrealÚconjr   r†   ÚlinalgÚvector_normÚH)r¸   Úmax_absÚaaÚvalue0rõ   Úvalue1rø   Úxs           r   Ú_lbr
  ø  s  € Ø	ˆG‰€AÜ	�‰�A˜Ÿ™›‘LÓ	!€BÜ—	‘	œ%Ÿ)™) B¨AÔ.°Ó2�I€FˆAÜ—	‘	œ%Ÿ)™) B¨AÔ.°Ó2�I€FˆAØ�‚ØŠa�ˆd‰G�L‰L‹N˜QÑˆØœŸ™×1Ñ1°1´u·|±|×7OÑ7OÐPQÓ7RÑ3RÐVW×VYÑVYÑ2YÓZÑZÐZà��!‘—	‘	“‰OˆØœŸ™×1Ñ1°!·#±#¸¼U¿\¹\×=UÑ=UÐVWÓ=XÑ9XÑ2YÓZÑZÐZr    c                 ó~   — | j                  t        d«      «      }t        j                  |dkD  t	        | |«      |«      S )z-Cheap lower bound for the spectral norm of A.r~   r   )r“   r”   r   Úwherer
  )r¸   r  s     r   r–   r–     s1   € à�f‰f”U˜5“\Ó"€GÜ�;‰;�w ‘{¤C¨¨7£O°WÓ=Ð=r    c           	      ó`  — | j                   }| j                  t        j                  ¬«      } |j                  t        j                  ¬«      }t        j                  j                  || j                  d| j                  d«      «      dd¬«      j                  | «      }|j                  |¬«      S )z
X @ inv(A)r   rÁ   TF)ÚupperÚleft)	r   r‹   r   r   r  Úsolve_triangularÚreshaperö   Ú
reshape_as)ÚXr¸   Ú
orig_dtypeÚouts       r   Ú_solve_triangular_rightr    s�   € à—‘€JØ	�‰”5—=‘=ˆÓ!€AØ	�‰”5—=‘=ˆÓ!€AÜ
�,‰,×
'Ñ
'¨¨1¯9©9°R¸¿¹À»Ó+DÈDÐW\Ð
'Ó
]×
hÑ
hÐijÓ
k€CØ�6‰6˜
ˆ6Ó#Ð#r    c           
      ó  — t        j                  | D �cg c]  }|j                  t        d«      «      ‘Œ c}«      }|j	                  «       dt        | «      z  z  }||z  }t        | «      D ]  \  }}|j                  ||   «       Œ y c c}w )Nr~   r	   )r   Ústackr“   r”   Úprodrƒ   rà   rˆ   )ÚQ_inru   ÚnormsÚgeometric_meanrõ   s        r   rQ   rQ     sv   € Ü�K‰K°tÖ<°!˜Ÿ™¤ e£Õ-Ò<Ó=€EØ—Z‘Z“\ a¬#¨d«)¡mÑ4€NØ˜UÑ"€EÜ˜$“ò ‰ˆˆ1Ø	�‰ˆu�Q‰xÕñùò =s   ”!Bc                 ó~   — t        j                  |d   g| D �cg c]  }|j                  «       ‘Œ c}¢| ¢|‘­Ž S c c}w )z.Precondition gradient G with preconditioner Q.rÁ   )r   Úeinsumr  )rp   r¨   r·   ru   s       r   rP   rP     s8   € ä�<‰<˜˜b™	ÐA°qÖ$9°! Q§V¡V¥XÒ$9ÐA¸AÐA¸qÒAÐAùÒ$9s   ™:
c                 ó�  — t        j                  | g|¢|‘­Ž }|j                  «       }t        t	        |«      «      }t        j
                  |j                  «       |dd  |d d z   «      }t        |«      D ]L  \  }}	|	j                  «       dk  r||	z  nt        ||	«      }||dz
  k  sŒ3t        j                  |||dz
  «      }ŒN ||fS )Nr	   r}   )
r   r  rŒ   rå   rã   Úpermuter  rà   r  Ú	transpose)
r²   r·   rp   r¶   r¸   Úorderr¦   r¹   rõ   ru   s
             r   rN   rN   !  s»   € Ü�‰�UÐ"˜QÐ" Ò"€AØ�E‰E‹G€EÜŒe�E‹lÓ€AÜ�M‰M˜!Ÿ&™&›( A a b E¨A¨b¨q¨E¡MÓ2€EÜ˜!“ò 9‰ˆˆ1ØŸU™U›W qš[�˜’	Ô.EÀeÈQÓ.OˆØˆu�q‰y‹=Ü—O‘O E¨1¨e°a©iÓ8‰Eð9ð ˆeˆ8€Or    c                 óÒ   — g }| D ]_  }t        j                  |||j                  «       «      }t        j                  ||j                  «       |«      }|j                  ||f«       Œa |S rk   )r   r  r  rÛ   )r³   r¸   r¹   rº   ÚexprGr»   r¼   s          r   rO   rO   -  s]   € Ø€EØò %ˆÜ—‘˜U A q§v¡v£xÓ0ˆÜ—‘˜U E§J¡J£L°%Ó8ˆØ�‰�e˜U�^Õ$ð%ð €Lr    )r9   g¸…ëQ¸ž?rÀ   iô  )r   rÁ   )NT)2rÅ   ÚloggingrØ   rI   r>   Útypingr   r   r   r   r   r   ÚnumpyrÝ   r   Ú
opt_einsumÚtorch.backends.opt_einsumÚbackendsÚenabledÚstrategyr=   ÚImportErrorÚ_dynamoÚconfigÚcache_size_limitrL   ÚAttributeErrorÚ_helpersr
   r   Ú_typesr   Ú	getLoggerrÂ   r�   r”   ÚTensorr   ÚoptimÚ	Optimizerr"   r‚   r…   r
  r–   r  rQ   rP   rN   rO   r<   r    r   ú<module>r8     s[  ðñó2 Û Û Û ß >× >ã Û ðãÛ$Ø(,€E‡N�N×ÑÔ%Ø)2€E‡N�N×ÑÔ&Ø€NðØ,5€E‡M�M×ÑÔ)Ø€J÷ 5Ý à
ˆ'×
Ñ
˜HÓ
%€ð
 ØØØñØðàðð ðð ð	ð
 ðð ‡\�\óô4mˆ5�;‰;× Ñ ô mó`	.ð( Øóe$òP
[ò>ò$òòBò
	óøðI ò Ø‚Nðûð ò Ø‚Jðús$   ¬A D Á-D ÄDÄDÄDÄD