U
    8‹Ñh@]  ã                   @   s|  d dl mZmZmZmZmZ d dlZd dlmZ ddlm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZ ddgZG dd„ deƒZd	d
e› de› de› de	› d�	 e_ee ee ee ee ee eeeeeeeeeedœdd„Zee ee ee ee ee eeeeeeeeeedœdd„Zeed�dee ee ee ee ee eee eeeeeeeeedœdd„ƒZdS )é    )ÚcastÚListÚOptionalÚTupleÚUnionN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_dispatch_sqrtÚ_foreach_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ
_get_valueÚ_maximize_docÚ_use_grad_for_differentiableÚ_view_as_realÚ	OptimizerÚParamsTÚRAdamÚradamc                       sp   e Zd Zddddddœeeeeef eeeee eeedœ
‡ fd	d
„Z‡ fdd„Z	dd„ Z
eddd„ƒZ‡  ZS )r   çü©ñÒMbP?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>r   FN)ÚforeachÚmaximizeÚ
capturableÚdifferentiable)
ÚparamsÚlrÚbetasÚepsÚweight_decayÚdecoupled_weight_decayr   r   r   r   c                   sÆ   d|kst d|› �ƒ‚d|ks,t d|› �ƒ‚d|d   krDdk sXn t d|d › �ƒ‚d|d   krpdk s„n t d|d › �ƒ‚d|ksšt d	|› �ƒ‚t|||||||	||
d
�	}tƒ  ||¡ d S )Nç        zInvalid learning rate: zInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: r   z#Invalid beta parameter at index 1: zInvalid weight_decay value: )	r!   r"   r#   r$   r   r   r   r%   r   )Ú
ValueErrorÚdictÚsuperÚ__init__)Úselfr    r!   r"   r#   r$   r%   r   r   r   r   Údefaults©Ú	__class__© ú?/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/optim/radam.pyr+      s,    ÷zRAdam.__init__c                    sÊ   t ƒ  |¡ | jD ]²}| dd ¡ | dd¡ | dd¡ | dd¡ | dd¡ |d D ]h}| j |g ¡}t|ƒdkrZt |d	 ¡sZt	|d	 ƒ}|d r®tj
|tƒ |jd
�ntj
|tƒ d�|d	< qZqd S )Nr   r   Fr   r%   r   r    r   Ústep©ÚdtypeÚdevice©r4   )r*   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchZ	is_tensorÚfloatÚtensorr   r5   )r,   r:   ÚgroupÚpZp_stateZstep_valr.   r0   r1   r7   B   s&    
ý  ÿûzRAdam.__setstate__c           
      C   sì   d}|d D ]Ú}|j d k	r|t |¡O }| |¡ |j jrBtdƒ‚| |j ¡ | j| }	t|	ƒdkr¼|d r€tjdt	ƒ |j
d�ntjdt	ƒ d	�|	d
< tj|tjd�|	d< tj|tjd�|	d< | |	d ¡ | |	d ¡ | |	d
 ¡ q|S )NFr    z'RAdam does not support sparse gradientsr   r   r0   r3   r&   r6   r2   )Zmemory_formatÚexp_avgÚ
exp_avg_sq)Úgradr=   Ú
is_complexÚappendZ	is_sparseÚRuntimeErrorr:   r<   Úzerosr   r5   r?   Z
zeros_likeZpreserve_format)
r,   r@   Úparams_with_gradÚgradsÚexp_avgsÚexp_avg_sqsÚstate_stepsÚhas_complexrA   r:   r0   r0   r1   Ú_init_groupV   s4    


ÿý ÿ
 ÿ
zRAdam._init_groupc                 C   sÈ   |   ¡  d}|dk	r.t ¡ � |ƒ }W 5 Q R X | jD ]Ž}g }g }g }g }g }ttttf |d ƒ\}	}
|  ||||||¡}t||||||	|
|d |d |d |d |d |d |d	 |d
 |d� q4|S )z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr"   r!   r$   r#   r   r   r   r   r%   )Úbeta1Úbeta2r!   r$   r#   r   r   r   r   r%   rN   )	Z _cuda_graph_capture_health_checkr=   Zenable_gradr8   r   r   r>   rO   r   )r,   ÚclosureZlossr@   rI   rJ   rK   rL   rM   rP   rQ   rN   r0   r0   r1   r2   y   sN    

     ÿðz
RAdam.step)r   r   r   r   F)N)Ú__name__Ú
__module__Ú__qualname__r   r>   r   Úboolr   r+   r7   rO   r   r2   Ú__classcell__r0   r0   r.   r1   r      s4        ù	ô
ô&#a  Implements RAdam algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \: \beta_1, \beta_2
                \text{ (betas)}, \: \theta_0 \text{ (params)}, \:f(\theta) \text{ (objective)}, \:
                \lambda \text{ (weightdecay)}, \:\textit{maximize}                               \\
            &\hspace{13mm} \epsilon \text{ (epsilon)}, \textit{decoupled\_weight\_decay}         \\
            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
                v_0 \leftarrow 0 \text{ ( second moment)},                                       \\
            &\hspace{18mm} \rho_{\infty} \leftarrow 2/(1-\beta_2) -1                      \\[-1.ex]
            &\rule{110mm}{0.4pt}  \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{6mm}\textbf{if} \: \textit{maximize}:                                       \\
            &\hspace{12mm}g_t           \leftarrow   -\nabla_{\theta} f_t (\theta_{t-1})         \\
            &\hspace{6mm}\textbf{else}                                                           \\
            &\hspace{12mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})          \\
            &\hspace{6mm} \theta_t \leftarrow \theta_{t-1}                                       \\
            &\hspace{6mm} \textbf{if} \: \lambda \neq 0                                          \\
            &\hspace{12mm}\textbf{if} \: \textit{decoupled\_weight\_decay}                       \\
            &\hspace{18mm} \theta_t \leftarrow \theta_{t} - \gamma \lambda \theta_{t}            \\
            &\hspace{12mm}\textbf{else}                                                          \\
            &\hspace{18mm} g_t \leftarrow g_t + \lambda \theta_{t}                               \\
            &\hspace{6mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
            &\hspace{6mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
            &\hspace{6mm}\widehat{m_t} \leftarrow   m_t/\big(1-\beta_1^t \big)                   \\
            &\hspace{6mm}\rho_t \leftarrow \rho_{\infty} -
                2 t \beta^t_2 /\big(1-\beta_2^t \big)                                    \\[0.1.ex]
            &\hspace{6mm}\textbf{if} \: \rho_t > 5                                               \\
            &\hspace{12mm} l_t \leftarrow \frac{\sqrt{ (1-\beta^t_2) }}{ \sqrt{v_t} +\epsilon  } \\
            &\hspace{12mm} r_t \leftarrow
      \sqrt{\frac{(\rho_t-4)(\rho_t-2)\rho_{\infty}}{(\rho_{\infty}-4)(\rho_{\infty}-2) \rho_t}} \\
            &\hspace{12mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t} r_t l_t        \\
            &\hspace{6mm}\textbf{else}                                                           \\
            &\hspace{12mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}                \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `On the variance of the adaptive learning rate and beyond`_.

    This implementation provides an option to use either the original weight_decay implementation as in Adam
    (where the weight_decay is applied to the gradient) or the one from AdamW (where weight_decay is applied
    to the weight) through the decoupled_weight_decay option. When decoupled_weight_decay is set to False
    (default), it uses the original Adam style weight decay, otherwise, it uses the AdamW style which
    corresponds more closely to the `author's implementation`_ in the RAdam paper. Further information
    about decoupled weight decay can be found in `Decoupled Weight Decay Regularization`_.

    a³  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        decoupled_weight_decay (bool, optional): whether to use decoupled weight
            decay as in AdamW to obtain RAdamW (default: False)
        z	
        a  

    .. _On the variance of the adaptive learning rate and beyond:
        https://arxiv.org/abs/1908.03265
    .. _author's implementation:
        https://github.com/LiyuanLucasLiu/RAdam
    .. _Decoupled Weight Decay Regularization:
        https://arxiv.org/abs/1711.05101

    )r    rJ   rK   rL   rM   rP   rQ   r!   r$   r#   r%   r   r   r   rN   c       
            s  t | ƒD �]ô\}}|s|| n||  }|| }|| ‰|| }tj ¡ s€|r€tƒ }|jj|jjkrp|jj|ks€td|› d�ƒ‚t |¡r²t 	|¡}t 	|¡}t 	|¡}t 	ˆ¡‰|d7 }|rÂ|nt
|ƒ}|dkrø|
rê| d||  ¡ n|j||d�}| |d| ¡ ˆ |¡j||d| d� d||  }d||  ‰ || }dd|  d ‰ˆd| ||  ˆ   ‰‡‡fdd	„}‡ ‡‡‡fd
d„}|�rÂt ˆdk|ƒ |ƒ  d¡}|j|| | dd� qˆdk�rì|j|| |ƒ  |ƒ  dd� q|j|| dd� qd S )NúIIf capturable=True, params and state_steps must be on supported devices: Ú.r   r   ©Úalpha)Úvalueé   c                      s,   ˆd ˆd  ˆ  ˆ d ˆ d  ˆ  d S )Né   r]   ç      à?r0   r0   )Úrho_infÚrho_tr0   r1   Ú_compute_rect9  s    ÿþýûz+_single_tensor_radam.<locals>._compute_rectc                     s.   ˆ  ¡ } ˆr|  ˆ¡} n
|  ˆ¡} ˆ d |  S )Nr_   )ÚsqrtÚaddÚadd_)Zexp_avg_sq_sqrt)Úbias_correction2r   r#   rC   r0   r1   Ú_compute_adaptive_lrA  s
    
z2_single_tensor_radam.<locals>._compute_adaptive_lrç      @r'   g      ð¿)Ú	enumerater=   Ú_utilsÚis_compilingr   r5   ÚtypeÚAssertionErrorrE   Zview_as_realr   Zmul_rd   Zlerp_Zaddcmul_Úwherere   )r    rJ   rK   rL   rM   rP   rQ   r!   r$   r#   r%   r   r   r   rN   ÚiÚparamrD   rB   Zstep_tÚcapturable_supported_devicesr2   Úbias_correction1Zbias_corrected_exp_avgrb   rg   Úupdater0   )rf   r   r#   rC   r`   ra   r1   Ú_single_tensor_radamú   sh    ÿ
þ
ý





 
 ÿ
ÿþýûrt   c       
             sä  t | ƒdkrd S |rtdƒ‚tj ¡ s`|r`tdd�‰t‡fdd„t| |ƒD ƒƒs`tdˆ› d�ƒ‚t 	| ||||g¡}| 
¡ D �]`\\}}}}}}|d jr¶tj|tjd	d
d�d	d� nt |d¡ |rÔt||||ƒ |rât |¡}ddˆ  d ‰|�rft ˆ|¡}t |¡ t |d¡ t ˆ|¡}t ||¡ t |d¡ t ||¡ t |¡ t |ˆ¡ |}n‡‡fdd„|D ƒ}|dk�rÈ|
�r t |dˆ|  ¡ n(|�r¸tj|||d� ntj|||d�}t ||dˆ  ¡ t |ˆ¡ t |||dˆ ¡ ~|�r2t |d¡}t |d¡}t ||¡ ~t |ˆ¡ ˆd ˆd  ‰t |ˆ¡}t ||¡ ~t |¡ dd„ t||ƒD ƒ}~~dd„ |D ƒ}t |ˆ¡ t ˆ |¡}t |¡ t |d¡ t ||¡ t |¡ t ˆ|¡}t |¡ t |d¡ t |¡ t |ˆ¡ t ||¡ ~t |¡ t ||¡ ~nf‡fdd„|D ƒ}dd„ |D ƒ}‡ fdd„|D ƒ}‡fdd„t||ƒD ƒ}‡‡fdd„t|||ƒD ƒ}t |¡}t ||	¡ t ||¡ t |¡ t ||¡ t |||¡ q|d S )Nr   z#_foreach ops don't support autogradF)Zsupports_xlac                 3   s.   | ]&\}}|j j|j jko$|j jˆ kV  qd S ©N)r5   rl   )Ú.0rA   r2   )rq   r0   r1   Ú	<genexpr>y  s   þz&_multi_tensor_radam.<locals>.<genexpr>rX   rY   r'   Úcpu)r5   rZ   r   r]   c                    s8   g | ]0}ˆd t |ƒ ˆ t |ƒ  dˆ t |ƒ    ‘qS )r]   r   ©r   ©rv   r2   )rQ   r`   r0   r1   Ú
<listcomp>®  s   ûÿ
þýÿz'_multi_tensor_radam.<locals>.<listcomp>r^   c                 S   s"   g | ]\}}t  |d k|d¡‘qS )rh   r&   ©r=   rn   )rv   Únra   r0   r0   r1   r{   Ý  s    c                 S   s   g | ]}t  |d kdd¡‘qS )r   r&   r'   r|   ©rv   Úrectr0   r0   r1   r{   â  s     c                    sD   g | ]<}|d kr<t |d |d  ˆ  ˆ d ˆ d  |  ƒnd‘qS )é   r^   r]   r   )r   )rv   ra   )r`   r0   r1   r{   ÷  s   	þúÿþýÿc                 S   s   g | ]}|d krd nd‘qS )r   r'   r0   r~   r0   r0   r1   r{     s     c                    s   g | ]}d ˆ t |ƒ  ‘qS )r   ry   rz   )rP   r0   r1   r{     s    c                    s    g | ]\}}ˆ | | d  ‘qS )éÿÿÿÿr0   )rv   r   Úbc)r!   r0   r1   r{     s    c                    s6   g | ].\}}}t d ˆ t|ƒ  ƒˆ| |  d ‘qS )r   r�   )r   r   )rv   r2   r   r‚   )rQ   r!   r0   r1   r{   
  s   ÿ)r<   rm   r=   rj   rk   r   ÚallÚzipr   Z"_group_tensors_by_device_and_dtypeÚvaluesZis_cpuZ_foreach_add_r?   r   Z_foreach_negZ_foreach_powZ_foreach_neg_Z_foreach_mul_Z_foreach_div_Z_foreach_addZ_foreach_lerp_Z_foreach_addcmul_Z_foreach_subZ_foreach_mulZ_foreach_sqrt_Z_foreach_sqrtZ_foreach_reciprocal_) r    rJ   rK   rL   rM   rP   rQ   r!   r$   r#   r%   r   r   r   rN   Zgrouped_tensorsZgrouped_paramsZgrouped_gradsZgrouped_exp_avgsZgrouped_exp_avg_sqsZgrouped_state_stepsÚ_rr   rf   Z
rho_t_listÚnumZsub2Zdenomr   Zunrect_step_sizeZunrectifiedÚbufferr0   )rP   rQ   rq   r!   r`   r1   Ú_multi_tensor_radam]  s   ÿý

üÿ	ú

  ÿ   ÿ


ú	
  ÿ  ÿ   ÿ
ÿ





	÷
ÿ
ÿ
þ

r‰   )Zsingle_tensor_fnF)r    rJ   rK   rL   rM   r%   r   r   r   rN   r   rP   rQ   r!   r$   r#   c                C   sŒ   t dd„ |D ƒƒstdƒ‚|dkr4t| |dd�\}}|rJtj ¡ rJtdƒ‚|r^tj ¡ s^t}nt}|| ||||||||||
||||	d� dS )	zpFunctional API that performs RAdam algorithm computation.

    See :class:`~torch.optim.RAdam` for details.
    c                 s   s   | ]}t |tjƒV  qd S ru   )Ú
isinstancer=   r   )rv   Útr0   r0   r1   rw   4  s     zradam.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Z	use_fusedz6torch.jit.script not supported with foreach optimizers)
rP   rQ   r!   r$   r#   r   r%   r   r   rN   )rƒ   rG   r
   r=   ZjitZis_scriptingr‰   rt   )r    rJ   rK   rL   rM   r%   r   r   r   rN   r   rP   rQ   r!   r$   r#   r†   Úfuncr0   r0   r1   r     s@    ÿ  ÿ
ñ)FNFFFF)Útypingr   r   r   r   r   r=   r   Z	optimizerr	   r
   r   r   r   r   r   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__r>   rV   rt   r‰   r   r0   r0   r0   r1   Ú<module>   s˜   @ 3ôóòñÍÿQðdð =	      óí