U
    8‹Ñh²c  ã                   @   s˜  d dl mZmZmZmZmZ d dlZd dlmZ ddlm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZ ddgZG dd„ deƒZd	d
e› de› de	› de› d�	 e_ee ee ee ee ee ee eeeeeeeeeeedœdd„Zee ee ee ee ee ee eeeeeeeeeeedœdd„Zeed�dee ee ee ee ee ee eee eeeeeeeeeedœdd„ƒZdS )é    )ÚcastÚListÚOptionalÚTupleÚUnionN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_dispatch_sqrtÚ_foreach_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ
_get_valueÚ_maximize_docÚ_stack_if_compilingÚ_use_grad_for_differentiableÚ_view_as_realÚ	OptimizerÚParamsTÚNAdamÚnadamc                       sr   e Zd Zddddddœeeeeef eeeeee eeed	œ‡ fd
d„Z‡ fdd„Z	dd„ Z
eddd„ƒZ‡  ZS )r   çü©ñÒMb`?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>r   çü©ñÒMbp?FN)ÚforeachÚmaximizeÚ
capturableÚdifferentiable)ÚparamsÚlrÚbetasÚepsÚweight_decayÚmomentum_decayÚdecoupled_weight_decayr   r   r    r!   c                   sÞ   d|kst d|› �ƒ‚d|ks,t d|› �ƒ‚d|d   krDdk sXn t d|d › �ƒ‚d|d   krpdk s„n t d|d › �ƒ‚d|ksšt d	|› �ƒ‚d|ks°t d
|› �ƒ‚t|||||||	||
|d�
}tƒ  ||¡ d S )Nç        zInvalid learning rate: zInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: r   z#Invalid beta parameter at index 1: zInvalid weight_decay value: zInvalid momentum_decay value: )
r#   r$   r%   r&   r'   r(   r   r   r    r!   )Ú
ValueErrorÚdictÚsuperÚ__init__)Úselfr"   r#   r$   r%   r&   r'   r(   r   r   r    r!   Údefaults©Ú	__class__© ú?/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/optim/nadam.pyr.      s2    özNAdam.__init__c                    s  t ƒ  |¡ | jD ]ø}| dd¡ | dd ¡ | dd¡ | dd¡ | dd¡ |d D ]®}| j |g ¡}t|ƒdkrZt |d	 ¡sÂt	|d	 ƒ}|d r®tj
|tƒ |jd
�ntj
|tƒ d�|d	< t |d ¡sZ|d }|d rôtj
|tƒ |jd
�ntj
|tƒ d�|d< qZqd S )Nr   Fr   r    r!   r(   r"   r   Ústep©ÚdtypeÚdevice©r7   Ú
mu_product)r-   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchZ	is_tensorÚfloatÚtensorr   r8   )r/   r>   ÚgroupÚpZp_stateZstep_valZmu_prod_valr1   r3   r4   r;   E   s<    
ý  ÿûý  ÿûzNAdam.__setstate__c                 C   s,  d}|d D �]}	|	j d k	r|t |	¡O }| |	¡ |	j jrDtdƒ‚| |	j ¡ | j|	 }
t|
ƒdkrî|d r‚tjdt	ƒ |	j
d�ntjdt	ƒ d	�|
d
< |d r²tjdt	ƒ |	j
d�ntjdt	ƒ d	�|
d< tj|	tjd�|
d< tj|	tjd�|
d< | |
d ¡ | |
d ¡ | |
d ¡ | |
d
 ¡ q|S )NFr"   z'NAdam does not support sparse gradientsr   r    r3   r6   r)   r9   r5   r*   r:   )Zmemory_formatÚexp_avgÚ
exp_avg_sq)ÚgradrA   Ú
is_complexÚappendZ	is_sparseÚRuntimeErrorr>   r@   Úzerosr   r8   rC   ZonesZ
zeros_likeZpreserve_format)r/   rD   Úparams_with_gradÚgradsÚexp_avgsÚexp_avg_sqsÚmu_productsÚstate_stepsÚhas_complexrE   r>   r3   r3   r4   Ú_init_groupc   s>    



ÿýÿý ÿ
 ÿ
zNAdam._init_groupc                 C   sÖ   |   ¡  d}|dk	r.t ¡ � |ƒ }W 5 Q R X | jD ]œ}g }g }g }g }g }g }	ttttf |d ƒ\}
}|  |||||||	¡}t||||||	|
||d |d |d |d |d |d |d	 |d
 |d |d� q4|S )z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr$   r#   r&   r'   r%   r   r(   r   r    r!   )Úbeta1Úbeta2r#   r&   r'   r%   r   r(   r   r    r!   rS   )	Z _cuda_graph_capture_health_checkrA   Zenable_gradr<   r   r   rB   rT   r   )r/   ÚclosureZlossrD   rM   rN   rO   rP   rQ   rR   rU   rV   rS   r3   r3   r4   r5   •   sV    

ù
îz
NAdam.step)r   r   r   r   r   F)N)Ú__name__Ú
__module__Ú__qualname__r   rB   r   Úboolr   r.   r;   rT   r   r5   Ú__classcell__r3   r3   r1   r4   r      s8         ø
ó
ó)2a¼  Implements NAdam algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma_t \text{ (lr)}, \: \beta_1,\beta_2 \text{ (betas)},
                \: \theta_0 \text{ (params)}, \: f(\theta) \text{ (objective)}                   \\
            &\hspace{13mm} \: \lambda \text{ (weight decay)}, \:\psi \text{ (momentum decay)}    \\
            &\hspace{13mm} \: \textit{decoupled\_weight\_decay}, \:\textit{maximize}             \\
            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
                v_0 \leftarrow 0 \text{ ( second moment)}                                 \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}\textbf{if} \: \textit{maximize}:                                       \\
            &\hspace{10mm}g_t           \leftarrow   -\nabla_{\theta} f_t (\theta_{t-1})         \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})          \\
            &\hspace{5mm} \theta_t \leftarrow \theta_{t-1}                                       \\
            &\hspace{5mm} \textbf{if} \: \lambda \neq 0                                          \\
            &\hspace{10mm}\textbf{if} \: \textit{decoupled\_weight\_decay}                       \\
            &\hspace{15mm} \theta_t \leftarrow \theta_{t-1} - \gamma \lambda \theta_{t-1}                    \\
            &\hspace{10mm}\textbf{else}                                                          \\
            &\hspace{15mm} g_t \leftarrow g_t + \lambda \theta_{t-1}                             \\
            &\hspace{5mm} \mu_t \leftarrow \beta_1 \big(1 - \frac{1}{2}  0.96^{t \psi} \big)     \\
            &\hspace{5mm} \mu_{t+1} \leftarrow \beta_1 \big(1 - \frac{1}{2} 0.96^{(t+1)\psi}\big)\\
            &\hspace{5mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
            &\hspace{5mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
            &\hspace{5mm}\widehat{m_t} \leftarrow \mu_{t+1} m_t/(1-\prod_{i=1}^{t+1}\mu_i)\\[-1.ex]
            & \hspace{11mm} + (1-\mu_t) g_t /(1-\prod_{i=1}^{t} \mu_{i})                         \\
            &\hspace{5mm}\widehat{v_t} \leftarrow   v_t/\big(1-\beta_2^t \big)                   \\
            &\hspace{5mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/
                \big(\sqrt{\widehat{v_t}} + \epsilon \big)                                       \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Incorporating Nesterov Momentum into Adam`_.
    a  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 2e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        momentum_decay (float, optional): momentum momentum_decay (default: 4e-3)
        decoupled_weight_decay (bool, optional): whether to use decoupled weight
            decay as in AdamW to obtain NAdamW (default: False)
        z	
        zÏ

    .. _Incorporating Nesterov Momentum into Adam:
        https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ
    .. _Decoupled Weight Decay Regularization:
        https://arxiv.org/abs/1711.05101

    )r"   rN   rO   rP   rQ   rR   rU   rV   r#   r&   r'   r%   r(   r   r    r!   rS   c                C   sb  t | ƒD �]R\}}|s|| n||  }|| }|| }|| }|| }t |¡rzt |¡}t |¡}t |¡}t |¡}tj ¡ sÎ|rÎtƒ }|jj|jj  kr®|jjkr¾n n|jj|ksÎt	d|› d�ƒ‚|d7 }|rà|}nt
|ƒ}d||  }|	dk�r&|�r| d||	  ¡ n|j||	d�}|ddd||
     }|ddd|d |
     }||9 }| |d| ¡ | |¡j||d| d	� | |¡ ¡ }|�s¦|�r| |¡}|| }|| d|  d|   }|| | d|   }| ||¡ | ||¡ qt
|ƒ| }| |¡ |j||| d|  dt
|ƒ  d	� |j||| | d|  d	� qd S )
NzVIf capturable=True, params, mu_products and state_steps must be on supported devices: Ú.r   r   ©Úalphar*   ç      à?ç¸…ëQ¸î?)Úvalue)Ú	enumeraterA   rI   Zview_as_realÚ_utilsÚis_compilingr   r8   ÚtypeÚAssertionErrorr   Zmul_ÚaddZlerp_Zaddcmul_ÚdivÚsqrtZaddcdiv_Zadd_)r"   rN   rO   rP   rQ   rR   rU   rV   r#   r&   r'   r%   r(   r   r    r!   rS   ÚiÚparamrH   rF   rG   r:   Zstep_tÚcapturable_supported_devicesr5   Zbias_correction2ÚmuÚmu_nextÚdenomZmu_product_nextr3   r3   r4   Ú_single_tensor_nadam  sp    




 ÿÿ

þ
ü	


  ÿ  ÿrq   c          "         s„  t | ƒdkrd S |rtdƒ‚tj ¡ sb|rbtdd�‰t‡fdd„t| ||ƒD ƒƒsbtdˆ› d�ƒ‚t 	| |||||g¡}| 
¡ D �]ü\\}}}}}}}|r¨t||||ƒ |r¶t |¡}|d jrÜtj|tjd	d
d�d	d� nt |d¡ |	dk�r6|�rt |dˆ|	  ¡ n(|�r&tj|||	d� ntj|||	d�}t ||dˆ  ¡ t |ˆ¡ t |||dˆ ¡ t |¡}|�r t |ˆ¡}t d|¡}t |d¡ t |d	¡ t |ˆ ¡ t |ˆ¡ t d|¡}t |d¡ t |d	¡ t |ˆ ¡ ~t ˆ|¡}t |d	¡ t |¡ t |¡ n:‡fdd„|D ƒ}‡ ‡fdd„|D ƒ}‡ ‡fdd„|D ƒ}t ||¡ t ||¡ t ||¡ ~|�r&t |d	¡ t |ˆ¡ t |d	¡}t |¡ t ||¡ |}~t ||¡}t |ˆ¡ t |d	¡ t ||¡ |} ~t ||¡}!t |!| |¡ t ||!|¡ q€t‡fdd„t||ƒD ƒƒ}t‡fdd„t||ƒD ƒƒ} t ||||¡ t |||| ¡ q€d S )Nr   z#_foreach ops don't support autogradF)Zsupports_xlac                 3   sD   | ]<\}}}|j j|j j  ko*|j jkn  o:|j jˆ kV  qd S ©N)r8   rf   )Ú.0rE   Úmpr5   )rm   r3   r4   Ú	<genexpr>Ž  s   þ$z&_multi_tensor_nadam.<locals>.<genexpr>zWIf capturable=True, params, mu_products, and state_steps must be on supported devices: r]   r*   Úcpu)r8   r^   r   ra   g      à¿c                    s    g | ]}t d ˆ t|ƒ  ƒ‘qS )r   )r   r   ©rs   r5   )rV   r3   r4   Ú
<listcomp>ç  s   ÿz'_multi_tensor_nadam.<locals>.<listcomp>c                    s(   g | ] }ˆ d ddt |ƒˆ     ‘qS )r*   r`   ra   ©r   rw   ©rU   r'   r3   r4   rx   ë  s   ÿc                    s,   g | ]$}ˆ d ddt |ƒd ˆ     ‘qS )r*   r`   ra   r   ry   rw   rz   r3   r4   rx   ï  s   þÿc                    s0   g | ](\}}t ˆ ƒd |  d t |ƒ  d ‘qS ©r*   éÿÿÿÿry   )rs   r:   rn   ©r#   r3   r4   rx      s   ÿc                    s0   g | ](\}}t ˆ ƒ| d t |ƒ|   d ‘qS r{   ry   )rs   r:   ro   r}   r3   r4   rx   &  s   ûÿþû)r@   rg   rA   rd   re   r   ÚallÚzipr   Z"_group_tensors_by_device_and_dtypeÚvaluesr   Z_foreach_negZis_cpuZ_foreach_add_rC   Z_foreach_mul_Z_foreach_addZ_foreach_lerp_Z_foreach_addcmul_Z_foreach_sqrtZ_foreach_mulZ_foreach_powZ_foreach_sub_Z_foreach_neg_Z_foreach_sqrt_Z_foreach_div_Z_foreach_subZ_foreach_addcdiv_r   )"r"   rN   rO   rP   rQ   rR   rU   rV   r#   r&   r'   r%   r(   r   r    r!   rS   Zgrouped_tensorsZgrouped_paramsZgrouped_gradsZgrouped_exp_avgsZgrouped_exp_avg_sqsZgrouped_mu_productsZgrouped_state_stepsÚ_Zexp_avg_sq_sqrtÚexponentZmusZmu_nextsZbias_correction_sqrtrp   Zstep_size_gradsZstep_size_expavgÚ	numeratorr3   )rU   rV   rm   r#   r'   r4   Ú_multi_tensor_nadamp  s   ÿ
ý

üÿ
ù
   ÿ

  ÿ
  ÿ  ÿ   ÿ


þþý

þÿ
ùÿ   ÿ   ÿr„   )Zsingle_tensor_fnF)r"   rN   rO   rP   rQ   rR   r(   r   r    r!   rS   r   rU   rV   r#   r&   r'   r%   c                C   sª   t dd„ |D ƒƒstdƒ‚t dd„ |D ƒƒs4tdƒ‚|dkrNt| |	dd�\}}|rdtj ¡ rdtd	ƒ‚|rxtj ¡ sxt}nt}|| |||||||||||||||	|
d
� dS )zpFunctional API that performs NAdam algorithm computation.

    See :class:`~torch.optim.NAdam` for details.
    c                 s   s   | ]}t |tjƒV  qd S rr   ©Ú
isinstancerA   r   ©rs   Útr3   r3   r4   ru   V  s     znadam.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsc                 s   s   | ]}t |tjƒV  qd S rr   r…   r‡   r3   r3   r4   ru   [  s     zPAPI has changed, `mu_products` argument must contain a list of singleton tensorsNF)Z	use_fusedz6torch.jit.script not supported with foreach optimizers)rU   rV   r#   r&   r'   r   r(   r%   r    r!   rS   )r~   rK   r
   rA   ZjitZis_scriptingr„   rq   )r"   rN   rO   rP   rQ   rR   r(   r   r    r!   rS   r   rU   rV   r#   r&   r'   r%   r�   Úfuncr3   r3   r4   r   9  sL    ÿÿ  ÿ
ï)FNFFFF) Útypingr   r   r   r   r   rA   r   Z	optimizerr	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__rB   r[   rq   r„   r   r3   r3   r3   r4   Ú<module>   s¤   D 6'óòñðÙÿDî_î J
      òë