U
    8‹ÑhRA  ã                   @   sn  d dl mZmZmZmZ d dlZd dlmZ ddlmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZ ddgZG dd„ deƒZd	d
e› de› de
› de› d�	 e_ee ee ee ee ee eeeeeeeeedœdd„Zee ee ee ee ee eeeeeeeeedœdd„Zeed�dee ee ee ee ee ee eeeeeeeeedœdd„ƒZdS )é    )ÚListÚOptionalÚTupleÚUnionN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_foreach_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ
_get_valueÚ_maximize_docÚ_use_grad_for_differentiableÚ_view_as_realÚ	OptimizerÚParamsTÚAdamaxÚadamaxc                       sl   e Zd Zdddddœeeeeef eeee eeedœ	‡ fd	d
„Z‡ fdd„Z	dd„ Z
eddd„ƒZ‡  ZS )r   çü©ñÒMb`?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>r   NF)ÚmaximizeÚdifferentiableÚ
capturable)	ÚparamsÚlrÚbetasÚepsÚweight_decayÚforeachr   r   r   c             
      sÄ   d|kst d|› �ƒ‚d|ks,t d|› �ƒ‚d|d   krDdk sXn t d|d › �ƒ‚d|d   krpdk s„n t d|d › �ƒ‚d|ksšt d	|› �ƒ‚t||||||||	d
�}
tƒ  ||
¡ d S )Nç        zInvalid learning rate: zInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: r   z#Invalid beta parameter at index 1: zInvalid weight_decay value: )r   r   r    r!   r"   r   r   r   )Ú
ValueErrorÚdictÚsuperÚ__init__)Úselfr   r   r   r    r!   r"   r   r   r   Údefaults©Ú	__class__© ú@/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/optim/adamax.pyr(      s*    ø
zAdamax.__init__c                    s¾   t ƒ  |¡ | jD ]¦}| dd ¡ | dd¡ | dd¡ | dd¡ |d D ]h}| j |g ¡}t|ƒdkrNt |d ¡sNt	|d ƒ}|d r¢tj
|tƒ |jd	�ntj
|tƒ d
�|d< qNqd S )Nr"   r   Fr   r   r   r   Ústep©ÚdtypeÚdevice©r1   )r'   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchZ	is_tensorÚfloatÚtensorr   r2   )r)   r7   ÚgroupÚpZp_stateZstep_valr+   r-   r.   r4   ?   s$    
ý  ÿûzAdamax.__setstate__c           
      C   sî   d}|d D ]Ü}|j d krq|t |¡O }| |¡ |j jrDtdƒ‚| |j ¡ | j| }	t|	ƒdkr¾|d r‚tjdt	ƒ |j
d�ntjdt	ƒ d	�|	d
< tj|tjd�|	d< tj|tjd�|	d< | |	d ¡ | |	d ¡ | |	d
 ¡ q|S )NFr   z(Adamax does not support sparse gradientsr   r   r-   r0   r#   r3   r/   )Zmemory_formatÚexp_avgÚexp_inf)Úgradr:   Ú
is_complexÚappendZ	is_sparseÚRuntimeErrorr7   r9   Úzerosr   r2   r<   Z
zeros_likeZpreserve_format)
r)   r=   Úparams_with_gradÚgradsÚexp_avgsÚexp_infsÚstate_stepsÚhas_complexr>   r7   r-   r-   r.   Ú_init_groupR   s6    


ÿý ÿ
 ÿ
zAdamax._init_groupc                 C   sÐ   |   ¡  d}|dk	r.t ¡ � |ƒ }W 5 Q R X | jD ]–}g }g }g }g }g }|d \}	}
|d }|d }|d }|d }|d }|d }|d	 }|  ||||||¡}t|||||||	|
|||||||d
� q4|S )z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r    r   r!   r"   r   r   r   )
r    Úbeta1Úbeta2r   r!   r"   r   r   r   rK   )Z _cuda_graph_capture_health_checkr:   Zenable_gradr5   rL   r   )r)   ÚclosureZlossr=   rF   rG   rH   rI   rJ   rM   rN   r    r   r!   r"   r   r   r   rK   r-   r-   r.   r/   u   sZ    

     ÿñzAdamax.step)r   r   r   r   N)N)Ú__name__Ú
__module__Ú__qualname__r   r;   r   r   Úboolr(   r4   rL   r   r/   Ú__classcell__r-   r-   r+   r.   r      s0        ù	õ
õ$#aÁ  Implements Adamax algorithm (a variant of Adam based on infinity norm).

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \beta_1, \beta_2
                \text{ (betas)},\theta_0 \text{ (params)},f(\theta) \text{ (objective)},
                \: \lambda \text{ (weight decay)},                                                \\
            &\hspace{13mm}    \epsilon \text{ (epsilon)}                                          \\
            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
                u_0 \leftarrow 0 \text{ ( infinity norm)}                                 \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm}if \: \lambda \neq 0                                                    \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda  \theta_{t-1}                            \\
            &\hspace{5mm}m_t      \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t               \\
            &\hspace{5mm}u_t      \leftarrow   \mathrm{max}(\beta_2 u_{t-1}, |g_{t}|+\epsilon)   \\
            &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \frac{\gamma m_t}{(1-\beta^t_1) u_t} \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Adam: A Method for Stochastic Optimization`_.
    a
  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 2e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        z	
        zd

    .. _Adam\: A Method for Stochastic Optimization:
        https://arxiv.org/abs/1412.6980

    )r   rG   rH   rI   rJ   r    rM   rN   r   r!   r   r   r   rK   c       	         C   s¬  t | ƒD �]œ\}}|| }|
s"|n| }|| }|| }|| }tj ¡ s€|r€tƒ }|jj|jjkrp|jj|ks€td|› d�ƒ‚|d7 }|	dkrž|j||	d�}t 	|¡rÐt 
|¡}t 
|¡}t 
|¡}t 
|¡}| |d| ¡ |�stj| |¡| ¡  |¡|d� nBt | |¡ d¡| ¡  |¡ d¡gd¡}| tj|ddd�¡ |�r||| d }| |¡ || }| ||¡ qd|t|ƒ  }|| }|j||| d	� qd S )
NúIIf capturable=True, params and state_steps must be on supported devices: Ú.r   r   ©Úalpha)ÚoutF)Zkeepdim)Úvalue)Ú	enumerater:   Ú_utilsÚis_compilingr   r2   ÚtypeÚAssertionErrorÚaddrB   Zview_as_realZlerp_ÚmaximumZmul_ÚabsZadd_ÚcatZ	unsqueezeZ
unsqueeze_Zcopy_ZamaxZdiv_Zaddcdiv_r   )r   rG   rH   rI   rJ   r    rM   rN   r   r!   r   r   r   rK   ÚiÚparamrA   r?   r@   Zstep_tÚcapturable_supported_devicesZnorm_bufZneg_bias_correctionÚdenomZbias_correctionZclrr-   r-   r.   Ú_single_tensor_adamaxÞ   sT    ÿ
þ
ý




ý"þ
rh   c       	            sð  |rt dƒ‚t| ƒdkrd S tj ¡ s`|r`tdd�‰t‡fdd„t| |ƒD ƒƒs`t dˆ› d�ƒ‚t 	| ||||g¡}| 
¡ D �]l\\}}}}}}|r¢t||||ƒ |
r°t |¡}|d jrÖtj|tjd	d
d�d	d� nt |d¡ |	dk�r|
�rtj|||	d� ntj|||	d�}t ||dˆ  ¡ t ||¡ |
�sN|	dk�rNt |¡}n
t |¡ t ||¡ t ||¡ |�r¶t ˆ |¡}t |d¡ t |ˆ¡ t ||¡}t |||¡ q|‡ fdd„|D ƒ}‡fdd„|D ƒ}t ||||¡ q|d S )Nz#_foreach ops don't support autogradr   F)Zsupports_xlac                 3   s.   | ]&\}}|j j|j jko$|j jˆ kV  qd S ©N)r2   r^   )Ú.0r>   r/   )rf   r-   r.   Ú	<genexpr>C  s   þz'_multi_tensor_adamax.<locals>.<genexpr>rU   rV   r$   Úcpu)r2   rW   r   c                    s   g | ]}d ˆ t |ƒ  ‘qS )r   ©r   )rj   r/   )rM   r-   r.   Ú
<listcomp>‰  s    z(_multi_tensor_adamax.<locals>.<listcomp>c                    s   g | ]}t ˆ ƒ| d  ‘qS )éÿÿÿÿrm   )rj   Zbc)r   r-   r.   rn   Œ  s     )r_   r9   r:   r\   r]   r   ÚallÚzipr   Z"_group_tensors_by_device_and_dtypeÚvaluesr   Z_foreach_negZis_cpuZ_foreach_add_r<   Z_foreach_addZ_foreach_lerp_Z_foreach_mul_Z_foreach_absZ_foreach_abs_Z_foreach_maximum_Z_foreach_powZ_foreach_sub_Z_foreach_div_Z_foreach_mulZ_foreach_addcdiv_)r   rG   rH   rI   rJ   r    rM   rN   r   r!   r   r   r   rK   Zgrouped_tensorsZgrouped_paramsZgrouped_gradsZgrouped_exp_avgsZgrouped_exp_infsZgrouped_state_stepsÚ_Zbias_correctionsrg   Z	step_sizer-   )rM   rf   r   r.   Ú_multi_tensor_adamax(  sŒ    ÿý

üÿ	ú
   ÿ

  ÿ
  ÿ

ÿ   ÿrt   )Zsingle_tensor_fnF)r   rG   rH   rI   rJ   r"   r   r   r   rK   r    rM   rN   r   r!   c
                C   s”   t j ¡ s$tdd„ |D ƒƒs$tdƒ‚|dkr>t| |dd�\}}|rTt j ¡ rTtdƒ‚|rht j ¡ sht}nt	}|| |||||
|||||||	|d� dS )	zrFunctional API that performs adamax algorithm computation.

    See :class:`~torch.optim.Adamax` for details.
    c                 s   s   | ]}t |tjƒV  qd S ri   )Ú
isinstancer:   r   )rj   Útr-   r-   r.   rk   ¬  s    zadamax.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Z	use_fusedz6torch.jit.script not supported with foreach optimizers)	r    rM   rN   r   r!   r   r   rK   r   )
r:   r\   r]   rp   rD   r	   ZjitZis_scriptingrt   rh   )r   rG   rH   rI   rJ   r"   r   r   r   rK   r    rM   rN   r   r!   rs   Úfuncr-   r-   r.   r   ’  sB    ÿÿ  ÿ
ò)NFFFF)Útypingr   r   r   r   r:   r   Z	optimizerr   r	   r
   r   r   r   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__r;   rS   rh   rt   r   r-   r-   r-   r.   Ú<module>   sŽ   < 
öõôóæÿ2ñKñj	     ôî