U
    8‹Ñh	w  ã                   @   sH  d dl mZmZmZmZmZ d dlZd dlmZ d dlm	Z	 ddl
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ ddgZG d	d„ deƒZd
de› de› de› de› de› d� e_ee ee ee ee ee ee ee ee ee e eee f e e eeeedœdd„Z!ee ee ee ee ee ee ee ee ee e eee f e e eeeedœdd„Z"ee ee ee ee ee ee ee ee ee e eee f e e eeeeddœdd„Z#ee!d�dee ee ee ee ee ee ee eeee ee ee eee e ee ef e e edœdd„ƒZ$dS )é    )ÚcastÚListÚOptionalÚTupleÚUnionN)ÚTensor)Ú$_get_fused_kernels_supported_devicesé   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_dispatch_sqrtÚ_foreach_docÚ
_fused_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ
_get_valueÚ_maximize_docÚ_stack_if_compilingÚ_use_grad_for_differentiableÚ_view_as_realÚ
DeviceDictÚ	OptimizerÚParamsTÚAdamWÚadamwc                       s€   e Zd Zdddddddœeeeef eeef eeeee	e eee	e dœ‡ fd	d
„Z
‡ fdd„Zdd„ Zeddd„ƒZ‡  ZS )r   çü©ñÒMbP?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>ç{®Gáz„?FN)ÚmaximizeÚforeachÚ
capturableÚdifferentiableÚfused)ÚparamsÚlrÚbetasÚepsÚweight_decayÚamsgradr!   r"   r#   r$   r%   c                   s8  d|kst d|› �ƒ‚t|tƒr0|r0|	s0t dƒ‚d|ksFt d|› �ƒ‚d|d   kr^dk srn t d|d › �ƒ‚d|d   krŠdk sžn t d	|d › �ƒ‚d|ks´t d
|› �ƒ‚t||||||||	|
|d�
}tƒ  ||¡ |�r4|
rðtdƒ‚d| _tƒ ‰ t	‡ fdd„| j
D ƒƒ�s&tdˆ › d�ƒ‚|�r4tdƒ‚d S )Nç        zInvalid learning rate: úElr as a Tensor is not supported for capturable=False and foreach=TruezInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: r	   z#Invalid beta parameter at index 1: zInvalid weight_decay value: )
r'   r(   r)   r*   r+   r"   r!   r#   r$   r%   z)`fused` does not support `differentiable`Tc                 3   s2   | ]*}|d  D ]}|j jˆ ko&t |¡V  qqdS )r&   N)ÚdeviceÚtypeÚtorchZis_floating_point)Ú.0ZpgÚp©Zfused_supported_devices© ú?/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/optim/adamw.pyÚ	<genexpr>S   s   
 þz!AdamW.__init__.<locals>.<genexpr>zX`fused=True` requires all the params to be floating point Tensors of supported devices: Ú.z0`fused` and `foreach` cannot be `True` together.)Ú
ValueErrorÚ
isinstancer   ÚdictÚsuperÚ__init__ÚRuntimeErrorZ_step_supports_amp_scalingr   ÚallÚparam_groups)Úselfr&   r'   r(   r)   r*   r+   r!   r"   r#   r$   r%   Údefaults©Ú	__class__r4   r6   r=      sP    ÿöþ

ÿzAdamW.__init__c                    sâ   t ƒ  |¡ | jD ]Ê}| dd¡ | dd¡ | dd ¡ | dd¡ | dd¡ | dd ¡}|d D ]t}| j |g ¡}t|ƒd	krft |d
 ¡sft	|d
 ƒ}|d s®|d rÆtj
|t|d�|jd�ntj
|tƒ d�|d
< qfqd S )Nr+   Fr!   r"   r#   r$   r%   r&   r   Ústep©Zis_fused©Údtyper/   ©rH   )r<   Ú__setstate__r@   Ú
setdefaultÚstateÚgetÚlenr1   Z	is_tensorÚfloatÚtensorr   r/   )rA   rL   Úgroupr%   r3   Zp_stateZstep_valrC   r5   r6   rJ   _   s,    
ûûýùzAdamW.__setstate__c	                 C   sz  d}	|d D �]f}
|
j d krq|	t |
¡O }	| |
¡ |
j jrFtdƒ‚| |
j ¡ | j|
 }t|ƒdkrè|d sx|d r”tjdt	|d d�|
j
d	�ntjd
t	ƒ d�|d< tj|
tjd�|d< tj|
tjd�|d< |rètj|
tjd�|d< | |d ¡ | |d ¡ |d �r| |d ¡ |d �r:|d j�r:tdƒ‚|d �rft|d tƒ�rf|d �sftdƒ‚| |d ¡ q|	S )NFr&   z'AdamW does not support sparse gradientsr   r#   r%   r5   rF   rG   r,   rI   rE   )Zmemory_formatÚexp_avgÚ
exp_avg_sqÚmax_exp_avg_sqr+   r$   zB`requires_grad` is not supported for `step` in differentiable moder"   r'   r-   )Úgradr1   Ú
is_complexÚappendZ	is_sparser>   rL   rN   Úzerosr   r/   rP   Z
zeros_likeZpreserve_formatZrequires_gradr:   r   )rA   rQ   Úparams_with_gradÚgradsr+   Úexp_avgsÚexp_avg_sqsÚmax_exp_avg_sqsÚstate_stepsÚhas_complexr3   rL   r5   r5   r6   Ú_init_groupv   sj    


	ûûýù
 ÿ
 ÿ
 ÿ

ÿÿþýÿzAdamW._init_groupc                 C   sð   |   ¡  d}|dk	r.t ¡ � |ƒ }W 5 Q R X | jD ]¶}g }g }g }g }g }g }	|d }
ttttf |d ƒ\}}|  ||||
||||	¡}t||||||	|
|||d |d |d |d |d |d	 |d
 |d t	| ddƒt	| ddƒ|d� q4|S )z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr+   r(   r'   r*   r)   r!   r"   r#   r$   r%   Ú
grad_scaleÚ	found_inf)r+   Úbeta1Úbeta2r'   r*   r)   r!   r"   r#   r$   r%   ra   rb   r_   )
Z _cuda_graph_capture_health_checkr1   Zenable_gradr@   r   r   rO   r`   r   Úgetattr)rA   ÚclosureZlossrQ   rY   rZ   r[   r\   r]   r^   r+   rc   rd   r_   r5   r5   r6   rE   ¿   s^    

ø

ìz
AdamW.step)r   r   r   r    F)N)Ú__name__Ú
__module__Ú__qualname__r   r   rO   r   r   Úboolr   r=   rJ   r`   r   rE   Ú__classcell__r5   r5   rC   r6   r      s8        ù	ó

ó@Ia  Implements AdamW algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{(lr)}, \: \beta_1, \beta_2
                \text{(betas)}, \: \theta_0 \text{(params)}, \: f(\theta) \text{(objective)},
                \: \epsilon \text{ (epsilon)}                                                    \\
            &\hspace{13mm}      \lambda \text{(weight decay)},  \: \textit{amsgrad},
                \: \textit{maximize}                                                             \\
            &\textbf{initialize} : m_0 \leftarrow 0 \text{ (first moment)}, v_0 \leftarrow 0
                \text{ ( second moment)}, \: \widehat{v_0}^{max}\leftarrow 0              \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\

            &\hspace{5mm}\textbf{if} \: \textit{maximize}:                                       \\
            &\hspace{10mm}g_t           \leftarrow   -\nabla_{\theta} f_t (\theta_{t-1})          \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm} \theta_t \leftarrow \theta_{t-1} - \gamma \lambda \theta_{t-1}         \\
            &\hspace{5mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
            &\hspace{5mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
            &\hspace{5mm}\widehat{m_t} \leftarrow   m_t/\big(1-\beta_1^t \big)                   \\
            &\hspace{5mm}\widehat{v_t} \leftarrow   v_t/\big(1-\beta_2^t \big)                   \\
            &\hspace{5mm}\textbf{if} \: amsgrad                                                  \\
            &\hspace{10mm}\widehat{v_t}^{max} \leftarrow \mathrm{max}(\widehat{v_t}^{max},
                \widehat{v_t})                                                                   \\
            &\hspace{10mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/
                \big(\sqrt{\widehat{v_t}^{max}} + \epsilon \big)                                 \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/
                \big(\sqrt{\widehat{v_t}} + \epsilon \big)                                       \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Decoupled Weight Decay Regularization`_.
    a�  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, Tensor, optional): learning rate (default: 1e-3). A tensor LR
            is not yet supported for all our implementations. Please use a float
            LR if you are not also specifying fused=True or capturable=True.
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay coefficient (default: 1e-2)
        amsgrad (bool, optional): whether to use the AMSGrad variant of this
            algorithm from the paper `On the Convergence of Adam and Beyond`_
            (default: False)
        z	
        z¿
    .. _Decoupled Weight Decay Regularization:
        https://arxiv.org/abs/1711.05101
    .. _On the Convergence of Adam and Beyond:
        https://openreview.net/forum?id=ryQu7f-RZ

    )r&   rZ   r[   r\   r]   r^   ra   rb   r+   rc   rd   r'   r*   r)   r!   r#   r$   r_   c       
   !      C   s¬  |d kr|d kst ‚tj ¡ r,t|tƒs,t ‚t| ƒD �]p\}}|sJ|| n||  }|| }|| }|| }tj ¡ s¬|r¬t	ƒ }|j
j|j
jkrœ|j
j|ks¬t d|› d�ƒ‚t |¡rôt |¡}t |¡}t |¡}|rêt || ¡||< t |¡}|d7 }| d||  ¡ | |d|	 ¡ | |
¡j||d|
 d� |�sD|�rø|}d|	|  }d|
|  }|| }| ¡ }| ¡ }|�rÐ|�r’||  ¡ }n|| }||  t ||¡¡ ||  ¡ ||   || ¡} n| ¡ ||   || ¡} | || ¡ nŠt|ƒ}d|	|  }d|
|  }|| }t|ƒ}|�r^tj|| ||| d� ||  ¡ |  |¡} n| ¡ |  |¡} |j|| | d� |r4t | | ¡r4t || ¡||< q4d S )NúIIf capturable=True, params and state_steps must be on supported devices: r8   r	   )Úvalue)Úout)ÚAssertionErrorr1   ÚjitÚis_scriptingr:   rO   Ú	enumerateÚ_utilsÚis_compilingr   r/   r0   rV   Zview_as_realZmul_Zlerp_Zaddcmul_ÚnegÚsqrtÚcloneZcopy_ÚmaximumZadd_Zaddcdiv_r   r   Zview_as_complex)!r&   rZ   r[   r\   r]   r^   ra   rb   r+   rc   rd   r'   r*   r)   r!   r#   r$   r_   ÚiÚparamrU   rR   rS   Zstep_tÚcapturable_supported_devicesrE   Úbias_correction1Úbias_correction2Ú	step_sizeZstep_size_negÚbias_correction2_sqrtrT   Zdenomr5   r5   r6   Ú_single_tensor_adamwB  sx    
ÿ
þ
ý




ÿþÿþr€   c       
            sÜ  t | ƒdkrd S tˆtƒr&|s&tdƒ‚tj ¡ sj|rjtdd�‰t‡fdd„t	| |ƒD ƒƒsjt
dˆ› d�ƒ‚|rvt
d	ƒ‚|d kr†|d ksŠt
‚t | |||||g¡}| ¡ D �],\\}}}}}}}|ræ|rØt|||||ƒ nt||||ƒ |rôt |¡}|d j�rtj|tjd
dd�d
d� nt |d¡ |dk�rFt |dˆ|  ¡ t ||dˆ  ¡ t |ˆ¡ t |||dˆ ¡ ~|�r>t ˆ |¡}t ˆ|¡}t |d¡ t |d¡ t |¡ t |ˆ¡ t |¡ t |¡ |}|}|�r t ||¡ t |¡}n
t |¡}t ||¡ t ||¡ t ||¡ t |||¡ q¨‡ fdd„|D ƒ}‡fdd„|D ƒ}t‡fdd„|D ƒƒ}dd„ |D ƒ}|�r¤t ||¡ t |¡}n
t |¡}t ||¡ t ||¡ t ||||¡ q¨d S )Nr   r-   F)Zsupports_xlac                 3   s.   | ]&\}}|j j|j jko$|j jˆ kV  qd S ©N)r/   r0   )r2   r3   rE   )r{   r5   r6   r7   Û  s   þz&_multi_tensor_adamw.<locals>.<genexpr>rl   r8   z#_foreach ops don't support autogradr.   Úcpu)r/   )Úalphar	   c                    s   g | ]}d ˆ t |ƒ  ‘qS ©r	   ©r   ©r2   rE   )rc   r5   r6   Ú
<listcomp>D  s    z'_multi_tensor_adamw.<locals>.<listcomp>c                    s   g | ]}d ˆ t |ƒ  ‘qS r„   r…   r†   )rd   r5   r6   r‡   G  s    c                    s   g | ]}ˆ | d  ‘qS )éÿÿÿÿr5   ©r2   Zbc)r'   r5   r6   r‡   K  s     c                 S   s   g | ]}t |ƒ‘qS r5   )r   r‰   r5   r5   r6   r‡   M  s    ) rN   r:   r   r>   r1   rs   rt   r   r?   Úzipro   r   Ú"_group_tensors_by_device_and_dtypeÚvaluesr   Z_foreach_negZis_cpuÚ_foreach_add_rP   Z_foreach_mul_Z_foreach_lerp_Z_foreach_addcmul_Z_foreach_powÚ_foreach_sub_Z_foreach_neg_Z_foreach_div_Z_foreach_reciprocal_Z_foreach_sqrt_Z_foreach_maximum_Z_foreach_sqrtZ_foreach_addcdiv_r   )r&   rZ   r[   r\   r]   r^   ra   rb   r+   rc   rd   r'   r*   r)   r!   r#   r$   r_   Úgrouped_tensorsÚdevice_paramsÚdevice_gradsÚdevice_exp_avgsÚdevice_exp_avg_sqsÚdevice_max_exp_avg_sqsÚdevice_state_stepsÚ_r|   r}   r~   r   Zexp_avg_sq_sqrtr5   )rc   rd   r{   r'   r6   Ú_multi_tensor_adamw¹  sÒ    ÿÿý

üÿ
ù
û   ÿ
  ÿ
   ÿ




ÿ
ÿÿ
ür—   )r&   rZ   r[   r\   r]   r^   ra   rb   r+   rc   rd   r'   r*   r)   r!   r#   r$   r_   Úreturnc       
          C   sf  | sd S |rt dƒ‚|d k	r&|j|ini }|d k	r<|j|ini }t|tƒrbt|jƒdkrb|j|ind }t | |||||g¡}| ¡ D ]Ü\\}}\\}}}}}}}d\}}|d k	rÆ| ||j	|dd�¡}|d k	rä| ||j	|dd�¡}|d k	�r||k�r| ||j	|dd�¡}t
 |d¡ t
j|||||||||	|
|||||d� |d k	r„t
 ||gt|ƒ ¡ q„d S )	Nz9Adam with fused=True does not support differentiable=Truer‚   )NNT)Únon_blocking)r/   r™   r	   )	r+   r'   rc   rd   r*   r)   r!   ra   rb   )r>   r/   r:   r   Ústrr   r‹   ÚitemsrK   Útor1   r�   Z_fused_adamw_rŽ   rN   ) r&   rZ   r[   r\   r]   r^   ra   rb   r+   rc   rd   r'   r*   r)   r!   r#   r$   r_   Zgrad_scale_dictZfound_inf_dictZlr_dictr�   r/   r–   r�   r‘   r’   r“   r”   r•   Zdevice_grad_scaleZdevice_found_infr5   r5   r6   Ú_fused_adamwd  s|    ÿÿ$ÿÿö ÿ ÿ ÿñ ÿr�   )Zsingle_tensor_fnF)r&   rZ   r[   r\   r]   r^   r"   r#   r$   r%   ra   rb   r_   r+   rc   rd   r'   r*   r)   r!   c                C   sü   t j ¡ s$tdd„ |D ƒƒs$tdƒ‚|	dkr\|dkr\t| |dd�\}}|r\t|tƒr\|s\d}|	dkrhd}	|dkrtd}|rŠt j 	¡ rŠtdƒ‚|	r t j 	¡ r tdƒ‚|	r´t j 	¡ s´t
}n|rÈt j 	¡ sÈt}nt}|| |||||||||||||||
||d	� dS )
zpFunctional API that performs AdamW algorithm computation.

    See :class:`~torch.optim.AdamW` for details.
    c                 s   s   | ]}t |tjƒV  qd S r�   )r:   r1   r   )r2   Útr5   r5   r6   r7   Ü  s    zadamw.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Z	use_fusedz6torch.jit.script not supported with foreach optimizersz4torch.jit.script not supported with fused optimizers)r+   rc   rd   r'   r*   r)   r!   r#   r$   ra   rb   r_   )r1   rs   rt   r?   r>   r   r:   r   rp   rq   r�   r—   r€   )r&   rZ   r[   r\   r]   r^   r"   r#   r$   r%   ra   rb   r_   r+   rc   rd   r'   r*   r)   r!   r–   Úfuncr5   r5   r6   r   ¾  s^    ÿÿ  ÿ
î)NFFNNNF)%Útypingr   r   r   r   r   r1   r   Ztorch.utils._foreach_utilsr   Z	optimizerr
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__rj   rO   r€   r—   r�   r   r5   r5   r5   r6   Ú<module>   sÜ   L a'ñðïîíÙÿF
íx
í -
ìZ
       ñ
é