U
    8‹ÑhPO  ã                   @   s²  d dl mZmZ d dlZd dlmZ d dlmZ ddlmZm	Z	m
Z
mZmZmZmZmZmZmZ ddgZG d	d„ deƒZd
de
› de› de	› d� e_dee ee ee ee ee ee ee eee eeeeeeedœdd„Zdd„ Zee ee ee ee ee ee eeeeeeeedœdd„Zee ee ee ee ee ee eeeeeeeedœdd„Zee ee ee ee ee ee eeeeeeeeddœdd„ZdS )é    )ÚListÚOptionalN)ÚTensor)Ú$_get_fused_kernels_supported_devicesé   )
Ú_default_to_fused_or_foreachÚ_differentiable_docÚ_foreach_docÚ_get_scalar_dtypeÚ
_get_valueÚ_maximize_docÚ_use_grad_for_differentiableÚ_view_as_realÚ	OptimizerÚParamsTÚAdagradÚadagradc                       sr   e Zd Zdddddœeeeeeeee eeee dœ
‡ fdd	„Z‡ fd
d„Zdd„ Z	dd„ Z
eddd„ƒZ‡  ZS )r   ç{®Gáz„?r   ç»½×Ùß|Û=NF)ÚmaximizeÚdifferentiableÚfused)
ÚparamsÚlrÚlr_decayÚweight_decayÚinitial_accumulator_valueÚepsÚforeachr   r   r   c                   s„  d|kst d|› �ƒ‚d|ks,t d|› �ƒ‚d|ksBt d|› �ƒ‚d|ksXt d|› �ƒ‚d|ksnt d|› �ƒ‚t||||||||	|
d�	}tƒ  ||¡ |
rð|	r¦tdƒ‚d	| _tƒ ‰ ˆ  d
¡ t‡ fdd„| j	D ƒƒsätdˆ › d�ƒ‚|rðtdƒ‚| j	D ]ˆ}|d D ]z}| j
| }|d �r6tjdt|d d�|jd�ntjdtƒ d�|d< t |¡�r`t||ƒn|}tj||tjd�|d< �qqöd S )Ng        zInvalid learning rate: zInvalid lr_decay value: zInvalid weight_decay value: z)Invalid initial_accumulator_value value: zInvalid epsilon value: )	r   r   r   r   r   r   r   r   r   z)`fused` does not support `differentiable`TZcudac                 3   s2   | ]*}|d  D ]}|j jˆ ko&t |¡V  qqdS )r   N)ÚdeviceÚtypeÚtorchZis_floating_point)Ú.0ZpgÚp©Zfused_supported_devices© úA/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/optim/adagrad.pyÚ	<genexpr>G   s   
 þz#Adagrad.__init__.<locals>.<genexpr>zX`fused=True` requires all the params to be floating point Tensors of supported devices: Ú.z0`fused` and `foreach` cannot be `True` together.r   r   r%   ©Zis_fused)Údtyper   ©r*   Ústep)Zmemory_formatÚsum)Ú
ValueErrorÚdictÚsuperÚ__init__ÚRuntimeErrorZ_step_supports_amp_scalingr   ÚremoveÚallÚparam_groupsÚstater!   Úzerosr
   r   ÚtensorÚ
is_complexÚcomplexZ	full_likeZpreserve_format)Úselfr   r   r   r   r   r   r   r   r   r   ÚdefaultsÚgroupr#   r6   Z
init_value©Ú	__class__r$   r&   r1      st    ÿ÷
þ
ÿ

ûýùÿý  ÿzAdagrad.__init__c                    sª   t ƒ  |¡ d }| jD ]4}| dd ¡ | dd¡ | dd¡ | dd ¡}qt| j ¡ ƒ}t|ƒdkovt 	|d d ¡}|s¦|D ]$}tj
t|d ƒt|d�d	�|d< q€d S )
Nr   r   Fr   r   r   r,   r)   r+   )r0   Ú__setstate__r5   Ú
setdefaultÚlistr6   ÚvaluesÚlenr!   Z	is_tensorr8   Úfloatr
   )r;   r6   r   r=   Zstate_valuesZstep_is_tensorÚsr>   r%   r&   r@   h   s"    

ÿ
 ÿzAdagrad.__setstate__c                 C   s4   | j D ](}|d D ]}| j| }|d  ¡  qqd S )Nr   r-   )r5   r6   Zshare_memory_)r;   r=   r#   r6   r%   r%   r&   Úshare_memory}   s    

zAdagrad.share_memoryc           
      C   s~   d\}}|d D ]d}|j d k	r||j jO }|t |¡O }| |¡ | |j ¡ | j| }	| |	d ¡ | |	d ¡ q||fS )N)FFr   r-   r,   )ÚgradÚ	is_sparser!   r9   Úappendr6   )
r;   r=   Úparams_with_gradÚgradsÚ
state_sumsÚstate_stepsÚhas_sparse_gradÚhas_complexr#   r6   r%   r%   r&   Ú_init_groupƒ   s    


zAdagrad._init_groupc           
      C   s´   d}|dk	r&t  ¡ � |ƒ }W 5 Q R X | jD ]‚}g }g }g }g }|  |||||¡\}}	t|||||d |d |d |d ||d |d |d |	|d	 t| d
dƒt| ddƒd� q,|S )z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r   r   r   Ú
grad_scaleÚ	found_inf)r   r   r   r   rO   r   r   r   rP   r   rR   rS   )r!   Zenable_gradr5   rQ   r   Úgetattr)
r;   ÚclosureZlossr=   rK   rL   rM   rN   rO   rP   r%   r%   r&   r,   ‘   sF    

    ÿ

ðzAdagrad.step)r   r   r   r   r   N)N)Ú__name__Ú
__module__Ú__qualname__r   rE   r   Úboolr1   r@   rG   rQ   r   r,   Ú__classcell__r%   r%   r>   r&   r      s6         ø
ôôPa[  Implements Adagrad algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)}, \: f(\theta)
                \text{ (objective)}, \: \lambda \text{ (weight decay)},                          \\
            &\hspace{12mm}    \tau \text{ (initial accumulator value)}, \: \eta\text{ (lr decay)}\\
            &\textbf{initialize} :  state\_sum_0 \leftarrow \tau                          \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm} \tilde{\gamma}    \leftarrow \gamma / (1 +(t-1) \eta)                  \\
            &\hspace{5mm} \textbf{if} \: \lambda \neq 0                                          \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1}                             \\
            &\hspace{5mm}state\_sum_t  \leftarrow  state\_sum_{t-1} + g^2_t                      \\
            &\hspace{5mm}\theta_t \leftarrow
                \theta_{t-1}- \tilde{\gamma} \frac{g_t}{\sqrt{state\_sum_t}+\epsilon}            \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Adaptive Subgradient Methods for Online Learning
    and Stochastic Optimization`_.
    aH  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        lr_decay (float, optional): learning rate decay (default: 0)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        initial_accumulator_value (float, optional): initial value of the
            sum of squares of gradients (default: 0)
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-10)
        z	
        aÒ  
        fused (bool, optional): whether the fused implementation (CPU only) is used.
            Currently, `torch.float64`, `torch.float32`, `torch.float16`, and `torch.bfloat16`
            are supported. (default: None). Please note that the fused implementations does not
            support sparse or complex gradients.
    .. _Adaptive Subgradient Methods for Online Learning and Stochastic
        Optimization: http://jmlr.org/papers/v12/duchi11a.html

    F)r   rL   rM   rN   r   rR   rS   rO   r   r   rP   r   r   r   r   r   c                C   sÔ   t dd„ |D ƒƒstdƒ‚|dkr<|dkr<t| |	dd�\}}|dkrHd}|dkrTd}|rjtj ¡ rjtdƒ‚|r€tj ¡ r€tdƒ‚|r”tj ¡ s”t}n|r¨tj ¡ s¨t}nt}|| ||||||||||	|
||d	� dS )
ztFunctional API that performs Adagrad algorithm computation.

    See :class:`~torch.optim.Adagrad` for details.
    c                 s   s   | ]}t |tjƒV  qd S ©N)Ú
isinstancer!   r   )r"   Útr%   r%   r&   r'     s     zadagrad.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Z	use_fusedz6torch.jit.script not supported with foreach optimizersz4torch.jit.script not supported with fused optimizers©
r   r   r   r   rO   r   r   rP   rR   rS   )	r4   r2   r   r!   ZjitZis_scriptingÚ_fused_adagradÚ_multi_tensor_adagradÚ_single_tensor_adagrad)r   rL   rM   rN   r   rR   rS   rO   r   r   rP   r   r   r   r   r   Ú_Úfuncr%   r%   r&   r   ó   sN    ÿ  ÿ
òc                 C   s   |   ¡ }t |||¡S r[   )Úsizer!   Zsparse_coo_tensor)rH   Úgrad_indicesrC   rd   r%   r%   r&   Ú_make_sparse=  s    rf   )r   rL   rM   rN   rR   rS   r   r   r   r   rO   r   r   rP   c             	   C   s†  |d kr|d kst ‚t| |||ƒD �]\\}}}}|d7 }t|ƒ}|sH|n| }|dkrr|jrdtdƒ‚|j||d�}|d|d |   }|jrö| ¡ }| ¡ }| ¡ }| 	t
||| d¡ƒ¡ | |¡}| ¡  ¡  	|	¡}|j	t
|||| ƒ| d� q"t |¡}|�r$t |¡}t |¡}t |¡}|j||dd� |�rH| ¡ |	 }n| ¡  	|	¡}|j||| d� |r"t |¡}t |¡}q"d S )Nr   r   z;weight_decay option is not compatible with sparse gradients©Úalphaé   ©Úvalue)ÚAssertionErrorÚzipr   rI   r2   ÚaddZcoalesceZ_indicesZ_valuesZadd_rf   ÚpowZsparse_maskZsqrt_r!   r9   Zview_as_realZaddcmul_ÚsqrtZaddcdiv_Zview_as_complex)r   rL   rM   rN   rR   rS   r   r   r   r   rO   r   r   rP   ÚparamrH   Z	state_sumZstep_tr,   Zclrre   Zgrad_valuesÚstdZ
std_valuesr9   r%   r%   r&   ra   B  sH    ÿ
 ÿ




ra   c                   sœ  |rt dƒ‚|d kr|d ks t ‚t| ƒdkr0d S t | |||g¡}| ¡ D �]J\\}}}}}|
optdd„ |D ƒƒ}|rœt||||ˆ |ˆ|	d|||||d� qJ|r¬t|||ƒ |rºt 	|¡}|d j
ràtj|tjddd	�dd
� nt |d¡ |dk�r|�rtj|||d
� ntj|||d
�}‡ ‡fdd„|D ƒ}tj|||dd� t |¡}t ||	¡ |dk�sj|�r|t ||¡ |}nt ||¡}t |||¡ qJd S )Nz#_foreach ops don't support autogradr   c                 s   s   | ]}|j V  qd S r[   )rI   )r"   rH   r%   r%   r&   r'      s    z(_multi_tensor_adagrad.<locals>.<genexpr>Tr^   g      ð?Úcpu)r   rg   r   c                    s&   g | ]}ˆ  d t |ƒd  ˆ   ‘qS )r   )r   )r"   r,   ©r   r   r%   r&   Ú
<listcomp>Ò  s    z)_multi_tensor_adagrad.<locals>.<listcomp>rj   )rl   rD   r   Ú"_group_tensors_by_device_and_dtyperC   Úanyra   r   r!   Z_foreach_negZis_cpuÚ_foreach_add_r8   Z_foreach_addZ_foreach_addcmul_Z_foreach_sqrtZ_foreach_mul_Z_foreach_mulZ_foreach_addcdiv_)r   rL   rM   rN   rR   rS   r   r   r   r   rO   r   r   rP   Zgrouped_tensorlistsÚdevice_paramsÚdevice_gradsÚdevice_state_sumsÚdevice_state_stepsrb   Zdevice_has_sparse_gradZ	minus_clrrr   Ú	numeratorr%   rt   r&   r`     s‚    
ÿû
ÿò

  ÿ
  ÿÿ
r`   )r   rL   rM   rN   rR   rS   r   r   r   r   rO   r   r   rP   Úreturnc                C   s>  | sd S |
s|rt dƒ‚|r$t dƒ‚|d k	r6|j|ind }|d k	rL|j|ind }t | |||g¡}| ¡ D ]Î\\}}\\}}}}}d\}}|d k	r¼|d k	r¼||kr´|j|dd�||< || }|d k	rî|d k	rî||kræ|j|dd�||< || }t |d¡ tj||||||||	|||d� |d k	rjt 	||gt
|ƒ ¡ qjd S )Nz5`fused` does not support sparse grad or complex paramz<adagrad with fused=True does not support differentiable=True)NNT)Znon_blockingr   )r   r   r   r   r   rR   rS   )r2   r   r   rv   ÚitemsÚtor!   rx   Z_fused_adagrad_Z_foreach_sub_rD   )r   rL   rM   rN   rR   rS   r   r   r   r   rO   r   r   rP   Zgrad_scale_dictZfound_inf_dictZgrouped_tensorsr   rb   ry   rz   r{   r|   Zdevice_grad_scaleZdevice_found_infr%   r%   r&   r_   å  sd    ÿÿ
ÿøõ ÿr_   )NNNFNFF)Útypingr   r   r!   r   Ztorch.utils._foreach_utilsr   Z	optimizerr   r   r	   r
   r   r   r   r   r   r   Ú__all__r   Ú__doc__rY   rE   r   rf   ra   r`   r_   r%   r%   r%   r&   Ú<module>   s²   0 *õôóæÿ9       óíJñ>ñgð