U
    8‹Ñh8K  ã                   @   sz  d dl mZmZ d dlZd dlmZ ddlmZmZmZm	Z	m
Z
mZmZmZmZmZmZmZ ddgZG dd„ deƒZd	d
e
› de› de› de› d�	 e_ee ee ee ee ee ee eeeeeeeeeedœdd„Zee ee ee ee ee ee eeeeeeeeeedœdd„Ze	ed�dee ee ee ee ee ee ee eeeeeeeeeedœdd„ƒZdS )é    )ÚListÚOptionalN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_foreach_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ_maximize_docÚ_use_grad_for_differentiableÚ_view_as_realÚ	OptimizerÚParamsTÚRMSpropÚrmspropc                       sZ   e Zd Zdeeeeeeee eedœ	‡ fdd	„Z‡ fd
d„Zdd„ Z	e
ddd„ƒZ‡  ZS )r   ç{®Gáz„?ç®Gáz®ï?ç:Œ0âŽyE>r   FN)	ÚparamsÚlrÚalphaÚepsÚweight_decayÚmomentumÚforeachÚmaximizeÚdifferentiablec                    sœ   d|kst d|› �ƒ‚d|ks,t d|› �ƒ‚d|ksBt d|› �ƒ‚d|ksXt d|› �ƒ‚d|ksnt d|› �ƒ‚t||||||||	|
|d�
}tƒ  ||¡ d S )Ng        zInvalid learning rate: zInvalid epsilon value: zInvalid momentum value: zInvalid weight_decay value: zInvalid alpha value: )
r   r   r   r   Úcenteredr   Ú
capturabler   r   r   )Ú
ValueErrorÚdictÚsuperÚ__init__)Úselfr   r   r   r   r   r   r    r!   r   r   r   Údefaults©Ú	__class__© úA/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/optim/rmsprop.pyr%      s.    özRMSprop.__init__c                    sÖ   t ƒ  |¡ | jD ]¾}| dd¡ | dd¡ | dd ¡ | dd¡ | dd¡ | dd¡ |d	 D ]h}| j |g ¡}t|ƒdkrft |d
 ¡sft	|d
 ƒ}|d rºtj
|tƒ |jd�ntj
|tƒ d�|d
< qfqd S )Nr   r   r    Fr   r   r   r!   r   Ústep©ÚdtypeÚdevice©r.   )r$   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchZ	is_tensorÚfloatÚtensorr   r/   )r&   r4   ÚgroupÚpZp_stateZstep_valr(   r*   r+   r1   @   s(    
ý  ÿûzRMSprop.__setstate__c                 C   s<  d}|d D �](}	|	j d krq|t |	¡O }| |	¡ |	j jrFtdƒ‚| |	j ¡ | j|	 }
t|
ƒdkrè|d r„tjdt	ƒ |	j
d�ntjdt	ƒ d�|
d	< tj|	tjd
�|
d< |d dkrÌtj|	tjd
�|
d< |d rètj|	tjd
�|
d< | |
d ¡ | |
d	 ¡ |d dk�r | |
d ¡ |d r| |
d ¡ q|S )NFr   z)RMSprop does not support sparse gradientsr   r!   r*   r-   r0   r,   )Zmemory_formatÚ
square_avgr   Zmomentum_bufferr    Úgrad_avg)Úgradr7   Ú
is_complexÚappendZ	is_sparseÚRuntimeErrorr4   r6   Úzerosr   r/   Z
zeros_likeZpreserve_format)r&   r:   Úparams_with_gradÚgradsÚsquare_avgsÚmomentum_buffer_listÚ	grad_avgsÚstate_stepsÚhas_complexr;   r4   r*   r*   r+   Ú_init_groupU   sH    



ÿý ÿ
 ÿ
 ÿ
zRMSprop._init_groupc                 C   s¾   |   ¡  d}|dk	r.t ¡ � |ƒ }W 5 Q R X | jD ]„}g }g }g }g }g }g }	|  |||||||	¡}
t||||||	|d |d |d |d |d |d |d |d	 |d
 |d |
d� q4|S )z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r    r   r   r   r!   )r   r   r   r   r   r    r   r   r   r!   rI   )Z _cuda_graph_capture_health_checkr7   Zenable_gradr2   rJ   r   )r&   ÚclosureZlossr:   rC   rD   rE   rG   rF   rH   rI   r*   r*   r+   r,   ˆ   sR    

ù
ïzRMSprop.step)
r   r   r   r   r   FFNFF)N)Ú__name__Ú
__module__Ú__qualname__r   r8   r   Úboolr%   r1   rJ   r   r,   Ú__classcell__r*   r*   r(   r+   r      s2             ôô'3a  Implements RMSprop algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \alpha \text{ (alpha)},\: \gamma \text{ (lr)},
                \: \theta_0 \text{ (params)}, \: f(\theta) \text{ (objective)}                   \\
            &\hspace{13mm}   \lambda \text{ (weight decay)},\: \mu \text{ (momentum)},\: centered\\
            &\textbf{initialize} : v_0 \leftarrow 0 \text{ (square average)}, \:
                \textbf{b}_0 \leftarrow 0 \text{ (buffer)}, \: g^{ave}_0 \leftarrow 0     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm}if \: \lambda \neq 0                                                    \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda  \theta_{t-1}                            \\
            &\hspace{5mm}v_t           \leftarrow   \alpha v_{t-1} + (1 - \alpha) g^2_t
                \hspace{8mm}                                                                     \\
            &\hspace{5mm} \tilde{v_t} \leftarrow v_t                                             \\
            &\hspace{5mm}if \: centered                                                          \\
            &\hspace{10mm} g^{ave}_t \leftarrow g^{ave}_{t-1} \alpha + (1-\alpha) g_t            \\
            &\hspace{10mm} \tilde{v_t} \leftarrow \tilde{v_t} -  \big(g^{ave}_{t} \big)^2        \\
            &\hspace{5mm}if \: \mu > 0                                                           \\
            &\hspace{10mm} \textbf{b}_t\leftarrow \mu \textbf{b}_{t-1} +
                g_t/ \big(\sqrt{\tilde{v_t}} +  \epsilon \big)                                   \\
            &\hspace{10mm} \theta_t \leftarrow \theta_{t-1} - \gamma \textbf{b}_t                \\
            &\hspace{5mm} else                                                                   \\
            &\hspace{10mm}\theta_t      \leftarrow   \theta_{t-1} -
                \gamma  g_t/ \big(\sqrt{\tilde{v_t}} + \epsilon \big)  \hspace{3mm}              \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to
    `lecture notes <https://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf>`_ by G. Hinton.
    and centered version `Generating Sequences
    With Recurrent Neural Networks <https://arxiv.org/pdf/1308.0850v5.pdf>`_.
    The implementation here takes the square root of the gradient average before
    adding epsilon (note that TensorFlow interchanges these two operations). The effective
    learning rate is thus :math:`\gamma/(\sqrt{v} + \epsilon)` where :math:`\gamma`
    is the scheduled learning rate and :math:`v` is the weighted moving average
    of the squared gradient.
    aŸ  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        momentum (float, optional): momentum factor (default: 0)
        alpha (float, optional): smoothing constant (default: 0.99)
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        centered (bool, optional) : if ``True``, compute the centered RMSProp,
            the gradient is normalized by an estimation of its variance
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        z	
        z

    )r   rD   rE   rG   rF   rH   r   r   r   r   r   r    r   r   r!   rI   c       
         C   s   t | ƒD �]�\}}|| }tj ¡ sZ|rZtƒ }|jj|jjkrJ|jj|ksZtd|› d�ƒ‚|| }|sj|n| }|| }|d7 }|	dkr–|j||	d�}t 	|¡}|rÂt 
|¡}t 
|¡}t 
|¡}| |¡j||d| d� |�r|| }|røt 
|¡}| |d| ¡ |j||dd� ¡ }n| ¡ }|�r8| |¡}n
| |¡}|
dk�rˆ|| }|�rdt 
|¡}| |
¡ ||¡ |j|| d� q|j||| d� qd S )NúIIf capturable=True, params and state_steps must be on supported devices: Ú.r   r   ©r   ©Úvalueéÿÿÿÿ)Ú	enumerater7   Ú_utilsÚis_compilingr   r/   ÚtypeÚAssertionErrorÚaddr?   Zview_as_realZmul_Zaddcmul_Zlerp_ZaddcmulZsqrt_ÚsqrtZadd_Zaddcdiv_)r   rD   rE   rG   rF   rH   r   r   r   r   r   r    r   r   r!   rI   ÚiÚparamr,   Úcapturable_supported_devicesr>   r<   Zis_complex_paramr=   ÚavgÚbufr*   r*   r+   Ú_single_tensor_rmsprop  sN    ÿ
þ
ý







rc   c       
            sj  t | ƒdkrd S |rtdƒ‚tj ¡ s\|r\tƒ ‰ t‡ fdd„t| |ƒD ƒƒs\tdˆ › d�ƒ‚t 	| |||||g¡}| 
¡ D �]è\\}}}}}}}|rÊ||g}|
dkr®| |¡ |r¼| |¡ t|f|žŽ  |rØt |¡}|d jrþtj|tjddd	�dd
� nt |d¡ |	dk�r<|�r,tj|||	d
� ntj|||	d
�}t ||¡ tj|||d| d� |�r t ||d| ¡ tj|||dd�}t |¡ t ||¡ nt |¡}t ||¡ |
dk�rt ||
¡ t |||¡ |�r
t|tjƒ�r
t || ¡}t ||¡ ntj||| d
� qz|�rPt|tjƒ�rPt || ¡ t |||¡ qztj|||| d� qzd S )Nr   z#_foreach ops don't support autogradc                 3   s.   | ]&\}}|j j|j jko$|j jˆ kV  qd S ©N)r/   rZ   )Ú.0r;   r,   ©r`   r*   r+   Ú	<genexpr>c  s   þz(_multi_tensor_rmsprop.<locals>.<genexpr>rQ   rR   g      ð?Úcpu)r/   rS   r   rT   rV   )r6   r[   r7   rX   rY   r   ÚallÚzipr   Z"_group_tensors_by_device_and_dtypeÚvaluesr@   r   Z_foreach_negZis_cpuZ_foreach_add_r9   Z_foreach_addZ_foreach_mul_Z_foreach_addcmul_Z_foreach_lerp_Z_foreach_addcmulZ_foreach_sqrt_Z_foreach_sqrtZ_foreach_addcdiv_Ú
isinstancer   Z_foreach_mulZ_foreach_div_)r   rD   rE   rG   rF   rH   r   r   r   r   r   r    r   r   r!   rI   Zgrouped_tensorsZgrouped_paramsZgrouped_gradsZgrouped_square_avgsZgrouped_grad_avgsZgrouped_momentum_buffer_listZgrouped_state_stepsÚ_Zstate_and_gradsra   Zmomentum_lrr*   rf   r+   Ú_multi_tensor_rmspropH  s¢    ý

üÿ÷



  ÿ
  ÿ   ÿ   ÿ


  ÿrn   )Zsingle_tensor_fnF)r   rD   rE   rG   rF   rH   r   r   r   r!   rI   r   r   r   r   r   r    c                C   s˜   t j ¡ s$tdd„ |D ƒƒs$tdƒ‚|dkr>t| |dd�\}}|rTt j ¡ rTtdƒ‚|rht j ¡ sht}nt	}|| |||||||||||||	||
d� dS )	zsFunctional API that performs rmsprop algorithm computation.
    See :class:`~torch.optim.RMSProp` for details.
    c                 s   s   | ]}t |tjƒV  qd S rd   )rl   r7   r   )re   Útr*   r*   r+   rg   ×  s    zrmsprop.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Z	use_fusedz6torch.jit.script not supported with foreach optimizers)
r   r   r   r   r   r    r   r!   r   rI   )
r7   rX   rY   ri   rA   r   ZjitZis_scriptingrn   rc   )r   rD   rE   rG   rF   rH   r   r   r   r!   rI   r   r   r   r   r   r    rm   Úfuncr*   r*   r+   r   »  sF    ÿÿ  ÿ
ð)NFFFF)Útypingr   r   r7   r   Z	optimizerr   r   r   r	   r
   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__r8   rO   rc   rn   r   r*   r*   r*   r+   Ú<module>   sš   8 *+ôóòñÕÿBïHïs
     óì