U
    8‹Ñh»>  ã                   @   sd  d dl mZmZmZmZ d dlZd dlmZ ddlmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZ ddgZG dd„ deƒZd	d
e› de› de› de
› d�	 e_ee ee ee ee ee eeeeeeeedœdd„Zee ee ee ee ee eeeeeeeedœdd„Zeed�dee ee ee ee ee eee eeeeeeedœdd„ƒZdS )é    )ÚAnyÚDictÚListÚOptionalN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_foreach_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ_maximize_docÚ_use_grad_for_differentiableÚ_view_as_realÚ	OptimizerÚParamsTÚAdadeltaÚadadeltac                       s�   e Zd Zdddddœeeeeeee eeedœ	‡ fd	d
„Z‡ fdd„Ze	e
ef ee ee ee ee ee dœdd„Zeddd„ƒZ‡  ZS )r   ç      ð?çÍÌÌÌÌÌì?ç�íµ ÷Æ°>r   NF)Ú
capturableÚmaximizeÚdifferentiable)	ÚparamsÚlrÚrhoÚepsÚweight_decayÚforeachr   r   r   c             
      s�   d|kst d|› �ƒ‚d|  kr*dks:n t d|› �ƒ‚d|ksPt d|› �ƒ‚d|ksft d|› �ƒ‚t||||||||	d�}
tƒ  ||
¡ d S )Ng        zInvalid learning rate: r   zInvalid rho value: zInvalid epsilon value: zInvalid weight_decay value: )r   r   r   r    r   r   r!   r   )Ú
ValueErrorÚdictÚsuperÚ__init__)Úselfr   r   r   r   r    r!   r   r   r   Údefaults©Ú	__class__© úB/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/optim/adadelta.pyr%      s&    ø
zAdadelta.__init__c                    s¾   t ƒ  |¡ | jD ]¦}| dd ¡ | dd¡ | dd¡ | dd¡ |d D ]h}| j |g ¡}t|ƒdkrNt |d ¡sNt	|d ƒ}|d r¢tj
|tƒ |jd	�ntj
|tƒ d
�|d< qNqd S )Nr!   r   Fr   r   r   r   Ústep©ÚdtypeÚdevice©r.   )r$   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchZ	is_tensorÚfloatÚtensorr   r/   )r&   r4   ÚgroupÚpZp_stateZstep_valr(   r*   r+   r1   <   s$    
ý  ÿûzAdadelta.__setstate__)r:   Úparams_with_gradÚgradsÚsquare_avgsÚ
acc_deltasÚstate_stepsc           
      C   sî   d}|d D ]Ü}|j d krq|t |¡O }| |¡ |j jrDtdƒ‚| |j ¡ | j| }	t|	ƒdkr¾|d r‚tjdt	ƒ |j
d�ntjdt	ƒ d�|	d	< tj|tjd
�|	d< tj|tjd
�|	d< | |	d ¡ | |	d ¡ | |	d	 ¡ q|S )NFr   z*Adadelta does not support sparse gradientsr   r   r*   r-   r0   r,   )Zmemory_formatÚ
square_avgÚ	acc_delta)Úgradr7   Ú
is_complexÚappendZ	is_sparseÚRuntimeErrorr4   r6   Úzerosr   r/   Z
zeros_likeZpreserve_format)
r&   r:   r<   r=   r>   r?   r@   Úhas_complexr;   r4   r*   r*   r+   Ú_init_groupO   s6    	


ÿý ÿ
 ÿ
zAdadelta._init_groupc                 C   sÎ   |   ¡  d}|dk	r.t ¡ � |ƒ }W 5 Q R X | jD ]”}g }g }g }g }g }|d |d |d |d |d |d |d |d	 f\}	}
}}}}}}|  ||||||¡}t||||||	|
|||||||d
� q4|S )z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r    r!   r   r   r   )	r   r   r   r    r!   r   r   r   rH   )Z _cuda_graph_capture_health_checkr7   Zenable_gradr2   rI   r   )r&   ÚclosureZlossr:   r<   r=   r>   r?   r@   r   r   r   r    r!   r   r   r   rH   r*   r*   r+   r,   z   sl    

ø÷     ÿòzAdadelta.step)r   r   r   r   N)N)Ú__name__Ú
__module__Ú__qualname__r   r8   r   Úboolr%   r1   r   Ústrr   r   r   rI   r   r,   Ú__classcell__r*   r*   r(   r+   r      s<        ù	õõ"
ù+a  Implements Adadelta algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)},
                \: f(\theta) \text{ (objective)}, \: \rho \text{ (decay)},
                \: \lambda \text{ (weight decay)}                                                \\
            &\textbf{initialize} :  v_0  \leftarrow 0 \: \text{ (square avg)},
                \: u_0 \leftarrow 0 \: \text{ (accumulate variables)}                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm}if \: \lambda \neq 0                                                    \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda  \theta_{t-1}                            \\
            &\hspace{5mm} v_t      \leftarrow v_{t-1} \rho + g^2_t (1 - \rho)                    \\
            &\hspace{5mm}\Delta x_t    \leftarrow   \frac{\sqrt{u_{t-1} +
                \epsilon }}{ \sqrt{v_t + \epsilon}  }g_t \hspace{21mm}                           \\
            &\hspace{5mm} u_t  \leftarrow   u_{t-1}  \rho +
                 \Delta x^2_t  (1 - \rho)                                                        \\
            &\hspace{5mm}\theta_t      \leftarrow   \theta_{t-1} - \gamma  \Delta x_t            \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `ADADELTA: An Adaptive Learning Rate Method`_.
    aá  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        rho (float, optional): coefficient used for computing a running average
            of squared gradients (default: 0.9). A higher value of `rho` will
            result in a slower average, which can be helpful for preventing
            oscillations in the learning process.
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-6).
        lr (float, optional): coefficient that scale delta before it is applied
            to the parameters (default: 1.0)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        z	
        zd

    .. _ADADELTA\: An Adaptive Learning Rate Method:
        https://arxiv.org/abs/1212.5701

    )r   r=   r>   r?   r@   r   r   r   r    r   r   r   rH   c                   sN  t j ¡ sD|rDtdd�‰ t‡ fdd„t| |ƒD ƒƒsDtdˆ › d�ƒ‚t| ||||ƒD ]ô\}}}}}|d7 }|	sr|n| }|dkrŽ|j||d	�}t  |¡r¶t  	|¡}t  	|¡}t  	|¡}| 
|¡j||d| d
� | |¡ ¡ }| |¡ ¡ }|
rø| ¡ }| |¡ 
|¡ | 
|¡j||d| d
� t  |¡�r8t  |¡}|j|| d	� qTd S )NF©Zsupports_xlac                 3   s.   | ]&\}}|j j|j jko$|j jˆ kV  qd S ©N©r/   Útype©Ú.0r;   r,   ©Zcapturable_supported_devicesr*   r+   Ú	<genexpr>  s   þz*_single_tensor_adadelta.<locals>.<genexpr>úIIf capturable=True, params and state_steps must be on supported devices: Ú.r   r   ©Úalpha©Úvalue)r7   Ú_utilsÚis_compilingr   ÚallÚzipÚAssertionErrorÚaddrD   Zview_as_realZmul_Zaddcmul_Zsqrt_ÚcloneZdiv_Zview_as_complexZadd_)r   r=   r>   r?   r@   r   r   r   r    r   r   r   rH   ÚparamrC   rA   rB   r,   ÚstdÚdeltar*   rW   r+   Ú_single_tensor_adadeltañ   sD    ÿý

ü    ÿ




ri   c                   sä  |
rt dƒ‚tj ¡ sP|rPtdd�‰ t‡ fdd„t| |ƒD ƒƒsPt dˆ › d�ƒ‚t| ƒdkr`d S t 	| ||||g¡}| 
¡ D �]`\\}}}}}}|r¢t||||ƒ |d jrÈtj|tjd	d
d�d	d� nt |d¡ |	rât |¡}|dk�r|	�rtj|||d� ntj|||d�}t ||¡ tj|||d| d� t ||¡}t |¡ t ||¡}t |¡ t ||¡ t ||¡ t ||¡ tj|||d| d� |�rÌt|tjƒ�rÌt || ¡ t ||¡ q|tj||| d� q|d S )Nz#_foreach ops don't support autogradFrQ   c                 3   s.   | ]&\}}|j j|j jko$|j jˆ kV  qd S rR   rS   rU   rW   r*   r+   rX   >  s   þz)_multi_tensor_adadelta.<locals>.<genexpr>rY   rZ   r   r   Úcpu)r/   r[   r   r]   )rc   r7   r_   r`   r   ra   rb   r6   r   Z"_group_tensors_by_device_and_dtypeÚvaluesr   Zis_cpuZ_foreach_add_r9   Z_foreach_negZ_foreach_addZ_foreach_mul_Z_foreach_addcmul_Z_foreach_sqrt_Z_foreach_div_Ú
isinstancer   )r   r=   r>   r?   r@   r   r   r   r    r   r   r   rH   Zgrouped_tensorsZdevice_paramsZdevice_gradsZdevice_square_avgsZdevice_acc_deltasZdevice_state_stepsÚ_rg   Zdeltasr*   rW   r+   Ú_multi_tensor_adadelta'  s„    ÿý

üÿ	ú
   ÿ
  ÿ

  ÿ   ÿ

rn   )Zsingle_tensor_fnF)r   r=   r>   r?   r@   r   r!   r   rH   r   r   r   r    r   c	                C   s’   t j ¡ s$tdd„ |D ƒƒs$tdƒ‚|dkr>t| |dd�\}}|rTt j ¡ rTtdƒ‚|rht j ¡ sht}nt	}|| |||||	|
||||||d� dS )	zvFunctional API that performs Adadelta algorithm computation.

    See :class:`~torch.optim.Adadelta` for details.
    c                 s   s   | ]}t |tjƒV  qd S rR   )rl   r7   r   )rV   Útr*   r*   r+   rX   ¡  s    zadadelta.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Z	use_fusedz6torch.jit.script not supported with foreach optimizers)r   r   r   r    r   r   r   rH   )
r7   r_   r`   ra   rF   r	   ZjitZis_scriptingrn   ri   )r   r=   r>   r?   r@   r   r!   r   rH   r   r   r   r    r   rm   Úfuncr*   r*   r+   r   †  s@    ÿÿ  ÿ
ó)FNFF)Útypingr   r   r   r   r7   r   Z	optimizerr   r	   r
   r   r   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__r8   rN   ri   rn   r   r*   r*   r*   r+   Ú<module>   s†   8 $óòñðäÿ7ò7ò_	    õï