U
    8‹ÑhÂH  ã                   @   sœ  d dl mZmZ d dlZd dlmZ d dlmZ ddlmZm	Z	m
Z
mZmZmZmZmZ ddgZG d	d„ deƒZd
de› de
› de	› de› d�	 d e_dee ee eee  eee ee ee ee eeeeeedœdd„Zee ee eee  ee ee eeeeeeedœdd„Zee ee eee  ee ee eeeeeeedœdd„Zee ee eee  ee ee eeeeeeeddœdd„ZdS )é    )ÚListÚOptionalN)ÚTensor)Ú$_get_fused_kernels_supported_devicesé   )Ú_default_to_fused_or_foreachÚ_differentiable_docÚ_foreach_docÚ
_fused_docÚ_maximize_docÚ_use_grad_for_differentiableÚ
DeviceDictÚ	OptimizerÚSGDÚsgdc                       sh   e Zd Zddddddœeeeeeee eee dœ‡ fdd„Z‡ fd	d
„Zdd„ Ze	ddd„ƒZ
‡  ZS )r   çü©ñÒMbP?r   FN)ÚmaximizeÚforeachÚdifferentiableÚfused)ÚlrÚmomentumÚ	dampeningÚweight_decayr   r   r   r   c                   sÚ   |dk rt d|› �ƒ‚|dk r,t d|› �ƒ‚|dk rBt d|› �ƒ‚t||||||||	|
d�	}|rx|dksp|dkrxt dƒ‚tƒ  ||¡ |
rÖd| _tƒ ‰ t‡ fd	d
„| jD ƒƒs¾tdˆ › d�ƒ‚|	rÊtdƒ‚|rÖtdƒ‚d S )Ng        zInvalid learning rate: zInvalid momentum value: zInvalid weight_decay value: )	r   r   r   r   Únesterovr   r   r   r   r   z8Nesterov momentum requires a momentum and zero dampeningTc                 3   s2   | ]*}|d  D ]}|j jˆ ko&t |¡V  qqdS )ÚparamsN)ÚdeviceÚtypeÚtorchZis_floating_point)Ú.0ZpgÚp©Zfused_supported_devices© ú=/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/optim/sgd.pyÚ	<genexpr>>   s   
 þzSGD.__init__.<locals>.<genexpr>zX`fused=True` requires all the params to be floating point Tensors of supported devices: Ú.z)`fused` does not support `differentiable`z0`fused` and `foreach` cannot be `True` together.)	Ú
ValueErrorÚdictÚsuperÚ__init__Z_step_supports_amp_scalingr   ÚallÚparam_groupsÚRuntimeError)Úselfr   r   r   r   r   r   r   r   r   r   Údefaults©Ú	__class__r!   r#   r)      sB    ÷þ
ÿzSGD.__init__c                    sX   t ƒ  |¡ | jD ]@}| dd¡ | dd¡ | dd ¡ | dd¡ | dd¡ qd S )Nr   Fr   r   r   r   )r(   Ú__setstate__r+   Ú
setdefault)r-   ÚstateÚgroupr/   r"   r#   r1   L   s    
zSGD.__setstate__c                 C   sh   d}|d D ]V}|j d k	r| |¡ | |j ¡ |j jr<d}|d dkr| j| }| | d¡¡ q|S )NFr   Tr   r   Úmomentum_buffer)ÚgradÚappendÚ	is_sparser3   Úget)r-   r4   r   ÚgradsÚmomentum_buffer_listÚhas_sparse_gradr    r3   r"   r"   r#   Ú_init_groupU   s    


zSGD._init_groupc                 C   sØ   d}|dk	r&t  ¡ � |ƒ }W 5 Q R X | jD ]¦}g }g }g }|  ||||¡}t||||d |d |d |d |d |d ||d |d	 t| d
dƒt| ddƒd� |d dkr,t||ƒD ]\}}	| j| }
|	|
d< q¶q,|S )z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r   r   r   Ú
grad_scaleÚ	found_inf)r   r   r   r   r   r   r<   r   r   r>   r?   r   r5   )r   Zenable_gradr+   r=   r   ÚgetattrÚzipr3   )r-   ÚclosureZlossr4   r   r:   r;   r<   r    r5   r3   r"   r"   r#   Ústepe   sF    

   ÿ

ò
zSGD.step)r   r   r   r   F)N)Ú__name__Ú
__module__Ú__qualname__ÚfloatÚboolr   r)   r1   r=   r   rC   Ú__classcell__r"   r"   r/   r#   r      s0        ù	ôô6	a  Implements stochastic gradient descent (optionally with momentum).

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)}, \: f(\theta)
                \text{ (objective)}, \: \lambda \text{ (weight decay)},                          \\
            &\hspace{13mm} \:\mu \text{ (momentum)}, \:\tau \text{ (dampening)},
            \:\textit{ nesterov,}\:\textit{ maximize}                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm}\textbf{if} \: \lambda \neq 0                                           \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda  \theta_{t-1}                            \\
            &\hspace{5mm}\textbf{if} \: \mu \neq 0                                               \\
            &\hspace{10mm}\textbf{if} \: t > 1                                                   \\
            &\hspace{15mm} \textbf{b}_t \leftarrow \mu \textbf{b}_{t-1} + (1-\tau) g_t           \\
            &\hspace{10mm}\textbf{else}                                                          \\
            &\hspace{15mm} \textbf{b}_t \leftarrow g_t                                           \\
            &\hspace{10mm}\textbf{if} \: \textit{nesterov}                                       \\
            &\hspace{15mm} g_t \leftarrow g_{t} + \mu \textbf{b}_t                             \\
            &\hspace{10mm}\textbf{else}                                                   \\[-1.ex]
            &\hspace{15mm} g_t  \leftarrow  \textbf{b}_t                                         \\
            &\hspace{5mm}\textbf{if} \: \textit{maximize}                                          \\
            &\hspace{10mm}\theta_t \leftarrow \theta_{t-1} + \gamma g_t                   \\[-1.ex]
            &\hspace{5mm}\textbf{else}                                                    \\[-1.ex]
            &\hspace{10mm}\theta_t \leftarrow \theta_{t-1} - \gamma g_t                   \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    Nesterov momentum is based on the formula from
    `On the importance of initialization and momentum in deep learning`__.
    aã  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-3)
        momentum (float, optional): momentum factor (default: 0)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        dampening (float, optional): dampening for momentum (default: 0)
        nesterov (bool, optional): enables Nesterov momentum (default: False)
        z	
        z
    aš  

    Example:
        >>> # xdoctest: +SKIP
        >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
        >>> optimizer.zero_grad()
        >>> loss_fn(model(input), target).backward()
        >>> optimizer.step()

    __ http://www.cs.toronto.edu/%7Ehinton/absps/momentum.pdf

    .. note::
        The implementation of SGD with Momentum/Nesterov subtly differs from
        Sutskever et al. and implementations in some other frameworks.

        Considering the specific case of Momentum, the update can be written as

        .. math::
            \begin{aligned}
                v_{t+1} & = \mu * v_{t} + g_{t+1}, \\
                p_{t+1} & = p_{t} - \text{lr} * v_{t+1},
            \end{aligned}

        where :math:`p`, :math:`g`, :math:`v` and :math:`\mu` denote the
        parameters, gradient, velocity, and momentum respectively.

        This is in contrast to Sutskever et al. and
        other frameworks which employ an update of the form

        .. math::
            \begin{aligned}
                v_{t+1} & = \mu * v_{t} + \text{lr} * g_{t+1}, \\
                p_{t+1} & = p_{t} - v_{t+1}.
            \end{aligned}

        The Nesterov version is analogously modified.

        Moreover, the initial value of the momentum buffer is set to the
        gradient value at the first step. This is in contrast to some other
        frameworks that initialize it to all zeros.

    F)r   Úd_p_listr;   r<   r   r   r>   r?   r   r   r   r   r   r   c                C   sÊ   |dkr6|dkr6t j ¡ s.t| ddd�\}}nd}d}|dkrBd}|dkrNd}|rdt j ¡ rdtdƒ‚|rzt j ¡ rztdƒ‚|rŽt j ¡ sŽt}n|r¢t j ¡ s¢t}nt}|| ||||	|
||||||d� dS )zlFunctional API that performs SGD algorithm computation.

    See :class:`~torch.optim.SGD` for details.
    NF)r   Z	use_fusedz6torch.jit.script not supported with foreach optimizersz4torch.jit.script not supported with fused optimizers)	r   r   r   r   r   r<   r   r>   r?   )r   ZjitZis_scriptingr   r,   Ú_multi_tensor_sgdÚ
_fused_sgdÚ_single_tensor_sgd)r   rJ   r;   r<   r   r   r>   r?   r   r   r   r   r   r   Úfuncr"   r"   r#   r   ô   sH    
  ÿô)r   r:   r;   r>   r?   r   r   r   r   r   r   r<   c                C   sÆ   |d kr|d kst ‚t| ƒD ]¤\}}|
s0|| n||  }|dkrP|j||d�}|dkr°|| }|d kr€t |¡ ¡ }|||< n| |¡j|d| d� |	r¬|j||d�}n|}|j|| d� qd S )Nr   ©Úalphar   )ÚAssertionErrorÚ	enumerateÚaddr   ÚcloneÚdetachÚmul_Úadd_)r   r:   r;   r>   r?   r   r   r   r   r   r   r<   ÚiÚparamr6   Úbufr"   r"   r#   rM   ;  s    
rM   c                C   s,  |d kr|d kst ‚t| ƒdkr$d S tj| ||gdd�}| ¡ D �]ä\\}}}}|odtdd„ |D ƒƒ}|
rtt |¡}|dkr¢|
r’tj|||d� ntj	|||d�}|dk�rªg }d}t
t|ƒƒD ](}|| d krÚd} qêqÀ| || ¡ qÀ|�rt ||¡ tj||d| d� n|g }t
t|ƒƒD ]j}|| d k�r\t || ¡ ¡  } ||< ||| < n$|| }| |¡j|| d| d� | |¡ �q"|	�r¦tj|||d� n|}|�sút|tjƒ�rætj ¡ �ræt || ¡}t ||¡ ntj||| d� q@t
t|ƒƒD ]}|| j|| | d� �qq@d S )	Nr   T©Zwith_indicesc                 s   s   | ]}|j V  qd S ©N)r8   )r   r6   r"   r"   r#   r$     s    z$_multi_tensor_sgd.<locals>.<genexpr>rO   Fr   )rQ   Úlenr   Ú"_group_tensors_by_device_and_dtypeÚvaluesÚanyr   Z_foreach_negZ_foreach_add_Z_foreach_addÚranger7   Z_foreach_mul_rT   rU   rV   rW   Ú
isinstancer   Ú_utilsZis_compilingZ_foreach_mul)r   r:   r;   r>   r?   r   r   r   r   r   r   r<   Úgrouped_tensorsÚdevice_paramsÚdevice_gradsÚdevice_momentum_buffer_listÚindicesZdevice_has_sparse_gradZbufsZall_states_with_momentum_bufferrX   rZ   Z
grads_x_lrr"   r"   r#   rK   c  sr     ÿü
ÿ
  ÿ
þÿrK   )r   r:   r;   r>   r?   r   r   r   r   r   r   r<   Úreturnc                C   s,  | sd S |rt dƒ‚|d k	r&|j|ini }|d k	r<|j|ini }|dk}tdd„ |D ƒƒo^| }|r„t|ƒD ]\}}t |¡||< qltj| ||gdd�}| ¡ D ]†\\}}\\}}}}d\}}|d k	rØ| 	|| 
|¡¡}|d k	rú|d k	rú| 	|| 
|¡¡}tj|||�rg n||||||	|
|||d� q d S )	Nz.`_fused_sgd` does not support sparse gradientsr   c                 s   s   | ]}|d kV  qd S r\   r"   )r   Útr"   r"   r#   r$   Ø  s     z_fused_sgd.<locals>.<genexpr>Fr[   )NN)	r   r   r   r   r   r   Úis_first_stepr>   r?   )r,   r   r*   rR   r   Z
empty_liker   r^   Úitemsr2   ÚtoZ_fused_sgd_)r   r:   r;   r>   r?   r   r   r   r   r   r   r<   Zgrad_scale_dictZfound_inf_dictZno_momentum_bufferrk   rX   Úgrd   r   Ú_re   rf   rg   Zdevice_grad_scaleZdevice_found_infr"   r"   r#   rL   ¼  sX    ÿÿÿ ÿý ÿôrL   )FNNNN)Útypingr   r   r   r   Ztorch.utils._foreach_utilsr   Z	optimizerr   r   r	   r
   r   r   r   r   Ú__all__r   Ú__doc__rH   rG   r   rM   rK   rL   r"   r"   r"   r#   Ú<module>   s¤   ( #	÷
öõôÝ1Ïÿe     ö
ïH
ó)
óZ
ò