U
    8‹Ñh£<  ã                   @   s|  d dl mZmZmZmZ d dlZd dlmZ ddlmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZ ddgZG dd„ deƒZd	e› d
e› d
e
› d
e› d�	e_ee ee ee ee ee ee eeeeeeeeedœdd„Zee ee ee ee ee ee eeeeeeeeedœdd„Zeed�dee ee ee ee ee ee ee eeeeeeeeedœdd„ƒZdS )é    )ÚListÚOptionalÚTupleÚUnionN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_foreach_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ
_get_valueÚ_maximize_docÚ_use_grad_for_differentiableÚ_view_as_realÚ	OptimizerÚParamsTÚASGDÚasgdc                       s\   e Zd Zdeeeeeeee eeedœ
‡ fd	d
„Z‡ fdd„Zdd„ Z	e
ddd„ƒZ‡  ZS )r   ç{®Gáz„?ç-Cëâ6?ç      è?ç    €„.Ar   NF)
ÚparamsÚlrÚlambdÚalphaÚt0Úweight_decayÚforeachÚmaximizeÚdifferentiableÚ
capturablec                    sX   d|kst d|› �ƒ‚d|ks,t d|› �ƒ‚t||||||||	|
d�	}tƒ  ||¡ d S )Ng        zInvalid learning rate: zInvalid weight_decay value: )	r   r   r   r   r    r!   r"   r#   r$   )Ú
ValueErrorÚdictÚsuperÚ__init__)Úselfr   r   r   r   r   r    r!   r"   r#   r$   Údefaults©Ú	__class__© ú>/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/optim/asgd.pyr(      s     ÷zASGD.__init__c                    sú   t ƒ  |¡ | jD ]â}| dd ¡ | dd¡ | dd¡ | dd¡ |d D ]¤}| j |g ¡}t|ƒdkrNt |d ¡sžt	|d ƒ}tj
|tƒ |jd	�|d< t |d
 ¡sÈtj
|d
 tƒ |jd	�|d
< t |d ¡sNtj
|d tƒ |jd	�|d< qNqd S )Nr!   r"   Fr#   r$   r   r   Ústep)ÚdtypeÚdeviceÚetaÚmu)r'   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchZ	is_tensorÚfloatÚtensorr   r1   )r)   r7   ÚgroupÚpZp_stateZstep_valr+   r-   r.   r4   :   s8    
  ÿ
  ÿ
  ÿzASGD.__setstate__c                 C   s
  d}|d D ]ø}	|	j d k	r|t |	¡O }| |	¡ |	j jrBtdƒ‚| |	j ¡ | j|	 }
t|
ƒdkrÌtjd|	j	t
ƒ d�|
d< tj|d |	j	t
ƒ d� ¡  ¡ |
d	< tjd|	j	t
ƒ d�|
d
< tj|	tjd�|
d< | |
d
 ¡ | |
d ¡ | |
d	 ¡ | |
d ¡ q|S )NFr   z&ASGD does not support sparse gradientsr   r-   )r1   r0   r/   r   r2   r3   )Zmemory_formatÚax)Úgradr:   Ú
is_complexÚappendZ	is_sparseÚRuntimeErrorr7   r9   Úzerosr1   r   Ú	as_tensorÚcloneÚdetachZonesZ
zeros_likeZpreserve_format)r)   r=   Úparams_with_gradÚgradsÚmusÚaxsÚetasÚstate_stepsÚhas_complexr>   r7   r-   r-   r.   Ú_init_groupR   sF    


  ÿ
  ÿÿ  ÿ
 ÿ
zASGD._init_groupc                 C   s¸   |   ¡  d}|dk	r.t ¡ � |ƒ }W 5 Q R X | jD ]~}g }g }g }g }g }g }	|  |||||||	¡}
t||||||	|d |d |d |d |d |d |d |d	 |d
 |
d� q4|S )z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r    r!   r"   r#   r$   )
r   r   r   r   r    r!   r"   r#   r$   rN   )Z _cuda_graph_capture_health_checkr:   Zenable_gradr5   rO   r   )r)   ÚclosureZlossr=   rH   rI   rJ   rK   rL   rM   rN   r-   r-   r.   r/   v   sP    

      ÿðz	ASGD.step)	r   r   r   r   r   NFFF)N)Ú__name__Ú
__module__Ú__qualname__r   r;   r   Úboolr(   r4   rO   r   r/   Ú__classcell__r-   r-   r+   r.   r      s2            õõ$ah  Implements Averaged Stochastic Gradient Descent.

    It has been proposed in `Acceleration of stochastic approximation by
    averaging`_.

    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        lambd (float, optional): decay term (default: 1e-4)
        alpha (float, optional): power for eta update (default: 0.75)
        t0 (float, optional): point at which to start averaging (default: 1e6)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        z	
        zx

    .. _Acceleration of stochastic approximation by averaging:
        https://dl.acm.org/citation.cfm?id=131098

    )r   rI   rK   rJ   rL   rM   r   r   r   r   r    r"   r#   r$   rN   c       	      
   C   s  t | ƒD �]\}}|| }|s"|n| }|| }|| }|| }|| }tj ¡ sª|rªtƒ }|jj|jj  krŠ|jj  krŠ|jjkršn n|jj|ksªtd|› d�ƒ‚t |¡rÒt 	|¡}t 	|¡}t 	|¡}|d7 }|
dkrð|j
||
d�}|�r| d||  ¡ |j||dd� n*t|ƒ}| d||  ¡ |j|| d� |�sX| ¡ dk�rp| | |¡ |¡¡ n
| |¡ |�rÀ| |d|| |  |	  ¡ | dt || t |¡¡ ¡ qt|ƒ}t |d|| |  |	  ¡}| |¡ t dtd|| ƒ ¡}| |¡ qd S )NúUIf capturable=True, params, mus, etas, and state_steps must be on supported devices: Ú.r   r   ©r   éÿÿÿÿ©Úvalue)Ú	enumerater:   Ú_utilsÚis_compilingr   r1   ÚtypeÚAssertionErrorrA   Zview_as_realÚaddZmul_Zaddcmul_r   Zadd_ÚitemÚsubZcopy_ÚmaximumZ	ones_likerE   Úmax)r   rI   rK   rJ   rL   rM   r   r   r   r   r    r"   r#   r$   rN   ÚiÚparamr@   r3   r?   r2   Zstep_tÚcapturable_supported_devicesZ	eta_valuer/   Znew_etaZnew_mur-   r-   r.   Ú_single_tensor_asgd¿   sX    þýü

û
ù




"
ri   c       	            sL  t | ƒdkrd S |rtdƒ‚tj ¡ sd|rdtdd�‰t‡fdd„t| |||ƒD ƒƒsdtdˆ› d�ƒ‚t 	| |||||g¡}| 
¡ D �]Â\\‰}\\}}}}}}}|r°t|||ƒ |r¾t |¡}|d jrätj|tjd	d
d�d	d� nt |d¡ |
dk�r8|�rtj|||
d� |}ntj|||
d�}tj||ˆd� ntj||ˆd�}tj|||dd� ~t ||¡}t |||¡ ~|�rt |ˆ¡}t |d	¡ t |¡ t ||¡ ~t |ˆ¡}t |ˆ¡ t |d¡ t |ˆ ¡ t |¡ t |ˆ¡ t ||¡ q‚‡ ‡‡‡fdd„|D ƒ}‡‡fdd„|D ƒ}t ||¡ t ||¡ q‚d S )Nr   z#_foreach ops don't support autogradF)Zsupports_xlac                 3   sT   | ]L\}}}}|j j|j j  ko:|j j  ko:|j jkn  oJ|j jˆ kV  qd S )N)r1   r_   )Ú.0r>   r3   r2   r/   )rh   r-   r.   Ú	<genexpr>'  s   
þ2z%_multi_tensor_asgd.<locals>.<genexpr>rV   rW   g      ð?Úcpu©r1   rX   r   rY   rZ   c                    s.   g | ]&}t jˆd ˆˆ |  ˆ   ˆd�‘qS ©r   rm   )r:   rE   ©rj   r/   )r   r1   r   r   r-   r.   Ú
<listcomp>ƒ  s   ÿz&_multi_tensor_asgd.<locals>.<listcomp>c                    s,   g | ]$}t jd td t|ƒˆ ƒ ˆ d�‘qS rn   )r:   rE   re   r   ro   )r1   r   r-   r.   rp   ‡  s   ÿ)r9   r`   r:   r]   r^   r   ÚallÚzipr   Z"_group_tensors_by_device_and_dtypeÚitemsr   Z_foreach_negZis_cpuZ_foreach_add_r<   Z_foreach_addZ_foreach_addcmul_Z_foreach_subZ_foreach_maximum_Z_foreach_reciprocal_Z_foreach_copy_Z_foreach_mulZ_foreach_mul_Z_foreach_pow_)r   rI   rK   rJ   rL   rM   r   r   r   r   r    r"   r#   r$   rN   Zgrouped_tensorsÚ_Zgrouped_paramsZgrouped_gradsZgrouped_axsZgrouped_musZgrouped_etasZgrouped_state_stepsZintermediateZnew_musZnew_etasr-   )r   rh   r1   r   r   r   r.   Ú_multi_tensor_asgd  sš    ÿý

üÿö

  ÿ
  ÿ  ÿ


þþru   )Zsingle_tensor_fnF)r   rI   rK   rJ   rL   rM   r!   r"   r#   r$   rN   r   r   r   r   r    c                C   sr   |dkrt | |dd�\}}|r0tj ¡ r0tdƒ‚|rDtj ¡ sDt}nt}|| |||||||||||||	|
d� dS )znFunctional API that performs asgd algorithm computation.

    See :class:`~torch.optim.ASGD` for details.
    NF)Z	use_fusedz6torch.jit.script not supported with foreach optimizers)	r   r   r   r   r    r"   r#   r$   rN   )r	   r:   ZjitZis_scriptingrC   ru   ri   )r   rI   rK   rJ   rL   rM   r!   r"   r#   r$   rN   r   r   r   r   r    rt   Úfuncr-   r-   r.   r   �  s8      ÿ
ñ)NFFFF)Útypingr   r   r   r   r:   r   Z	optimizerr   r	   r
   r   r   r   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__r;   rT   ri   ru   r   r-   r-   r-   r.   Ú<module>   s�   < óòñð
ðMð 
     óí