U
    8‹ÑhzE  ã                   @   sR   d dl mZ d dlZddlmZmZ dgZddd„Zddd„ZG dd„ deƒZ	dS )é    )ÚOptionalNé   )Ú	OptimizerÚParamsTÚLBFGSc                 C   sä   |d k	r|\}}n| |kr"| |fn|| f\}}|| d||  | |   }	|	d ||  }
|
dkrÔ|
  ¡ }| |krœ|||  || |	 || d|     }n(| | | || |	 || d|     }tt||ƒ|ƒS || d S d S )Né   é   r   g       @)ÚsqrtÚminÚmax)Úx1Úf1Zg1Zx2Úf2Zg2ÚboundsZ
xmin_boundZ
xmax_boundÚd1Z	d2_squareZd2Zmin_pos© r   ú?/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/optim/lbfgs.pyÚ_cubic_interpolate
   s    
	*(r   ç-Cëâ6?çÍÌÌÌÌÌì?ç•Ö&è.>é   c           !   	   C   s   |  ¡  ¡ }|jtjd�}| |||ƒ\}}d}| |¡}d|||f\}}}}d}d}||
k �r�|||| |  ks‚|dkr°||kr°||g}||g}||jtjd�g}||g}�q�t |ƒ| | krÜ|g}|g}|g}d}�q�|dk�r||g}||g}||jtjd�g}||g}�q�|d||   }|d }|}t||||||||fd�}|}|}|jtjd�}|}| |||ƒ\}}|d7 }| |¡}|d7 }qT||
k�r²d|g}||g}||g}d}|d |d	 k�rÌd
nd\}}|�sü||
k �rüt |d |d  ƒ| |	k �r�qüt|d |d |d |d |d |d ƒ}dt|ƒt|ƒ  } tt|ƒ| |t|ƒ ƒ| k �rÎ|�s†|t|ƒk�s†|t|ƒk�rÈt |t|ƒ ƒt |t|ƒ ƒk �r¶t|ƒ|  }nt|ƒ|  }d}nd}nd}| |||ƒ\}}|d7 }| |¡}|d7 }|||| |  k�s ||| k�rj|||< |||< |jtjd�||< |||< |d |d k�r`d
nd\}}nŽt |ƒ| | k�r„d}nJ||| ||   dk�rÎ|| ||< || ||< || ||< || ||< |||< |||< |jtjd�||< |||< �qÔ|| }|| }|| }||||fS )N©Zmemory_formatr   r   FTg{®Gáz„?é
   )r   éÿÿÿÿ)r   r   )r   r   gš™™™™™¹?)Úabsr   ÚcloneÚtorchÚcontiguous_formatÚdotr   r
   )!Úobj_funcÚxÚtÚdÚfÚgÚgtdZc1Úc2Útolerance_changeZmax_lsZd_normZf_newZg_newÚls_func_evalsZgtd_newZt_prevZf_prevZg_prevZgtd_prevÚdoneZls_iterZbracketZ	bracket_fZ	bracket_gZbracket_gtdZmin_stepZmax_stepÚtmpZinsuf_progressZlow_posZhigh_posZepsr   r   r   Ú_strong_wolfe&   sÄ    

$
      ÿ


ú ""
$ r,   c                
       s~   e Zd ZdZdeeeee eeeee dœ‡ fd	d
„Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Ze ¡ dd„ ƒZ‡  ZS )r   a\  Implements L-BFGS algorithm.

    Heavily inspired by `minFunc
    <https://www.cs.ubc.ca/~schmidtm/Software/minFunc.html>`_.

    .. warning::
        This optimizer doesn't support per-parameter options and parameter
        groups (there can be only one).

    .. warning::
        Right now all parameters have to be on a single device. This will be
        improved in the future.

    .. note::
        This is a very memory intensive optimizer (it requires additional
        ``param_bytes * (history_size + 1)`` bytes). If it doesn't fit in memory
        try reducing the history size, or use a different algorithm.

    Args:
        params (iterable): iterable of parameters to optimize. Parameters must be real.
        lr (float): learning rate (default: 1)
        max_iter (int): maximal number of iterations per optimization step
            (default: 20)
        max_eval (int): maximal number of function evaluations per optimization
            step (default: max_iter * 1.25).
        tolerance_grad (float): termination tolerance on first order optimality
            (default: 1e-7).
        tolerance_change (float): termination tolerance on function
            value/parameter changes (default: 1e-9).
        history_size (int): update history size (default: 100).
        line_search_fn (str): either 'strong_wolfe' or None (default: None).
    r   é   NçH¯¼šò×z>r   éd   )ÚparamsÚlrÚmax_iterÚmax_evalÚtolerance_gradr(   Úhistory_sizeÚline_search_fnc	           
   	      sh   |d kr|d d }t |||||||d�}	tƒ  ||	¡ t| jƒdkrNtdƒ‚| jd d | _d | _d S )Né   é   )r1   r2   r3   r4   r(   r5   r6   r   z>LBFGS doesn't support per-parameter options (parameter groups)r   r0   )ÚdictÚsuperÚ__init__ÚlenÚparam_groupsÚ
ValueErrorÚ_paramsÚ_numel_cache)
Úselfr0   r1   r2   r3   r4   r(   r5   r6   Údefaults©Ú	__class__r   r   r;   Ö   s$    ù	ÿzLBFGS.__init__c                 C   s&   | j d kr tdd„ | jD ƒƒ| _ | j S )Nc                 s   s,   | ]$}t  |¡rd | ¡  n| ¡ V  qdS )r   N)r   Ú
is_complexÚnumel©Ú.0Úpr   r   r   Ú	<genexpr>ø   s   ÿzLBFGS._numel.<locals>.<genexpr>)r@   Úsumr?   ©rA   r   r   r   Ú_numelö   s
    
þ
zLBFGS._numelc                 C   s„   g }| j D ]l}|jd kr,| | ¡ ¡ ¡ }n&|jjrF|j ¡  d¡}n|j d¡}t 	|¡rlt 
|¡ d¡}| |¡ q
t |d¡S )Nr   r   )r?   ZgradÚnewrF   Zzero_Z	is_sparseZto_denseÚviewr   rE   Úview_as_realÚappendÚcat)rA   ZviewsrI   rO   r   r   r   Ú_gather_flat_gradÿ   s    


zLBFGS._gather_flat_gradc                 C   sh   d}| j D ]H}t |¡r"t |¡}| ¡ }|j|||| …  |¡|d� ||7 }q
||  ¡ ksdt‚d S )Nr   ©Úalpha)	r?   r   rE   rP   rF   Úadd_Zview_asrM   ÚAssertionError)rA   Z	step_sizeÚupdateÚoffsetrI   rF   r   r   r   Ú	_add_grad  s    


 
zLBFGS._add_gradc                 C   s   dd„ | j D ƒS )Nc                 S   s   g | ]}|j tjd �‘qS )r   )r   r   r   rG   r   r   r   Ú
<listcomp>  s     z&LBFGS._clone_param.<locals>.<listcomp>)r?   rL   r   r   r   Ú_clone_param  s    zLBFGS._clone_paramc                 C   s$   t | j|ƒD ]\}}| |¡ qd S ©N)Úzipr?   Úcopy_)rA   Zparams_datarI   Zpdatar   r   r   Ú
_set_param  s    zLBFGS._set_paramc                 C   s0   |   ||¡ t|ƒ ƒ}|  ¡ }|  |¡ ||fS r]   )rZ   ÚfloatrS   r`   )rA   Úclosurer!   r"   r#   ÚlossÚ	flat_gradr   r   r   Ú_directional_evaluate  s
    

zLBFGS._directional_evaluatec           &   	      s¢  t ˆjƒdkst‚t ¡ ˆ ƒ‰ ˆjd }|d }|d }|d }|d }|d }|d }|d	 }	ˆjˆjd  }
|
 d
d¡ |
 dd¡ ˆ ƒ }t|ƒ}d}|
d
  d7  < ˆ 	¡ }| 
¡  ¡ |k}|rÊ|S |
 d¡}|
 d¡}|
 d¡}|
 d¡}|
 d¡}|
 d¡}|
 d¡}|
 d¡}d}||k �r^|d7 }|
d  d7  < |
d dk�rj| ¡ }g }g }g }d}�nN| |¡}| |¡}| |¡}|dk�rît |ƒ|	k�r¾| d¡ | d¡ | d¡ | |¡ | |¡ | d| ¡ || |¡ }t |ƒ}d|
k�rdg|	 |
d< |
d }| ¡ }t|d ddƒD ]8}||  |¡||  ||< |j|| ||  d� �q.t ||¡ }} t|ƒD ]6}||  | ¡||  }!| j|| || |! d� �q€|dk�rÒ|jtjd�}n
| |¡ |}|
d dk�r
tdd| 
¡  ¡  ƒ| }n|}| |¡}"|"| k�r(�q^d}#|dk	�rš|dk�rJtdƒ‚n2ˆ ¡ }$‡ ‡fdd„}%t|%|$|||||"ƒ\}}}}#ˆ ||¡ | 
¡  ¡ |k}nPˆ ||¡ ||k�rêt ¡ � tˆ ƒ ƒ}W 5 Q R X ˆ 	¡ }| 
¡  ¡ |k}d}#||#7 }|
d
  |#7  < ||k�r�q^||k�r�q^|�r(�q^| |¡ 
¡  ¡ |k�rD�q^t
|| ƒ|k �r�q^�q||
d< ||
d< ||
d< ||
d< ||
d< ||
d< ||
d< ||
d< |S )z¦Perform a single optimization step.

        Args:
            closure (Callable): A closure that reevaluates the model
                and returns the loss.
        r   r   r1   r2   r3   r4   r(   r6   r5   Z
func_evalsÚn_iterr#   r"   Úold_dirsÚold_stpsÚroÚH_diagÚprev_flat_gradÚ	prev_lossg»½×Ùß|Û=g      ð?ÚalNr   rT   r   Zstrong_wolfez only 'strong_wolfe' is supportedc                    s   ˆ  ˆ | ||¡S r]   )re   )r!   r"   r#   ©rb   rA   r   r   r    ²  s    zLBFGS.step.<locals>.obj_func)r<   r=   rW   r   Zenable_gradÚstater?   Ú
setdefaultra   rS   r   r   ÚgetÚnegÚsubÚmulr   ÚpoprQ   ÚrangerV   r   r   r_   r
   rK   ÚRuntimeErrorr\   r,   rZ   )&rA   rb   Úgroupr1   r2   r3   r4   r(   r6   r5   ro   Z	orig_lossrc   Zcurrent_evalsrd   Zopt_condr#   r"   rg   rh   ri   rj   rk   rl   rf   ÚyÚsZysZnum_oldrm   ÚqÚiÚrZbe_ir&   r)   Zx_initr    r   rn   r   Ústep&  sì    

























      ÿ



z
LBFGS.step)r   r-   Nr.   r   r/   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ra   Úintr   Ústrr;   rM   rS   rZ   r\   r`   re   r   Zno_gradr~   Ú__classcell__r   r   rC   r   r   ´   s4   $       ÷÷ 	)N)r   r   r   r   )
Útypingr   r   Z	optimizerr   r   Ú__all__r   r,   r   r   r   r   r   Ú<module>   s   
       ÿ
 