U
    8‹Ñh U  ã                	   @   s"  d Z ddlZddlZddlmZ ddlZddlmZ dCdd„ZdDdd„Z	dEd	d
„Z
dd„ Zdd„ ZdFdd„ZdGeeeeej edœdd„ZdHeeeeej edœdd„ZdIeeeeeeej edœdd„Zeeedœdd „Zeed!œd"d#„Zeed!œd$d%„Zd&d'„ ZdJd)d*„Zd+d,„ ZdKeeeej ed-œd.d/„ZdLeeeej ed-œd0d1„Zd2d3„ ZdMeeeeeej d6œd7d8„ZdNeeeeeej d6œd9d:„ZdOeej d;œd<d=„ZdPeej d;œd?d@„Z dAdB„ Z!e!eƒZ"e!eƒZ#e!eƒZ$e!eƒZ%e!eƒZ&e!eƒZ'e!eƒZ(e!eƒZ)e!eƒZ*e!eƒZ+e!e ƒZ,dS )QzHThis file contains utilities for initializing neural network parameters.é    N)ÚTensor)ÚOptionalc              
   C   s0   t  ¡ � | j|||d�W  5 Q R £ S Q R X d S ©N©Ú	generator)ÚtorchÚno_gradÚuniform_©ÚtensorÚaÚbr   © r   ú;/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/nn/init.pyÚ_no_grad_uniform_   s    
r   c              
   C   s0   t  ¡ � | j|||d�W  5 Q R £ S Q R X d S r   )r   r   Únormal_©r   ÚmeanÚstdr   r   r   r   Ú_no_grad_normal_   s    
r   c           	   
   C   sÎ   dd„ }||d|  k s(||d|  kr6t jddd� t ¡ �† ||| | ƒ}||| | ƒ}| jd| d d| d |d� |  ¡  |  |t d¡ ¡ |  	|¡ | j
||d	� | W  5 Q R £ S Q R X d S )
Nc                 S   s   dt  | t  d¡ ¡ d S )Nç      ð?ç       @)ÚmathÚerfÚsqrt)Úxr   r   r   Únorm_cdf   s    z(_no_grad_trunc_normal_.<locals>.norm_cdfé   zjmean is more than 2 std from [a, b] in nn.init.trunc_normal_. The distribution of values may be incorrect.©Ú
stacklevelé   r   r   )ÚminÚmax)ÚwarningsÚwarnr   r   r	   Zerfinv_Úmul_r   r   Zadd_Zclamp_)	r   r   r   r   r   r   r   ÚlÚur   r   r   Ú_no_grad_trunc_normal_   s     þ
 
r(   c              
   C   s*   t  ¡ � |  |¡W  5 Q R £ S Q R X d S ©N)r   r   Zfill_©r   Úvalr   r   r   Ú_no_grad_fill_;   s    
r,   c              
   C   s(   t  ¡ � |  ¡ W  5 Q R £ S Q R X d S r)   )r   r   Úzero_©r   r   r   r   Ú_no_grad_zero_@   s    
r/   c                 C   sÂ   dddddddg}| |ks"| dkr&d	S | d
kr2dS | dkrDt  d¡S | dkr¤|dkrZd}n4t|tƒsnt|tƒsxt|tƒr~|}ntd|› d�ƒ‚t  dd	|d   ¡S | dkr°dS td| › �ƒ‚dS )aÞ  Return the recommended gain value for the given nonlinearity function.

    The values are as follows:

    ================= ====================================================
    nonlinearity      gain
    ================= ====================================================
    Linear / Identity :math:`1`
    Conv{1,2,3}D      :math:`1`
    Sigmoid           :math:`1`
    Tanh              :math:`\frac{5}{3}`
    ReLU              :math:`\sqrt{2}`
    Leaky Relu        :math:`\sqrt{\frac{2}{1 + \text{negative\_slope}^2}}`
    SELU              :math:`\frac{3}{4}`
    ================= ====================================================

    .. warning::
        In order to implement `Self-Normalizing Neural Networks`_ ,
        you should use ``nonlinearity='linear'`` instead of ``nonlinearity='selu'``.
        This gives the initial weights a variance of ``1 / N``,
        which is necessary to induce a stable fixed point in the forward pass.
        In contrast, the default gain for ``SELU`` sacrifices the normalization
        effect for more stable gradient flow in rectangular layers.

    Args:
        nonlinearity: the non-linear function (`nn.functional` name)
        param: optional parameter for the non-linear function

    Examples:
        >>> gain = nn.init.calculate_gain('leaky_relu', 0.2)  # leaky_relu with negative_slope=0.2

    .. _Self-Normalizing Neural Networks: https://papers.nips.cc/paper/2017/hash/5d44ee6f2c3f71b73125876103c8f6c4-Abstract.html
    ZlinearZconv1dZconv2dZconv3dZconv_transpose1dZconv_transpose2dZconv_transpose3dZsigmoidr    Útanhg«ªªªªªú?Zrelur   Ú
leaky_reluNç{®Gáz„?znegative_slope z not a valid numberr   Zselug      è?zUnsupported nonlinearity )r   r   Ú
isinstanceÚboolÚintÚfloatÚ
ValueError)ÚnonlinearityÚparamZ
linear_fnsZnegative_sloper   r   r   Úcalculate_gainE   s"    "
r:   ç        r   )r   r   r   r   Úreturnc                 C   s4   t j | ¡r&t jjt| f| |||d�S t| |||ƒS )a«  Fill the input Tensor with values drawn from the uniform distribution.

    :math:`\mathcal{U}(a, b)`.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        a: the lower bound of the uniform distribution
        b: the upper bound of the uniform distribution
        generator: the torch Generator to sample from (default: None)

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.uniform_(w)
    r
   )r   Ú	overridesÚhas_torch_function_variadicÚhandle_torch_functionr	   r   r
   r   r   r   r	   }   s         ÿr	   )r   r   r   r   r<   c                 C   s4   t j | ¡r&t jjt| f| |||d�S t| |||ƒS )aÁ  Fill the input Tensor with values drawn from the normal distribution.

    :math:`\mathcal{N}(\text{mean}, \text{std}^2)`.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        mean: the mean of the normal distribution
        std: the standard deviation of the normal distribution
        generator: the torch Generator to sample from (default: None)

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.normal_(w)
    r   )r   r=   r>   r?   r   r   r   r   r   r   r   ˜   s         ÿr   ç       Àr   )r   r   r   r   r   r   r<   c                 C   s   t | |||||d�S )a  Fill the input Tensor with values drawn from a truncated normal distribution.

    The values are effectively drawn from the
    normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)`
    with values outside :math:`[a, b]` redrawn until they are within
    the bounds. The method used for generating the random values works
    best when :math:`a \leq \text{mean} \leq b`.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        mean: the mean of the normal distribution
        std: the standard deviation of the normal distribution
        a: the minimum cutoff value
        b: the maximum cutoff value
        generator: the torch Generator to sample from (default: None)

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.trunc_normal_(w)
    r   )r(   )r   r   r   r   r   r   r   r   r   Útrunc_normal_²   s    rA   )r   r+   r<   c                 C   s,   t j | ¡r"t jjt| f| |d�S t| |ƒS )zþFill the input Tensor with the value :math:`\text{val}`.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        val: the value to fill the tensor with

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.constant_(w, 0.3)
    r*   )r   r=   r>   r?   Ú	constant_r,   r*   r   r   r   rB   Ñ   s    rB   )r   r<   c                 C   s
   t | dƒS )z¾Fill the input Tensor with the scalar value `1`.

    Args:
        tensor: an n-dimensional `torch.Tensor`

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.ones_(w)
    r   )r,   r.   r   r   r   Úones_á   s    
rC   c                 C   s   t | ƒS )z¿Fill the input Tensor with the scalar value `0`.

    Args:
        tensor: an n-dimensional `torch.Tensor`

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.zeros_(w)
    )r/   r.   r   r   r   Úzeros_î   s    
rD   c              	   C   sB   |   ¡ dkrtdƒ‚t ¡ � tj| j| | jdœŽ W 5 Q R X | S )a=  Fill the 2-dimensional input `Tensor` with the identity matrix.

    Preserves the identity of the inputs in `Linear` layers, where as
    many inputs are preserved as possible.

    Args:
        tensor: a 2-dimensional `torch.Tensor`

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.eye_(w)
    r   ú,Only tensors with 2 dimensions are supported)ÚoutÚrequires_grad)Ú
ndimensionr7   r   r   ÚeyeÚshaperG   r.   r   r   r   Úeye_û   s
    
 rK   r    c                 C   s&  |   ¡ }|dkrtdƒ‚|  ¡ }|d | dkr8tdƒ‚|d | }t||d ƒ}t ¡ �Â |  ¡  t|ƒD ]ª}t|ƒD ]œ}|dkr¦d| || | ||  d¡d f< qx|dkrÜd| || | ||  d¡d |  d¡d f< qxd| || | ||  d¡d |  d¡d |  d¡d f< qxqlW 5 Q R X | S )	aF  Fill the {3, 4, 5}-dimensional input `Tensor` with the Dirac delta function.

    Preserves the identity of the inputs in `Convolutional`
    layers, where as many input channels are preserved as possible. In case
    of groups>1, each group of channels preserves identity

    Args:
        tensor: a {3, 4, 5}-dimensional `torch.Tensor`
        groups (int, optional): number of groups in the conv layer (default: 1)
    Examples:
        >>> w = torch.empty(3, 16, 5, 5)
        >>> nn.init.dirac_(w)
        >>> w = torch.empty(3, 24, 5, 5)
        >>> nn.init.dirac_(w, 3)
    )é   é   é   z5Only tensors with 3, 4, or 5 dimensions are supportedr   z!dim 0 must be divisible by groupsr    rL   r   rM   )rH   r7   Úsizer!   r   r   r-   Úrange)r   ÚgroupsÚ
dimensionsZsizesZout_chans_per_grpZmin_dimÚgÚdr   r   r   Údirac_  s2    
"ÿÿÿ ÿrU   c                 C   sp   |   ¡ }|dk rtdƒ‚|  d¡}|  d¡}d}|   ¡ dkrX| jdd … D ]}||9 }qJ|| }|| }||fS )Nr   zNFan in and fan out can not be computed for tensor with fewer than 2 dimensionsr    r   )Zdimr7   rO   rJ   )r   rR   Znum_input_fmapsZnum_output_fmapsZreceptive_field_sizeÚsÚfan_inÚfan_outr   r   r   Ú_calculate_fan_in_and_fan_out<  s    


rY   )r   Úgainr   r<   c                 C   sD   t | ƒ\}}|t dt|| ƒ ¡ }t d¡| }t| | ||ƒS )aì  Fill the input `Tensor` with values using a Xavier uniform distribution.

    The method is described in `Understanding the difficulty of training
    deep feedforward neural networks` - Glorot, X. & Bengio, Y. (2010).
    The resulting tensor will have values sampled from
    :math:`\mathcal{U}(-a, a)` where

    .. math::
        a = \text{gain} \times \sqrt{\frac{6}{\text{fan\_in} + \text{fan\_out}}}

    Also known as Glorot initialization.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        gain: an optional scaling factor
        generator: the torch Generator to sample from (default: None)

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.xavier_uniform_(w, gain=nn.init.calculate_gain('relu'))
    r   ç      @)rY   r   r   r6   r   )r   rZ   r   rW   rX   r   r   r   r   r   Úxavier_uniform_O  s    r\   c                 C   s4   t | ƒ\}}|t dt|| ƒ ¡ }t| d||ƒS )aÔ  Fill the input `Tensor` with values using a Xavier normal distribution.

    The method is described in `Understanding the difficulty of training deep feedforward
    neural networks` - Glorot, X. & Bengio, Y. (2010). The resulting tensor
    will have values sampled from :math:`\mathcal{N}(0, \text{std}^2)` where

    .. math::
        \text{std} = \text{gain} \times \sqrt{\frac{2}{\text{fan\_in} + \text{fan\_out}}}

    Also known as Glorot initialization.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        gain: an optional scaling factor
        generator: the torch Generator to sample from (default: None)

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.xavier_normal_(w)
    r   r;   )rY   r   r   r6   r   )r   rZ   r   rW   rX   r   r   r   r   Úxavier_normal_n  s    r]   c                 C   sH   |  ¡ }ddg}||kr,td|› d|› �ƒ‚t| ƒ\}}|dkrD|S |S )NrW   rX   zMode z" not supported, please use one of )Úlowerr7   rY   )r   ÚmodeZvalid_modesrW   rX   r   r   r   Ú_calculate_correct_fan�  s    r`   rW   r1   ©r   r   r_   r8   r   c           	   
   C   s¢   t j | ¡r(t jjt| f| ||||d�S d| jkr@t d¡ | S t| |ƒ}t	||ƒ}|t
 |¡ }t
 d¡| }t  ¡ �  | j| ||d�W  5 Q R £ S Q R X dS )að  Fill the input `Tensor` with values using a Kaiming uniform distribution.

    The method is described in `Delving deep into rectifiers: Surpassing
    human-level performance on ImageNet classification` - He, K. et al. (2015).
    The resulting tensor will have values sampled from
    :math:`\mathcal{U}(-\text{bound}, \text{bound})` where

    .. math::
        \text{bound} = \text{gain} \times \sqrt{\frac{3}{\text{fan\_mode}}}

    Also known as He initialization.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        a: the negative slope of the rectifier used after this layer (only
            used with ``'leaky_relu'``)
        mode: either ``'fan_in'`` (default) or ``'fan_out'``. Choosing ``'fan_in'``
            preserves the magnitude of the variance of the weights in the
            forward pass. Choosing ``'fan_out'`` preserves the magnitudes in the
            backwards pass.
        nonlinearity: the non-linear function (`nn.functional` name),
            recommended to use only with ``'relu'`` or ``'leaky_relu'`` (default).
        generator: the torch Generator to sample from (default: None)

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.kaiming_uniform_(w, mode='fan_in', nonlinearity='relu')
    ra   r   ú,Initializing zero-element tensors is a no-opr[   r   N)r   r=   r>   r?   Úkaiming_uniform_rJ   r#   r$   r`   r:   r   r   r   r	   )	r   r   r_   r8   r   ÚfanrZ   r   Úboundr   r   r   rc   —  s&    #ù	




rc   c              
   C   sj   d| j krt d¡ | S t| |ƒ}t||ƒ}|t |¡ }t ¡ � | j	d||d�W  5 Q R £ S Q R X dS )aØ  Fill the input `Tensor` with values using a Kaiming normal distribution.

    The method is described in `Delving deep into rectifiers: Surpassing
    human-level performance on ImageNet classification` - He, K. et al. (2015).
    The resulting tensor will have values sampled from
    :math:`\mathcal{N}(0, \text{std}^2)` where

    .. math::
        \text{std} = \frac{\text{gain}}{\sqrt{\text{fan\_mode}}}

    Also known as He initialization.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        a: the negative slope of the rectifier used after this layer (only
            used with ``'leaky_relu'``)
        mode: either ``'fan_in'`` (default) or ``'fan_out'``. Choosing ``'fan_in'``
            preserves the magnitude of the variance of the weights in the
            forward pass. Choosing ``'fan_out'`` preserves the magnitudes in the
            backwards pass.
        nonlinearity: the non-linear function (`nn.functional` name),
            recommended to use only with ``'relu'`` or ``'leaky_relu'`` (default).
        generator: the torch Generator to sample from (default: None)

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.kaiming_normal_(w, mode='fan_out', nonlinearity='relu')
    r   rb   r   N)
rJ   r#   r$   r`   r:   r   r   r   r   r   )r   r   r_   r8   r   rd   rZ   r   r   r   r   Úkaiming_normal_Ï  s    #




rf   r   c           
   	   C   sÐ   |   ¡ dk rtdƒ‚|  ¡ dkr$| S |  d¡}|  ¡ | }|  ||¡jdd|d�}||k rb| ¡  tj 	|¡\}}t 
|d¡}| ¡ }	||	9 }||k rž| ¡  t ¡ �  |  |¡ |¡ |  |¡ W 5 Q R X | S )a   Fill the input `Tensor` with a (semi) orthogonal matrix.

    Described in `Exact solutions to the nonlinear dynamics of learning in deep
    linear neural networks` - Saxe, A. et al. (2013). The input tensor must have
    at least 2 dimensions, and for tensors with more than 2 dimensions the
    trailing dimensions are flattened.

    Args:
        tensor: an n-dimensional `torch.Tensor`, where :math:`n \geq 2`
        gain: optional scaling factor
        generator: the torch Generator to sample from (default: None)

    Examples:
        >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_LAPACK)
        >>> w = torch.empty(3, 5)
        >>> nn.init.orthogonal_(w)
    r   z4Only tensors with 2 or more dimensions are supportedr   r    r   )rH   r7   ZnumelrO   Únewr   Zt_r   ZlinalgZqrZdiagÚsignr   Zview_asZcopy_r%   )
r   rZ   r   ÚrowsÚcolsZ	flattenedÚqÚrrT   Úphr   r   r   Úorthogonal_ü  s&    

rn   r2   c           
   	   C   sˆ   |   ¡ dkrtdƒ‚| j\}}tt || ¡ƒ}t ¡ �F | jd||d� t	|ƒD ]&}t 
|¡}|d|… }	d| |	|f< qRW 5 Q R X | S )aŽ  Fill the 2D input `Tensor` as a sparse matrix.

    The non-zero elements will be drawn from the normal distribution
    :math:`\mathcal{N}(0, 0.01)`, as described in `Deep learning via
    Hessian-free optimization` - Martens, J. (2010).

    Args:
        tensor: an n-dimensional `torch.Tensor`
        sparsity: The fraction of elements in each column to be set to zero
        std: the standard deviation of the normal distribution used to generate
            the non-zero values
        generator: the torch Generator to sample from (default: None)

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.sparse_(w, sparsity=0.1)
    r   rE   r   r   N)rH   r7   rJ   r5   r   Úceilr   r   r   rP   Zrandperm)
r   Zsparsityr   r   ri   rj   Z	num_zerosZcol_idxZrow_indicesZzero_indicesr   r   r   Úsparse_/  s    


rp   c                    sF   ˆ j ‰ˆd d… ‰‡ ‡‡fdd„}dˆ› dˆ› dˆ› d�|_ˆ|_ |S )Néÿÿÿÿc                     s(   t jdˆ› dˆ› d�tdd� ˆ | |ŽS )Nz	`nn.init.z)` is now deprecated in favor of `nn.init.z`.r   r   )r#   r$   ÚFutureWarning)ÚargsÚkwargs©ÚmethÚnew_nameZold_namer   r   Údeprecated_initZ  s    ýz(_make_deprecate.<locals>.deprecated_initz
    z_(...)

    .. warning::
        This method is now deprecated in favor of :func:`torch.nn.init.z"`.

    See :func:`~torch.nn.init.z` for details.)Ú__name__Ú__doc__)rv   rx   r   ru   r   Ú_make_deprecateV  s    ÿüü
r{   )N)N)N)N)r;   r   N)r;   r   N)r;   r   r@   r   N)r    )r   N)r   N)r   rW   r1   N)r   rW   r1   N)r    N)r2   N)-rz   r   r#   r   r   Útypingr   Z	_Optionalr   r   r(   r,   r/   r:   r6   Ú	Generatorr	   r   rA   rB   rC   rD   rK   rU   rY   r\   r]   r`   Ústrrc   rf   rn   rp   r{   ÚuniformÚnormalZconstantrI   ZdiracZxavier_uniformZxavier_normalZkaiming_uniformZkaiming_normalZ
orthogonalÚsparser   r   r   r   Ú<module>   sà   


#
:   üû   üû     úù
,   ÿ  þ!  ýü    ûû:    ûû/  ýý6  üü'