U
    8‹ÑhÝ2  ã                   @   sê   d Z ddlmZmZ ddlmZ ddlmZ ddlZddl	m
Z
mZmZ ddlmZ ddlmZmZmZmZ dd	lmZ d
dddgZej e¡ ej e
¡ ej e¡ G dd„ deƒZG dd„ dejƒZedœdd
„Zedœdd„ZdS )zCDefines bias subclasses that work with scaled_dot_product_attentioné    )ÚautoÚIntEnum)ÚOptional)ÚwarnN)Úcan_use_efficient_attentionÚcan_use_flash_attentionÚ
SDPAParams)Ú_raise_kernel_warnings)Ú_calculate_scaleÚ_input_requires_gradÚ_postprocess_flash_outputÚ_validate_sdpa_input)Úscaled_dot_product_attentionÚcausal_upper_leftÚcausal_lower_rightÚCausalVariantÚ
CausalBiasc                   @   s   e Zd ZdZeƒ Zeƒ ZdS )r   a#  
    Enum for causal variants used in attention mechanisms.

    Defines two types of causal biases:

    `UPPER_LEFT`: Represents upper-left triangular bias for standard causal attention.
    The equivalent pytorch code for constructing this bias is:

    .. code-block:: python

        torch.tril(torch.ones(size, dtype=torch.bool))

    For instance, with `shape=(3,4)`, the materialized bias tensor will be:

    .. code-block:: text

        [[1, 0, 0, 0],
         [1, 1, 0, 0],
         [1, 1, 1, 0]]


    `LOWER_RIGHT`: Represents lower-right triangular bias, the include values are aligned to the lower
    right corner of the matrix.

    The equivalent pytorch code for constructing this bias is:

    .. code-block:: python

        diagonal_offset = size[1] - size[0]
        torch.tril(
            torch.ones(size, dtype=torch.bool),
            diagonal=diagonal_offset,
        )

    For instance, with `shape=(3,4)`, the materialized bias tensor will be:

    .. code-block:: text

        [[1, 1, 0, 0],
         [1, 1, 1, 0],
         [1, 1, 1, 1]]

    Note that these variants are equivalent to each other when the sequence lengths of the query and key/value
    tensors are equal since the triangular matrix is square.

    .. warning:: This enum is a prototype and subject to change.
    N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Ú
UPPER_LEFTÚLOWER_RIGHT© r   r   úE/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/nn/attention/bias.pyr      s   0c                   @   s¨   e Zd ZdZeeedœdd„Zejej	dœdd„Z
ejej	dœdd	„Zdeej ej	dœdd„Zedej	ej	ej	d eeee ej	dœdd„ƒZeddd„ƒZdd„ Zd
S )r   a  
    A bias representing causal attention patterns. For an overview of the bias structure, see the :class:`CausalVariant` enum.

    This class is used for defining causal (triangular) attention biases. For construing the bias, there exist
    two factory functions: :func:`causal_upper_left` and :func:`causal_lower_right`.

    Example:

    .. code-block:: python

        from torch.nn.attention.bias import causal_lower_right

        bsz, num_heads, seqlen_q, seqlen_kv, head_dim = 32, 8, 4, 12, 8

        # Create a lower-right causal bias
        attn_bias = causal_lower_right(seqlen_q, seqlen_kv)

        q = torch.randn(bsz, num_heads, seqlen_q, head_dim, device="cuda", dtype=torch.float16)
        k = torch.randn(bsz, num_heads, seqlen_kv, head_dim, device="cuda", dtype=torch.float16)
        v = torch.randn(bsz, num_heads, seqlen_kv, head_dim, device="cuda", dtype=torch.float16)

        out = F.scaled_dot_product_attention(q, k, v, attn_bias)

    .. warning:: This class is a prototype and subject to change.
    )ÚvariantÚ	seq_len_qÚ
seq_len_kvc                 C   s>   t |tƒst‚|| _|| _|| _||kr:|tjkr:tdƒ dS )aÞ  
        Initializes the CausalBias instance with a specified variant and sequence lengths.

        Args:
            variant (CausalVariant): The type of causal bias to use (either UPPER_LEFT or LOWER_RIGHT).
            seq_len_q (int): The sequence length of the query tensor.
            seq_len_kv (int): The sequence length of the key/value tensor.

        Raises a warning if the LOWER_RIGHT variant is used with seq_len_q > seq_len_kv, as it may produce NaNs.
        zTLower right causal bias will produce NaNs in the output when seq_len_q > seq_len_kv!N)Ú
isinstancer   ÚAssertionErrorr   r   r   r   r   )Úselfr   r   r   r   r   r   Ú__init__n   s    ÿzCausalBias.__init__)ÚdeviceÚreturnc                 C   s   t  t j| j| j|t jd�¡S )zUpper left causal bias©r"   Zdtype)ÚtorchÚtrilÚonesr   r   Úbool©r    r"   r   r   r   Ú_upper_left‚   s    ÿzCausalBias._upper_leftc                 C   s.   | j | j }tjtj| j| j |tjd�|d�S )zLower right causal biasr$   )Zdiagonal)r   r   r%   r&   r'   r(   )r    r"   Zdiagonal_offsetr   r   r   Ú_lower_rightˆ   s       ÿüzCausalBias._lower_rightNc                 C   sB   |dkrt  d¡}| jtjkr(|  |¡S | jtjkr>|  |¡S dS )a˜  
        Materializes the causal bias into a tensor form.

        Depending on the variant, this method generates either an upper-left or lower-right
        triangular matrix to represent the causal bias.

        Args:
            device (Optional[torch.device]): The device on which to create the tensor. Defaults to CPU.

        Returns:
            torch.Tensor: The materialized bias tensor.
        NÚcpu)r%   r"   r   r   r   r*   r   r+   r)   r   r   r   Ú_materialize’   s    

zCausalBias._materializeç        F)ÚqueryÚkeyÚvalueÚ	attn_maskÚ	dropout_pÚ	is_causalÚscaler#   c                 C   sâ  |rt dƒ‚|j|jks$|jtjkr:t| ||d|d|d�S |jtjk�rÎt| ||d|||ƒ t	| ||d||ƒ}t
|ƒ�r4|  d¡d dk}|  d¡}	t|	|ƒ}
|�r
tjj | dd|  d¡d  f¡} tjj |dd| d¡d  f¡}tjj |dd| d¡d  f¡}tjjj| |||dd|
d	�d }t||	ƒS t|ƒ�r¦d}t| ||ƒ�rTd}tjjj|  d
d¡| d
d¡| d
d¡ddddd|t|jƒ||dd�d  d
d¡S t|ƒ t| ||| | j¡|d|d�S nt d|j› �ƒ‚dS )a·  
        Handles the logic for computing attention with the specified causal bias.

        Args:
            query (Tensor): Query tensor; shape :math:`(N, ..., L, E)`.
            key (Tensor): Key tensor; shape :math:`(N, ..., S, E)`.
            value (Tensor): Value tensor; shape :math:`(N, ..., S, Ev)`.
            attn_mask (CausalBias): The type of causal attention to apply.
                A boolean mask where a value of True indicates that the element *should* take part in attention.
                A float mask of the same type as query, key, value that is added to the attention score.
            dropout_p (float): Dropout probability; if greater than 0.0, dropout is applied
            is_causal (bool): If true, assumes upper left causal attention masking and errors if both attn_mask and is_causal
                are set.
            scale (optional float): Scaling factor applied prior to softmax. If None, the default value is set
                to :math:`\frac{1}{\sqrt{E}}`.

        Returns:
            output (Tensor): Attention output; shape :math:`(N, ..., L, Ev)`.

        Raises:
            ValueError: If the causal bias variant is not a CausalVariant type.

        z.CausalBias should not be used with causal=TrueNT)r2   r3   r4   r5   éÿÿÿÿé   r   F)r4   Zreturn_debug_maskr5   é   é   )
ZbiasZcu_seqlens_qZcu_seqlens_kZmax_seqlen_qZmax_seqlen_kr3   Zcustom_mask_typeÚcompute_log_sumexpr5   Zseqlen_kz<CausalBias.variant must be a CausalVariant type, but found: )Ú
ValueErrorr   r   r   r   r   r   r   r   r   r   Úsizer
   r%   ÚnnÚ
functionalÚpadÚopsZatenZ#_scaled_dot_product_flash_attentionr   r   r   Z_efficient_attention_forwardZ	transposeÚintr	   r-   r"   )r/   r0   r1   r2   r3   r4   r5   Zsdpa_paramsZneeds_paddingZog_head_sizeZog_scaleÚoutr:   r   r   r   Ú	_dispatch¦   s–    !
ÿ
þù	


"""ùø	




óò ò
ù

ÿzCausalBias._dispatchr   c                 C   s.   |dkri }|t jjjkr"tdƒ‚| j||ŽS )zjDefines the behavior of torch.nn.functional.scaled_dot_product_attention when the attn_bias is an AttnBiasNz5CausalBias only supports scaled_dot_product_attention)r%   r=   r>   r   ÚNotImplementedErrorrC   )ÚclsÚfuncÚtypesÚargsÚkwargsr   r   r   Ú__torch_function__  s    ÿzCausalBias.__torch_function__c                 C   s   |   ¡  ¡ S )N)r-   Ú__repr__)r    r   r   r   rK     s    zCausalBias.__repr__)N)r.   FN)r   N)r   r   r   r   r   rA   r!   r%   r"   ÚTensorr*   r+   r   r-   ÚstaticmethodÚfloatr(   rC   ÚclassmethodrJ   rK   r   r   r   r   r   S   s,   
   ùøi
)r#   c                  G   s*   t | ƒdkstdƒ‚| \}}ttj||ƒS )a&  
    Creates an upper-left triangular causal bias.

    This function generates a upper-left triangular matrix to represent causal attention bias with a
    diagonal offset set so that the inclusive values are aligned to the upper left corner of the matrix.
    This equivalent to the `is_causal=True` argument in `scaled_dot_product_attention`.

    The equivalent pytorch code for constructing this bias is:

    .. code-block:: python

        torch.tril(torch.ones(size, dtype=torch.bool))

    For instance, with `shape=(3,4)`, the materialized bias tensor will be:

    .. code-block:: text

        [[1, 0, 0, 0],
         [1, 1, 0, 0],
         [1, 1, 1, 0]]

    Args:
        size: The size of the bias matrix.

    Returns:
        CausalBias: The UPPER_LEFT triangular causal bias variant.
    r9   z*causal_upper_left only supports 2D tensors)Úlenr   r   r   r   ©r<   r   r   r   r   r   r     s    c                  G   s*   t | ƒdkstdƒ‚| \}}ttj||ƒS )a:  
    Creates a lower-right triangular causal bias.

    This function generates a lower-right triangular matrix to represent causal attention bias with a
    diagonal offset set so that the inclusive values are aligned to the lower right corner of the matrix.

    The equivalent pytorch code for constructing this bias is:

    .. code-block:: python

        diagonal_offset = size[1] - size[0]
        torch.tril(
            torch.ones(size, dtype=torch.bool),
            diagonal=diagonal_offset,
        )

    For instance, with `shape=(3,4)`, the materialized bias tensor will be:

    .. code-block:: text

        [[1, 1, 0, 0],
         [1, 1, 1, 0],
         [1, 1, 1, 1]]

    Args:
        size: The size of the bias matrix.

    Returns:
        CausalBias: The LOWER_RIGHT triangular causal bias variant.
    r9   z+causal_lower_right only supports 2D tensors)rP   r   r   r   r   rQ   r   r   r   r   @  s    )r   Úenumr   r   Útypingr   Úwarningsr   r%   Ztorch.backends.cudar   r   r   Ztorch.nn.attentionr	   Ztorch.nn.attention._utilsr
   r   r   r   Ztorch.nn.functionalr   Ú__all__Z_dynamoZallow_in_graphr   rL   r   r   r   r   r   r   r   Ú<module>   s"   5 M!