U
    CŒÑhT'  ã                   @   s†   d dl mZmZ d dlZdgZejejdœdd„ZG dd„ dejjƒZ	G d	d
„ d
ejjƒZ
G dd„ dejjƒZG dd„ dejjƒZdS )é    )ÚOptionalÚTupleNÚ	Conformer)ÚlengthsÚreturnc                 C   sF   | j d }tt | ¡ ¡ ƒ}tj|| j| jd� ||¡|  	d¡k}|S )Nr   )ÚdeviceÚdtypeé   )
ÚshapeÚintÚtorchÚmaxÚitemZaranger   r   ÚexpandZ	unsqueeze)r   Z
batch_sizeÚ
max_lengthZpadding_mask© r   úI/tmp/user/2202/pip-unpacked-wheel-p89lpdx5/torchaudio/models/conformer.pyÚ_lengths_to_padding_mask	   s    
 ÿþr   c                	       sH   e Zd ZdZdeeeeeeddœ‡ fdd„Zej	ej	dœd	d
„Z
‡  ZS )Ú_ConvolutionModulea  Conformer convolution module.

    Args:
        input_dim (int): input dimension.
        num_channels (int): number of depthwise convolution layer input channels.
        depthwise_kernel_size (int): kernel size of depthwise convolution layer.
        dropout (float, optional): dropout probability. (Default: 0.0)
        bias (bool, optional): indicates whether to add bias term to each convolution layer. (Default: ``False``)
        use_group_norm (bool, optional): use GroupNorm rather than BatchNorm. (Default: ``False``)
    ç        FN)Ú	input_dimÚnum_channelsÚdepthwise_kernel_sizeÚdropoutÚbiasÚuse_group_normr   c                    sÌ   t ƒ  ¡  |d d dkr"tdƒ‚tj |¡| _tj tjj|d| ddd|d�tjj	dd�tjj|||d|d d ||d�|r�tjj
d|d�n
tj |¡tj ¡ tjj||ddd|d	�tj |¡¡| _d S )
Nr	   é   r   z<depthwise_kernel_size must be odd to achieve 'SAME' padding.)ÚstrideÚpaddingr   )Zdim)r   r   Úgroupsr   )Z
num_groupsr   )Zkernel_sizer   r   r   )ÚsuperÚ__init__Ú
ValueErrorr   ÚnnÚ	LayerNormÚ
layer_normÚ
SequentialZConv1dZGLUZ	GroupNormZBatchNorm1dÚSiLUÚDropoutÚ
sequential)Úselfr   r   r   r   r   r   ©Ú	__class__r   r   r!      sJ    	
ú
ù
ÿ
ú
áz_ConvolutionModule.__init__©Úinputr   c                 C   s,   |   |¡}| dd¡}|  |¡}| dd¡S )z�
        Args:
            input (torch.Tensor): with shape `(B, T, D)`.

        Returns:
            torch.Tensor: output, with shape `(B, T, D)`.
        r	   r   )r%   Ú	transposer)   )r*   r.   Úxr   r   r   ÚforwardM   s    

z_ConvolutionModule.forward)r   FF)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚfloatÚboolr!   r   ÚTensorr1   Ú__classcell__r   r   r+   r   r      s      ùø/r   c                       sB   e Zd ZdZd
eeeddœ‡ fdd„Zejejdœdd	„Z	‡  Z
S )Ú_FeedForwardModulezÍPositionwise feed forward layer.

    Args:
        input_dim (int): input dimension.
        hidden_dim (int): hidden dimension.
        dropout (float, optional): dropout probability. (Default: 0.0)
    r   N)r   Ú
hidden_dimr   r   c                    s`   t ƒ  ¡  tj tj |¡tjj||dd�tj ¡ tj |¡tjj||dd�tj |¡¡| _	d S )NT)r   )
r    r!   r   r#   r&   r$   ZLinearr'   r(   r)   )r*   r   r;   r   r+   r   r   r!   d   s    



úz_FeedForwardModule.__init__r-   c                 C   s
   |   |¡S )z—
        Args:
            input (torch.Tensor): with shape `(*, D)`.

        Returns:
            torch.Tensor: output, with shape `(*, D)`.
        )r)   )r*   r.   r   r   r   r1   o   s    z_FeedForwardModule.forward)r   )r2   r3   r4   r5   r   r6   r!   r   r8   r1   r9   r   r   r+   r   r:   [   s   r:   c                
       sf   e Zd ZdZdeeeeeeeddœ‡ fdd„Zej	ej	dœd	d
„Z
ej	eej	 ej	dœdd„Z‡  ZS )ÚConformerLayera¢  Conformer layer that constitutes Conformer.

    Args:
        input_dim (int): input dimension.
        ffn_dim (int): hidden layer dimension of feedforward network.
        num_attention_heads (int): number of attention heads.
        depthwise_conv_kernel_size (int): kernel size of depthwise convolution layer.
        dropout (float, optional): dropout probability. (Default: 0.0)
        use_group_norm (bool, optional): use ``GroupNorm`` rather than ``BatchNorm1d``
            in the convolution module. (Default: ``False``)
        convolution_first (bool, optional): apply the convolution module ahead of
            the attention module. (Default: ``False``)
    r   FN)r   Úffn_dimÚnum_attention_headsÚdepthwise_conv_kernel_sizer   r   Úconvolution_firstr   c                    sˆ   t ƒ  ¡  t|||d�| _tj |¡| _tjj|||d�| _	tj 
|¡| _t||||d|d�| _t|||d�| _tj |¡| _|| _d S )N)r   T)r   r   r   r   r   r   )r    r!   r:   Úffn1r   r#   r$   Úself_attn_layer_normZMultiheadAttentionÚ	self_attnr(   Úself_attn_dropoutr   Úconv_moduleÚffn2Úfinal_layer_normr@   )r*   r   r=   r>   r?   r   r   r@   r+   r   r   r!   ‰   s     

ú	zConformerLayer.__init__r-   c                 C   s2   |}|  dd¡}|  |¡}|  dd¡}|| }|S )Nr   r	   )r/   rE   )r*   r.   Úresidualr   r   r   Ú_apply_convolution¨   s    
z!ConformerLayer._apply_convolution)r.   Úkey_padding_maskr   c                 C   sš   |}|   |¡}|d | }| jr*|  |¡}|}|  |¡}| j||||dd�\}}|  |¡}|| }| jsr|  |¡}|}|  |¡}|d | }|  |¡}|S )a
  
        Args:
            input (torch.Tensor): input, with shape `(T, B, D)`.
            key_padding_mask (torch.Tensor or None): key padding mask to use in self attention layer.

        Returns:
            torch.Tensor: output, with shape `(T, B, D)`.
        g      à?F)ÚqueryÚkeyÚvaluerJ   Zneed_weights)rA   r@   rI   rB   rC   rD   rF   rG   )r*   r.   rJ   rH   r0   Ú_r   r   r   r1   °   s.    	


û




zConformerLayer.forward)r   FF)r2   r3   r4   r5   r   r6   r7   r!   r   r8   rI   r   r1   r9   r   r   r+   r   r<   z   s       ø÷r<   c                
       sX   e Zd ZdZd
eeeeeeeedœ‡ fdd„Zej	ej	e
ej	ej	f dœdd	„Z‡  ZS )r   a(  Conformer architecture introduced in
    *Conformer: Convolution-augmented Transformer for Speech Recognition*
    :cite:`gulati2020conformer`.

    Args:
        input_dim (int): input dimension.
        num_heads (int): number of attention heads in each Conformer layer.
        ffn_dim (int): hidden layer dimension of feedforward networks.
        num_layers (int): number of Conformer layers to instantiate.
        depthwise_conv_kernel_size (int): kernel size of each Conformer layer's depthwise convolution layer.
        dropout (float, optional): dropout probability. (Default: 0.0)
        use_group_norm (bool, optional): use ``GroupNorm`` rather than ``BatchNorm1d``
            in the convolution module. (Default: ``False``)
        convolution_first (bool, optional): apply the convolution module ahead of
            the attention module. (Default: ``False``)

    Examples:
        >>> conformer = Conformer(
        >>>     input_dim=80,
        >>>     num_heads=4,
        >>>     ffn_dim=128,
        >>>     num_layers=4,
        >>>     depthwise_conv_kernel_size=31,
        >>> )
        >>> lengths = torch.randint(1, 400, (10,))  # (batch,)
        >>> input = torch.rand(10, int(lengths.max()), input_dim)  # (batch, num_frames, input_dim)
        >>> output = conformer(input, lengths)
    r   F)r   Ú	num_headsr=   Ú
num_layersr?   r   r   r@   c	           	   	      s:   t ƒ  ¡  tj ‡ ‡‡‡‡‡‡fdd„t|ƒD ƒ¡| _d S )Nc                    s"   g | ]}t ˆˆˆˆˆˆˆ d �‘qS ))r   r   r@   )r<   )Ú.0rN   ©r@   r?   r   r=   r   rO   r   r   r   Ú
<listcomp>  s   
÷ùz&Conformer.__init__.<locals>.<listcomp>)r    r!   r   r#   Z
ModuleListÚrangeÚconformer_layers)	r*   r   rO   r=   rP   r?   r   r   r@   r+   rR   r   r!   õ   s    

öÿzConformer.__init__)r.   r   r   c                 C   s:   t |ƒ}| dd¡}| jD ]}|||ƒ}q| dd¡|fS )aX  
        Args:
            input (torch.Tensor): with shape `(B, T, input_dim)`.
            lengths (torch.Tensor): with shape `(B,)` and i-th element representing
                number of valid frames for i-th batch element in ``input``.

        Returns:
            (torch.Tensor, torch.Tensor)
                torch.Tensor
                    output frames, with shape `(B, T, input_dim)`
                torch.Tensor
                    output lengths, with shape `(B,)` and i-th element representing
                    number of valid frames for i-th batch element in output frames.
        r   r	   )r   r/   rU   )r*   r.   r   Zencoder_padding_maskr0   Zlayerr   r   r   r1     s
    
zConformer.forward)r   FF)r2   r3   r4   r5   r   r6   r7   r!   r   r8   r   r1   r9   r   r   r+   r   r   ×   s   $   ÷÷)Útypingr   r   r   Ú__all__r8   r   r#   ÚModuler   r:   r<   r   r   r   r   r   Ú<module>   s   	I]