U
    CŒÑh†“  ã                	   @   s  d dl Z d dlmZmZmZ d dlZdgZejejdœdd„Zdejejejejejeej eej dœdd	„Z	e
ejjd
œdd„Zee
 eeee  dœdd„Zee ee eejejdœdd„ZG dd„ dejjƒZG dd„ dejjƒZG dd„ dejjƒZG dd„ deƒZdS )é    N)ÚListÚOptionalÚTupleÚEmformer)ÚlengthsÚreturnc                 C   sF   | j d }tt | ¡ ¡ ƒ}tj|| j| jd� ||¡|  	d¡k}|S )Nr   )ÚdeviceÚdtypeé   )
ÚshapeÚintÚtorchÚmaxÚitemZaranger   r	   ÚexpandÚ	unsqueeze)r   Ú
batch_sizeÚ
max_lengthÚpadding_mask© r   úH/tmp/user/2202/pip-unpacked-wheel-p89lpdx5/torchaudio/models/emformer.pyÚ_lengths_to_padding_mask
   s    
 ÿþr   )Ú	utteranceÚright_contextÚsummaryr   ÚmemsÚleft_context_keyr   c                 C   sŒ   |  d¡|   d¡ |  d¡ }|  d¡}|dkr6d }nR|t |¡ ¡  |  d¡ }	|d k	rd|  d¡nd}
||  d¡ |	 |
 }t|d�}|S )Nr   r
   )r   )Úsizer   r   r   r   )r   r   r   r   r   r   ÚTÚBr   Úright_context_blocks_lengthZleft_context_blocks_lengthZklengthsr   r   r   Ú_gen_padding_mask   s    

r!   )Ú
activationr   c                 C   sH   | dkrt j ¡ S | dkr$t j ¡ S | dkr6t j ¡ S td| › �ƒ‚d S )NÚreluZgeluZsiluzUnsupported activation )r   ÚnnZReLUZGELUZSiLUÚ
ValueError)r"   r   r   r   Ú_get_activation_module'   s    


r&   )Úweight_init_scale_strategyÚ
num_layersr   c                 C   s`   | d krdd„ t |ƒD ƒS | dkr4dd„ t |ƒD ƒS | dkrNdd„ t |ƒD ƒS td| › �ƒ‚d S )Nc                 S   s   g | ]}d ‘qS ©Nr   )Ú.0Ú_r   r   r   Ú
<listcomp>4   s     z*_get_weight_init_gains.<locals>.<listcomp>Ú	depthwisec                 S   s   g | ]}d t  |d ¡ ‘qS )ç      ð?r
   ©ÚmathÚsqrt©r*   Ú	layer_idxr   r   r   r,   6   s     Zconstantc                 S   s   g | ]}d t  d¡ ‘qS )r.   é   r/   r2   r   r   r   r,   8   s     z-Unsupported weight_init_scale_strategy value )Úranger%   )r'   r(   r   r   r   Ú_get_weight_init_gains2   s    r6   )Ú
col_widthsÚcol_maskÚnum_rowsr   r   c                    s@   t | ƒt |ƒkrtdƒ‚‡ ‡fdd„t| |ƒD ƒ}tj|dd�S )Nz0Length of col_widths must match that of col_maskc                    s4   g | ],\}}|r t jˆ|ˆ d �nt jˆ|ˆ d �‘qS )©r   )r   ZonesÚzeros)r*   Z	col_widthZis_ones_col©r   r9   r   r   r,   C   s   þÿz-_gen_attention_mask_block.<locals>.<listcomp>r
   ©Údim)Úlenr%   Úzipr   Úcat)r7   r8   r9   r   Z
mask_blockr   r<   r   Ú_gen_attention_mask_block=   s    ürB   c                       s<  e Zd ZdZdeeeee eedœ‡ fdd„Ze	j
e	j
ee	j
e	j
f d	œd
d„Ze	j
e	j
ee	j
 e	j
dœdd„Zde	j
e	j
e	j
e	j
e	j
e	j
ee	j
 ee	j
 ee	j
e	j
e	j
e	j
f dœ	dd„Ze	j
e	j
e	j
e	j
e	j
e	j
ee	j
e	j
f dœdd„Ze	jje	j
e	j
e	j
e	j
e	j
e	j
e	j
ee	j
e	j
e	j
e	j
f dœdd„ƒZ‡  ZS )Ú_EmformerAttentiona_  Emformer layer attention module.

    Args:
        input_dim (int): input dimension.
        num_heads (int): number of attention heads in each Emformer layer.
        dropout (float, optional): dropout probability. (Default: 0.0)
        weight_init_gain (float or None, optional): scale factor to apply when initializing
            attention module parameters. (Default: ``None``)
        tanh_on_mem (bool, optional): if ``True``, applies tanh to memory elements. (Default: ``False``)
        negative_inf (float, optional): value to use for negative infinity in attention weights. (Default: -1e8)
    ç        NFç    „×—Á©Ú	input_dimÚ	num_headsÚdropoutÚweight_init_gainÚtanh_on_memÚnegative_infc                    sÐ   t ƒ  ¡  || dkr,td|› d|› d�ƒ‚|| _|| _|| _|| _|| _| j| j d | _t	j
j|d| dd�| _t	j
j||dd�| _t	j
j||dd�| _|rÌt	j
jj| jj|d	� t	j
jj| jj|d	� d S )
Nr   zinput_dim (z") is not a multiple of num_heads (z).g      à¿r4   T)Zbias)Zgain)ÚsuperÚ__init__r%   rG   rH   rI   rK   rL   Úscalingr   r$   ÚLinearÚemb_to_key_valueÚemb_to_queryÚout_projÚinitZxavier_uniform_Zweight)ÚselfrG   rH   rI   rJ   rK   rL   ©Ú	__class__r   r   rN   Y   s    	
z_EmformerAttention.__init__)Úinputr   r   c           
      C   sX   |j \}}}| d¡d }|d || … }t ||g¡}|  |¡jddd�\}}	||	fS )Nr   r
   r4   ©Úchunksr>   )r   r   r   rA   rQ   Úchunk)
rU   rX   r   r   r+   Úsummary_lengthZright_ctx_utterance_blockZmems_right_ctx_utterance_blockÚkeyÚvaluer   r   r   Ú_gen_key_valuew   s    z!_EmformerAttention._gen_key_value)Úattention_weightsÚattention_maskr   r   c                 C   sº   |  ¡ }| | d¡| j¡}| d¡}| d¡| j }|d k	r†| || j|d¡}| | d¡ d¡ tj	¡| j¡}| || j |d¡}tj
jj|dd� |¡}tj
jj|t | jƒ| jd�S )Nr   r
   éÿÿÿÿr4   r=   )ÚpÚtraining)ÚfloatZmasked_fillr   rL   r   rH   ÚviewÚtor   Úboolr$   Z
functionalZsoftmaxZtype_asrI   rd   )rU   r`   ra   r   Zattention_weights_floatr   r   Úattention_probsr   r   r   Ú_gen_attention_probs   s    
 ÿz'_EmformerAttention._gen_attention_probs)	r   r   r   r   r   ra   r   Úleft_context_valr   c	                    s  |  d¡‰ |  d¡|  d¡ |  d¡ }	ˆ t |||g¡¡}
ˆ t |||g¡¡jddd�\}}|d k	rö|d k	rö|	t |¡ ¡  |  d¡ }t |d |  d¡| … |||  d¡| d … g¡}t |d |  d¡| … |||  d¡| d … g¡}‡ ‡fdd„|
||fD ƒ\}}}t |ˆj	 | 
dd¡¡}t||||||ƒ}ˆ |||¡}t ||¡}|jˆ ˆj |	ˆjˆj fk�r„tdƒ‚| 
dd¡ ¡  |	ˆ ˆj¡}ˆ |¡}|  d¡}|d |	| … }||	| d … }ˆj�rèt |¡}ntj|dd	d
�}||||fS )Nr
   r   r4   rY   c                    s4   g | ],}|  ¡  d ˆ ˆj ˆjˆj ¡ dd¡‘qS )rb   r   r
   )Ú
contiguousrf   rH   rG   Ú	transpose)r*   Ztensor©r   rU   r   r   r,   ¸   s   ÿz4_EmformerAttention._forward_impl.<locals>.<listcomp>z+Computed attention has incorrect dimensionsiöÿÿÿé
   )Úminr   )r   rR   r   rA   rQ   r[   r   r   ZbmmrO   rm   r!   rj   r   rH   rG   ÚAssertionErrorrl   rf   rS   rK   ÚtanhÚclamp)rU   r   r   r   r   r   ra   r   rk   r   Úqueryr]   r^   r    Zreshaped_queryZreshaped_keyZreshaped_valuer`   r   ri   Ú	attentionZoutput_right_context_memsr\   Úoutput_right_contextÚoutput_memsr   rn   r   Ú_forward_impl’   sP    
$ýÿýÿ	þ
ý

z _EmformerAttention._forward_impl)r   r   r   r   r   ra   r   c           
      C   s,   |   ||||||¡\}}}	}	||dd… fS )ac  Forward pass for training.

        B: batch size;
        D: feature dimension of each frame;
        T: number of utterance frames;
        R: number of right context frames;
        S: number of summary elements;
        M: number of memory elements.

        Args:
            utterance (torch.Tensor): utterance frames, with shape `(T, B, D)`.
            lengths (torch.Tensor): with shape `(B,)` and i-th element representing
                number of valid frames for i-th batch element in ``utterance``.
            right_context (torch.Tensor): right context frames, with shape `(R, B, D)`.
            summary (torch.Tensor): summary elements, with shape `(S, B, D)`.
            mems (torch.Tensor): memory elements, with shape `(M, B, D)`.
            attention_mask (torch.Tensor): attention mask for underlying attention module.

        Returns:
            (Tensor, Tensor):
                Tensor
                    output frames corresponding to utterance and right_context, with shape `(T + R, B, D)`.
                Tensor
                    updated memory elements, with shape `(M, B, D)`.
        Nrb   )rx   )
rU   r   r   r   r   r   ra   Úoutputrw   r+   r   r   r   ÚforwardÛ   s    "z_EmformerAttention.forward)r   r   r   r   r   r   rk   r   c              
   C   sÔ   |  d¡|  d¡ |  d¡ }|  d¡|  d¡ |  d¡ |  d¡ }	t ||	¡jtj|jd�}
d|
dd|  d¡…f< | j||||||
||d�\}}}}||||  d¡|  d¡ d… ||  d¡|  d¡ d… fS )a½  Forward pass for inference.

        B: batch size;
        D: feature dimension of each frame;
        T: number of utterance frames;
        R: number of right context frames;
        S: number of summary elements;
        M: number of memory elements.

        Args:
            utterance (torch.Tensor): utterance frames, with shape `(T, B, D)`.
            lengths (torch.Tensor): with shape `(B,)` and i-th element representing
                number of valid frames for i-th batch element in ``utterance``.
            right_context (torch.Tensor): right context frames, with shape `(R, B, D)`.
            summary (torch.Tensor): summary elements, with shape `(S, B, D)`.
            mems (torch.Tensor): memory elements, with shape `(M, B, D)`.
            left_context_key (torch.Tensor): left context attention key computed from preceding invocation.
            left_context_val (torch.Tensor): left context attention value computed from preceding invocation.

        Returns:
            (Tensor, Tensor, Tensor, and Tensor):
                Tensor
                    output frames corresponding to utterance and right_context, with shape `(T + R, B, D)`.
                Tensor
                    updated memory elements, with shape `(M, B, D)`.
                Tensor
                    attention key computed for left context and utterance.
                Tensor
                    attention value computed for left context and utterance.
        r   ©r	   r   Trb   N)r   rk   )r   r   r;   rg   rh   r   rx   )rU   r   r   r   r   r   r   rk   Z	query_dimZkey_dimra   ry   rw   r]   r^   r   r   r   Úinfer   s&    )(øüz_EmformerAttention.infer)rD   NFrE   )NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   re   r   rh   rN   r   ÚTensorr   r_   rj   rx   rz   ÚjitÚexportr|   Ú__classcell__r   r   rV   r   rC   L   sd       ùù"
û  ÷öKø%÷rC   c                       sþ  e Zd ZdZd'eeeeeeeeee eedœ‡ fd	d
„Z	eee
j ee
j dœdd„Zee
j ee
je
je
jf dœdd„Ze
je
jee
jee
j ee
j dœdd„Ze
je
je
je
jdœdd„Ze
je
jee
je
jf dœdd„Ze
je
je
jee
je
jf dœdd„Ze
je
je
je
jee
j ee
je
jf dœdd„Ze
je
je
je
jeee
j  ee
je
jee
j f dœd d!„Ze
je
je
je
je
jee
je
je
jf dœd"d#„Ze
jje
je
je
jeee
j  e
jee
je
jee
j e
jf d$œd%d&„ƒZ‡  ZS )(Ú_EmformerLayera$  Emformer layer that constitutes Emformer.

    Args:
        input_dim (int): input dimension.
        num_heads (int): number of attention heads.
        ffn_dim: (int): hidden layer dimension of feedforward network.
        segment_length (int): length of each input segment.
        dropout (float, optional): dropout probability. (Default: 0.0)
        activation (str, optional): activation function to use in feedforward network.
            Must be one of ("relu", "gelu", "silu"). (Default: "relu")
        left_context_length (int, optional): length of left context. (Default: 0)
        max_memory_size (int, optional): maximum number of memory elements to use. (Default: 0)
        weight_init_gain (float or None, optional): scale factor to apply when initializing
            attention module parameters. (Default: ``None``)
        tanh_on_mem (bool, optional): if ``True``, applies tanh to memory elements. (Default: ``False``)
        negative_inf (float, optional): value to use for negative infinity in attention weights. (Default: -1e8)
    rD   r#   r   NFrE   )rG   rH   Úffn_dimÚsegment_lengthrI   r"   Úleft_context_lengthÚmax_memory_sizerJ   rK   rL   c              
      sÐ   t ƒ  ¡  t||||	|
|d�| _tj |¡| _tjj||dd�| _	t
|ƒ}tj tj |¡tj ||¡|tj |¡tj ||¡tj |¡¡| _tj |¡| _tj |¡| _|| _|| _|| _|| _|dk| _d S )NrF   T©Zkernel_sizeZstrideZ	ceil_moder   )rM   rN   rC   ru   r   r$   ZDropoutrI   Ú	AvgPool1dÚ	memory_opr&   Z
SequentialZ	LayerNormrP   Úpos_ffÚlayer_norm_inputÚlayer_norm_outputrˆ   r‡   r‰   rG   Úuse_mem)rU   rG   rH   r†   r‡   rI   r"   rˆ   r‰   rJ   rK   rL   Zactivation_modulerV   r   r   rN   R  s6    
ú


úz_EmformerLayer.__init__)r   r   r   c                 C   sb   t j| j|| j|d�}t j| j|| j|d�}t j| j|| j|d�}t jd|t j|d�}||||gS )Nr:   r
   r{   )r   r;   r‰   rG   rˆ   Zint32)rU   r   r   Zempty_memoryr   rk   Úpast_lengthr   r   r   Ú_init_state€  s
    z_EmformerLayer._init_state)Ústater   c                 C   s„   |d d d   ¡ }t| j|ƒ}t| jt || j ¡ƒ}|d | j| d … }|d | j| d … }|d | j| d … }|||fS )Né   r   r
   r4   )r   rp   rˆ   r‰   r0   Úceilr‡   )rU   r“   r‘   Zpast_left_context_lengthZpast_mem_lengthÚpre_memsÚlc_keyÚlc_valr   r   r   Ú_unpack_state‡  s    z_EmformerLayer._unpack_state)Únext_kÚnext_vÚupdate_lengthr   r“   r   c                 C   s’   t  |d |g¡}t  |d |g¡}t  |d |g¡| j d … |d< ||jd | j d … |d< ||jd | j d … |d< |d | |d< |S )Nr
   r4   r   r”   )r   rA   r‰   r   rˆ   )rU   rš   r›   rœ   r   r“   Znew_kZnew_vr   r   r   Ú_pack_state�  s    "z_EmformerLayer._pack_state)Ú	rc_outputr   r   r   c                 C   s4   |   |¡t ||g¡ }|  |¡| }|  |¡}|S r)   )rI   r   rA   r�   r�   )rU   rž   r   r   Úresultr   r   r   Ú_process_attention_output   s    
z(_EmformerLayer._process_attention_output)r   r   r   c                 C   s8   |   t ||g¡¡}|| d¡d … |d | d¡… fS ©Nr   )rŽ   r   rA   r   )rU   r   r   rŽ   r   r   r   Ú_apply_pre_attention_layer_norm«  s    þz._EmformerLayer._apply_pre_attention_layer_normc                 C   s2   |   |||¡}|| d¡d … |d | d¡… fS r¡   )r    r   )rU   rž   r   r   r   r   r   Ú_apply_post_attention_ffn´  s    z(_EmformerLayer._apply_post_attention_ffn)r   r   r   r   ra   r   c           	      C   sp   |d krt dƒ‚| jr6|  | ddd¡¡ ddd¡}nt d¡j|j|jd�}| j	||||||d�\}}||fS )Nz;attention_mask must be not None when for_inference is Falser
   r4   r   r{   )r   r   r   r   r   ra   )
r%   r�   rŒ   Úpermuter   Úemptyrg   r	   r   ru   )	rU   r   r   r   r   ra   r   rž   Únext_mr   r   r   Ú_apply_attention_forwardº  s     ú
z'_EmformerLayer._apply_attention_forward)r   r   r   r   r“   r   c              	   C   s¼   |d kr| j | d¡|jd�}|  |¡\}}}| jr`|  | ddd¡¡ ddd¡}	|	d d… }	nt d¡j	|j
|jd�}	| jj||||	|||d�\}
}}}|  ||| d¡||¡}|
||fS )Nr
   r:   r4   r   r{   )r   r   r   r   r   r   rk   )r’   r   r   r™   r�   rŒ   r¤   r   r¥   rg   r	   ru   r|   r�   )rU   r   r   r   r   r“   r–   r—   r˜   r   rž   r¦   rš   r›   r   r   r   Ú_apply_attention_inferÓ  s$    ù	z%_EmformerLayer._apply_attention_inferc                 C   sB   |   ||¡\}}|  |||||¡\}}	|  |||¡\}
}|
||	fS )a1  Forward pass for training.

        B: batch size;
        D: feature dimension of each frame;
        T: number of utterance frames;
        R: number of right context frames;
        M: number of memory elements.

        Args:
            utterance (torch.Tensor): utterance frames, with shape `(T, B, D)`.
            lengths (torch.Tensor): with shape `(B,)` and i-th element representing
                number of valid frames for i-th batch element in ``utterance``.
            right_context (torch.Tensor): right context frames, with shape `(R, B, D)`.
            mems (torch.Tensor): memory elements, with shape `(M, B, D)`.
            attention_mask (torch.Tensor): attention mask for underlying attention module.

        Returns:
            (Tensor, Tensor, Tensor):
                Tensor
                    encoded utterance frames, with shape `(T, B, D)`.
                Tensor
                    updated right context frames, with shape `(R, B, D)`.
                Tensor
                    updated memory elements, with shape `(M, B, D)`.
        )r¢   r§   r£   )rU   r   r   r   r   ra   Úlayer_norm_utteranceÚlayer_norm_right_contextrž   rw   Úoutput_utterancerv   r   r   r   rz   ï  s    $
ýûz_EmformerLayer.forward)r   r   r   r“   r   r   c                 C   sF   |   ||¡\}}|  |||||¡\}}	}
|  |||¡\}}|||
|	fS )a2  Forward pass for inference.

        B: batch size;
        D: feature dimension of each frame;
        T: number of utterance frames;
        R: number of right context frames;
        M: number of memory elements.

        Args:
            utterance (torch.Tensor): utterance frames, with shape `(T, B, D)`.
            lengths (torch.Tensor): with shape `(B,)` and i-th element representing
                number of valid frames for i-th batch element in ``utterance``.
            right_context (torch.Tensor): right context frames, with shape `(R, B, D)`.
            state (List[torch.Tensor] or None): list of tensors representing layer internal state
                generated in preceding invocation of ``infer``.
            mems (torch.Tensor): memory elements, with shape `(M, B, D)`.

        Returns:
            (Tensor, Tensor, List[torch.Tensor], Tensor):
                Tensor
                    encoded utterance frames, with shape `(T, B, D)`.
                Tensor
                    updated right context frames, with shape `(R, B, D)`.
                List[Tensor]
                    list of tensors representing layer internal state
                    generated in current invocation of ``infer``.
                Tensor
                    updated memory elements, with shape `(M, B, D)`.
        )r¢   r¨   r£   )rU   r   r   r   r“   r   r©   rª   rž   rw   Úoutput_stater«   rv   r   r   r   r|     s    )
ý    ÿ
z_EmformerLayer.infer)rD   r#   r   r   NFrE   )r}   r~   r   r€   r   re   Ústrr   rh   rN   r   r   r   r�   r’   r   r™   r�   r    r¢   r£   r§   r¨   rz   r‚   rƒ   r|   r„   r   r   rV   r   r…   ?  s’          ôô.&ùû þ
  þùùù/ùr…   c                       sØ   e Zd Zdejjeeeedœ‡ fdd„Zejejdœdd„Z	eee
e dœd	d
„Zejejdœdd„Zejejeejejf dœdd„Zejjdejejee
e
ej   eejeje
e
ej  f dœdd„ƒZ‡  ZS )Ú_EmformerImplr   )Úemformer_layersr‡   rˆ   Úright_context_lengthr‰   c                    sJ   t ƒ  ¡  |dk| _tjj||dd�| _|| _|| _|| _	|| _
|| _d S )Nr   TrŠ   )rM   rN   r�   r   r$   r‹   rŒ   r¯   rˆ   r°   r‡   r‰   )rU   r¯   r‡   rˆ   r°   r‰   rV   r   r   rN   P  s    

ýz_EmformerImpl.__init__)rX   r   c                 C   s‚   |j d }t || j | j ¡}g }t|d ƒD ].}|d | j }|| j }| |||… ¡ q0| ||| j d … ¡ t |¡S ©Nr   r
   )	r   r0   r•   r°   r‡   r5   Úappendr   rA   )rU   rX   r   Únum_segsZright_context_blocksÚseg_idxÚstartÚendr   r   r   Ú_gen_right_contextf  s    

z _EmformerImpl._gen_right_context)r´   Úutterance_lengthr   c              
   C   sÈ   t  || j ¡}| j}| j}|| }|| }t|| j | dƒ}t|d | j |ƒ}	| j| }
| jr¨t|| j dƒ}|d }||| || |||
| ||	| ||	 g	}n|||
| ||	| ||	 g}|S r±   )	r0   r•   r‡   r°   rˆ   r   rp   r�   r‰   )rU   r´   r¸   r³   ÚrcÚlcZrc_startZrc_endZ	seg_startZseg_endZ	rc_lengthZm_startZ
mem_lengthr7   r   r   r   Ú_gen_attention_mask_col_widthsq  s:    
÷ú	z,_EmformerImpl._gen_attention_mask_col_widthsc              	   C   s*  |  d¡}t || j ¡}g }g }g }| jr`d}dd„ t|ƒD ƒ}dd„ t|ƒD ƒ}	|||g}
n"d}dd„ t|ƒD ƒ}d }	||g}
t|ƒD ]z}|  ||¡}t||| j|j	ƒ}| 
|¡ t||t| j||| j  ƒ|j	ƒ}| 
|¡ |	d k	rŠt||	d|j	ƒ}| 
|¡ qŠdt d	d„ |
D ƒ¡  tj¡}|S )
Nr   é	   c                 S   s   g | ]}|d k‘qS ))r
   é   é   r   ©r*   Úidxr   r   r   r,      s     z5_EmformerImpl._gen_attention_mask.<locals>.<listcomp>c                 S   s   g | ]}|d k‘qS ))r½   r¾   r   r¿   r   r   r   r,   ¢  s     é   c                 S   s   g | ]}|d k‘qS ))r
   r½   r   r¿   r   r   r   r,   §  s     r
   c                 S   s   g | ]}t  |¡‘qS r   )r   rA   )r*   Úmaskr   r   r   r,   Â  s     )r   r0   r•   r‡   r�   r5   r»   rB   r°   r   r²   rp   r   rA   rg   rh   )rU   rX   r¸   r³   Zrc_maskZ
query_maskZsummary_maskZnum_colsZrc_q_cols_maskZs_cols_maskZmasks_to_concatr´   r7   Zrc_mask_blockZquery_mask_blockZsummary_mask_blockra   r   r   r   Ú_gen_attention_mask•  sL    
   ÿ
þù	
 z!_EmformerImpl._gen_attention_mask)rX   r   r   c           	      C   s¶   |  ddd¡}|  |¡}|d| d¡| j … }|  |¡}| jrf|  |  ddd¡¡  ddd¡dd… nt d¡j	|j
|jd�}|}| jD ]}||||||ƒ\}}}qˆ|  ddd¡|fS )aG  Forward pass for training and non-streaming inference.

        B: batch size;
        T: max number of input frames in batch;
        D: feature dimension of each frame.

        Args:
            input (torch.Tensor): utterance frames right-padded with right context frames, with
                shape `(B, T + right_context_length, D)`.
            lengths (torch.Tensor): with shape `(B,)` and i-th element representing
                number of valid utterance frames for i-th batch element in ``input``.

        Returns:
            (Tensor, Tensor):
                Tensor
                    output frames, with shape `(B, T, D)`.
                Tensor
                    output lengths, with shape `(B,)` and i-th element representing
                    number of valid frames for i-th batch element in output frames.
        r
   r   r4   Nrb   r{   )r¤   r·   r   r°   rÃ   r�   rŒ   r   r¥   rg   r	   r   r¯   )	rU   rX   r   r   r   ra   r   ry   Úlayerr   r   r   rz   Å  s    

ÿ(ý
z_EmformerImpl.forwardN)rX   r   Ústatesr   c                 C   s$  |  d¡| j| j kr:td| j| j › d|  d¡› d�ƒ‚| ddd¡}|  d¡| j }||d… }|d|… }tj|| j dd�}| jr¨|  | ddd¡¡ ddd¡nt 	d¡j
|j|jd	�}|}	g }
t| jƒD ]<\}}| |	|||dkrðdn|| |¡\}	}}}|
 |¡ qÒ|	 ddd¡||
fS )
a±  Forward pass for streaming inference.

        B: batch size;
        D: feature dimension of each frame.

        Args:
            input (torch.Tensor): utterance frames right-padded with right context frames, with
                shape `(B, segment_length + right_context_length, D)`.
            lengths (torch.Tensor): with shape `(B,)` and i-th element representing
                number of valid frames for i-th batch element in ``input``.
            states (List[List[torch.Tensor]] or None, optional): list of lists of tensors
                representing internal state generated in preceding invocation of ``infer``. (Default: ``None``)

        Returns:
            (Tensor, Tensor, List[List[Tensor]]):
                Tensor
                    output frames, with shape `(B, segment_length, D)`.
                Tensor
                    output lengths, with shape `(B,)` and i-th element representing
                    number of valid frames for i-th batch element in output frames.
                List[List[Tensor]]
                    output states; list of lists of tensors representing internal state
                    generated in current invocation of ``infer``.
        r
   zIPer configured segment_length and right_context_length, expected size of z# for dimension 1 of input, but got Ú.r   r4   N)rp   r{   )r   r‡   r°   r%   r¤   r   rs   r�   rŒ   r¥   rg   r	   r   Ú	enumerater¯   r|   r²   )rU   rX   r   rÅ   Zright_context_start_idxr   r   Zoutput_lengthsr   ry   Zoutput_statesr3   rÄ   r¬   r   r   r   r|   è  s2    ÿÿ ýûz_EmformerImpl.infer)r   r   r   )N)r}   r~   r   r   r$   Ú
ModuleListr   rN   r�   r·   r   r»   rÃ   r   rz   r‚   rƒ   r   r|   r„   r   r   rV   r   r®   O  s,      úú$0"# üûr®   c                       sD   e Zd ZdZdeeeeeeeeeeee eedœ‡ fd	d
„Z	‡  Z
S )r   a_  Emformer architecture introduced in
    *Emformer: Efficient Memory Transformer Based Acoustic Model for Low Latency Streaming Speech Recognition*
    :cite:`shi2021emformer`.

    See Also:
        * :func:`~torchaudio.models.emformer_rnnt_model`,
          :func:`~torchaudio.models.emformer_rnnt_base`: factory functions.
        * :class:`torchaudio.pipelines.RNNTBundle`: ASR pipelines with pretrained model.

    Args:
        input_dim (int): input dimension.
        num_heads (int): number of attention heads in each Emformer layer.
        ffn_dim (int): hidden layer dimension of each Emformer layer's feedforward network.
        num_layers (int): number of Emformer layers to instantiate.
        segment_length (int): length of each input segment.
        dropout (float, optional): dropout probability. (Default: 0.0)
        activation (str, optional): activation function to use in each Emformer layer's
            feedforward network. Must be one of ("relu", "gelu", "silu"). (Default: "relu")
        left_context_length (int, optional): length of left context. (Default: 0)
        right_context_length (int, optional): length of right context. (Default: 0)
        max_memory_size (int, optional): maximum number of memory elements to use. (Default: 0)
        weight_init_scale_strategy (str or None, optional): per-layer weight initialization scaling
            strategy. Must be one of ("depthwise", "constant", ``None``). (Default: "depthwise")
        tanh_on_mem (bool, optional): if ``True``, applies tanh to memory elements. (Default: ``False``)
        negative_inf (float, optional): value to use for negative infinity in attention weights. (Default: -1e8)

    Examples:
        >>> emformer = Emformer(512, 8, 2048, 20, 4, right_context_length=1)
        >>> input = torch.rand(128, 400, 512)  # batch, num_frames, feature_dim
        >>> lengths = torch.randint(1, 200, (128,))  # batch
        >>> output, lengths = emformer(input, lengths)
        >>> input = torch.rand(128, 5, 512)
        >>> lengths = torch.ones(128) * 5
        >>> output, lengths, states = emformer.infer(input, lengths, None)
    rD   r#   r   r-   FrE   )rG   rH   r†   r(   r‡   rI   r"   rˆ   r°   r‰   r'   rK   rL   c                    sV   t ||ƒ‰
tj ‡ ‡‡‡‡‡‡‡‡‡	‡
fdd„t|ƒD ƒ¡}tƒ j|ˆˆ|	ˆd� d S )Nc                    s.   g | ]&}t ˆˆˆˆˆˆ ˆˆˆ
| ˆ	ˆd �‘qS ))rI   r"   rˆ   r‰   rJ   rK   rL   )r…   r2   ©r"   rI   r†   rG   rˆ   r‰   rL   rH   r‡   rK   Zweight_init_gainsr   r   r,   ]  s   óõz%Emformer.__init__.<locals>.<listcomp>)rˆ   r°   r‰   )r6   r   r$   rÈ   r5   rM   rN   )rU   rG   rH   r†   r(   r‡   rI   r"   rˆ   r°   r‰   r'   rK   rL   r¯   rV   rÉ   r   rN   K  s    
òÿûzEmformer.__init__)rD   r#   r   r   r   r-   FrE   )r}   r~   r   r€   r   re   r­   r   rh   rN   r„   r   r   rV   r   r   &  s0   +        òò)N)r0   Útypingr   r   r   r   Ú__all__r�   r   r!   r­   r$   ÚModuler&   r   re   r6   rh   r   rB   rC   r…   r®   r   r   r   r   r   Ú<module>   s<    úù   þ t   X