U
    CŒÑhµ5  ã                   @   s`  d dl Z d dlZd dlmZmZ d dlmZ d dlmZ d dl	m
Z
mZmZ d dlZd dlZd dlmZ d dlmZmZmZ g Zde e ej¡j¡ Zed	d
e ƒZdd„ ZG dd„ dejjƒZ G dd„ dejjƒZ!G dd„ deƒZ"G dd„ deƒZ#G dd„ dejje"ƒZ$G dd„ de#ƒZ%eG dd„ dƒƒZ&e&deedd�ddd d!d"d#d$d%d&d d'�Z'd(e'_(dS ))é    N)ÚABCÚabstractmethod)Ú	dataclass)Úpartial)ÚCallableÚListÚTuple)Úmodule_utils)Úemformer_rnnt_baseÚRNNTÚRNNTBeamSearché(   é
   gš™™™™™©?c                 C   s@   t  | | tjk ¡| | tjk< | | tjk tj | | tjk< | S ©N)ÚtorchÚlogÚmathÚe©Úx© r   úP/tmp/user/2202/pip-unpacked-wheel-p89lpdx5/torchaudio/pipelines/rnnt_pipeline.pyÚ_piecewise_linear_log   s    r   c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )Ú_FunctionalModulec                    s   t ƒ  ¡  || _d S r   )ÚsuperÚ__init__Ú
functional)Úselfr   ©Ú	__class__r   r   r      s    
z_FunctionalModule.__init__c                 C   s
   |   |¡S r   )r   ©r   Úinputr   r   r   Úforward   s    z_FunctionalModule.forward©Ú__name__Ú
__module__Ú__qualname__r   r"   Ú__classcell__r   r   r   r   r      s   r   c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )Ú_GlobalStatsNormalizationc              	      s\   t ƒ  ¡  t|ƒ�}t | ¡ ¡}W 5 Q R X |  dt |d ¡¡ |  dt |d ¡¡ d S )NÚmeanÚ	invstddev)	r   r   ÚopenÚjsonÚloadsÚreadZregister_bufferr   Útensor)r   Zglobal_stats_pathÚfZblobr   r   r   r   $   s
    

z"_GlobalStatsNormalization.__init__c                 C   s   || j  | j S r   )r)   r*   r    r   r   r   r"   -   s    z!_GlobalStatsNormalization.forwardr#   r   r   r   r   r(   #   s   	r(   c                   @   s.   e Zd Zeejeejejf dœdd„ƒZdS )Ú_FeatureExtractor©r!   Úreturnc                 C   s   dS )áX  Generates features and length output from the given input tensor.

        Args:
            input (torch.Tensor): input tensor.

        Returns:
            (torch.Tensor, torch.Tensor):
            torch.Tensor:
                Features, with shape `(length, *)`.
            torch.Tensor:
                Length, with shape `(1,)`.
        Nr   r    r   r   r   Ú__call__2   s    z_FeatureExtractor.__call__N)r$   r%   r&   r   r   ÚTensorr   r5   r   r   r   r   r1   1   s   r1   c                   @   s$   e Zd Zeee edœdd„ƒZdS )Ú_TokenProcessor)Útokensr3   c                 K   s   dS )zÊDecodes given list of tokens to text sequence.

        Args:
            tokens (List[int]): list of tokens to decode.

        Returns:
            str:
                Decoded text sequence.
        Nr   )r   r8   Úkwargsr   r   r   r5   C   s    z_TokenProcessor.__call__N)r$   r%   r&   r   r   ÚintÚstrr5   r   r   r   r   r7   B   s   r7   c                       sJ   e Zd ZdZejjddœ‡ fdd„Zeje	ejejf dœdd„Z
‡  ZS )	Ú_ModuleFeatureExtractorz›``torch.nn.Module``-based feature extraction pipeline.

    Args:
        pipeline (torch.nn.Module): module that implements feature extraction logic.
    N)Úpipeliner3   c                    s   t ƒ  ¡  || _d S r   )r   r   r=   )r   r=   r   r   r   r   W   s    
z _ModuleFeatureExtractor.__init__r2   c                 C   s$   |   |¡}t |jd g¡}||fS )r4   r   )r=   r   r/   Úshape)r   r!   ÚfeaturesÚlengthr   r   r   r"   [   s    
z_ModuleFeatureExtractor.forward)r$   r%   r&   Ú__doc__r   ÚnnÚModuler   r6   r   r"   r'   r   r   r   r   r<   P   s   r<   c                   @   s8   e Zd ZdZeddœdd„Zd
ee eedœdd	„Z	dS )Ú_SentencePieceTokenProcessorztSentencePiece-model-based token processor.

    Args:
        sp_model_path (str): path to SentencePiece model.
    N)Úsp_model_pathr3   c                 C   sJ   t  d¡stdƒ‚dd l}|j|d�| _| j ¡ | j ¡ | j ¡ h| _	d S )NÚsentencepiecez2SentencePiece is not available. Please install it.r   )Z
model_file)
r	   Zis_module_availableÚRuntimeErrorrF   ZSentencePieceProcessorÚsp_modelZunk_idZeos_idZpad_idÚpost_process_remove_list)r   rE   Zspmr   r   r   r   t   s    
ýz%_SentencePieceTokenProcessor.__init__T)r8   Úlstripr3   c                    sH   ‡ fdd„|dd… D ƒ}d  ˆ j |¡¡ dd¡}|r@| ¡ S |S dS )aX  Decodes given list of tokens to text sequence.

        Args:
            tokens (List[int]): list of tokens to decode.
            lstrip (bool, optional): if ``True``, returns text sequence with leading whitespace
                removed. (Default: ``True``).

        Returns:
            str:
                Decoded text sequence.
        c                    s   g | ]}|ˆ j kr|‘qS r   )rI   )Ú.0Ztoken_index©r   r   r   Ú
<listcomp>�   s    
 z9_SentencePieceTokenProcessor.__call__.<locals>.<listcomp>é   NÚ u   â–�ú )ÚjoinrH   Zid_to_pieceÚreplacerJ   )r   r8   rJ   Zfiltered_hypo_tokensZoutput_stringr   rL   r   r5   �   s    

ÿz%_SentencePieceTokenProcessor.__call__)T)
r$   r%   r&   rA   r;   r   r   r:   Úboolr5   r   r   r   r   rD   m   s   rD   c                   @   sL  e Zd ZU dZG dd„ deƒZG dd„ deƒZee	d< e
g ef e	d< ee	d< ee	d	< ee	d
< ee	d< ee	d< ee	d< ee	d< ee	d< ee	d< ee	d< edœdd„Zeedœdd„ƒZeedœdd„ƒZeedœdd„ƒZeedœdd„ƒZeedœdd„ƒZeedœdd „ƒZedœd!d"„Zedœd#d$„Zedœd%d&„Zedœd'd(„Zd)S )*Ú
RNNTBundleu«  Dataclass that bundles components for performing automatic speech recognition (ASR, speech-to-text)
    inference with an RNN-T model.

    More specifically, the class provides methods that produce the featurization pipeline,
    decoder wrapping the specified RNN-T model, and output token post-processor that together
    constitute a complete end-to-end ASR inference pipeline that produces a text sequence
    given a raw waveform.

    It can support non-streaming (full-context) inference as well as streaming inference.

    Users should not directly instantiate objects of this class; rather, users should use the
    instances (representing pre-trained models) that exist within the module,
    e.g. :data:`torchaudio.pipelines.EMFORMER_RNNT_BASE_LIBRISPEECH`.

    Example
        >>> import torchaudio
        >>> from torchaudio.pipelines import EMFORMER_RNNT_BASE_LIBRISPEECH
        >>> import torch
        >>>
        >>> # Non-streaming inference.
        >>> # Build feature extractor, decoder with RNN-T model, and token processor.
        >>> feature_extractor = EMFORMER_RNNT_BASE_LIBRISPEECH.get_feature_extractor()
        100%|â–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆ| 3.81k/3.81k [00:00<00:00, 4.22MB/s]
        >>> decoder = EMFORMER_RNNT_BASE_LIBRISPEECH.get_decoder()
        Downloading: "https://download.pytorch.org/torchaudio/models/emformer_rnnt_base_librispeech.pt"
        100%|â–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆ| 293M/293M [00:07<00:00, 42.1MB/s]
        >>> token_processor = EMFORMER_RNNT_BASE_LIBRISPEECH.get_token_processor()
        100%|â–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆ| 295k/295k [00:00<00:00, 25.4MB/s]
        >>>
        >>> # Instantiate LibriSpeech dataset; retrieve waveform for first sample.
        >>> dataset = torchaudio.datasets.LIBRISPEECH("/home/librispeech", url="test-clean")
        >>> waveform = next(iter(dataset))[0].squeeze()
        >>>
        >>> with torch.no_grad():
        >>>     # Produce mel-scale spectrogram features.
        >>>     features, length = feature_extractor(waveform)
        >>>
        >>>     # Generate top-10 hypotheses.
        >>>     hypotheses = decoder(features, length, 10)
        >>>
        >>> # For top hypothesis, convert predicted tokens to text.
        >>> text = token_processor(hypotheses[0][0])
        >>> print(text)
        he hoped there would be stew for dinner turnips and carrots and bruised potatoes and fat mutton pieces to [...]
        >>>
        >>>
        >>> # Streaming inference.
        >>> hop_length = EMFORMER_RNNT_BASE_LIBRISPEECH.hop_length
        >>> num_samples_segment = EMFORMER_RNNT_BASE_LIBRISPEECH.segment_length * hop_length
        >>> num_samples_segment_right_context = (
        >>>     num_samples_segment + EMFORMER_RNNT_BASE_LIBRISPEECH.right_context_length * hop_length
        >>> )
        >>>
        >>> # Build streaming inference feature extractor.
        >>> streaming_feature_extractor = EMFORMER_RNNT_BASE_LIBRISPEECH.get_streaming_feature_extractor()
        >>>
        >>> # Process same waveform as before, this time sequentially across overlapping segments
        >>> # to simulate streaming inference. Note the usage of ``streaming_feature_extractor`` and ``decoder.infer``.
        >>> state, hypothesis = None, None
        >>> for idx in range(0, len(waveform), num_samples_segment):
        >>>     segment = waveform[idx: idx + num_samples_segment_right_context]
        >>>     segment = torch.nn.functional.pad(segment, (0, num_samples_segment_right_context - len(segment)))
        >>>     with torch.no_grad():
        >>>         features, length = streaming_feature_extractor(segment)
        >>>         hypotheses, state = decoder.infer(features, length, 10, state=state, hypothesis=hypothesis)
        >>>     hypothesis = hypotheses[0]
        >>>     transcript = token_processor(hypothesis[0])
        >>>     if transcript:
        >>>         print(transcript, end=" ", flush=True)
        he hoped there would be stew for dinner turn ips and car rots and bru 'd oes and fat mut ton pieces to [...]
    c                   @   s   e Zd ZdZdS )zRNNTBundle.FeatureExtractorz:Interface of the feature extraction part of RNN-T pipelineN©r$   r%   r&   rA   r   r   r   r   ÚFeatureExtractorâ   s   rV   c                   @   s   e Zd ZdZdS )zRNNTBundle.TokenProcessorz7Interface of the token processor part of RNN-T pipelineNrU   r   r   r   r   ÚTokenProcessorå   s   rW   Ú
_rnnt_pathÚ_rnnt_factory_funcÚ_global_stats_pathÚ_sp_model_pathÚ_right_paddingÚ_blankÚ_sample_rateÚ_n_fftÚ_n_melsÚ_hop_lengthÚ_segment_lengthÚ_right_context_length)r3   c                 C   s6   |   ¡ }tj | j¡}t |¡}| |¡ | ¡  |S r   )	rY   Ú
torchaudioÚutilsÚdownload_assetrX   r   ÚloadZload_state_dictÚeval)r   ÚmodelÚpathZ
state_dictr   r   r   Ú
_get_modelõ   s    

zRNNTBundle._get_modelc                 C   s   | j S )zSSample rate (in cycles per second) of input waveforms.

        :type: int
        )r^   rL   r   r   r   Úsample_rateý   s    zRNNTBundle.sample_ratec                 C   s   | j S )z7Size of FFT window to use.

        :type: int
        )r_   rL   r   r   r   Ún_fft  s    zRNNTBundle.n_fftc                 C   s   | j S )z`Number of mel spectrogram features to extract from input waveforms.

        :type: int
        )r`   rL   r   r   r   Ún_mels  s    zRNNTBundle.n_melsc                 C   s   | j S )zdNumber of samples between successive frames in input expected by model.

        :type: int
        )ra   rL   r   r   r   Ú
hop_length  s    zRNNTBundle.hop_lengthc                 C   s   | j S )zTNumber of frames in segment in input expected by model.

        :type: int
        )rb   rL   r   r   r   Úsegment_length  s    zRNNTBundle.segment_lengthc                 C   s   | j S )zcNumber of frames in right contextual block in input expected by model.

        :type: int
        )rc   rL   r   r   r   Úright_context_length%  s    zRNNTBundle.right_context_lengthc                 C   s   |   ¡ }t|| jƒS )zOConstructs RNN-T decoder.

        Returns:
            RNNTBeamSearch
        )rk   r   r]   )r   ri   r   r   r   Úget_decoder-  s    zRNNTBundle.get_decoderc                    s^   t j ˆ j¡}ttj t jj	ˆ j
ˆ jˆ jˆ jd�tdd„ ƒtdd„ ƒt|ƒt‡ fdd„ƒ¡ƒS )zzConstructs feature extractor for non-streaming (full-context) ASR.

        Returns:
            FeatureExtractor
        ©rl   rm   rn   ro   c                 S   s   |   dd¡S ©NrN   r   ©Z	transposer   r   r   r   Ú<lambda>B  ó    z2RNNTBundle.get_feature_extractor.<locals>.<lambda>c                 S   s   t | t ƒS r   ©r   Ú_gainr   r   r   r   rv   C  rw   c                    s   t jj | dddˆ jf¡S )Nr   )r   rB   r   Úpadr\   r   rL   r   r   rv   E  rw   ©rd   re   rf   rZ   r<   r   rB   Z
SequentialZ
transformsZMelSpectrogramrl   rm   rn   ro   r   r(   ©r   Ú
local_pathr   rL   r   Úget_feature_extractor6  s       ÿ

ùÿz RNNTBundle.get_feature_extractorc              	   C   sP   t j | j¡}ttj t jj	| j
| j| j| jd�tdd„ ƒtdd„ ƒt|ƒ¡ƒS )zvConstructs feature extractor for streaming (simultaneous) ASR.

        Returns:
            FeatureExtractor
        rs   c                 S   s   |   dd¡S rt   ru   r   r   r   r   rv   U  rw   z<RNNTBundle.get_streaming_feature_extractor.<locals>.<lambda>c                 S   s   t | t ƒS r   rx   r   r   r   r   rv   V  rw   r{   r|   r   r   r   Úget_streaming_feature_extractorI  s       ÿ

úÿz*RNNTBundle.get_streaming_feature_extractorc                 C   s   t j | j¡}t|ƒS )zQConstructs token processor.

        Returns:
            TokenProcessor
        )rd   re   rf   r[   rD   r|   r   r   r   Úget_token_processor[  s    zRNNTBundle.get_token_processorN)r$   r%   r&   rA   r1   rV   r7   rW   r;   Ú__annotations__r   r   r:   rk   Úpropertyrl   rm   rn   ro   rp   rq   r   rr   r~   r   r€   r   r   r   r   rT   ˜   s@   
H	rT   z(models/emformer_rnnt_base_librispeech.pti  )Znum_symbolsz2pipeline-assets/global_stats_rnnt_librispeech.jsonz.pipeline-assets/spm_bpe_4096_librispeech.modelé   i   i€>  i�  éP   é    é   )rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   aì  ASR pipeline based on Emformer-RNNT,
pretrained on *LibriSpeech* dataset :cite:`7178964`,
capable of performing both streaming and non-streaming inference.

The underlying model is constructed by :py:func:`torchaudio.models.emformer_rnnt_base`
and utilizes weights trained on LibriSpeech using training script ``train.py``
`here <https://github.com/pytorch/audio/tree/main/examples/asr/emformer_rnnt>`__ with default arguments.

Please refer to :py:class:`RNNTBundle` for usage instructions.
))r,   r   Úabcr   r   Zdataclassesr   Ú	functoolsr   Útypingr   r   r   r   rd   Ztorchaudio._internalr	   Ztorchaudio.modelsr
   r   r   Ú__all__Úlog10ZiinfoZint16ÚmaxZ_decibelÚpowry   r   rB   rC   r   r(   r1   r7   r<   rD   rT   ZEMFORMER_RNNT_BASE_LIBRISPEECHrA   r   r   r   r   Ú<module>   sJ   	+ M
ô