U
    CŒÑhV<  ã                   @   s²   d dl Z d dlmZmZmZ d dlZd dlm  mZ	 d dlmZm
Z
 dddddgZG d	d„ dejƒZG d
d„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZdS )é    N)ÚListÚOptionalÚTuple)ÚnnÚTensorÚResBlockÚ	MelResNetÚ	Stretch2dÚUpsampleNetworkÚWaveRNNc                       s:   e Zd ZdZd
eddœ‡ fdd„Zeedœdd	„Z‡  ZS )r   af  ResNet block based on *Efficient Neural Audio Synthesis* :cite:`kalchbrenner2018efficient`.

    Args:
        n_freq: the number of bins in a spectrogram. (Default: ``128``)

    Examples
        >>> resblock = ResBlock()
        >>> input = torch.rand(10, 128, 512)  # a random spectrogram
        >>> output = resblock(input)  # shape: (10, 128, 512)
    é€   N)Ún_freqÚreturnc                    sR   t ƒ  ¡  t tj||ddd�t |¡tjdd�tj||ddd�t |¡¡| _d S )Né   F©Úin_channelsÚout_channelsÚkernel_sizeÚbiasT©Zinplace)ÚsuperÚ__init__r   Ú
SequentialÚConv1dÚBatchNorm1dÚReLUÚresblock_model)Úselfr   ©Ú	__class__© úG/tmp/user/2202/pip-unpacked-wheel-p89lpdx5/torchaudio/models/wavernn.pyr      s    

ûzResBlock.__init__©Úspecgramr   c                 C   s   |   |¡| S )zéPass the input through the ResBlock layer.
        Args:
            specgram (Tensor): the input sequence to the ResBlock layer (n_batch, n_freq, n_time).

        Return:
            Tensor shape: (n_batch, n_freq, n_time)
        )r   ©r   r#   r    r    r!   Úforward(   s    	zResBlock.forward)r   ©	Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úintr   r   r%   Ú__classcell__r    r    r   r!   r      s   c                       sB   e Zd ZdZdeeeeeddœ‡ fdd„Zeed	œd
d„Z‡  ZS )r   a�  MelResNet layer uses a stack of ResBlocks on spectrogram.

    Args:
        n_res_block: the number of ResBlock in stack. (Default: ``10``)
        n_freq: the number of bins in a spectrogram. (Default: ``128``)
        n_hidden: the number of hidden dimensions of resblock. (Default: ``128``)
        n_output: the number of output dimensions of melresnet. (Default: ``128``)
        kernel_size: the number of kernel size in the first Conv1d layer. (Default: ``5``)

    Examples
        >>> melresnet = MelResNet()
        >>> input = torch.rand(10, 128, 512)  # a random spectrogram
        >>> output = melresnet(input)  # shape: (10, 128, 508)
    é
   r   é   N)Ún_res_blockr   Ún_hiddenÚn_outputr   r   c                    sf   t ƒ  ¡  ‡ fdd„t|ƒD ƒ}tjtj|ˆ |dd�t ˆ ¡tjdd�f|tjˆ |dd�fžŽ | _d S )	Nc                    s   g | ]}t ˆ ƒ‘qS r    )r   )Ú.0Ú_©r0   r    r!   Ú
<listcomp>I   s     z&MelResNet.__init__.<locals>.<listcomp>Fr   Tr   r   )r   r   r   )	r   r   Úranger   r   r   r   r   Úmelresnet_model)r   r/   r   r0   r1   r   Z	ResBlocksr   r4   r!   r   D   s    

ýûzMelResNet.__init__r"   c                 C   s
   |   |¡S )zÿPass the input through the MelResNet layer.
        Args:
            specgram (Tensor): the input sequence to the MelResNet layer (n_batch, n_freq, n_time).

        Return:
            Tensor shape: (n_batch, n_output, n_time - kernel_size + 1)
        )r7   r$   r    r    r!   r%   S   s    	zMelResNet.forward)r-   r   r   r   r.   r&   r    r    r   r!   r   4   s            ÿ    þc                       s:   e Zd ZdZeeddœ‡ fdd„Zeedœdd„Z‡  ZS )	r	   a‘  Upscale the frequency and time dimensions of a spectrogram.

    Args:
        time_scale: the scale factor in time dimension
        freq_scale: the scale factor in frequency dimension

    Examples
        >>> stretch2d = Stretch2d(time_scale=10, freq_scale=5)

        >>> input = torch.rand(10, 100, 512)  # a random spectrogram
        >>> output = stretch2d(input)  # shape: (10, 500, 5120)
    N)Ú
time_scaleÚ
freq_scaler   c                    s   t ƒ  ¡  || _|| _d S ©N)r   r   r9   r8   )r   r8   r9   r   r    r!   r   m   s    
zStretch2d.__init__r"   c                 C   s   |  | jd¡  | jd¡S )zþPass the input through the Stretch2d layer.

        Args:
            specgram (Tensor): the input sequence to the Stretch2d layer (..., n_freq, n_time).

        Return:
            Tensor shape: (..., n_freq * freq_scale, n_time * time_scale)
        éþÿÿÿéÿÿÿÿ)Zrepeat_interleaver9   r8   r$   r    r    r!   r%   s   s    
zStretch2d.forwardr&   r    r    r   r!   r	   _   s   c                	       sP   e Zd ZdZdee eeeeeddœ‡ fdd„Zeeeef d	œd
d„Z	‡  Z
S )r
   añ  Upscale the dimensions of a spectrogram.

    Args:
        upsample_scales: the list of upsample scales.
        n_res_block: the number of ResBlock in stack. (Default: ``10``)
        n_freq: the number of bins in a spectrogram. (Default: ``128``)
        n_hidden: the number of hidden dimensions of resblock. (Default: ``128``)
        n_output: the number of output dimensions of melresnet. (Default: ``128``)
        kernel_size: the number of kernel size in the first Conv1d layer. (Default: ``5``)

    Examples
        >>> upsamplenetwork = UpsampleNetwork(upsample_scales=[4, 4, 16])
        >>> input = torch.rand(10, 128, 10)  # a random spectrogram
        >>> output = upsamplenetwork(input)  # shape: (10, 128, 1536), (10, 128, 1536)
    r-   r   r.   N)Úupsample_scalesr/   r   r0   r1   r   r   c                    sÔ   t ƒ  ¡  d}|D ]}||9 }q|| _|d d | | _t|||||ƒ| _t|dƒ| _g }	|D ]d}
t|
dƒ}tj	ddd|
d d fd|
fdd�}t
jj |jd|
d d  ¡ |	 |¡ |	 |¡ q^tj|	Ž | _d S )Nr   é   r   F)r   r   r   Úpaddingr   ç      ð?)r   r   Útotal_scaleÚindentr   Úresnetr	   Úresnet_stretchr   ZConv2dÚtorchÚinitZ	constant_ZweightÚappendr   Úupsample_layers)r   r=   r/   r   r0   r1   r   rA   Úupsample_scaleZ	up_layersZscaleZstretchÚconvr   r    r!   r   ‘   s,    	


    ÿ
zUpsampleNetwork.__init__r"   c                 C   sf   |   |¡ d¡}|  |¡}| d¡}| d¡}|  |¡}| d¡dd…dd…| j| j …f }||fS )a¿  Pass the input through the UpsampleNetwork layer.

        Args:
            specgram (Tensor): the input sequence to the UpsampleNetwork layer (n_batch, n_freq, n_time)

        Return:
            Tensor shape: (n_batch, n_freq, (n_time - kernel_size + 1) * total_scale),
                          (n_batch, n_output, (n_time - kernel_size + 1) * total_scale)
        where total_scale is the product of all elements in upsample_scales.
        r   N)rC   Ú	unsqueezerD   ÚsqueezerH   rB   )r   r#   Zresnet_outputZupsampling_outputr    r    r!   r%   °   s    



&zUpsampleNetwork.forward)r-   r   r   r   r.   )r'   r(   r)   r*   r   r+   r   r   r   r%   r,   r    r    r   r!   r
   €   s         ùøc                       s~   e Zd ZdZdee eeeeeeeeeddœ‡ fdd	„Zeeed
œdd„Ze	j
jdeee eeee f dœdd„ƒZ‡  ZS )r   aW  WaveRNN model from *Efficient Neural Audio Synthesis* :cite:`wavernn`
    based on the implementation from `fatchord/WaveRNN <https://github.com/fatchord/WaveRNN>`_.

    The original implementation was introduced in *Efficient Neural Audio Synthesis*
    :cite:`kalchbrenner2018efficient`. The input channels of waveform and spectrogram have to be 1.
    The product of `upsample_scales` must equal `hop_length`.

    See Also:
        * `Training example <https://github.com/pytorch/audio/tree/release/0.12/examples/pipeline_wavernn>`__
        * :class:`torchaudio.pipelines.Tacotron2TTSBundle`: TTS pipeline with pretrained model.

    Args:
        upsample_scales: the list of upsample scales.
        n_classes: the number of output classes.
        hop_length: the number of samples between the starts of consecutive frames.
        n_res_block: the number of ResBlock in stack. (Default: ``10``)
        n_rnn: the dimension of RNN layer. (Default: ``512``)
        n_fc: the dimension of fully connected layer. (Default: ``512``)
        kernel_size: the number of kernel size in the first Conv1d layer. (Default: ``5``)
        n_freq: the number of bins in a spectrogram. (Default: ``128``)
        n_hidden: the number of hidden dimensions of resblock. (Default: ``128``)
        n_output: the number of output dimensions of melresnet. (Default: ``128``)

    Example
        >>> wavernn = WaveRNN(upsample_scales=[5,5,8], n_classes=512, hop_length=200)
        >>> waveform, sample_rate = torchaudio.load(file)
        >>> # waveform shape: (n_batch, n_channel, (n_time - kernel_size + 1) * hop_length)
        >>> specgram = MelSpectrogram(sample_rate)(waveform)  # shape: (n_batch, n_channel, n_freq, n_time)
        >>> output = wavernn(waveform, specgram)
        >>> # output shape: (n_batch, n_channel, (n_time - kernel_size + 1) * hop_length, n_classes)
    r-   é   r.   r   N)r=   Ú	n_classesÚ
hop_lengthr/   Ún_rnnÚn_fcr   r   r0   r1   r   c                    s:  t ƒ  ¡  || _|d r |d n|d | _|| _|
d | _|| _|| _tt	 
| j¡ƒ| _d}|D ]}||9 }q`|| jkrŒtd|› d|› �ƒ‚t||||	|
|ƒ| _t || j d |¡| _tj||dd�| _tj|| j |dd�| _tjdd�| _tjdd�| _t || j |¡| _t || j |¡| _t || j¡| _d S )	Nr>   r   é   z/Expected: total_scale == hop_length, but found z != T)Zbatch_firstr   )r   r   r   Ú_padrP   Ún_auxrO   rN   r+   ÚmathÚlog2Ún_bitsÚ
ValueErrorr
   Úupsampler   ZLinearÚfcZGRUÚrnn1Úrnn2r   Úrelu1Úrelu2Úfc1Úfc2Úfc3)r   r=   rN   rO   r/   rP   rQ   r   r   r0   r1   rA   rI   r   r    r!   r   è   s,    



zWaveRNN.__init__)Úwaveformr#   r   c                    s  |  d¡dkrtdƒ‚|  d¡dkr,tdƒ‚| d¡| d¡ }}|  d¡}tjd|ˆ j|j|jd�}tjd|ˆ j|j|jd�}ˆ  |¡\}}| 	dd¡}| 	dd¡}‡ fdd„t
d	ƒD ƒ}|d
d
…d
d
…|d |d …f }|d
d
…d
d
…|d |d …f }	|d
d
…d
d
…|d |d …f }
|d
d
…d
d
…|d |d …f }tj| d¡||gdd�}ˆ  |¡}|}ˆ  ||¡\}}|| }|}tj||	gdd�}ˆ  ||¡\}}|| }tj||
gdd�}ˆ  |¡}ˆ  |¡}tj||gdd�}ˆ  |¡}ˆ  |¡}ˆ  |¡}| d¡S )a  Pass the input through the WaveRNN model.

        Args:
            waveform: the input waveform to the WaveRNN layer (n_batch, 1, (n_time - kernel_size + 1) * hop_length)
            specgram: the input spectrogram to the WaveRNN layer (n_batch, 1, n_freq, n_time)

        Return:
            Tensor: shape (n_batch, 1, (n_time - kernel_size + 1) * hop_length, n_classes)
        r   z*Require the input channel of waveform is 1z*Require the input channel of specgram is 1r   )ÚdtypeÚdevicer>   c                    s   g | ]}ˆ j | ‘qS r    ©rT   ©r2   Úi©r   r    r!   r5   .  s     z#WaveRNN.forward.<locals>.<listcomp>r.   Né   rR   r<   ©Zdim)ÚsizerX   rL   rE   ÚzerosrP   rc   rd   rY   Z	transposer6   ÚcatrK   rZ   r[   r\   r_   r]   r`   r^   ra   )r   rb   r#   Z
batch_sizeÚh1Úh2ÚauxZaux_idxZa1Za2Úa3Za4ÚxÚresr3   r    rh   r!   r%     sB    
""""





zWaveRNN.forward)r#   Úlengthsr   c                    s  |j }|j}tjj |ˆjˆjf¡}ˆ |¡\}‰ |dk	rF|ˆjj }g }| 	¡ \}}}tj
d|ˆjf||d�}	tj
d|ˆjf||d�}
tj
|df||d�}‡ ‡fdd„tdƒD ƒ}t|ƒD �]8‰|dd…dd…ˆf }‡fdd„|D ƒ\}}}}tj|||gdd�}ˆ |¡}ˆ | d¡|	¡\}}	||	d	  }tj||gdd�}ˆ | d¡|
¡\}}
||
d	  }tj||gdd�}t ˆ |¡¡}tj||gdd�}t ˆ |¡¡}ˆ |¡}tj|dd�}t |d¡ ¡ }d
| d
ˆj d  d }| |¡ q¼t |¡ dd
d	¡|fS )a¾  Inference method of WaveRNN.

        This function currently only supports multinomial sampling, which assumes the
        network is trained on cross entropy loss.

        Args:
            specgram (Tensor):
                Batch of spectrograms. Shape: `(n_batch, n_freq, n_time)`.
            lengths (Tensor or None, optional):
                Indicates the valid length of each audio in the batch.
                Shape: `(batch, )`.
                When the ``specgram`` contains spectrograms with different durations,
                by providing ``lengths`` argument, the model will compute
                the corresponding valid output lengths.
                If ``None``, it is assumed that all the audio in ``waveforms``
                have valid length. Default: ``None``.

        Returns:
            (Tensor, Optional[Tensor]):
            Tensor
                The inferred waveform of size `(n_batch, 1, n_time)`.
                1 stands for a single channel.
            Tensor or None
                If ``lengths`` argument was provided, a Tensor of shape `(batch, )`
                is returned.
                It indicates the valid length in time axis of the output Tensor.
        Nr   )rd   rc   c                    s6   g | ].}ˆ d d …ˆj | ˆj |d  …d d …f ‘qS )Nr   re   rf   )rp   r   r    r!   r5   x  s     z!WaveRNN.infer.<locals>.<listcomp>rR   c                    s"   g | ]}|d d …d d …ˆ f ‘qS r:   r    )r2   Úa)rg   r    r!   r5   ~  s     rj   r   r>   r@   )rd   rc   rE   r   Ú
functionalÚpadrS   rY   rA   rk   rl   rP   r6   rm   rZ   r[   rK   r\   ÚFZrelur_   r`   ra   ZsoftmaxZmultinomialÚfloatrW   rG   ÚstackZpermute)r   r#   rt   rd   rc   ÚoutputZb_sizer3   Zseq_lenrn   ro   rr   Z	aux_splitZm_tZa1_tZa2_tZa3_tZa4_tÚinpZlogitsZ	posteriorr    )rp   rg   r   r!   ÚinferK  s@    

zWaveRNN.infer)r-   rM   rM   r.   r   r   r   )N)r'   r(   r)   r*   r   r+   r   r   r%   rE   ZjitÚexportr   r   r}   r,   r    r    r   r!   r   Ç   s0   %       õô*9)rU   Útypingr   r   r   rE   Ztorch.nn.functionalr   rv   rx   r   Ú__all__ÚModuler   r   r	   r
   r   r    r    r    r!   Ú<module>   s   û	#+!G