U
    ‹Ñh3   ã                   @   s’   d dl Z ddlmZ ddlmZ ddlmZ dd„ Zeejejejdœd	d
„ƒZeejejejdœdd„ƒZ	G dd„ de j
jƒZG dd„ dƒZdS )é    Né   )Újit)Úlanguage)Únext_power_of_2c                 C   s4   | dkrdS | dkrdS | dkr$dS | dkr0dS d	S )
Né€   é   é   é   i   é   i   é   é   © )Únr   r   úL/tmp/user/2202/pip-unpacked-wheel-sn6cy74c/triton/ops/blocksparse/softmax.pyÚ	num_warps   s    r   )ÚROW_SIZEÚ
BLOCK_SIZEÚIS_DENSEc                  C   sô  t  d¡}t  d¡}t  d¡}|t  d¡ | }t  d|
¡| }t  d|
¡| }||| d  }t  |d ¡}t  |d ¡}|| }||| | | 7 }||| | 7 }|rºt  d|
¡}nH|dt  d¡ t  d¡ |  }t j|| | ||k dd�}|| | }||k }t j|| | |tdƒ d�}| t j¡}|}||9 }|d k	�r¦||| 7 }||| 7 }|| d | }|dk||k @ }t j|||  | |dd�}||7 }| t j¡}t  ||k|	@ tdƒ |¡}t  	|¡}t j
| | | ||d� d S )Nr   r   r	   ©ÚmaskÚotherÚinfç        ©r   )ÚtlÚ
program_idÚnum_programsÚarangeÚloadÚfloatÚtoÚfloat32ÚwhereÚsoftmaxÚstore) ÚOutÚAZ	stride_xzÚLUTÚRÚextentÚ	stride_zrÚ	stride_hrÚscaleÚ	is_causalr   r   r   ÚhÚmÚzÚhmÚlane_nÚblock_nÚheaderÚsizeÚoffsetZoff_aÚnsÚoff_lutÚstart_nr   ÚaÚoutÚoff_loÚmask_loÚ
rel_logitsr   r   r   Ú_blocksparse_softmax_fwd   sB    


 

r?   c           '      C   s"  t  d¡}t  d¡}t  d¡}|t  d¡ | }t  d|¡| }t  d|¡| }||| d  }t  |d ¡}t  |d ¡}|| | | }||| | 7 }||k }|||  | }|||  | }|rÖt  d|¡}nD|dt  d¡ t  d¡ |  }t j|| | |dd�} | | | }t j|| |dd�}!|! t j¡}!t j|| |dd�}"|" t j¡}"t  ||k|@ |!|!k@ d|!¡}!|!|"t  |!|" d¡  }#|d k	�rò|||
 7 }||| 7 }|	| d | }$|$dk|$|	k @ |@ }%t j	|||	  |$ |#|%d� |#| }#| ||  | }&t j	|&| |#|d� d S )Nr   r   r	   r   r   r   )
r   r   r   r   r   r    r!   r"   Úsumr$   )'ZDAZ
stride_zdxZDOutZstride_zdoutr%   Zstride_zoutr,   r'   ZDRr)   r*   r+   Z	stride_err-   r   r   r   r.   r/   r0   r1   r2   r3   r4   r5   r6   Zoff_mnr   ZAsZDOutsr7   r8   r9   r:   ÚdoutÚdar<   r=   ZDAsr   r   r   Ú_blocksparse_softmax_bwdJ   sD    


 
rC   c                   @   s0   e Zd Zedd„ ƒZedd„ ƒZedd„ ƒZdS )Ú_softmaxc              	   C   sâ   t jg t j| jd�}| ¡ }t| jd ƒD ]*}t  || |d d …d d …f  d¡f¡}q*|| }t  	|¡}t j
|d d… dd�|dd …< | jdd�d d …df }t j||fdd� d¡}	t  |	|f¡ t j¡ |¡}
|
t| ¡ ƒfS )	N©ÚdtypeÚdevicer   éÿÿÿÿ)Zdimr   F)Úas_tupler	   )ÚtorchZtensorZint64rG   ÚcloneÚrangeÚshapeÚcatr@   Z
zeros_likeZcumsumZnonzeroÚstackÚviewÚtypeZint32r    ÚintÚmax)ÚlayoutÚblockrG   Ú_emptyZsizesr.   Ztotal_sizesÚoffsetsÚcolumnsr4   Úlutr   r   r   Úmake_lut†   s    (
z_softmax.make_lutc
                 C   s  |d k	r,t |tjƒr,|jjdks$t‚| ¡ }|jd }
|d |d | |
g}|d krXdn|j}|d krjdn| ¡ }t 	|¡}t
| ||| d¡|||d |d |d |||t|ƒ|	t|ƒd� |  ||¡ || _|| _|| _|| _|| _|| _|j| _|	| _|| _|S )NÚcpur   r   )r   r   r   r   rH   ©r   r   r   r   )Ú
isinstancerJ   ZTensorrG   rQ   ÚAssertionErrorÚitemrM   ÚstrideÚ
empty_liker?   r   r   Zsave_for_backwardÚspdimsrU   Úmaxlutr,   Ú	rel_shapeÚrel_stridesrF   Ú	rel_dtypeÚis_denser-   )Úctxr:   r,   r>   r-   rb   rU   rY   rc   rg   ÚMÚgridrd   re   r;   r   r   r   Úforward—   sF    

      øz_softmax.forwardc                 C   sô   | j \}}d }| jd r.tj| j| j|jd�}|jd }| jd | jd | j	 |f}t 
|¡}t| || d¡|| d¡|| d¡| j||| jd | jd | jd | jd | j| j	t| jƒ| jt| jƒd� |d d |d d d d d d d d d d d d d d fS )Nr   rE   r   r   rH   r	   r\   )Zsaved_tensorsZneeds_input_gradrJ   Úzerosrd   rf   rG   rM   rb   rU   ra   rC   r`   r,   re   r-   r   rc   rg   r   )rh   rA   r;   rY   Zdrri   rj   rB   r   r   r   Úbackwardº   s8    



       õz_softmax.backwardN)Ú__name__Ú
__module__Ú__qualname__ÚstaticmethodrZ   rk   rm   r   r   r   r   rD   „   s   

"rD   c                   @   s(   e Zd Zd	dd„Zddddœdd„ZdS )
r#   Fc                 C   s8   |j | _|| _|| _t | j| j|¡\| _| _|| _d S )N)	rM   rb   rT   rU   rD   rZ   rY   rc   rg   )ÚselfrT   rU   rG   rg   r   r   r   Ú__init__Ø   s
    zsoftmax.__init__g      ð?N)r,   r>   r-   c                C   sL   |d k	r$|j |j kr$td|j › �ƒ‚t ||||| j| j| j| j| j¡	}|S )Nz$relative position embedding must be )	rF   Ú
ValueErrorrD   Úapplyrb   rU   rY   rc   rg   )rr   r:   r,   r>   r-   r   r   r   Ú__call__ß   s    ÿzsoftmax.__call__)F)rn   ro   rp   rs   rv   r   r   r   r   r#   Ö   s   
r#   )rJ   Ú r   r   r   r   r   Z	constexprr?   rC   ZautogradZFunctionrD   r#   r   r   r   r   Ú<module>   s    û5÷9R