U
    ‹Ñh)$  ã                   @   sˆ  d dl Z ddlmZmZmZmZmZ ddlmZ ddl	m
Z
mZ e je je je jgZdd„ Zd	d
„ Zdd„ Zdd„ Zeedddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�geƒ  dddge
eddœd�edd d!„ iƒeejejejejejejejejejejd"œ
d#d$„ƒƒƒZG d%d&„ d&e jjƒZejZdS )'é    Né   )ÚConfigÚautotuneÚcdivÚ
heuristicsÚjit)Úlanguageé   )Úearly_config_pruneÚestimate_matmul_timec                 C   s   dt | ƒkrtjS | S )NZfp8)ÚstrÚtorchÚfloat16)Úa© r   ú?/tmp/user/2202/pip-unpacked-wheel-sn6cy74c/triton/ops/matmul.pyÚupcast_if_fp8
   s    r   c                 C   sb   t | ƒ} t |ƒ}| |kr| S | tks(t‚|tks4t‚tD ]$}| |krL|  S ||kr8|   S q8d S ©N)r   Ú_ordered_datatypesÚAssertionError)r   ÚbÚdr   r   r   Úget_higher_dtype   s    r   c                    s   ‡ fdd„S )Nc                    s   | ˆ    ¡ S r   )Zzero_)Únargs©Únamer   r   Ú<lambda>!   ó    zinit_to_zero.<locals>.<lambda>r   r   r   r   r   Úinit_to_zero    s    r   c                  C   sŒ   g } dD ]~}dD ]t}dD ]j}dD ]`}|dkr0dnd}|   t|||dd	œ||d
�¡ dD ](}|   t||||d	œ||tdƒd�¡ qVq qqq| S )N)r   é   é   é   é   )é   é    )r$   é@   )r$   r%   é€   é   r%   r   r    r	   ©ÚBLOCK_MÚBLOCK_NÚBLOCK_KÚSPLIT_K©Ú
num_stagesÚ	num_warps)r   r    é   r#   ÚC)r.   r/   Zpre_hook)Úappendr   r   )Úconfigsr.   Zblock_mZblock_kZblock_nr/   Zsplit_kr   r   r   Úget_configs_io_bound$   s*     ÿÿ  ÿÿr4   r&   r'   r$   r(   r   r0   r-   r%   r    r!   ÚMÚNÚKé
   )r
   Z
perf_modelZtop_k)r3   ÚkeyZprune_configs_byÚEVEN_Kc                 C   s   | d | d | d   dkS )Nr7   r+   r,   r   r   )Úargsr   r   r   r   U   r   r   )
Ú	acc_dtypeÚinput_precisionÚfp8_fast_accumr)   r*   r+   ÚGROUP_Mr,   r:   ÚAB_DTYPEc           +      C   s  t  d¡}t  d¡}t  ||¡}t  ||¡}|| }|| }t|||  |ƒ}|| ||  }|| | }|| t  d|¡ }|| t  d|¡ } t  t  || |¡|¡}!t  t  | | |¡|¡}"|| t  d|¡ }#| |!d d …d f | |#d d d …f |   } ||#d d …d f | |"d d d …f |	   }t j||f|d�}$tdt  ||| ¡ƒD ]ô}%|�rvt  	| ¡}&t  	|¡}'nb||%||   }(t jd|j
jd�})t j	| |#d d d …f |(k |)d�}&t j	||#d d …d f |(k |)d�}'|d k	�rö|& |¡}&|' |¡}'|�rt j|&|'|$||d�}$n|$t j|&|'||d�7 }$| || | 7 } ||| | 7 }�qV|$ |j
j¡}$|| t  d|¡ }|| t  d|¡ } ||d d …d f |
 | d d d …f |   }||k d d …d f | |k d d d …f @ }*|dk�ròt j||$|*d� nt j||$|*d� d S )Nr   r	   )Údtype)r	   r	   )ÚmaskÚother)Z	out_dtyper=   )rB   )ÚtlZ
program_idr   ÚminZarangeZmax_contiguousZmultiple_ofÚzerosÚrangeÚloadrA   Z
element_tyÚtoÚdotÚstoreZ
atomic_add)+ÚAÚBr1   r5   r6   r7   Z	stride_amZ	stride_akZ	stride_bkZ	stride_bnZ	stride_cmZ	stride_cnr<   r=   r>   r)   r*   r+   r?   r,   r:   r@   ÚpidZpid_zZgrid_mZgrid_nÚwidthZgroup_idZ
group_sizeZpid_mZpid_nZrmZrnZramZrbnZrkÚaccÚkr   r   Zk_remainingZ_0rB   r   r   r   Ú_kernel6   sR    -

,,
  


,(
rR   c                   @   s.   e Zd ZeZi Zedd„ ƒZeddd„ƒZdS )Ú_matmulc                    sì  | j }|  d¡dkr*|  d¡dkr*|  ¡ } | d¡dkrN| d¡dkrN| ¡ }| jd |jd ksjtdƒ‚| j\‰ }|j\}‰t| j|jƒ}	|d kr˜|	}tjˆ ˆf||d�}
tj	tj
tj	ftjtj
tjftj
tj
ftjtjfi}|d krö||	 d }nFt|tjƒ�stdƒ‚||| j k�s$tdƒ‚|||j k�s<tdƒ‚dd	„ }||ƒ}||	ƒ}	||ƒ}| jtjtjfk�rˆ|jtjtjfk�rˆd }	‡ ‡fd
d„}t| | ||
ˆ ˆ||  d¡|  d¡| d¡| d¡|
 d¡|
 d¡|||d|	d� |
S )Nr   r	   zincompatible dimensions)ÚdevicerA   zacc_dtype must be a torch.dtypez+acc_dtype not compatible with the type of az+acc_dtype not compatible with the type of bc                 S   s   t tt| ƒ d¡d ƒS )NÚ.éÿÿÿÿ)ÚgetattrrD   r   Úsplit)Útyr   r   r   Ú
to_tl_type¾   s    z!_matmul._call.<locals>.to_tl_typec                    s$   t ˆ | d ƒt ˆ| d ƒ | d fS )Nr)   r*   r,   )r   )ZMETA©r5   r6   r   r   r   É   r   z_matmul._call.<locals>.<lambda>r0   )r<   r=   r>   r?   r@   )rT   ZstrideÚ
contiguousÚshaper   r   rA   r   Úemptyr   Úfloat32Úbfloat16Úint8Zint32Ú
isinstancerD   Z
float8e4nvZfloat8e5rR   )r   r   r<   r=   r>   Úoutput_dtyperT   r7   Ú_Zab_dtypeÚcZsupported_acc_dtypesrZ   Zgridr   r[   r   Ú_call›   sj    

 
  
   þ(         ø	z_matmul._callNTc                 C   s   t j||||||d�S )N)r<   r=   r>   rc   )rS   rf   )Úctxr   r   r<   r=   r>   rc   r   r   r   ÚforwardÕ   s    ÿz_matmul.forward)NNTN)	Ú__name__Ú
__module__Ú__qualname__rR   ÚkernelZ_locksÚstaticmethodrf   rh   r   r   r   r   rS   –   s   
9rS   )r   Ú r   r   r   r   r   r   rD   Zmatmul_perf_modelr
   r   ra   r   r`   r_   r   r   r   r   r4   Z	constexprrR   ZautogradZFunctionrS   ÚapplyÚmatmulr   r   r   r   Ú<module>   sl   ìëýè ÿ     ø>E