U
    ‹Ñh¥¿  ã                   @   s¾  d dl Z d dlmZ d dlZd dlZd dlZd dlmZ	 d dl
mZ ddlmZ d dlmZ ddlmZ dd	lmZ G d
d„ dƒZG dd„ dƒZedd�G dd„ dƒƒZdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zejeejgd�Zejeej gd�Z!ejeej"gd�Z#G dd„ dƒZ$G dd „ d ƒZ%d!d"„ Z&d#d$„ Z'd%d&„ Z(G d'd(„ d(ƒZ)G d)d*„ d*e)ƒZ*G d+d,„ d,e)ƒZ+d-d.„ Z,d/d0„ Z-d1d2„ Z.d3d4„ Z/e%ƒ Z0d5d6d7d8d9d:gZ1G d;d<„ d<ƒZ2G d=d>„ d>ƒZ3dS )?é    N)ÚTuple)Ú	dataclassé   )ÚInterpreterError)Úpartialé   )Úinterpreter)Úirc                   @   s4   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ ZdS )ÚTensorHandlec                 C   s   || _ || _i | _dS )a  
            data: numpy array
            dtype: triton type, either pointer_type or scalar_type.
            we don't store block_type here because the shape information is already availale in the data field
            attr: a dictionary of attributes
        N)ÚdataÚdtypeÚattr)Úselfr   r   © r   úH/tmp/user/2202/pip-unpacked-wheel-sn6cy74c/triton/runtime/interpreter.pyÚ__init__   s    zTensorHandle.__init__c                 C   s   t | j ¡ ƒS ©N)Úboolr   Úall©r   r   r   r   Ú__bool__   s    zTensorHandle.__bool__c                 C   s   | j }t|dƒr|j}q|S )NÚ
element_ty)r   Úhasattrr   )r   r   r   r   r   Úget_element_ty    s    
zTensorHandle.get_element_tyc                 C   s   t | j ¡ | jƒS r   )r
   r   Úcopyr   r   r   r   r   Úclone&   s    zTensorHandle.clonec                 C   s   || j |< d S r   )r   )r   ÚkeyÚvaluer   r   r   Úset_attr)   s    zTensorHandle.set_attrN)Ú__name__Ú
__module__Ú__qualname__r   r   r   r   r   r   r   r   r   r
      s
   r
   c                   @   s   e Zd Zdd„ Zdd„ ZdS )ÚBlockPointerHandlec                 C   s(   || _ || _|| _|| _|| _|| _d S r   )ÚbaseÚshapeÚstridesÚoffsetsÚtensor_shapeÚorder)r   r#   r$   r%   r&   r'   r(   r   r   r   r   /   s    zBlockPointerHandle.__init__c           
      C   sà   | j  ¡ }|jd }| j}t | j j| j¡}tj| jtd�}t	t
|ƒƒD ]~}dgt
|ƒ }|| ||< | j| jt || ¡  |¡}	|||	 | j| j  tj¡ }||krHt ||	| j| jk ¡}qHt|| j jjƒ}||fS )Né   ©r   r   )r#   r   Úprimitive_bitwidthr'   ÚnpÚbroadcast_tor   Zonesr   ÚrangeÚlenr&   ÚarangeÚreshaper%   ÚastypeÚuint64Úlogical_andr$   r
   r   Úscalar)
r   Úboundary_checkÚdtype_ttZn_bytesr'   ÚptrsÚmasksZdimZ
bcast_dimsÚoffr   r   r   Úmaterialize_pointers7   s    

  z'BlockPointerHandle.materialize_pointersN)r   r    r!   r   r;   r   r   r   r   r"   -   s   r"   T)Úfrozenc                   @   sr   e Zd ZU dZeed< dZeed< dZe	ed< dZ
eed< dZeed< d	Ze	ed
< dZee	 ed< dZeed< dS )ÚInterpreterOptionsNÚextern_libsFÚdebugÚarchTÚallow_fp8e4nvÚallow_fp8e4b15Útf32Údefault_dot_input_precision)rC   Ztf32x3ZieeeÚallowed_dot_input_precisionsr   Úmax_num_imprecise_acc_default)r   r    r!   r>   ÚdictÚ__annotations__r?   r   r@   ÚstrrA   rB   rD   rE   r   rF   Úintr   r   r   r   r=   H   s   
r=   c                 C   sD   | t jkrt jS | t jkr t jS | t jkr0t jS | t jkr@t jS | S r   )	r,   Úuint8Úint8Úuint16Úint16Úuint32Úint32r3   Úint64r*   r   r   r   Ú_get_signed_np_dtypeT   s    



rR   c              &   C   sR  t | tjƒrt tj¡S tjt t¡tjt tj¡tj	t tj	¡tj
t tj
¡tjt tj¡tjt tj¡tjt tj¡tjt tj¡tjt tj¡tjt tj¡tjt tj¡tjt tj¡tjt tj¡tjt tj¡tjt tj¡tjt tj¡tjt tj¡tjt tj¡i}t | tjƒ�rJt | jtjƒ�r@t tj¡S || j S ||  S r   )Ú
isinstanceÚtlÚpointer_typer,   r   r3   Úint1r   Úfloat16Úfloat32Úfloat64rL   rK   rN   rM   rP   rO   rQ   Úbfloat16Úfloat8e5Úfloat8e5b16Ú
float8e4nvÚ
float8e4b8Úfloat8e4b15Ú
block_typer   )Ztt_dtypeZnp_typesr   r   r   Ú_get_np_dtype`   sX      
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ì
ra   c                 C   s¾  t td|j› �ƒ}t td|j› �ƒ}tj|  ¡ |d�}||jd ? d@ }|j|j d }|j|j d }	|d|j> d @ }
|j}|j}||j? d|> d @  tj¡}|dk}t 	|¡�r@tj
|tjd�}t|jƒD ]"}|
|? d@ }|j| ||dk< qÒ|
dk}d||  ||< || |||@ < |
| || > d|j> d @ |
|< t dt || | d|	> d ¡¡}| |¡}| |¡}|j|jk�râ|
|j|j ? d|j> d @ }|tjjk�rÖ|
d|j|j d > @ }||dk }| |¡}n$|
 |¡|j|j > d|j> d @ }|dk}t 	|¡�r–||j? d|> d @  tj¡}|dk}||@ }tj
|tjd�}d| || |  ||< || || ? d|j||  > B ||< ||jd > ||j> B |B }| | j¡S )NÚuintr*   r   r   )Úgetattrr,   r+   Z
frombufferÚtobytesZfp_mantissa_widthZexponent_biasr2   rP   ÚanyÚ
zeros_liker.   ÚmaximumÚminimumÚ_irZROUNDING_MODEZRTNEr1   r$   )ÚinputZinput_dtypeZoutput_dtypeÚrounding_modeZinput_uint_dtypeZoutput_unint_dtypeZ	input_binÚsignZinput_exponent_widthZoutput_exponent_widthZsignificandZ
bias_inputZbias_outputÚexponentZsubnormal_indexZbit_posÚiZ	bit_indexZzero_significand_indexZexponent_outputZsign_outputZsignificand_outputÚcut_offZnon_zero_exponent_indexÚshiftÚoutputr   r   r   Ú_convert_float€   sl    ÿ$

ÿ
ÿþÿÿÿrr   c                 C   s
   t  | ¡S r   )ÚmathÚerf©Úxr   r   r   Ú_erfÀ   s    rw   c                 C   s   t | ƒt |ƒ d? S )Né@   )rJ   )ÚaÚbr   r   r   Ú
_umulhi_64Å   s    r{   )Zotypesc                   @   s   e Zd Zedd„ ƒZdS )ÚExtraFunctionsc                 C   s   t  | | j||¡|¡S r   )rT   ÚtensorÚcreate_fp_to_fpÚhandle)rj   Údst_tyZfp_downcast_roundingÚ_builderr   r   r   Ú_convert_custom_typesÒ   s    z$ExtraFunctions._convert_custom_typesN)r   r    r!   Ústaticmethodr‚   r   r   r   r   r|   Ð   s   r|   c                   @   sb  e Zd Zejjejjejjejjejjejjejj	ejj	iZ
ejjejjejjejjejjejjejjejjejjejjejjejjejjejjejjejjejjejjejjejji
Zddœdd„Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Z dd„ Z!dd„ Z"dd„ Z#dd„ Z$dd„ Z%dd „ Z&d!d"„ Z'd#d$„ Z(d%d&„ Z)d'd(„ Z*d)d*„ Z+d+d,„ Z,d-d.„ Z-d/d0„ Z.d1d2„ Z/d3d4„ Z0d5d6„ Z1d7d8„ Z2d9d:„ Z3d;d<„ Z4d=d>„ Z5d?d@„ Z6dAdB„ Z7dCdD„ Z8dEdF„ Z9dGdH„ Z:dIdJ„ Z;dKdL„ Z<dMdN„ Z=dOdP„ Z>dQdR„ Z?dSdT„ Z@dUdV„ ZAdWdX„ ZBdYdX„ ZCdZdX„ ZDd[dX„ ZEd\dX„ ZFd]dX„ ZGd^dX„ ZHd_d`„ ZIdadb„ ZJdcdd„ ZKdedX„ ZLdfdX„ ZMdgdX„ ZNdhdX„ ZOdidX„ ZPdjdX„ ZQdkdX„ ZRdldX„ ZSdmdX„ ZTdndX„ ZUdodX„ ZVdpdX„ ZWdqdX„ ZXdrdX„ ZYdsdX„ ZZdtdX„ Z[dudX„ Z\dvdX„ Z]dwdX„ Z^dxdX„ Z_dydX„ Z`dzdX„ Zad{dX„ Zbd|dX„ Zcd}dX„ Zdd~dX„ ZeddX„ Zfd€dX„ Zgd�dX„ Zhd‚dX„ ZidƒdX„ Zjd„dX„ Zkd…dX„ Zld†dX„ Zmd‡dX„ ZndˆdX„ Zod‰dX„ ZpdŠdX„ Zqd‹dX„ ZrdŒdX„ Zsd�dX„ ZtdŽdX„ Zud�dX„ Zvd�dX„ Zwd‘dX„ Zxd’dX„ Zyd“dX„ Zzd”dX„ Z{d•d–„ Z|d—d˜„ Z}d™dš„ Z~d›dœ„ Zd�dX„ Z€dždX„ Z�dŸd „ Z‚d¡d¢„ Zƒd£d¤„ Z„d¥dX„ Z…d¦dX„ Z†d§dX„ Z‡d¨dX„ Zˆd©dX„ Z‰dªdX„ ZŠd«dX„ Z‹d¬dX„ ZŒd­dX„ Z�d®dX„ ZŽd¯dX„ Z�d°d±„ Z�d²d³„ Z‘d´dX„ Z’dµd¶„ Z“d·d¸„ Z”d¹dº„ Z•d»d¼„ Z–d½d¾„ Z—d¿dÀ„ Z˜dÁdÂ„ Z™dÃdÄ„ ZšdÅdÆ„ Z›dÇdÈ„ ZœdÉdÊ„ Z�dËdÌ„ ZždÍdÎ„ ZŸdÏdÐ„ Z dÑdÒ„ Z¡dÓdÔ„ Z¢dÕdÖ„ Z£d×dØ„ Z¤dÙdÚ„ Z¥dÛdÜ„ Z¦dÝdÞ„ Z§dßdà„ Z¨dádâ„ Z©dãdä„ Zªdådæ„ Z«dS )çÚInterpreterBuilderN©Úreturnc                 C   s$   d | _ tƒ | _i | _tj| jd< d S )NZconvert_custom_types)r@   r=   ÚoptionsZcodegen_fnsr|   r‚   r   r   r   r   r   ì   s    zInterpreterBuilder.__init__c                 C   sR   || j d k stdƒ‚|| j d k s,tdƒ‚|| j d k sBtdƒ‚|||f| _d S )Nr   zx >= grid_dim[0]r   zy >= grid_dim[1]r   zz >= grid_dim[2])Úgrid_dimÚ
ValueErrorÚgrid_idx©r   rv   ÚyÚzr   r   r   Úset_grid_idxò   s    zInterpreterBuilder.set_grid_idxc                 C   s   |||f| _ d S r   )rˆ   )r   ZnxÚnyZnzr   r   r   Úset_grid_dimû   s    zInterpreterBuilder.set_grid_dimc                 C   s   t jS r   )rT   rW   r   r   r   r   Úget_half_ty   s    zInterpreterBuilder.get_half_tyc                 C   s   t jS r   )rT   rZ   r   r   r   r   Úget_bf16_ty  s    zInterpreterBuilder.get_bf16_tyc                 C   s   t jS r   )rT   rX   r   r   r   r   Úget_float_ty  s    zInterpreterBuilder.get_float_tyc                 C   s   t jS r   )rT   rY   r   r   r   r   Úget_double_ty	  s    z InterpreterBuilder.get_double_tyc                 C   s   t jS r   )rT   rL   r   r   r   r   Úget_int8_ty  s    zInterpreterBuilder.get_int8_tyc                 C   s   t jS r   )rT   rK   r   r   r   r   Úget_uint8_ty  s    zInterpreterBuilder.get_uint8_tyc                 C   s   t jS r   )rT   rN   r   r   r   r   Úget_int16_ty  s    zInterpreterBuilder.get_int16_tyc                 C   s   t jS r   )rT   rM   r   r   r   r   Úget_uint16_ty  s    z InterpreterBuilder.get_uint16_tyc                 C   s   t jS r   )rT   rP   r   r   r   r   Úget_int32_ty  s    zInterpreterBuilder.get_int32_tyc                 C   s   t jS r   )rT   rO   r   r   r   r   Úget_uint32_ty  s    z InterpreterBuilder.get_uint32_tyc                 C   s   t jS r   )rT   rQ   r   r   r   r   Úget_int64_ty  s    zInterpreterBuilder.get_int64_tyc                 C   s   t jS r   )rT   r3   r   r   r   r   Úget_uint64_ty!  s    z InterpreterBuilder.get_uint64_tyc                 C   s   t jS r   )rT   r]   r   r   r   r   Úget_fp8e4nv_ty$  s    z!InterpreterBuilder.get_fp8e4nv_tyc                 C   s   t jS r   )rT   r_   r   r   r   r   Úget_fp8e4b15_ty'  s    z"InterpreterBuilder.get_fp8e4b15_tyc                 C   s   t jS r   )rT   r^   r   r   r   r   Úget_fp8e4b8_ty*  s    z!InterpreterBuilder.get_fp8e4b8_tyc                 C   s   t jS r   )rT   r[   r   r   r   r   Úget_fp8e5_ty-  s    zInterpreterBuilder.get_fp8e5_tyc                 C   s   t jS r   )rT   r\   r   r   r   r   Úget_fp8e5b16_ty0  s    z"InterpreterBuilder.get_fp8e5b16_tyc                 C   s   t  ||¡S r   )rT   rU   )r   Zelt_tyZ
addr_spacer   r   r   Ú
get_ptr_ty3  s    zInterpreterBuilder.get_ptr_tyc                 C   s   t  ||¡S r   )rT   r`   )r   r   r$   r   r   r   Úget_block_ty6  s    zInterpreterBuilder.get_block_tyc                 C   s   t tj|gtjd�tjƒS ©Nr*   )r
   r,   ÚarrayZbool_rT   rV   ©r   r   r   r   r   Úget_int19  s    zInterpreterBuilder.get_int1c                 C   s   t tj|gtjd�tjƒS r¤   )r
   r,   r¥   rK   rT   r¦   r   r   r   Ú	get_uint8<  s    zInterpreterBuilder.get_uint8c                 C   s   t tj|gtjd�tjƒS r¤   )r
   r,   r¥   rL   rT   r¦   r   r   r   Úget_int8?  s    zInterpreterBuilder.get_int8c                 C   s   t tj|gtjd�tjƒS r¤   )r
   r,   r¥   rM   rT   r¦   r   r   r   Ú
get_uint16B  s    zInterpreterBuilder.get_uint16c                 C   s   t tj|gtjd�tjƒS r¤   )r
   r,   r¥   rN   rT   r¦   r   r   r   Ú	get_int16E  s    zInterpreterBuilder.get_int16c                 C   s   t tj|gtjd�tjƒS r¤   )r
   r,   r¥   rO   rT   r¦   r   r   r   Ú
get_uint32H  s    zInterpreterBuilder.get_uint32c                 C   s   t tj|gtjd�tjƒS r¤   )r
   r,   r¥   rP   rT   r¦   r   r   r   Ú	get_int32K  s    zInterpreterBuilder.get_int32c                 C   s   t tj|gtjd�tjƒS r¤   )r
   r,   r¥   r3   rT   r¦   r   r   r   Ú
get_uint64N  s    zInterpreterBuilder.get_uint64c                 C   s   t tj|gtjd�tjƒS r¤   )r
   r,   r¥   rQ   rT   r¦   r   r   r   Ú	get_int64Q  s    zInterpreterBuilder.get_int64c                 C   s   t tj|gtjd�tjƒS r¤   )r
   r,   r¥   rW   rT   r¦   r   r   r   Úget_fp16T  s    zInterpreterBuilder.get_fp16c                 C   s   t tj|gtjd�tjƒS r¤   )r
   r,   r¥   rX   rT   r¦   r   r   r   Úget_fp32W  s    zInterpreterBuilder.get_fp32c                 C   s   t tj|gtjd�tjƒS r¤   )r
   r,   r¥   rY   rT   r¦   r   r   r   Úget_fp64Z  s    zInterpreterBuilder.get_fp64c                 C   s   t tjdgt|ƒd�|ƒS ©Nr   r*   )r
   r,   r¥   ra   )r   Útyper   r   r   Úget_null_value]  s    z!InterpreterBuilder.get_null_valuec                 C   s2   | j d krtdƒ‚ttj| j | gtjd�tjƒS )Nzgrid_idx is Noner*   )rŠ   r‰   r
   r,   r¥   rP   rT   ©r   Úaxisr   r   r   Úcreate_get_program_ida  s    
z(InterpreterBuilder.create_get_program_idc                 C   s    t tj| j| gtjd�tjƒS r¤   )r
   r,   r¥   rˆ   rP   rT   r¶   r   r   r   Úcreate_get_num_programsf  s    z*InterpreterBuilder.create_get_num_programsc                 C   s0   t tj|jtd�tjƒ}d }|  ||||||¡S r¤   )r
   r,   Ú	ones_liker   r   rT   rV   Úcreate_masked_load)r   ÚptrÚ_0Ú_1Úis_volatileÚmaskÚotherr   r   r   Úcreate_loadj  s    zInterpreterBuilder.create_loadc                 C   s*   t tj|jtd�tjƒ}|  |||d d ¡S r¤   )r
   r,   rº   r   r   rT   rV   Úcreate_masked_store)r   r¼   Úvalr½   r¾   rÀ   r   r   r   Úcreate_storeo  s    zInterpreterBuilder.create_storec           
      C   sN   |  ¡ }t|ƒ}|d kr.ttj|j|d�|ƒ}t |j|j|j|¡}	t|	|ƒS r¤   )r   ra   r
   r,   rf   r   Ú_interpreterÚload)
r   r8   rÀ   rÁ   Úcache_modifierÚeviction_policyr¿   r7   Údtype_npÚretr   r   r   r»   s  s    z%InterpreterBuilder.create_masked_loadc                 C   s   t  |j|j|j¡S r   )rÆ   Ústorer   )r   r8   r   rÀ   rÈ   rÉ   r   r   r   rÃ   {  s    z&InterpreterBuilder.create_masked_storec                 C   sx   |j j}|j}|tjkr"|tjks6|tjkr\|tjkr\t|j||d ƒ t|ƒ¡}t	||jƒS t	|j 
t|ƒ¡|jƒS d S r   )r   r5   rT   rZ   rX   rr   r   Úviewra   r
   r2   )r   ÚsrcÚdst_typeÚsrc_element_typeÚdst_element_typer   r   r   r   Ú	cast_impl  s    ÿÿzInterpreterBuilder.cast_implc                 C   s   |   ||¡S r   ©rÒ   ©r   rÎ   rÏ   r   r   r   Ú<lambda>‰  ó    zInterpreterBuilder.<lambda>c                 C   s   |   ||¡S r   rÓ   rÔ   r   r   r   rÕ   Š  rÖ   c                 C   s   |   ||¡S r   rÓ   rÔ   r   r   r   rÕ   ‹  rÖ   c                 C   s   |   ||¡S r   rÓ   rÔ   r   r   r   rÕ   Œ  rÖ   c                 C   s   |   ||¡S r   rÓ   rÔ   r   r   r   rÕ   �  rÖ   c                 C   s   |   ||¡S r   rÓ   rÔ   r   r   r   rÕ   Ž  rÖ   c                 C   s   |   ||¡S r   rÓ   )r   rÎ   rÏ   Ú	is_signedr   r   r   rÕ   �  rÖ   c                 C   s4   |j j}|j}t|j|||ƒ t|ƒ¡}t||jƒS r   )r   r5   rr   r   rÍ   ra   r
   )r   rÎ   rÏ   rk   rÐ   rÑ   r   r   r   r   r~   ‘  s    z"InterpreterBuilder.create_fp_to_fpc                 C   s   t |j t|ƒ¡|jƒS r   )r
   r   rÍ   ra   r5   rÔ   r   r   r   Úcreate_bitcast—  s    z!InterpreterBuilder.create_bitcastc                 C   s   t ||j|jƒ|jjƒS r   ©r
   r   r   r5   )r   ÚlhsÚrhsÚopr   r   r   Ú	binary_op›  s    zInterpreterBuilder.binary_opc                 C   s   |   ||tj¡S r   ©rÝ   r,   Úadd©r   rÚ   rÛ   r   r   r   rÕ   ž  rÖ   c                 C   s   |   ||tj¡S r   ©rÝ   r,   Úmultiplyrà   r   r   r   rÕ   Ÿ  rÖ   c                 C   s   |   ||tj¡S r   ©rÝ   r,   Údividerà   r   r   r   rÕ      rÖ   c                 C   s   |   ||tj¡S r   )rÝ   r,   Ú	remainderrà   r   r   r   rÕ   ¡  rÖ   c                 C   s   |   ||tj¡S r   ©rÝ   r,   Úsubtractrà   r   r   r   rÕ   ¢  rÖ   c                 C   s   |   ||tj¡S r   rá   rà   r   r   r   rÕ   £  rÖ   c                 C   s   |   ||tj¡S r   rã   rà   r   r   r   rÕ   ¤  rÖ   c                 C   s   |   ||¡S r   ©Úcreate_idivrà   r   r   r   rÕ   ¥  rÖ   c                 C   s   |   ||¡S r   rè   rà   r   r   r   rÕ   ¦  rÖ   c                 C   s   |   ||tj¡S r   ©rÝ   r,   Úfmodrà   r   r   r   rÕ   ¨  rÖ   c                 C   s   |   ||tj¡S r   rê   rà   r   r   r   rÕ   ©  rÖ   c                 C   s   |   ||tj¡S r   rÞ   rà   r   r   r   rÕ   ª  rÖ   c                 C   s   |   ||tj¡S r   ræ   rà   r   r   r   rÕ   «  rÖ   c                 C   s   |   ||tj¡S r   )rÝ   r,   Z
left_shiftrà   r   r   r   rÕ   ¬  rÖ   c                 C   s   |   ||tj¡S r   )rÝ   r,   Úright_shiftrà   r   r   r   rÕ   ­  rÖ   c                 C   s   |   ||tj¡S r   ©rÝ   r,   rh   rà   r   r   r   rÕ   ®  rÖ   c                 C   s   |   ||tj¡S r   rí   rà   r   r   r   rÕ   ¯  rÖ   c                 C   s   |   ||tj¡S r   rí   rà   r   r   r   rÕ   °  rÖ   c                 C   s   |   ||tj¡S r   rí   rà   r   r   r   rÕ   ±  rÖ   c                 C   s   |   ||tj¡S r   ©rÝ   r,   rg   rà   r   r   r   rÕ   ²  rÖ   c                 C   s   |   ||tj¡S r   rî   rà   r   r   r   rÕ   ³  rÖ   c                 C   s   |   ||tj¡S r   rî   rà   r   r   r   rÕ   ´  rÖ   c                 C   s   |   ||tj¡S r   rî   rà   r   r   r   rÕ   µ  rÖ   c                 C   s   |   ||tj¡S r   ©rÝ   r,   Z
less_equalrà   r   r   r   rÕ   ¶  rÖ   c                 C   s   |   ||tj¡S r   ©rÝ   r,   Zlessrà   r   r   r   rÕ   ·  rÖ   c                 C   s   |   ||tj¡S r   ©rÝ   r,   Zgreater_equalrà   r   r   r   rÕ   ¸  rÖ   c                 C   s   |   ||tj¡S r   ©rÝ   r,   Zgreaterrà   r   r   r   rÕ   ¹  rÖ   c                 C   s   |   ||tj¡S r   rï   rà   r   r   r   rÕ   º  rÖ   c                 C   s   |   ||tj¡S r   rð   rà   r   r   r   rÕ   »  rÖ   c                 C   s   |   ||tj¡S r   rñ   rà   r   r   r   rÕ   ¼  rÖ   c                 C   s   |   ||tj¡S r   rò   rà   r   r   r   rÕ   ½  rÖ   c                 C   s   |   ||tj¡S r   ©rÝ   r,   Úequalrà   r   r   r   rÕ   ¾  rÖ   c                 C   s   |   ||tj¡S r   ©rÝ   r,   Ú	not_equalrà   r   r   r   rÕ   ¿  rÖ   c                 C   s   |   ||tj¡S r   rð   rà   r   r   r   rÕ   À  rÖ   c                 C   s   |   ||tj¡S r   rò   rà   r   r   r   rÕ   Á  rÖ   c                 C   s   |   ||tj¡S r   rï   rà   r   r   r   rÕ   Â  rÖ   c                 C   s   |   ||tj¡S r   rñ   rà   r   r   r   rÕ   Ã  rÖ   c                 C   s   |   ||tj¡S r   ró   rà   r   r   r   rÕ   Ä  rÖ   c                 C   s   |   ||tj¡S r   rõ   rà   r   r   r   rÕ   Å  rÖ   c                 C   s   |   ||tj¡S r   rð   rà   r   r   r   rÕ   Æ  rÖ   c                 C   s   |   ||tj¡S r   rò   rà   r   r   r   rÕ   Ç  rÖ   c                 C   s   |   ||tj¡S r   rï   rà   r   r   r   rÕ   È  rÖ   c                 C   s   |   ||tj¡S r   rñ   rà   r   r   r   rÕ   É  rÖ   c                 C   s   |   ||tj¡S r   ró   rà   r   r   r   rÕ   Ê  rÖ   c                 C   s   |   ||tj¡S r   rõ   rà   r   r   r   rÕ   Ë  rÖ   c                 C   s   |   ||tj¡S r   )rÝ   r,   Zbitwise_andrà   r   r   r   rÕ   Ì  rÖ   c                 C   s   |   ||tj¡S r   )rÝ   r,   Zbitwise_xorrà   r   r   r   rÕ   Í  rÖ   c                 C   s   |   ||tj¡S r   )rÝ   r,   Z
bitwise_orrà   r   r   r   rÕ   Î  rÖ   c                 C   s&   t |jt |j|j¡ |j |jjƒS r   )r
   r   r,   rë   r   r5   rà   r   r   r   ré   Ð  s    zInterpreterBuilder.create_idivc                 C   sD   t |jjƒ}t |jjƒ}|j |¡|_|j |¡|_|  ||tj¡S r   )rR   r   r   r2   rÝ   r,   rì   )r   rÚ   rÛ   Z	lhs_dtypeZ	rhs_dtyper   r   r   Úcreate_ashrÖ  s
    zInterpreterBuilder.create_ashrc                 C   s”   |j j}|tjks|tjkr4tt|j |j ƒ|jjƒS ttd|j	d d › �ƒ}|j  
|¡}|j  
|¡}t ||¡|j	d ? }t| 
|¡|jjƒS d S )Nrb   r)   r   )r   r   r,   rQ   r3   r
   Únp_umulhi_u64r5   rc   Úitemsizer2   râ   )r   rÚ   rÛ   r   Zcompute_dtypeZlhs_dataZrhs_dataZret_datar   r   r   Úcreate_umulhiÞ  s    z InterpreterBuilder.create_umulhic                 C   s   t ||j|j|jƒ|jjƒS r   rÙ   )r   rÚ   rÛ   rÁ   rÜ   r   r   r   Ú
ternary_opê  s    zInterpreterBuilder.ternary_opc                 C   s   |   |||tj¡S r   )rû   r,   Zclip)r   ÚargÚloÚhiZpropagate_nansr   r   r   rÕ   í  rÖ   c                 C   s   |   |||tj¡S r   )rû   r,   Úwhere)r   ÚcondrÚ   rÛ   r   r   r   rÕ   î  rÖ   c                 C   s   t |j|j |j |jjƒS r   rÙ   r‹   r   r   r   Ú
create_fmað  s    zInterpreterBuilder.create_fmac                 C   s   t ||jƒ|jjƒS r   rÙ   )r   rü   rÜ   r   r   r   Úunary_opô  s    zInterpreterBuilder.unary_opc                 C   sZ   |j }|jd }ttd|j› �ƒ}|j |¡}d|> d }||@  t|ƒ¡}t||j jƒS )Nr   rb   )	r   r+   rc   r,   r   rÍ   ra   r
   r5   )r   rü   r7   Zmask_bitwidthZnp_uint_dtyper   rÀ   rË   r   r   r   Úcreate_fabs÷  s    
zInterpreterBuilder.create_fabsc                 C   s   |   |tj¡S r   )r  r,   Úcos©r   rü   r   r   r   rÕ     rÖ   c                 C   s   |   |tj¡S r   )r  r,   Úexpr  r   r   r   rÕ     rÖ   c                 C   s   |   |tj¡S r   )r  r,   Zexp2r  r   r   r   rÕ     rÖ   c                 C   s   |   |tj¡S r   )r  r,   Úabsr  r   r   r   rÕ     rÖ   c                 C   s   |   |tj¡S r   )r  r,   Úfloorr  r   r   r   rÕ     rÖ   c                 C   s   |   |tj¡S r   )r  r,   Úceilr  r   r   r   rÕ     rÖ   c                 C   s   |   |tj¡S r   )r  r,   Úlogr  r   r   r   rÕ     rÖ   c                 C   s   |   |tj¡S r   )r  r,   Úlog2r  r   r   r   rÕ     rÖ   c                 C   s   |   |tj¡S r   ©r  r,   Úsqrtr  r   r   r   rÕ   	  rÖ   c                 C   s   |   |tj¡S r   r  r  r   r   r   rÕ   
  rÖ   c                 C   s   |   |tj¡S r   )r  r,   Úsinr  r   r   r   rÕ     rÖ   c                 C   s0   |j jtjkrt|j ƒnt|j ƒ}t||jjƒS r   )r   r   r,   rX   Únp_erf_fp32Únp_erf_fp64r
   r5   )r   rü   rË   r   r   r   Ú
create_erf  s    "zInterpreterBuilder.create_erfc                 C   s   t dt |j¡ |jjƒS )Nr   )r
   r,   r  r   r   r5   r  r   r   r   Úcreate_rsqrt  s    zInterpreterBuilder.create_rsqrtc                 C   s   t |j |¡|jjƒS r   )r
   r   r1   r   r5   )r   rü   r$   Zallow_reorderr   r   r   rÕ     rÖ   c                 C   s   t t |j|¡|jjƒS r   )r
   r,   Ú	transposer   r   r5   )r   rü   Úpermr   r   r   Úcreate_trans  s    zInterpreterBuilder.create_transc                 C   s�   |j }|j }|jjdkr"|j ¡ s8|jjdkrl|j ¡ rlt||jtjd ƒ tj¡}t||jtjd ƒ tj¡}t	tj
|||j jd�|j  |jjƒS )Nr)   r*   )r   r   r+   Zis_floatingrr   rT   rW   rÍ   r,   r
   Úmatmulr5   )r   ry   rz   ÚdZinput_precisionZmax_num_imprecise_accZa_dataZb_datar   r   r   Ú
create_dot  s    
ÿÿzInterpreterBuilder.create_dotc                 C   s   t tj||tjd�tjƒS r¤   )r
   r,   r0   rP   rT   )r   ÚstartÚstopr   r   r   Úcreate_make_range#  s    z$InterpreterBuilder.create_make_rangec                 C   s"   t tj|j|d|fd�d tjƒS )Nr   )Úbinsr.   )r
   r,   Z	histogramr   rT   rP   )r   r   r  r   r   r   Úcreate_histogram&  s    z#InterpreterBuilder.create_histogramc                 C   s<   |  ¡ }|j}td|d ƒ}t|j||j tj¡  |jƒS )Nr   r)   )	r   r+   Úmaxr
   r   r2   r,   r3   r   )r   r¼   Úoffsetr7   Zelement_bitwidthZelement_bytewidthr   r   r   Úcreate_addptr+  s    z InterpreterBuilder.create_addptrc                 C   sœ   |  |¡\}}| ¡ }	t|	ƒ}
|d kr,d }n\|tjjkrPttj|j	|
d�|	ƒ}n8|tjj
krzttj|j	tdƒ|
d�|	ƒ}ntd|› �ƒ‚|  ||||||¡S )Nr*   Únanzunsupported padding option )r;   r   ra   ri   ZPADDING_OPTIONZPAD_ZEROr
   r,   rf   r   ZPAD_NANZ	full_likeÚfloatr‰   r»   )r   r¼   r6   Zpadding_optionrÈ   rÉ   r¿   r8   r9   r7   rÊ   rÁ   r   r   r   Úcreate_tensor_pointer_load2  s    z-InterpreterBuilder.create_tensor_pointer_loadc                 C   s    |  |¡\}}|  |||||¡S r   )r;   rÃ   )r   r¼   r   r6   rÈ   rÉ   r8   r9   r   r   r   Úcreate_tensor_pointer_storeA  s    z.InterpreterBuilder.create_tensor_pointer_storec                 C   s   t t |j|¡|jjƒS r   )r
   r,   Úexpand_dimsr   r   r5   )r   rü   r·   r   r   r   Úcreate_expand_dimsE  s    z%InterpreterBuilder.create_expand_dimsc                 C   s   t t |j|¡|jjƒS r   )r
   r,   r-   r   r   r5   ©r   rü   r$   r   r   r   Úcreate_broadcastH  s    z#InterpreterBuilder.create_broadcastc                 C   s   t |j tj¡|jƒS r   ©r
   r   r2   r,   r3   r5   ©r   rÄ   r€   r   r   r   Úcreate_int_to_ptrK  s    z$InterpreterBuilder.create_int_to_ptrc                 C   s   t |j tj¡|jƒS r   r)  r*  r   r   r   Úcreate_ptr_to_intN  s    z$InterpreterBuilder.create_ptr_to_intc                 C   s   t t |j|jg¡|jjƒS r   )r
   r,   Zconcatenater   r   r5   rà   r   r   r   Ú
create_catQ  s    zInterpreterBuilder.create_catc                 C   s    t tj|j|jgdd�|jjƒS )Néÿÿÿÿ©r·   )r
   r,   Ústackr   r   r5   rà   r   r   r   Úcreate_joinT  s    zInterpreterBuilder.create_joinc                 C   s(   t |jd |jjƒt |jd |jjƒfS )N).r   ).r   rÙ   )r   rÄ   r   r   r   Úcreate_splitX  s    zInterpreterBuilder.create_splitc                 C   sZ   t |jtjƒr4ttj||jd t|jƒd�|jj	ƒS ttj||jt|jƒd�|jj	ƒS d S r³   )
rS   r   rT   r`   r
   r,   Úfullr   ra   r5   r'  r   r   r   Úcreate_splat\  s    &zInterpreterBuilder.create_splatc                 C   sB   || j krtd|› �ƒ‚| j | }tt |j|j|j|¡|jjƒS )Núunsupported semantic )Úir_sem_to_interpreter_semr‰   r
   rÆ   Z
atomic_casr   r   r5   )r   r¼   ÚcmprÄ   ÚsemÚscoper   r   r   Úcreate_atomic_casb  s    

z$InterpreterBuilder.create_atomic_casc                 C   sf   || j krtd|› �ƒ‚|| jkr0td|› �ƒ‚| j | }| j| }tt ||j|j|j|¡|jjƒS )Nzunsupported rmwOp r5  )	Úir_rmw_op_to_interpreter_rmw_opr‰   r6  r
   rÆ   Z
atomic_rmwr   r   r5   )r   ZrmwOpr¼   rÄ   rÀ   r8  r9  r   r   r   Úcreate_atomic_rmwh  s    



z$InterpreterBuilder.create_atomic_rmwc                 C   s   t dƒ‚d S )Nz4extern_elementwise not supported in interpreter mode©ÚNotImplementedError)r   ZlibNameZlibPathÚsymbolZargListZretTypeÚisPurer   r   r   Úcreate_extern_elementwiseq  s    z,InterpreterBuilder.create_extern_elementwisec                 C   s   t dƒ‚d S )Nz,inline_asm not supported in interpreter moder=  )r   Z	inlineAsmÚconstraintsÚvaluesr´   r@  Úpackr   r   r   Úcreate_inline_asmt  s    z$InterpreterBuilder.create_inline_asmc                 C   s†   d| j d › d| j d › d| j d › d�}|r<|d|› �7 }|rTtjdd	d
„ id� |D ]}t|d|j› � ƒ qX|r‚tjd d� d S )Nú(r   z, r   r   ú)ú r   c                 S   s   d| d›�S )NÚ0xZ02xr   ru   r   r   r   rÕ   }  rÖ   z1InterpreterBuilder.create_print.<locals>.<lambda>)Ú	formatter)rŠ   r,   Zset_printoptionsÚprintr   )r   ÚprefixÚhexrC  Úmsgr   r   r   r   Úcreate_printw  s    *zInterpreterBuilder.create_printc                 C   s&   |s"t |› d|› d|› d|› �ƒ‚d S )Nz in ú:©ÚAssertionError)r   Ú	conditionÚmessageÚfileNameÚfuncNameZlineNor   r   r   Úcreate_assertƒ  s    z InterpreterBuilder.create_assertc                 C   s   d S r   r   r   r   r   r   Úcreate_barrier‡  s    z!InterpreterBuilder.create_barrierc                 C   s    dd„ |D ƒ}t ||||||ƒS )Nc                 S   s   g | ]}|  ¡ ‘qS r   ©r   ©Ú.0r  r   r   r   Ú
<listcomp>�  s     z<InterpreterBuilder.create_make_block_ptr.<locals>.<listcomp>)r"   )r   r#   r$   r%   r&   r'   r(   Únew_offsetsr   r   r   Úcreate_make_block_ptr‹  s    z(InterpreterBuilder.create_make_block_ptrc                 C   sv   t |jƒt |ƒkrtdƒ‚dd„ |jD ƒ}t|j|j|j||j|jƒ}t	t |ƒƒD ]}|j|  j
|| j
7  _
qR|S )Nz len(ptr.offsets) != len(offsets)c                 S   s   g | ]}|  ¡ ‘qS r   rY  rZ  r   r   r   r\  ”  s     z5InterpreterBuilder.create_advance.<locals>.<listcomp>)r/   r&   r‰   r"   r#   r$   r%   r'   r(   r.   r   )r   r¼   r&   r]  rË   rn   r   r   r   Úcreate_advance�  s    z!InterpreterBuilder.create_advancec                 C   s<   t |ƒ}d|jkr*ttjdd|d�|jƒS td|› �ƒ‚d S )NrJ   r   r.  r*   zunsupported type )ra   Únamer
   r,   r3  r5   Ú	TypeError)r   r´   Znp_typer   r   r   Úget_all_ones_valueš  s    
z%InterpreterBuilder.get_all_ones_value)¬r   r    r!   ri   ZMEM_SEMANTICZACQUIRErÆ   ZRELEASEZRELAXEDZACQUIRE_RELEASEr6  Z	ATOMIC_OPZADDZRMW_OPZFADDZMINZUMINÚMAXZUMAXÚANDÚORZXORZXCHGr;  r   rŽ   r�   r‘   r’   r“   r”   r•   r–   r—   r˜   r™   rš   r›   rœ   r�   rž   rŸ   r    r¡   r¢   r£   r§   r¨   r©   rª   r«   r¬   r­   r®   r¯   r°   r±   r²   rµ   r¸   r¹   rÂ   rÅ   r»   rÃ   rÒ   Zcreate_si_to_fpZcreate_ui_to_fpZcreate_fp_to_siZcreate_fp_to_uiZcreate_fp_extZcreate_fp_truncZcreate_int_castr~   rØ   rÝ   Zcreate_faddZcreate_fmulZcreate_fdivZcreate_fremZcreate_fsubZ
create_mulZcreate_precise_divfZcreate_sdivZcreate_udivZcreate_sremZcreate_uremZ
create_addZ
create_subZ
create_shlZcreate_lshrZcreate_minsiZcreate_minuiZcreate_minimumfZcreate_minnumfZcreate_maxsiZcreate_maxuiZcreate_maximumfZcreate_maxnumfZcreate_icmpSLEZcreate_icmpSLTZcreate_icmpSGEZcreate_icmpSGTZcreate_icmpULEZcreate_icmpULTZcreate_icmpUGEZcreate_icmpUGTZcreate_icmpEQZcreate_icmpNEZcreate_fcmpOLTZcreate_fcmpOGTZcreate_fcmpOLEZcreate_fcmpOGEZcreate_fcmpOEQZcreate_fcmpONEZcreate_fcmpULTZcreate_fcmpUGTZcreate_fcmpULEZcreate_fcmpUGEZcreate_fcmpUEQZcreate_fcmpUNEZ
create_andZ
create_xorZ	create_orré   r÷   rú   rû   Zcreate_clampfZcreate_selectr  r  r  Z
create_cosZ
create_expZcreate_exp2Zcreate_iabsZcreate_floorZcreate_ceilZ
create_logZcreate_log2Zcreate_precise_sqrtZcreate_sqrtZ
create_sinr  r  Zcreate_reshaper  r  r  r  r   r#  r$  r&  r(  r+  r,  r-  r1  r2  r4  r:  r<  rA  rE  rO  rW  rX  r^  r_  rb  r   r   r   r   r„   ×   sd      ü          ö	

		
r„   c                    s"   |dœ‡ fdd„
}t | ||ƒ d S )N)Úmemberc                    s    | |dd„ |  ¡ D ƒdˆ i—ŽS )Nc                 S   s   i | ]\}}|d kr||“qS )r�   r   ©r[  ÚkÚvr   r   r   Ú
<dictcomp>¤  s   þ z1_patch_attr.<locals>.<lambda>.<locals>.<dictcomp>r�   )Úitems)rf  ÚargsÚkwargs©Úbuilderr   r   rÕ   £  s   ÿÿýz_patch_attr.<locals>.<lambda>)Úsetattr)Úobjr`  rf  ro  Ú
new_memberr   rn  r   Ú_patch_attr¢  s    rs  c                 C   s2   t  | ¡D ]"\}}tj |¡r
t| |||ƒ q
d S r   )ÚinspectÚ
getmembersrT   ÚcoreÚ
is_builtinrs  )Úpkgro  r`  rf  r   r   r   Ú_patch_builtinª  s    ry  c                    sJ   dd„ ‰ dd„ }dd„ | _ ‡ fdd„| _dd„ | _d	d„ | _t|ƒ| _d S )
Nc                 S   s   | j j}|jdkrt|ƒS dS )Nr   T)r   r   Úsizer   )r   r   r   r   r   Ú	_get_bool²  s    z%_patch_lang_tensor.<locals>._get_boolc                 S   s&   t j tt | jj¡| jjƒ| jj	¡S r   )
rT   rv  r}   r
   r,   r  r   r   r   r5   r   r   r   r   Ú_get_transpose¸  s    z*_patch_lang_tensor.<locals>._get_transposec                 S   s   t | jjƒS r   )rJ   r   r   r   r   r   r   rÕ   »  rÖ   z$_patch_lang_tensor.<locals>.<lambda>c                    s   ˆ | ƒS r   r   r   ©r{  r   r   rÕ   ¼  rÖ   c                 S   s   t | jjƒS r   )Úreprr   r   r   r   r   r   rÕ   ½  rÖ   c                 S   s   t | jjƒS r   )rI   r   r   r   r   r   r   rÕ   ¾  rÖ   )Ú	__index__r   Ú__repr__Ú__str__ÚpropertyÚT)r}   r|  r   r}  r   Ú_patch_lang_tensor°  s    


r„  c                   @   s<   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ ZdS )ÚReduceScanOpIneterfacec                 C   s   || _ || _d S r   )r·   Ú
combine_fn)r   r·   r†  r   r   r   r   Ä  s    zReduceScanOpIneterface.__init__c                 C   s,   |d k	r(|t |ƒkr(td|› d|› �ƒ‚d S )Nzaxis z out of bounds for shape )r/   r‰   )r   r$   r·   r   r   r   Ú
check_axisÈ  s    z!ReduceScanOpIneterface.check_axisc                 C   s>   |D ]4}t |tjjƒs(tdt|ƒ› �ƒ‚|  |j| j¡ qd S )Nzinput must be a tensor, got )	rS   rT   rv  r}   r‰   r´   r‡  r$   r·   )r   rj   rü   r   r   r   Úcheck_tensorÌ  s    z#ReduceScanOpIneterface.check_tensorc                 C   sN   t |dƒr |jr t ||j¡}ntj|gt|ƒd�}|}tj t	||j
ƒ|¡S )Nr$   r*   )r   r$   rT   r`   r,   r¥   ra   rv  r}   r
   r5   )r   rË   r   Zret_typer   r   r   Ú	to_tensorÒ  s
    z ReduceScanOpIneterface.to_tensorc                 C   s$   t |tƒs|f}|  |¡ |  |¡S r   )rS   Útuplerˆ  Ú
apply_impl©r   rj   r   r   r   ÚapplyÚ  s    

zReduceScanOpIneterface.applyc                 C   s   t dƒ‚d S )Nzapply_impl not implementedr=  rŒ  r   r   r   r‹  à  s    z!ReduceScanOpIneterface.apply_implN)	r   r    r!   r   r‡  rˆ  r‰  r�  r‹  r   r   r   r   r…  Â  s   r…  c                       sF   e Zd Z‡ fdd„Zdd„ Zdd„ Zddd	„Zd
d„ Zdd„ Z‡  Z	S )Ú	ReduceOpsc                    s   t ƒ  ||¡ || _d S r   )Úsuperr   Ú	keep_dims)r   r·   r†  r�  ©Ú	__class__r   r   r   æ  s    zReduceOps.__init__c                 C   sN   g }|D ]8}|d k	r |  |¡ qd}|  |  |jj ¡ |j¡¡ qt|ƒ|fS )Nr   )Úappendr‰  r   r   Úflattenr   rŠ  )r   rj   r·   rË   r   r   r   r   Úunravelê  s    zReduceOps.unravelc                    s@  ˆj }ˆ ˆ ˆj ¡\‰ }g }g }ˆ d jjj}|d|… ||d d …  }ˆ D ],}| |jj¡ | tj||jjjd�¡ qNt	|d j
ƒD �]}	t |	|¡‰ˆd|… ˆ|d d …  ‰t‡ ‡‡fdd„t|ƒD ƒƒ}
ˆ| dk�rt	t|ƒƒD ]}|
| jj ¡ || ˆ< qðqŠt‡ ‡‡fdd„t|ƒD ƒƒ}ˆjj||
žŽ }t|tƒ�sP|fn|}t	t|ƒƒD ]:}t|| tjjƒ�rˆ|| jj ¡ n|| || ˆ< �q`qŠg }t|ƒD ]v\}	}ˆj�rô|d k	�rÒt ||¡}n t	t|ƒƒD ]}t |d¡}�qÞn|d k�r| ¡ }| ˆ |ˆ |	 j¡¡ �qªt|ƒdk�r8|d S t|ƒS )Nr   r   r*   c                 3   s(   | ] \}}ˆ  |ˆ ˆ | j¡V  qd S r   ©r‰  r   ©r[  Úiir  )rj   Úinput_indexr   r   r   Ú	<genexpr>  s     z+ReduceOps.generic_reduce.<locals>.<genexpr>c                 3   s(   | ] \}}ˆ  |ˆ ˆ | j¡V  qd S r   r–  ©r[  ZoiÚo)rj   Úoutput_indexr   r   r   rš  	  s     )r·   r•  r   r   r$   r“  r,   Úzerosr   r.   rz  Úunravel_indexrŠ  Ú	enumerater/   Úitemr†  ÚfnrS   rT   rv  r}   r�  r%  r‰  )r   rj   Zoriginal_axisr·   Ú
input_dataÚoutput_dataZinput_shapeZoutput_shaperü   rn   Zinput_tupleÚjÚ	acc_tupleÚcombine_fn_retrË   r   Ú_r   )rj   r™  r�  r   r   Úgeneric_reduceô  sJ     ÿÿ

zReduceOps.generic_reduceNc                 C   s¢   t |tƒr|d n|}d }d }|rB|  ||jj| j| jd�|j¡}|rf|  ||jj| j| jd�tj	¡}|d k	r~|d k	r~||fS |d k	rŠ|S |d k	r–|S t
dƒ‚d S )Nr   ©r·   Zkeepdimsz-val_reduce_op and idx_reduce_op are both None)rS   rŠ  r‰  r   r   r·   r�  r   rT   rP   r‰   )r   rj   Úval_reduce_opÚidx_reduce_oprÄ   Úidxr   r   r   Úmin_max  s      zReduceOps.min_maxc                 C   s"   |   tj|jj| j| jd�|j¡S )Nrª  )r‰  r,   Úsumr   r   r·   r�  r   rŒ  r   r   r   r¯  1  s    zReduceOps.sumc                 C   s¾   | j tjjkr&| j|d tjtjd�S | j tjjkrL| j|d tj	tj
d�S | j tjjkrp| j|d tj	d d�S | j tjjkr”| j|d tjd d�S | j tjjkr°|  |d ¡S |  |¡S d S )Nr   )r«  r¬  )r†  rT   ÚstandardZ_argmin_combine_tie_break_leftr®  r,   ÚminZargminZ_argmax_combine_tie_break_leftr  ZargmaxZ_elementwise_maxZ_elementwise_minÚ_sum_combiner¯  r©  rŒ  r   r   r   r‹  4  s    zReduceOps.apply_impl)N)
r   r    r!   r   r•  r©  r®  r¯  r‹  Ú__classcell__r   r   r‘  r   rŽ  ä  s   
+
rŽ  c                       s<   e Zd Z‡ fdd„Zdd„ Zdd„ Zdd„ Zd	d
„ Z‡  ZS )ÚScanOpsc                    s   t ƒ  ||¡ || _d S r   )r�  r   Úreverse)r   r·   r†  rµ  r‘  r   r   r   F  s    zScanOps.__init__c                 C   s"   | j tj|jj| jd�|jd�gS ©Nr/  r*   )r‰  r,   Úcumsumr   r   r·   r   rŒ  r   r   r   r·  J  s    zScanOps.cumsumc                 C   s"   | j tj|jj| jd�|jd�gS r¶  )r‰  r,   Úcumprodr   r   r·   r   rŒ  r   r   r   r¸  M  s    zScanOps.cumprodc                    s¢  g }g }ˆd j jj}ˆD ],}| |j j¡ | tj||j jjd�¡ qt|d jƒD �]}t 	||¡‰ t
‡ ‡‡fdd„t|ƒD ƒƒ}ˆ ˆj dkrÀtt|ƒƒD ]}|| j j ¡ || ˆ < q qVt
‡ ‡fdd„ttˆ ƒƒD ƒƒ‰t
‡‡‡fdd„t|ƒD ƒƒ}	ˆjj|	|žŽ }
t|
t
ƒ�s |
fn|
}	tt|ƒƒD ]:}t|	| tjjƒ�rX|	| j j ¡ n|	| || ˆ < �q0qVg }t|ƒD ]"\}}| ˆ |ˆ| j¡¡ �qz|S )Nr   r*   c                 3   s(   | ] \}}ˆ  |ˆ  ˆ| j¡V  qd S r   r–  r—  )Úindexrj   r   r   r   rš  [  s     z'ScanOps.generic_scan.<locals>.<genexpr>c                 3   s,   | ]$}|ˆj krˆ | d  nˆ | V  qdS )r   Nr/  )r[  rn   )r¹  r   r   r   rš  a  s     c                 3   s(   | ] \}}ˆ  |ˆ ˆ | j¡V  qd S r   r–  r›  )rj   Ú
prev_indexr   r   r   rš  b  s     )r   r   r$   r“  r,   rž  r   r.   rz  rŸ  rŠ  r   r·   r/   r¡  r†  r¢  rS   rT   rv  r}   r‰  )r   rj   r£  r¤  r$   rü   rn   r   r¥  r¦  r§  rË   r   )r¹  rj   rº  r   r   Úgeneric_scanP  s6      ÿÿzScanOps.generic_scanc              	   C   sÈ   g }| j r:|D ](}| |  tj|jj| jd�|j¡¡ qn|}| j	t
jjkr\|  |d ¡}n(| j	t
jjkrz|  |d ¡}n
|  |¡}| j r¬|D ]}tj|jj| jd�|j_qŽt|ƒdkrÀ|d pÆt|ƒS )Nr/  r   r   )rµ  r“  r‰  r,   Zflipr   r   r·   r   r†  rT   r°  r²  r·  Z_prod_combiner¸  r»  r/   rŠ  )r   rj   Z	new_inputrü   rË   r   r   r   r‹  n  s    (
zScanOps.apply_impl)	r   r    r!   r   r·  r¸  r»  r‹  r³  r   r   r‘  r   r´  D  s
   r´  c                  C   s4   ddd„} ddd„}| t _|t _| t j_|t j_d S )NFc                 [   s   t |||ƒ | ¡S r   )rŽ  r�  )rj   r·   r†  r�  rm  r   r   r   Ú_new_reduce†  s    z'_patch_reduce_scan.<locals>._new_reducec                 [   s   t |||ƒ | ¡S r   )r´  r�  )rj   r·   r†  rµ  rm  r   r   r   Ú	_new_scan‰  s    z%_patch_reduce_scan.<locals>._new_scan)F)F)rT   ÚreduceZassociative_scanrv  )r¼  r½  r   r   r   Ú_patch_reduce_scan‚  s    

r¿  c                 C   sx   dd„ }ddd„}ddd„}dd	„ }|| _ || _|| _t| _|| j_t|d
d�| _t|dd�| _	t|dd�| _
tƒ  d S )Nc                 S   sP  | j dkr| ¡ S | j dkr$| ¡ S | j dkr6| ¡ S | j dkrH| ¡ S | j dkrZ| ¡ S | j dkrl| ¡ S | j dkr~| ¡ S | j dkr�| ¡ S | j d	kr¢| 	¡ S | j d
kr´| 
¡ S | j dkrÆ| ¡ S | j dkrØ| ¡ S | j dk� rì| ¡ S | j dk�r | ¡ S | j dk�r| ¡ S | j dk�r(| ¡ S | j dk�r<| ¡ S td| › d�ƒ‚d S )NÚvoidrV   rL   rK   rN   rM   rP   rO   rQ   r3   Zfp8e5Zfp8e4nvZfp8e4b15Zfp16Zbf16Zfp32Zfp64zfail to convert z to ir type)r`  Zget_void_tyZget_int1_tyr•   r–   r—   r˜   r™   rš   r›   rœ   r    r�   rž   r‘   r’   r“   r”   r‰   )r   ro  r   r   r   Ú
_new_to_ir”  sF    











z$_patch_lang_core.<locals>._new_to_irc                 [   s6   |d krd}|d kr d|  }}n
| | }}t |||ƒS )Nr   r   )r.   )Zarg1Zarg2Ústeprm  r  Úendr   r   r   Ú
_new_range¼  s    
z$_patch_lang_core.<locals>._new_rangeÚ c                 S   s   | st |ƒ‚d S r   rQ  )r   rN  r   r   r   Ú_new_static_assertÅ  s    z,_patch_lang_core.<locals>._new_static_assertc                 S   sn   t | tjƒs| S t |ttfƒs$|gn|}dd„ |D ƒ}t|ƒtdt| jƒƒkr\td|› �ƒ‚| j	 
||¡ | S )Nc                 S   s"   g | ]}t |tjƒr|jn|‘qS r   )rS   rT   Ú	constexprr   ©r[  ri  r   r   r   r\  Î  s     z7_patch_lang_core.<locals>._set_attr.<locals>.<listcomp>r   z$len(values) != len(input.shape) for )rS   rT   r}   ÚlistrŠ  r/   r  r$   r‰   r   r   )rj   rC  r`  r   r   r   Ú	_set_attrÈ  s    z#_patch_lang_core.<locals>._set_attrztt.divisiblity©r`  ztt.contiguityztt.constancy)NN)rÅ  )r.   Zstatic_rangeZstatic_assertrK  Zstatic_printr   Zto_irr   Zmultiple_ofZmax_contiguousZmax_constancyr¿  )ÚlangrÁ  rÄ  rÆ  rÊ  r   r   r   Ú_patch_lang_core’  s    (
	
rÍ  c                 C   s€   dd„ | j  ¡ D ƒ}t|ƒdks(tdƒ‚t|d tƒ t|d jtƒ |d tkrbt|d jtƒ t	|d jƒ t
|d ƒ d S )Nc                 S   s"   g | ]\}}|t t jfkr|‘qS r   )rT   rv  )r[  r¨  r   r   r   r   r\  á  s      z_patch_lang.<locals>.<listcomp>r   z:triton.language must be visible from within jit'd functionr   )Ú__globals__rk  r/   rR  ry  Úinterpreter_builderr}   rT   rs   r„  rÍ  )r¢  rÌ  r   r   r   Ú_patch_langà  s    rÐ  c                 C   s<  t | tƒrât tjjj tjjj 	| ¡¡¡}t
j}d|   krFdk rRn nt
j}nnd|   krfdk rrn nt
j}nNd|   kr†dk r’n nt
j}n.d|   kr¦dk r²n nt
j}ntd| › �ƒ‚tt
j| g|d�|ƒ}t ||¡S t| d	ƒ�r8t tjjj tjjj 	| ¡¡¡}tt
j|  ¡ gt
jd�|ƒ}t ||¡S | S )
Ni   €l        l        lûÿÿÿ         l            l            zUnsupported integer value r*   Údata_ptr)rS   rJ   rT   Z	str_to_tyÚtritonZruntimeÚjitZJITFunctionZ_type_ofZ_key_ofr,   rP   rO   rQ   r3   r‰   r
   r¥   r}   r   rÑ  )rü   Útyr   r   r   r   r   Ú_implicit_cvtì  s&    
""rÕ  Z	num_warpsZ
num_stagesZnum_ctasZenable_fp_fusionÚgridZmaxnregc                   @   s,   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	S )
ÚGridExecutorc                    sN   ddl m‰ || _|| _|| _‡fdd„|j ¡ D ƒ‰ ‡ fdd„|D ƒ| _d S )Nr   ©Ú_normalize_tyc                    s   i | ]\}}|ˆ |ƒ“qS r   r   )r[  r`  rÔ  rØ  r   r   rj    s      z)GridExecutor.__init__.<locals>.<dictcomp>c                    s   g | ]}ˆ   |¡d kr|‘qS )rÇ  )Úget)r[  r`  )rH   r   r   r\    s      z)GridExecutor.__init__.<locals>.<listcomp>)rÓ  rÙ  r¢  Ú	arg_namesrÖ  rH   rk  Ú
constexprs)r   r¢  rÛ  rÖ  r   )rH   rÙ  r   r     s    zGridExecutor.__init__c                 C   sp   g }|D ](}t |dƒr&| | ¡ ¡ q| |¡ qi }| ¡ D ](\}}t |dƒr^| ¡ ||< q>|||< q>||fS ©NrÑ  )r   r“  Úcpurk  )r   Úargs_devrm  Úargs_hstrü   Ú
kwargs_hstr   r   r   r   r   Ú_init_args_hst  s    


zGridExecutor._init_args_hstc           
      C   sr   t ||ƒD ](\}}t|dƒr
|j | |j¡j¡ q
| ¡ D ]0\}}|| }	t|dƒr<|j |	 |j¡j¡ q<d S rÝ  )Úzipr   r   Zcopy_ÚtoZdevicerk  )
r   rß  rà  rm  rá  Zarg_devZarg_hstr   Z	kwarg_devZ	kwarg_hstr   r   r   Ú_restore_args_dev$  s    

zGridExecutor._restore_args_devc              
      sN  dd„ |  ¡ D ƒ}| dd¡r"d S ˆ  ||¡\}}tˆ jƒ tjˆ jf|ž|Ž}‡ fdd„|  ¡ D ƒ}tˆ jƒrzˆ  |¡nˆ j}t	|ƒdks”t
dƒ‚|ddt	|ƒ   }tj|Ž  zTt|d	 ƒD ]B}t|d
 ƒD ]0}t|d ƒD ]}	t |||	¡ ˆ jf |Ž qàqÐqÀW n2 tk
�r8 }
 ztt|
ƒƒ|
‚W 5 d }
~
X Y nX ˆ  ||||¡ d S )Nc                 S   s   i | ]\}}|t kr||“qS r   )ÚRESERVED_KWSrg  r   r   r   rj  1  s       z)GridExecutor.__call__.<locals>.<dictcomp>ZwarmupFc                    s(   i | ] \}}||ˆ j kr|nt|ƒ“qS r   )rÜ  rÕ  )r[  r`  rü   r   r   r   rj  ;  s      é   z#grid must have at most 3 dimensions)r   r   r   r   )rk  Úpoprâ  rÐ  r¢  rt  ÚgetcallargsÚcallablerÖ  r/   rR  rÏ  r�   r.   rŽ   Ú	Exceptionr   r~  rå  )r   rß  rm  rà  rá  rl  rÖ  rv   rŒ   r�   Úer   r   r   Ú__call__/  s(    

 zGridExecutor.__call__N)r   r    r!   r   râ  rå  rí  r   r   r   r   r×  	  s   	r×  c                   @   s6   e Zd Zddœdd„Zedd„ ƒZ dd„ Zd	d
„ ZdS )ÚInterpretedFunctionNr…   c                    s<   |ˆ _ ‡ fdd„}|ˆ _t |¡}dd„ |j ¡ D ƒˆ _d S )Nc                     s   |d }t ˆ jˆ j|ƒ| |ŽS )NrÖ  ©r×  r¢  rÛ  )rl  rm  rÖ  r   r   r   ÚrunR  s    z)InterpretedFunction.__init__.<locals>.runc                 S   s   g | ]
}|j ‘qS r   rË  rÈ  r   r   r   r\  X  s     z0InterpretedFunction.__init__.<locals>.<listcomp>)r¢  rð  rt  Ú	signatureÚ
parametersrC  rÛ  )r   r¢  rð  rñ  r   r   r   r   O  s
    
zInterpretedFunction.__init__c                 C   s   | j jS r   )r¢  r   r   r   r   r   r   Z  s    zInterpretedFunction.__name__c                 C   s   t | j| j|ƒS r   rï  )r   rÖ  r   r   r   Ú__getitem__^  s    zInterpretedFunction.__getitem__c              
   O   sN   t | jƒ z| j||ŽW S  tk
rH } ztt|ƒƒ|‚W 5 d }~X Y nX d S r   )rÐ  r¢  rë  r   r~  )r   rl  rm  rì  r   r   r   rí  a  s
    
zInterpretedFunction.__call__)r   r    r!   r   r‚  ró  rí  r   r   r   r   rî  M  s
   
rî  )4rt  Útypingr   rs   Znumpyr,   rÒ  Ztriton.languageÚlanguagerT   Zdataclassesr   Úerrorsr   Ú	functoolsr   Z_C.libtritonr   rÆ   r	   ri   r
   r"   r=   rR   ra   rr   rw   r{   Z	vectorizerX   r  rY   r  r3   rø   r|   r„   rs  ry  r„  r…  rŽ  r´  r¿  rÍ  rÐ  rÕ  rÏ  ræ  r×  rî  r   r   r   r   Ú<module>   sR    @   N"`>ND