U
    ‹Ñh” ã                   @  s  d dl mZ d dlmZmZmZmZmZ ddlm	Z	 ddl
mZ ddl
mZ edƒZG d	d
„ d
eƒZddddœdd„Zddddœdd„Zddddœdd„Zdddddœdd„Zdddddœdd„Z�ddddd!d"œd#d$„Zddddd%œd&d'„Zddddd%œd(d)„Zddddd%œd*d+„Zddddd%œd,d-„Zddddd%œd.d/„Zdddddd0œd1d2„Zddddd%œd3d4„Zddd5dd6œd7d8„Zddd5dd6œd9d:„Zdddd5dd;œd<d=„Z dddd!d%œd>d?„Z!ddddd%œd@dA„Z"ddddd%œdBdC„Z#ddddd%œdDdE„Z$ddddd%œdFdG„Z%ddddd%œdHdI„Z&dddJœdKdL„Z'ddddd%œdMdN„Z(ddddd%œdOdP„Z)ddddd%œdQdR„Z*dddSœdTdU„Z+ddddVœdWdX„Z,ddddVœdYdZ„Z-dd[d\œd]d^„Z.ddddd%œd_d`„Z/ddddd%œdadb„Z0ddddd%œdcdd„Z1ddddd%œdedf„Z2ddddd%œdgdh„Z3ddddd%œdidj„Z4dddddkœdldm„Z5dnddddoœdpdq„Z6ddndddrœdsdt„Z7ddndddduœdvdw„Z8dddddxœdydz„Z9dddddd{œd|d}„Z:ddddd~œdd€„Z;ddd!d�œd‚dƒ„Z<dd„ddd…œd†d‡„Z=ddndddˆœd‰dŠ„Z>ddddd"œd‹dŒ„Z?d�dŽœd�d�„Z@ddddd‘œd’d“„ZA�ddddd�dd•œd–d—„ZBd˜d™„ ZCdšd›„ ZDdœd�„ ZEdždŸ„ ZFd d¡„ ZGd¢d£„ ZHd¤d¥„ ZId¦d§„ ZJd¨d©„ ZKddªdªd«d¬d¬d¬dddd­œ
d®d¯„ZLdd¬d¬ddd°œd±d²„ZMddddd³œd´dµ„ZNd¶d·„ ZOd¸d¹„ ZPdddªd¬d¬dddºœd»d¼„ZQdddd¬d¬ddd½œd¾d¿„ZRdddd¬ddÀdÁœdÂdÃ„ZSdddd¬d¬dddÄœdÅdÆ„ZTdddd¬d¬dddÄœdÇdÈ„ZUdddd¬d¬dddÄœdÉdÊ„ZVdddd¬d¬dddÄœdËdÌ„ZWdddd¬d¬dddÄœdÍdÎ„ZXdddd¬d¬dddÄœdÏdÐ„ZYdddd¬d¬dddÄœdÑdÒ„ZZdÓdÔ„ Z[dddd�dddddÕœdÖd×„Z\ddddddØœdÙdÚ„Z]dÛdÜ„ Z^dÝdddÞdßœdàdá„Z_dÝddddÞdâœdãdä„Z`dddddåœdædç„Zaddnddèœdédê„Zbddnddèœdëdì„Zcddnddèœdídî„Zddddïœdðdñ„Zed¬dòddddóœdôdõ„Zfdd¬d¬ddddöœd÷dø„Zgdùdú„ Zh�ddûdü„Ziddddýœdþdÿ„Zjddddýœ�d �d„Zkd”S (  é    )Úannotations)ÚListÚOptionalÚSequenceÚTupleÚTypeVaré   )Úiré   )Úcore)ÚmathÚTc                      s   e Zd Z‡ fdd„Z‡  ZS )ÚIncompatibleTypeErrorImplc                   s@   || _ || _d| j  ¡  d | j ¡  | _tt| ƒ | j¡ d S )Nzinvalid operands of type ú and )Útype_aÚtype_bÚ__repr__ÚmessageÚsuperr   Ú__init__)Úselfr   r   ©Ú	__class__© úF/tmp/user/2202/pip-unpacked-wheel-sn6cy74c/triton/language/semantic.pyr      s    z"IncompatibleTypeErrorImpl.__init__)Ú__name__Ú
__module__Ú__qualname__r   Ú__classcell__r   r   r   r   r      s   r   Úintz
ir.builderz	tl.tensor)ÚaxisÚbuilderÚreturnc                 C  s*   | dkrt d| › �ƒ‚t | | ¡tj¡S )N©r   r
   r   z+program_id axis must be 0, 1, or 2 but got )Ú
ValueErrorÚtlÚtensorZcreate_get_program_idÚint32©r    r!   r   r   r   Ú
program_id   s    r)   c                 C  s*   | dkrt d| › �ƒ‚t | | ¡tj¡S )Nr#   z-num_programs axis must be 0, 1, or 2 but got )r$   r%   r&   Zcreate_get_num_programsr'   r(   r   r   r   Únum_programs    s    r*   ztl.dtype)Úa_tyÚb_tyr"   c                 C  s„   | j }|j }| j}|j}||kr0||kr,| S |S |tjjjkrN||krJ| S |S |tjjjkrl||krh|S | S td|› d|› �ƒ‚d S )Nzunexpected signedness r   )Úint_bitwidthÚint_signednessr%   ÚdtypeZ
SIGNEDNESSZUNSIGNEDÚ	TypeError)r+   r,   Za_rankZb_rankZa_snZb_snr   r   r   Úinteger_promote_impl+   s    r1   Úbool)r+   r,   Ú
div_or_modr"   c                 C  sà   |   ¡ s|  ¡ rtjS |  ¡ s&| ¡ r,tjS |  ¡ s<| ¡ rL|rFtjS tjS |  ¡ s\| ¡ r‚|rftjS |  ¡ r|| ¡ r|tjS tjS |  	¡ r’| 	¡ s¦t
d| › d|› �ƒ‚|rÖ| j|jkrÖt
d|  ¡  d | ¡  d ƒ‚t| |ƒS )Núunexpected type r   zCannot use /, #, or % with úx because they have different signedness;this is unlikely to result in a useful answer. Cast them to the same signedness.)Zis_fp64r%   Úfloat64Úis_fp32Úfloat32Úis_fp16Úfloat16Úis_bf16Úbfloat16Úis_intr0   r.   r   r1   )r+   r,   r3   r   r   r   Úcomputation_type_impl;   s*    ÿr>   ÚNone)r   r   Úallow_ptr_ar"   c                 C  sF   |   ¡ rB|st| |ƒ‚|  ¡ r0| |kr0t| |ƒ‚| ¡ rBt| |ƒ‚d S ©N)Úis_ptrr   Úis_floating)r   r   r@   r   r   r   Úcheck_ptr_type_implc   s    

rD   FTzTuple[tl.tensor, tl.tensor])ÚlhsÚrhsr!   r"   c           
      C  sx   t | ||ƒ\} }| jj}|jj}t|||ƒ t|||ƒ |rp| ¡ sp| ¡ spt|||ƒ}	t| |	|ƒ} t||	|ƒ}| |fS rA   )Úbroadcast_impl_valueÚtypeÚscalarrD   rB   r>   Úcast)
rE   rF   r!   Zallow_lhs_ptrZallow_rhs_ptrZarithmetic_checkr3   Z
lhs_sca_tyZ
rhs_sca_tyÚ
ret_sca_tyr   r   r   Úbinary_op_type_checking_implo   s    rL   )ÚinputÚotherr!   r"   c                 C  sÞ   t | ||ddƒ\} }| jj}|jj}| ¡ r<| ¡ r<tdƒ‚| ¡ rf| ¡ sf||  } }| jj}|jj}| ¡ rˆt | | j|j¡| j¡S | 	¡ rªt | 
| j|j¡| j¡S | ¡ rÌt | | j|j¡| j¡S td|› �ƒ‚d S )NTzcannot add pointers togetherr4   )rL   rH   rI   rB   r0   r%   r&   Úcreate_addptrÚhandlerC   Zcreate_faddr=   Z
create_add©rM   rN   r!   Úinput_scalar_tyÚother_scalar_tyr   r   r   Úadd€   s     
rT   c                 C  sš   t | ||ddƒ\} }| jj}| ¡ rDt | | jt||ƒj¡| j¡S | 	¡ rft | 
| j|j¡| j¡S | ¡ rˆt | | j|j¡| j¡S td|› �ƒ‚d S )NTFr4   )rL   rH   rI   rB   r%   r&   rO   rP   ÚminusrC   Zcreate_fsubr=   Z
create_subr0   ©rM   rN   r!   Ú	scalar_tyr   r   r   Úsub˜   s     rX   c                 C  sn   t | ||ƒ\} }| jj}| ¡ r:t | | j|j¡| j¡S | ¡ r\t | 	| j|j¡| j¡S t
d|› �ƒ‚d S ©Nr4   )rL   rH   rI   rC   r%   r&   Zcreate_fmulrP   r=   Z
create_mulr0   rV   r   r   r   Úmul§   s    rZ   c                 C  sò   t | ||ddddƒ\} }| jj}|jj}| ¡ rF| ¡ rFt|||ƒ}n’| ¡ rd| ¡ rdt| ||ƒ} nt| ¡ r’| ¡ r’t| tj|ƒ} t|tj|ƒ}nF| ¡ rÊ| ¡ rÊ|j|jkr¼t|||ƒ}qØt| ||ƒ} nt	d|› �ƒ‚t 
| | j|j¡| j¡S ©NFTr4   )rL   rH   rI   rC   r=   rJ   r%   r8   Zfp_mantissa_widthr0   r&   Úcreate_fdivrP   rQ   r   r   r   Útruediv³   s     r]   c                 C  s¨   t | ||ddddƒ\} }| jj}|jj}| ¡ r–| ¡ r–t||ƒ}t| ||ƒ} t|||ƒ}| ¡ r|t | 	| j
|j
¡| j¡S t | | j
|j
¡| j¡S td|› �ƒ‚d S r[   )rL   rH   rI   r=   r1   rJ   Úis_int_signedr%   r&   Zcreate_sdivrP   Zcreate_udivr0   )rM   rN   r!   rR   rS   Úret_tyr   r   r   ÚfloordivÍ   s    
r`   )rM   rN   Úieee_roundingr!   r"   c                 C  s^   | j j}|j j}| ¡ r | ¡ s(tdƒ‚t| ||ddddƒ\} }| | j|j¡}t || j ¡S )Nz4both operands of fdiv must have floating scalar typeFT)	rH   rI   rC   r0   rL   r\   rP   r%   r&   )rM   rN   ra   r!   rR   rS   Úretr   r   r   ÚfdivÜ   s    rc   c              	   C  sÞ   t | ||ddddƒ\} }| jj}|jj}| ¡ r\t| ttjt| |d|ƒ|d�||ƒ|ƒ}|S | 	¡ rÌ|j
|j
kr�td| ¡  d | ¡  d ƒ‚| ¡ r²t | | j|j¡| j¡S t | | j|j¡| j¡S td|› �ƒ‚d S )NFT©Z_builderzCannot mod z by r5   r4   )rL   rH   rI   rC   rX   rZ   r   Úfloorrc   r=   r.   r0   r   r^   r%   r&   Zcreate_sremrP   Zcreate_urem)rM   rN   r!   rW   rS   rb   r   r   r   Úmodæ   s    ( rf   ztl.PropagateNan)ÚxÚyÚpropagate_nanr!   c                 C  sÐ   t | ||ƒ\} }| j}| ¡ rz|tjjkrDt | | j|j¡| j	¡S |tjj
krjt | | j|j¡| j	¡S td|› �ƒ‚nR| ¡ rœt | | j|j¡| j	¡S | ¡ r¾t | | j|j¡| j	¡S td|› �ƒ‚d S ©NzUnexpected propagate_nan úUnexpected dtype )rL   r/   rC   r%   ÚPropagateNanÚALLr&   Zcreate_minimumfrP   rH   ÚNONEZcreate_minnumfr$   r^   Zcreate_minsiÚis_int_unsignedZcreate_minuir0   ©rg   rh   ri   r!   r/   r   r   r   Úminimum  s    rq   c                 C  sÐ   t | ||ƒ\} }| j}| ¡ rz|tjjkrDt | | j|j¡| j	¡S |tjj
krjt | | j|j¡| j	¡S td|› �ƒ‚nR| ¡ rœt | | j|j¡| j	¡S | ¡ r¾t | | j|j¡| j	¡S td|› �ƒ‚d S rj   )rL   r/   rC   r%   rl   rm   r&   Zcreate_maximumfrP   rH   rn   Zcreate_maxnumfr$   r^   Zcreate_maxsiro   Zcreate_maxuir0   rp   r   r   r   Úmaximum  s    rr   )rg   ÚminÚmaxri   r!   c                 C  sr   t |||ƒ\}}t | ||ƒ\} }t | ||ƒ\} }| j}| ¡ r^t | | j|j|j|¡| j¡S td|› d�ƒ‚d S )Nrk   z(. Only floating point clamp is supported)	rL   r/   rC   r%   r&   Zcreate_clampfrP   rH   r0   )rg   rs   rt   ri   r!   r/   r   r   r   Úclamp%  s     ru   c                 C  sz   t | ||dddƒ\} }| jj}|jj}| ¡ r6| ¡ s@t||ƒ‚t||ƒ}||kr^t| ||ƒ} ||krrt|||ƒ}| |fS )NF)rL   rH   rI   r=   r   r1   rJ   )rM   rN   r!   Úinput_sca_tyZother_sca_tyrK   r   r   r   Úbitwise_op_type_checking_impl6  s    

rw   c                 C  s*   t | ||ƒ\} }t | | j|j¡| j¡S rA   )rw   r%   r&   Z
create_andrP   rH   ©rM   rN   r!   r   r   r   Úand_E  s    ry   c                 C  s*   t | ||ƒ\} }t | | j|j¡| j¡S rA   )rw   r%   r&   Z	create_orrP   rH   rx   r   r   r   Úor_J  s    rz   c                 C  s*   t | ||ƒ\} }t | | j|j¡| j¡S rA   )rw   r%   r&   Z
create_xorrP   rH   rx   r   r   r   Úxor_O  s    r{   c                 C  sD   | j  ¡ st| t d¡|ƒ} |j  ¡ s8t|t d¡|ƒ}t| ||ƒS ©NÚint1)rH   Úis_int1Úbitcastr%   r/   ry   rx   r   r   r   Úlogical_andT  s
    

r€   c                 C  sD   | j  ¡ st| t d¡|ƒ} |j  ¡ s8t|t d¡|ƒ}t| ||ƒS r|   )rH   r~   r   r%   r/   rz   rx   r   r   r   Ú
logical_or\  s
    

r�   ©rM   r!   c                 C  s&   | j  ¡ st| t d¡|ƒ} t| |ƒS r|   )rH   r~   r   r%   r/   Úinvertr‚   r   r   r   Únot_d  s    
r„   c                 C  s*   t | ||ƒ\} }t | | j|j¡| j¡S rA   )rw   r%   r&   Zcreate_lshrrP   rH   rx   r   r   r   Úlshrj  s    r…   c                 C  s*   t | ||ƒ\} }t | | j|j¡| j¡S rA   )rw   r%   r&   Zcreate_ashrrP   rH   rx   r   r   r   Úashro  s    r†   c                 C  s*   t | ||ƒ\} }t | | j|j¡| j¡S rA   )rw   r%   r&   Z
create_shlrP   rH   rx   r   r   r   Úshlt  s    r‡   )rM   r"   c                 C  s   | S rA   r   ©rM   r   r   r   Úplus~  s    r‰   )rM   r!   r"   c                 C  sH   | j j}| ¡ r$td| ¡  d ƒ‚t | | |¡¡|¡}t	|| |ƒS )Nz$wrong type argument to unary minus (ú))
rH   rI   rB   r$   r   r%   r&   Úget_null_valueÚto_irrX   )rM   r!   rv   Ú_0r   r   r   rU   ‚  s
    rU   c                 C  sP   | j j}| ¡ s| ¡ r,td| ¡  d ƒ‚t | | 	|¡¡|¡}t
| ||ƒS )Nz%wrong type argument to unary invert (rŠ   )rH   rI   rB   rC   r$   r   r%   r&   Zget_all_ones_valuerŒ   r{   )rM   r!   rv   Z_1r   r   r   rƒ   Š  s
    rƒ   ztl.block_type)Úvr"   c                 C  s&   | j  ¡ stjS | j j}t tj|¡S rA   )rH   Úis_blockr%   r}   ÚshapeÚ
block_type)rŽ   r�   r   r   r   Ú
_bool_like•  s    
r’   c                 C  s–   t | ||ƒ\} }| jj}| ¡ r<t | | j|j¡t| ƒ¡S | 	¡ r„| 
¡ rht | | j|j¡t| ƒ¡S t | | j|j¡t| ƒ¡S td|› �ƒ‚d S rY   )rL   rH   rI   rC   r%   r&   Zcreate_fcmpOGTrP   r’   r=   r^   Zcreate_icmpSGTZcreate_icmpUGTr0   rV   r   r   r   Úgreater_thanœ  s    r“   c                 C  s–   t | ||ƒ\} }| jj}| ¡ r<t | | j|j¡t| ƒ¡S | 	¡ r„| 
¡ rht | | j|j¡t| ƒ¡S t | | j|j¡t| ƒ¡S td|› �ƒ‚d S rY   )rL   rH   rI   rC   r%   r&   Zcreate_fcmpOGErP   r’   r=   r^   Zcreate_icmpSGEZcreate_icmpUGEr0   rV   r   r   r   Úgreater_equal«  s    r”   c                 C  s–   t | ||ƒ\} }| jj}| ¡ r<t | | j|j¡t| ƒ¡S | 	¡ r„| 
¡ rht | | j|j¡t| ƒ¡S t | | j|j¡t| ƒ¡S td|› �ƒ‚d S rY   )rL   rH   rI   rC   r%   r&   Zcreate_fcmpOLTrP   r’   r=   r^   Zcreate_icmpSLTZcreate_icmpULTr0   rV   r   r   r   Ú	less_thanº  s    r•   c                 C  s–   t | ||ƒ\} }| jj}| ¡ r<t | | j|j¡t| ƒ¡S | 	¡ r„| 
¡ rht | | j|j¡t| ƒ¡S t | | j|j¡t| ƒ¡S td|› �ƒ‚d S rY   )rL   rH   rI   rC   r%   r&   Zcreate_fcmpOLErP   r’   r=   r^   Zcreate_icmpSLEZcreate_icmpULEr0   rV   r   r   r   Ú
less_equalÉ  s    r–   c                 C  sr   t | ||ƒ\} }| jj}| ¡ r<t | | j|j¡t| ƒ¡S | 	¡ r`t | 
| j|j¡t| ƒ¡S td|› �ƒ‚d S rY   )rL   rH   rI   rC   r%   r&   Zcreate_fcmpOEQrP   r’   r=   Zcreate_icmpEQr0   rV   r   r   r   ÚequalØ  s    r—   c                 C  sr   t | ||ƒ\} }| jj}| ¡ r<t | | j|j¡t| ƒ¡S | 	¡ r`t | 
| j|j¡t| ƒ¡S td|› �ƒ‚d S rY   )rL   rH   rI   rC   r%   r&   Zcreate_fcmpUNErP   r’   r=   Zcreate_icmpNEr0   rV   r   r   r   Ú	not_equalä  s    r˜   )ÚstartÚendr!   r"   c                 C  sœ   t | tƒrt |tƒstdƒ‚t| d? ƒ}t|d? ƒ}|s<|rDtdƒ‚|| krTtdƒ‚||  }||d @ dkrttdƒ‚|g}t tj|¡}t | | |¡|¡S )Nz/arange's arguments must be of type tl.constexpré    zarange must fit in int32z=arange's end argument must be greater than the start argumentr
   r   z#arange's range must be a power of 2)	Ú
isinstancer   r$   r2   r%   r‘   r'   r&   Zcreate_make_range)r™   rš   r!   Zis_start_int64Zis_end_int64Úranger�   r_   r   r   r   Úarangeõ  s    rž   z	List[int])r�   r/   r!   r"   c                 C  sŠ   t |tjƒr.|jjdks tdƒ‚t|||ƒ}nP|d kr>tdƒ‚|dkrX| | 	|¡¡}nt
|d|j› �ƒ}||ƒ}t ||¡}t|| |ƒS )Nr
   zonly accepts size-1 tensorz2dtype must be specified when value is not a tensorr   Úget_)rœ   r%   r&   ÚnumelÚvalueÚAssertionErrorrJ   r$   r‹   rŒ   ÚgetattrÚnameÚsplat)r�   r¡   r/   r!   Zget_value_fnr   r   r   Úfull  s    r¦   )r¡   r�   r!   r"   c                 C  sF   | j  ¡ rtdƒ‚t|ƒdkr"| S t | j|¡}t | | j	|¡|¡S )NzCannot splat a block tensorr   )
rH   r�   r¢   Úlenr%   r‘   r/   r&   Úcreate_splatrP   )r¡   r�   r!   r_   r   r   r   r¥     s
    r¥   )rM   Ú	dst_shapeÚcan_reorderr!   r"   c                 C  sR   d}|D ]}||9 }q| j j|kr*tdƒ‚t | j j|¡}t | | j||¡|¡S )Nr
   z:reshape() cannot change total number of elements in tensor)	rH   r    r$   r%   r‘   rI   r&   Zcreate_reshaperP   )rM   r©   rª   r!   r    Úsr_   r   r   r   Úreshape%  s    
r¬   )rM   r    r!   r"   c                 C  sZ   dd„ | j D ƒ}| |d¡ | j ¡ s4t| ||d�S t | jj|¡}t | 	| j
|¡|¡S )Nc                 S  s   g | ]}t  |¡‘qS r   ©r%   Ú_constexpr_to_value)Ú.0rg   r   r   r   Ú
<listcomp>0  s     zexpand_dims.<locals>.<listcomp>r
   )r�   r!   )r�   ÚinsertrH   r�   r¥   r%   r‘   rI   r&   Úcreate_expand_dimsrP   )rM   r    r!   r©   r_   r   r   r   Úexpand_dims/  s    
r³   )rE   rF   rª   r!   r"   c                 C  sX   |st dƒ‚t| jƒdkst ‚t | jj| jd |jd  g¡}t | | j	|j	¡|¡S )Nz;current implementation of `cat` always may reorder elementsr
   r   )
r¢   r§   r�   r%   r‘   rH   rI   r&   Z
create_catrP   )rE   rF   rª   r!   Úret_typer   r   r   Úcat:  s    "rµ   )ÚaÚbr!   r"   c                 C  s¦   t | ||ƒ\} }| jg k}|r6t| d|ƒ} t|d|ƒ}t| jd tjƒrTt d¡}nd}| j|g }t | jj|¡}t 	| 
| j|j¡|¡}|r¢t|dgd|d�}|S )Nr   éÿÿÿÿr   F©rª   r!   )rG   r�   r³   rœ   r%   Ú	constexprr‘   rH   rI   r&   Zcreate_joinrP   r¬   )r¶   r·   r!   Z
was_rank_1ZtwoÚ	new_shaper´   rb   r   r   r   ÚjoinA  s    
r¼   )r¶   r!   r"   c                 C  sp   t | jƒdkst‚t | jd ¡dks*t‚| jd d… }t | jj|¡}| | j	¡\}}t 
||¡t 
||¡fS )Nr   r¸   r   )r§   r�   r¢   r%   r®   r‘   rH   rI   Zcreate_splitrP   r&   )r¶   r!   r»   r´   ZoutLHSZoutRHSr   r   r   ÚsplitZ  s    

þr½   z
Tuple[int])rM   Údimsr!   r"   c                   s~   t ˆ jƒt |ƒkrtdƒ‚tdd„ |D ƒƒttt |ƒƒƒkrJtd|› �ƒ‚t ˆ jj	‡ fdd„|D ƒ¡}t 
| ˆ j|¡|¡S )Nz5permute dims must have the same length as input shapec                 s  s   | ]}t  |¡V  qd S rA   r­   ©r¯   Údr   r   r   Ú	<genexpr>j  s     zpermute.<locals>.<genexpr>z?permute dims must be a permutation of 0, 1, ..., n-1, but were c                   s   g | ]}ˆ j | ‘qS r   ©r�   r¿   rˆ   r   r   r°   m  s     zpermute.<locals>.<listcomp>)r§   r�   r$   ÚsortedÚlistr�   r%   r‘   rH   rI   r&   Zcreate_transrP   )rM   r¾   r!   r´   r   rˆ   r   Úpermuteg  s    "rÅ   )rM   r�   r!   r"   c                 C  sÞ   | j  ¡ s.t | j |¡}t | | j|¡|¡S | j  ¡ }t|ƒt|ƒkr\t	d|› d|› �ƒ‚||krh| S t
|ƒD ]F\}}|| |krp|dkrpt	d|| › d|› d|› d|› d|› �
ƒ‚qpt | j j|¡}t | | j|¡|¡S )Nz!Cannot broadcast, rank mismatch: ú, r
   z3Cannot broadcast, the expanded size of the tensor (z ) must match the existing size (z) at non-singleton dimension ú: )rH   r�   r%   r‘   r&   r¨   rP   Úget_block_shapesr§   r$   Ú	enumeraterI   Úcreate_broadcast)rM   r�   r!   r_   Z	src_shapeÚiÚitemr   r   r   Úbroadcast_impl_shapeq  s    

,rÍ   c              	   C  sd  | j }|j }| ¡ rJ| ¡ sJt |j|j¡}t | |j| 	¡ ¡|¡}�n| ¡ sˆ| ¡ rˆt |j|j¡}t | | j| 	¡ ¡|¡} �nÔ| ¡ �r\| ¡ �r\| 	¡ }| 	¡ }t
|ƒt
|ƒk �rtt
|ƒt
|ƒƒD ]8}t | | jd¡t |jdg| ¡¡} | j }| 	¡ }qÐn`t
|ƒt
|ƒk �rltt
|ƒt
|ƒƒD ]:}t | |jd¡t |jdg| ¡¡}|j }| 	¡ }�q0t
|ƒt
|ƒk�s‚t‚g }t|ƒD ]p\}	}
||	 }|
dk�r´| |¡ nH|dk�sÈ||
k�rÔ| |
¡ n(tdt|	ƒ d t|
ƒ d t|ƒ ƒ‚�qŽ||k�r.t |j|¡}t | | j|¡|¡} ||k�r\t |j|¡}t | |j|¡|¡}| |fS )Nr   r
   z?Cannot make_shape_compatible: incompatible dimensions at index rÇ   r   )rH   r�   r%   r‘   rI   r�   r&   r¨   rP   rÈ   r§   r�   r²   r¢   rÉ   Úappendr$   ÚstrrÊ   )rE   rF   r!   Zlhs_tyZrhs_tyZ	lhs_shapeZ	rhs_shapeÚ_Ú	ret_shaperË   ÚleftÚrightr_   r   r   r   rG   ƒ  sf    ÿÿ
ÿÿÿÿÿ


rG   zOptional[str]©Zrounding_modec                 C  s@   | d krd S | dkrt jjS | dkr,t jjS td| › d�ƒ‚d S )NZrtneZrtzzInvalid rounding mode: z0. Supported rounding modes are 'rtne' and 'rtz'.)r	   ÚROUNDING_MODEÚRTNEZRTZr$   rÔ   r   r   r   Ú_str_to_rounding_mode½  s    r×   )rM   Údst_tyr!   r"   c                 C  s¢   | j }| ¡ r"t |j| j  ¡ ¡}||kr.| S |j}|j}| ¡ sJ| ¡ rVt| ||ƒS |j}|j}||kr†t	dt
|ƒ d t
|ƒ ƒ‚t | | j| |¡¡|¡S )Nz!Cannot bitcast data-type of size z to data-type of size )rH   r�   r%   r‘   rI   rÈ   rB   rJ   Úprimitive_bitwidthr$   rÏ   r&   Úcreate_bitcastrP   rŒ   )rM   rØ   r!   Úsrc_tyÚ
src_sca_tyÚ
dst_sca_tyZsrc_bitsZdst_bitsr   r   r   r   Ç  s     ÿr   N)rM   rØ   r!   Úfp_downcast_roundingr"   c                 C  s²  | j }t|tjƒr|j}t|tjƒr*|j}| ¡ rFt |j| j  ¡ ¡}||krR| S |j}|j}t	|ƒ}d}| 
¡ rª| 
¡ rª|j|jk rª|d kr˜tjj}qÎ|tjjkrÎd}n$|d k	rÎtdt|ƒ d t|ƒ ƒ‚| ¡ sÞ| ¡ rî|jjsîtdƒ‚| ¡ �s| ¡ �r2|j d¡d k	�stdƒ‚|jd | |||d�S | ¡ �rF| 
¡ �s`| 
¡ �rZ| ¡ �s`|�r~t | | j| |¡|¡|¡S | ¡ �r’| ¡ �r¦| ¡ �r¼| ¡ �s¼t t | tj!|ƒ||ƒS | 
¡ �oÚ| 
¡ �oÚ|j|jk}|�rþt | "| j| |¡¡|¡S | 
¡ �o| 
¡ �o|j|jk }	|	�r@t | #| j| |¡¡|¡S | $¡ �rØ| $¡ �rØ|j%|j%k�sp|j&|j&k�rØ| '¡ �o‚| (¡  }
| (¡ �rº| j) |¡}t | *|¡| j)¡}t+| ||ƒS t | ,| j| |¡|
¡|¡S | -¡ �rd| $¡ �rd| (¡ �r"| j) |¡}t | *|¡| j)¡}t+| ||ƒS | '¡ �rHt | .| j| |¡¡|¡S t | /| j| |¡¡|¡S | $¡ �rÄ| -¡ �rÄ| (¡ �sŒ| '¡ �s¨t | 0| j| |¡¡|¡S t | 1| j| |¡¡|¡S | 2¡ �r4| $¡ �r4|j%}|d	k�rt | 3| j| |¡¡|¡S |d
k�r4t+t | tj4|ƒt | 5d¡tj4¡|ƒS | $¡ �rd| 2¡ �rdt | 6| j| |¡¡|¡S | 2¡ �r”| 2¡ �r”t | 7| j| |¡¡|¡S d�s®td| › d|› �ƒ‚d S )NFTz]fp_downcast_rounding should be set only for truncating fp conversions. Source scalar type is z and destination type is z4fp8e4nv data type is not supported on CUDA arch < 89Zconvert_custom_typesz0target doesn't provide conversion for this type.rd   é@   r
   r   zcannot cast z to )8rH   rœ   r%   rº   r¡   r�   r‘   rI   rÈ   r×   rC   rÙ   r	   rÕ   rÖ   r$   rÏ   Ú
is_fp8e4nvÚoptionsÚallow_fp8e4nvr¢   Úis_fp8e4b15Zcodegen_fnsÚgetÚis_fp8r&   Zcreate_fp_to_fprP   rŒ   r9   r7   r;   rJ   r8   Zcreate_fp_truncZcreate_fp_extr=   r-   r.   r^   Úis_boolr/   r‹   r˜   Úcreate_int_castZis_standard_floatingZcreate_fp_to_siZcreate_fp_to_uiZcreate_ui_to_fpZcreate_si_to_fprB   Zcreate_ptr_to_intÚint64Ú	get_int64Zcreate_int_to_ptrrÚ   )rM   rØ   r!   rÞ   rÛ   rÜ   rÝ   Zuse_custom_roundingZtruncate_fpZext_fpZsign_extendÚtyr�   Zbitwidthr   r   r   rJ   Ú  sÊ    
ÿ 
 ÿÿÿÿÿÿÿÿþÿÿ
ÿ
þ
ÿ
þ
ÿ
ÿ




&rJ   c                 C  sD   t jj}| r@| dkrt jj}n"| dkr0t jj}ntd| › d�ƒ‚|S )Nz.caú.cgúCache modifier ú not supported)r	   ÚCACHE_MODIFIERrn   ÚCAÚCGr$   ©Úcache_modifierÚcacher   r   r   Ú_str_to_load_cache_modifierQ  s    

rô   c                 C  sh   t jj}| rd| dkrt jj}nF| dkr0t jj}n4| dkrBt jj}n"| dkrTt jj}ntd| › d�ƒ‚|S )Nz.wbrë   z.csz.wtrì   rí   )r	   rî   rn   ZWBrð   ÚCSZWTr$   rñ   r   r   r   Ú_str_to_store_cache_modifier]  s    



rö   c                 C  sD   t jj}| r@| dkrt jj}n"| dkr0t jj}ntd| › d�ƒ‚|S )NZ
evict_lastZevict_firstzEviction policy rí   )r	   ZEVICTION_POLICYÚNORMALZ
EVICT_LASTZEVICT_FIRSTr$   )Úeviction_policyÚevictionr   r   r   Ú_str_to_eviction_policym  s    

rú   c                 C  s@   d }| r<| dkrt jj}n"| dkr,t jj}ntd| › d�ƒ‚|S )NÚzeroÚnanzPadding option rí   )r	   ÚPADDING_OPTIONZPAD_ZEROÚPAD_NANr$   )Úpadding_optionÚpaddingr   r   r   Ú_str_to_padding_optiony  s    

r  c                 C  sh   t jj}| rd| dkrt jj}nF| dkr0t jj}n4| dkrBt jj}n"| dkrTt jj}ntd| › d�ƒ‚|S )NÚacquireÚreleaseZacq_relZrelaxedúMemory semantic rí   )r	   ZMEM_SEMANTICZACQUIRE_RELEASEZACQUIREZRELEASEZRELAXEDr$   )Z
sem_optionÚsemr   r   r   Ú_str_to_sem…  s    



r  c                 C  sV   t jj}| rR| dkrt jj}n4| dkr0t jj}n"| dkrBt jj}ntd| › d�ƒ‚|S )NZgpuZctaÚsysr  rí   )r	   ZMEM_SYNC_SCOPEZGPUZCTAZSYSTEMr$   )Zscope_optionÚscoper   r   r   Ú_str_to_scope•  s    


r	  c                 C  sŒ   | rˆt | dƒs| g} dd„ | D ƒ} | D ],}t|tƒrNd|  krLt|ƒk s&n t‚q&t| ƒdksdt‚t| ƒtt| ƒƒks€tdƒ‚t| ƒS dS )NÚ__iter__c                 S  s"   g | ]}t |tjƒr|jn|‘qS r   ©rœ   r%   rº   r¡   ©r¯   Úelemr   r   r   r°   §  s     z0_canonicalize_boundary_check.<locals>.<listcomp>r   z'Duplicate dimension in `boundary_check`r   )Úhasattrrœ   r   r§   r¢   ÚsetrÃ   )Úboundary_checkÚblock_shapeZdimr   r   r   Ú_canonicalize_boundary_check£  s    
*r  c	              
   C  s„   |d k	s|d k	rt dƒ‚| jjj}	|	tjks4tdƒ‚|	 ¡ rP|tjj	krPt dƒ‚| jj}
t
||
 ¡ ƒ}t | | j|||||¡|
¡S )NúK`mask` and `other` arguments cannot be specified for loading block pointersú3`tl.int1` should be rewrited in `tl.make_block_ptr`z@Padding option `nan` is not supported for integer block pointers)r$   rH   Ú
element_tyr%   r}   r¢   r=   r	   rý   rþ   r  rÈ   r&   Zcreate_tensor_pointer_loadrP   )ÚptrÚmaskrN   r  r   ró   rù   Úis_volatiler!   Úelt_tyrØ   r   r   r   Ú_load_block_pointer°  s    
 ÿr  c	              
   C  sˆ  | j j ¡ s"td| j  ¡ › d�ƒ‚|d kr:|d k	r:tdƒ‚|sB|rJtdƒ‚| j  ¡ s€|rj|j  ¡ rjtdƒ‚|r€|j  ¡ r€tdƒ‚| j  ¡ r¾|d k	r¤t|| j  ¡ |ƒ}|d k	r¾t|| j  ¡ |ƒ}| j j}	|	j}
|
t	j
kröt	j}
t	 |
|	j¡}	t| |	|ƒ} |d k	�rt||
|ƒ}| j  ¡ �r0| j  ¡ }t	 |
|¡}n|
}|d k�rXt	 | | j|||¡|¡S t	 | | j|j|�rt|jnd |||¡|¡S d S )NúUnsupported ptr type z in `tl.load`z)`other` cannot be provided without `mask`zä`padding_option` or `boundary_check` argument is not supported for loading a tensor ofpointers or loading a scalar. Because the compiler does not know the boundary; please use block pointers (defined by `make_block_ptr`) insteadúEMask argument cannot be block type if pointer argument is not a blockzFOther argument cannot be block type if pointer argument is not a block)rH   rI   rB   r$   r   r�   rÍ   rÈ   r  r%   r}   Úint8Úpointer_typeÚaddress_spacerJ   r‘   r&   Zcreate_loadrP   Zcreate_masked_load)r  r  rN   r  r   ró   rù   r  r!   Úptr_tyr  r�   rØ   r   r   r   Ú_load_legacyÆ  sH    





ÿþr!  zOptional[tl.tensor]r   rÏ   )
r  r  rN   r  rÿ   rò   rø   r  r!   r"   c	              
   C  sb   t |ƒ}	t|ƒ}
t|ƒ}| j ¡ rF| jj ¡ rFt| |||||	|
||ƒ	S t| |||||	|
||ƒ	S d S rA   )	rô   rú   r  rH   rB   r  r�   r  r!  )r  r  rN   r  rÿ   rò   rø   r  r!   ró   rù   r   r   r   r   Úload   s    r"  )Údesc_ptrrò   rø   r!   r"   c                 C  s<   t ||dd�}| | j|| |¡t|ƒt|ƒ¡}t ||¡S ©NF©Úrequire_i64)Ú_convert_to_ir_valuesZcreate_descriptor_loadrP   rŒ   rô   rú   r%   r&   )r#  Úoffsetsrò   rø   rH   r!   rg   r   r   r   Údescriptor_load  s    þr)  )r#  r¡   r!   r"   c                 C  s*   t ||dd�}t | | j|j|¡tj¡S r$  )r'  r%   r&   Zcreate_descriptor_storerP   Úvoid)r#  r¡   r(  r!   r   r   r   Údescriptor_store  s    r+  c           	   	   C  sò   |d k	rt dƒ‚| jj ¡ }|j ¡ s2t|||ƒ}|j ¡ sDtdƒ‚||j ¡ ksntd|› d|j ¡ › d�ƒ‚| jjj|jjks td| jjj› d|jj› d�ƒ‚| jjj}|tjks¼tdƒ‚t	||ƒ}t
|||ƒ}t | | j|j|||¡tj¡S )	Nr  z-Value argument must be block type or a scalarzBlock shape(z) and value shape(z
) mismatchzBlock element type(z) and value element type(r  )r$   rH   r  rÈ   r�   rÍ   r¢   r%   r}   r  rJ   r&   Zcreate_tensor_pointer_storerP   r*  )	r  Úvalr  r  ró   rù   r!   r  r  r   r   r   Ú_store_block_pointer  s"    
ÿ2

ÿr-  c           	   	   C  s4  | j j ¡ s"td| j  ¡ › d�ƒ‚|r.tdƒ‚| j  ¡ s`|j  ¡ rJtdƒ‚|r`|j  ¡ r`tdƒ‚| j  ¡ r–t|| j  ¡ |ƒ}|d k	r–t|| j  ¡ |ƒ}| j j}|j}|t	j
krÎt	j}t	 ||j¡}t| ||ƒ} t|||ƒ}|süt	 | | j|j||¡t	j¡S |j j ¡ �stdƒ‚t	 | | j|j|j||¡t	j¡S )Nr  z in `tl.store`zÑ`boundary_check` argument is not supported for storing a tensor of pointers or storing a scalar. Because the compiler does not know the boundary; please use block pointers (defined by `make_block_ptr`) insteadzFValue argument cannot be block type if pointer argument is not a blockr  z"Mask must have boolean scalar type)rH   rI   rB   r$   r   r�   rÍ   rÈ   r  r%   r}   r  r  r  rJ   r&   Zcreate_storerP   r*  ræ   Zcreate_masked_store)	r  r,  r  r  ró   rù   r!   r   r  r   r   r   Ú_store_legacy;  s2    



r.  )r  r,  r  rò   rø   r!   r"   c           	      C  sp   t |ƒ}t|ƒ}| j ¡ s&| jj ¡ r.tdƒ‚| j ¡ rX| jj ¡ rXt	| ||||||ƒS t
| ||||||ƒS d S )Nú"Cannot store to a constant pointer)rö   rú   rH   Úis_constrI   r$   rB   r  r�   r-  r.  )	r  r,  r  r  rò   rø   r!   ró   rù   r   r   r   Ústoreg  s    r1  )r  Úcmpr,  r  r  r!   r"   c              	   C  sN   t |ƒ}t|ƒ}| jjj}|jdkr,tdƒ‚t | 	| j
|j
|j
||¡|j¡S )N)é   r›   rß   z9atomic_cas only supports elements with width {16, 32, 64})r  r	  rH   rI   r  rÙ   r$   r%   r&   Zcreate_atomic_casrP   )r  r2  r,  r  r  r!   r  r   r   r   Ú
atomic_cas}  s    

r4  z&Tuple[tl.tensor, tl.tensor, tl.tensor])r  r,  r  Úopr!   r"   c                 C  sF  | j j ¡ std| j  ¡  ƒ‚| j  ¡ s4| j j ¡ r<tdƒ‚| j jj}|tjkrh|dkrhtd| d ƒ‚|tj	tj
tjtjfkr˜td| d t|ƒ ƒ‚| j  ¡ rÖ|d k	r¼t|| j  ¡ |ƒ}|d k	rÖt|| j  ¡ |ƒ}t|| j jj|ƒ}|�s<| d¡}tj	}| j  ¡ �r0| || j  ¡ ¡}t tj	| j  ¡ ¡}t ||¡}| ||fS )Nz)Pointer argument of store instruction is r/  rT   Zatomic_z does not support fp16z does not support T)rH   rI   rB   r$   r   r0  r  r%   r:   r}   r  Zint16r<   rÏ   r�   rÍ   rÈ   rJ   Zget_int1r¨   r‘   r&   )r  r,  r  r5  r!   r  Zmask_irZmask_tyr   r   r   Úatom_red_typechecking_impl†  s.    


r6  )r  r,  r  r  r  r!   r"   c                 C  sº  t | ||d|ƒ\} }}t|ƒ}t|ƒ}|jj}| ¡ rŽ| ¡ rft | 	t
jj| j|j|j||¡|j¡S t | 	t
jj| j|j|j||¡|j¡S |tjtjhkr¬td|› �ƒ‚tg d||ƒ}|tjkrÊtjntj}t|||ƒ}	t| t |d¡|ƒ}
|tjk�rtjntj}t|||ƒ}t| t |d¡|ƒ}t|||ƒ}t|||ƒ}t | 	t
jj|
j|	jt|||ƒj||¡|	j¡}t | 	t
jj|j|jt|||ƒj||¡|j¡}t||||ƒ}t|||ƒS )Nrt   z#atomic_max not supported for dtype ç        r
   )r6  r  r	  rH   rI   r=   r^   r%   r&   Úcreate_atomic_rmwr	   Ú	ATOMIC_OPÚMAXrP   ÚUMAXr8   r6   r0   r¦   r'   rè   r   r  Úuint32Úuint64r”   r•   ry   ÚUMINÚwhere©r  r,  r  r  r  r!   Úsca_tyrû   Zi_typeZi_valZi_ptrZui_typeZui_valZui_ptrÚposÚnegZpos_retZneg_retrb   r   r   r   Ú
atomic_max¡  sV     ÿ ÿ  ÿþ  ÿþrD  c                 C  sº  t | ||d|ƒ\} }}t|ƒ}t|ƒ}|jj}| ¡ rŽ| ¡ rft | 	t
jj| j|j|j||¡|j¡S t | 	t
jj| j|j|j||¡|j¡S |tjtjhkr¬td|› �ƒ‚tg d||ƒ}|tjkrÊtjntj}t|||ƒ}	t| t |d¡|ƒ}
|tjk�rtjntj}t|||ƒ}t| t |d¡|ƒ}t|||ƒ}t|||ƒ}t | 	t
jj|
j|	jt|||ƒj||¡|	j¡}t | 	t
jj|j|jt|||ƒj||¡|j¡}t||||ƒ}t|||ƒS )Nrs   z#atomic_min not supported for dtype r7  r
   )r6  r  r	  rH   rI   r=   r^   r%   r&   r8  r	   r9  ZMINrP   r>  r8   r6   r0   r¦   r'   rè   r   r  r<  r=  r”   r•   ry   r;  r?  r@  r   r   r   Ú
atomic_minÈ  sV     ÿ ÿ  ÿþ  ÿþrE  c              
   C  sj   t | ||d|ƒ\} }}t|ƒ}t|ƒ}|jj}| ¡ r>tjjntjj	}t
 | || j|j|j||¡|j¡S )NrT   )r6  r  r	  rH   rI   rC   r	   r9  ZFADDZADDr%   r&   r8  rP   )r  r,  r  r  r  r!   rA  r5  r   r   r   Ú
atomic_addï  s    rF  c              
   C  sN   t | ||d|ƒ\} }}t|ƒ}t|ƒ}t | tjj| j	|j	|j	||¡|j
¡S )NÚand)r6  r  r	  r%   r&   r8  r	   r9  ÚANDrP   rH   ©r  r,  r  r  r  r!   r   r   r   Ú
atomic_andø  s     ÿrJ  c              
   C  sN   t | ||d|ƒ\} }}t|ƒ}t|ƒ}t | tjj| j	|j	|j	||¡|j
¡S )NÚor)r6  r  r	  r%   r&   r8  r	   r9  ÚORrP   rH   rI  r   r   r   Ú	atomic_or   s     ÿrM  c              
   C  sN   t | ||d|ƒ\} }}t|ƒ}t|ƒ}t | tjj| j	|j	|j	||¡|j
¡S )NÚxor)r6  r  r	  r%   r&   r8  r	   r9  ZXORrP   rH   rI  r   r   r   Ú
atomic_xor  s     ÿrO  c              
   C  sN   t | ||d|ƒ\} }}t|ƒ}t|ƒ}t | tjj| j	|j	|j	||¡|j
¡S )NZxchg)r6  r  r	  r%   r&   r8  r	   r9  ZXCHGrP   rH   rI  r   r   r   Úatomic_xchg  s     ÿrP  c                 C  sH   |   ¡ |jjks(td|jj› d| › �ƒ‚|  ¡ } | dkr<d} ttj| ƒS )Nzinput_precision must be one of z. Got ZTF32X3ZTF32x3)Úlowerrá   Zallowed_dot_input_precisionsr¢   Úupperr£   r	   ZINPUT_PRECISION)Úinput_precisionr!   r   r   r   Ú_str_to_dot_input_precision  s    ÿrT  )rE   rF   ÚaccrS  Úmax_num_imprecise_accÚ	out_dtyper!   r"   c              
   C  s  dd„ }| j  ¡ r|j  ¡ s t‚|| j|j|jƒ | j ¡ sF|j ¡ rbt| tj|ƒ} t|tj|ƒ}|d krr|jj	}t
||ƒ}t| jƒ}t|jƒ}	||	  kr¤dksÖn ||	  krºdksÖn td| j› d|j› d�ƒ‚| jd j|jd	 jk�s&td
| j› d|j› d| jd j› d|jd	 j› d�	ƒ‚| jd	 jdk�r\| jd jdk�r\|jd jdk�svtd| j› d|j› d�ƒ‚| j j ¡ �rÈ| j jtjk�sœtdƒ‚| jd jdk�s¶tdƒ‚| d¡}
tj}nd| ¡ �rÜtdƒ‚nP| j j ¡ �sø| j j ¡ �r
| d¡}
tj}n"| ¡ �r| d¡n| d¡}
|}| j jd	 }|j jd }|dk�rZ| j jd nd }t ||�rt|||gn||g¡}|d k�rª| |
|�rž|||gn||g¡}n|j}|j |k�sÀt‚|d k�rð| j ¡ �rì|j ¡ �rì|jj}nd}t |  | j|j|||¡|¡S )Nc                 S  s€  |j sT|  ¡ s| ¡ rtdƒ‚|  ¡ r2| ¡ r2d S | |ksPtd| › d|› d�ƒ‚�n(|  ¡ sd| ¡ r¤| |ks‚td| › d|› d�ƒ‚|  ¡ s¢|  ¡ s¢td| › d�ƒ‚nØ|  ¡ s´| ¡ rð|jrÆd	d
dg}nd	d
g}dd„ }|| |dƒ |||dƒ nŒ|  ¡ �s&|  	¡ �s&|  
¡ �s&|  ¡ �s&td| › �ƒ‚| ¡ �s\| 	¡ �s\| 
¡ �s\| ¡ �s\td|› �ƒ‚| |k�s|td| › d|› d�ƒ‚d S )Nz1Dot op does not support fp8e4nv on CUDA arch < 90zFirst input (z) and second input (z) must have the same dtype!z0Both operands must be same type. First operand (z) and second operand (rŠ   z:Both operands must be either int8 or uint8. Operand type (Zfp8e4nvZfp8e5Zfp8e4b15c                   s@   t ‡ fdd„|D ƒƒs<d |¡}td|› d|› dˆ › d�ƒ‚d S )Nc                 3  s    | ]}t ˆ d |› �ƒƒ V  qdS )Úis_N)r£   )r¯   Z
dtype_name©r/   r   r   rÁ   =  s     zLdot.<locals>.assert_dtypes_valid.<locals>._validate_dtype.<locals>.<genexpr>rÆ   zOnly supports z. z (rŠ   )Úanyr¼   r¢   )r/   Úallowed_typesZoperand_nameZsupported_typesr   rY  r   Ú_validate_dtype<  s    
z9dot.<locals>.assert_dtypes_valid.<locals>._validate_dtypezFirst operandzSecond operandzUnsupported dtype )râ   rà   r¢   rå   r=   Zis_int8Zis_uint8Zallow_fp8e4b15r9   r;   r7   r~   )Z	lhs_dtypeZ	rhs_dtyperá   r[  r\  r   r   r   Úassert_dtypes_valid*  s4    ÿ"
ÿ*ÿ*ÿz dot.<locals>.assert_dtypes_validr   é   z+Both inputs must be either 2D or 3D; (lhs: z	 vs rhs: rŠ   r¸   éþÿÿÿzFirst input shape (z) and second input shape z= are not compatible for matmul (second index of first shape (z0) must be equal to first index of second shape (r3  z0All non-batch values in both first input shape (z) and second input shape (z) must be >= 16!zonly int8 supported!r
   r›   zsmall blocks not supported!r   zhout_dtype=bfloat16 is unsupported. Please use out_dtype=float32/float16 and cast with `.to(tl.bfloat16)`)!rH   r�   r¢   r/   rá   rã   rJ   r%   r:   Zdefault_dot_input_precisionrT  r§   r�   r¡   rI   r=   r  Ú	get_int32r'   r;   r$   r7   Zget_fp32r8   r9   Zget_fp16r‘   r¨   rP   rå   Zmax_num_imprecise_acc_defaultr&   Z
create_dot)rE   rF   rU  rS  rV  rW  r!   r]  Zlhs_rankZrhs_rankr�   Zret_scalar_tyÚMÚNÚBr_   Z
acc_handler   r   r   Údot'  sf     


Fÿ0ÿ$ÿþ

ÿ
 
"

ÿrd  )Ú	conditionrg   rh   r!   r"   c                 C  s˜   t | tj|ƒ} | j ¡ rHt| ||ƒ\} }t|||ƒ\}}t| ||ƒ\} }t|||ddƒ\}}| j ¡ svt| ||ƒ\} }|j}t | | j	|j	|j	¡|¡S )NT)
rJ   r%   r}   rH   r�   rG   rL   r&   Zcreate_selectrP   )re  rg   rh   r!   rÐ   r_   r   r   r   r?  ‡  s    

r?  c                 C  s"   |rt  ||¡}n|}t  | |¡S rA   )r%   r‘   r&   )rg   rW   rÑ   Zres_tyr   r   r   Úwrap_tensorš  s    rf  zSequence[tl.tensor]zTuple[tl.tensor, ...])Úinputsr    r!   r"   c                   sÊ   ˆ d kr"t ‡fdd„ˆD ƒƒ‰d‰ ˆd jj‰tˆƒ}ˆ |k sNtd|› d�ƒ‚‡ fdd„tˆƒD ƒ‰t‡fdd„ˆD ƒƒs‚td	ƒ‚ˆ d
d„ ˆD ƒˆ ¡‰|ˆƒ ˆ ¡  t ‡‡‡fdd„t	tˆƒƒD ƒƒS )Nc                 3  s$   | ]}t ||jjgd ˆ d�V  qdS )Tr¹   N)r¬   r    r¡   ©r¯   Út©r!   r   r   rÁ   ¥  s     zreduction.<locals>.<genexpr>r   z&reduction axis must be < inputs rank (rŠ   c                   s   g | ]\}}|ˆ kr|‘qS r   r   )r¯   rË   r«   )r    r   r   r°   «  s      zreduction.<locals>.<listcomp>c                 3  s   | ]}|j jˆ kV  qd S rA   )rH   r�   rh  rÂ   r   r   rÁ   ¬  s     z-all reduction inputs must have the same shapec                 S  s   g | ]
}|j ‘qS r   ©rP   rh  r   r   r   r°   ®  s     c                 3  s(   | ] }t ˆ |¡ˆ | jjˆƒV  qd S rA   ©rf  Z
get_resultrH   rI   ©r¯   rË   )rg  Ú	reduce_oprÑ   r   r   rÁ   ²  s     )
ÚtuplerH   r�   r§   r¢   rÉ   ÚallZcreate_reduceÚverifyr�   )rg  r    Úregion_builder_fnr!   Úrankr   )r    r!   rg  rn  rÑ   r�   r   Ú	reduction£  s    rt  )rg  r    Úreverser!   r"   c                   sº   ˆ d j j‰tˆƒ}| |  kr*|k sBn td|› d|› d�ƒ‚|dk rR||7 }ˆ D ]}|j jˆksVtdƒ‚qV| dd„ ˆ D ƒ||¡‰|ˆƒ ˆ ¡  t‡ ‡‡fdd	„ttˆ ƒƒD ƒƒS )
Nr   z
scan axis z must be < inputs rank (rŠ   z(all scan inputs must have the same shapec                 S  s   g | ]
}|j ‘qS r   rk  rh  r   r   r   r°   Ç  s     z$associative_scan.<locals>.<listcomp>c                 3  s(   | ] }t ˆ |¡ˆ | jjˆƒV  qd S rA   rl  rm  ©rg  Zscan_opr�   r   r   rÁ   Ë  s     z#associative_scan.<locals>.<genexpr>)rH   r�   r§   r¢   Zcreate_scanrq  ro  r�   )rg  r    rr  ru  r!   rs  ri  r   rv  r   Úassociative_scanº  s    .rw  )rM   Únum_binsr!   r"   c                 C  sJ   t | jƒdkstdƒ‚| j ¡ s(tdƒ‚t | | j|¡t 	tj
|f¡¡S )Nr
   z histogram only supports 1D inputz%histogram only supports integer input)r§   r�   r¢   r/   r=   r%   r&   Zcreate_histogramrP   r‘   r'   )rM   rx  r!   r   r   r   Ú	histogramÓ  s    ry  )rg   Úvaluesr"   c                 C  s@   t dt| jƒƒt|ƒkr tdƒ‚| j dt || j ¡ ¡¡ | S )Nr
   zAShape of input to multiple_of does not match the length of valuesztt.divisibility)	rt   r§   r�   r$   rP   Úset_attrr	   Ú	make_attrÚget_context©rg   rz  r   r   r   Úmultiple_ofÜ  s    r  c                 C  s:   t | jƒt |ƒkrtdƒ‚| j dt || j ¡ ¡¡ | S )NzDShape of input to max_contiguous does not match the length of valuesztt.contiguity©r§   r�   r$   rP   r{  r	   r|  r}  r~  r   r   r   Úmax_contiguousã  s    r�  c                 C  s:   t | jƒt |ƒkrtdƒ‚| j dt || j ¡ ¡¡ | S )NzCShape of input to max_constancy does not match the length of valuesztt.constancyr€  r~  r   r   r   Úmax_constancyê  s    r‚  )r!   r"   c                 C  s   t  |  ¡ t j¡S rA   )r%   r&   Zcreate_barrierr*  rj  r   r   r   Údebug_barrierñ  s    rƒ  zList[tl.tensor])ÚprefixÚargsÚhexr!   r"   c                 C  sx   |   d¡s|r| d7 } |   d¡s4|r4| d d… d } t| ƒdkrR|  d¡sRd|  } dd„ |D ƒ}t | | ||¡tj¡S )Nú rÇ   r¸   r   c                 S  s   g | ]
}|j ‘qS r   rk  )r¯   Úargr   r   r   r°   ÿ  s     z device_print.<locals>.<listcomp>)Úendswithr§   Ú
startswithr%   r&   Zcreate_printr*  )r„  r…  r†  r!   Únew_argsr   r   r   Údevice_printõ  s    rŒ  )ÚcondÚmsgÚ	file_nameÚlinenor!   r"   c              	   C  sP   | j }| ¡ s2t |jd¡}t | | jd¡|¡} t | | j||||¡tj	¡S )N)r
   )
rH   r�   r%   r‘   rI   r&   r¨   rP   Zcreate_assertr*  )r�  rŽ  r�  Ú	func_namer�  r!   Zcond_tyr   r   r   Údevice_assert  s
    r’  c                 C  s4  t |tƒrt |¡}t |tjƒr’|rZd|j  kr:dk sNn td|j› d�ƒ‚|  |j¡S d|j  krpdk s„n td|j› d�ƒ‚|  |j¡S n†t |tjƒ�r|j	jdks´td	ƒ‚|j
 ¡ sÆtd
ƒ‚|j
tjkrð|rð|  |j|  ¡ |j
 ¡ ¡S |j
tjk�r|�sd�stdƒ‚|jS d�s0tdt|ƒ› �ƒ‚d S )Nlûÿÿÿ         l            z@Block pointers only support 64 bit `shape/strides`, got a value z which is out of the rangeé   €ì        zFBlock pointers only support 32 bit `offsets/block_shape`, got a value r
   z*Expected a scalar in shape/strides/offsetsz8Expected an integer scalar type in shape/strides/offsetsFzzBlock pointers only support 32 bit `offsets/block_shape`, add a `.to(tl.int32)` or use regular indexing for 64 bit supportz3Unsupported element type in shape/strides/offsets: )rœ   r   r%   rº   r¡   r¢   ré   r`  r&   r    r/   r=   rè   rç   rP   Zget_int64_tyr^   r'   rH   )r!   r  r&  r   r   r   Ú_convert_elem_to_ir_value  s"    

**r•  c                   s,   t |dƒr‡ ‡fdd„|D ƒS tˆ |ˆƒgS )Nr
  c                   s   g | ]}t ˆ |ˆƒ‘qS r   )r•  r  ©r!   r&  r   r   r°   %  s     z)_convert_to_ir_values.<locals>.<listcomp>)r  r•  )r!   Ú	list_liker&  r   r–  r   r'  #  s    
r'  )Úbaser!   r"   c              	     s:  t ||ƒ}t ||ƒ}t ||dd�}| j ¡ r8| jj ¡ r@tdƒ‚| jjtjkrht| t 	tj
| jj¡|ƒ} tˆ dƒsxˆ g‰ dd„ ˆ D ƒ‰ tdd„ ˆ D ƒƒs td	ƒ‚t|dƒs°|g}d
d„ |D ƒ}t|ƒttt|ƒƒƒksÞtdƒ‚t‡ fdd„||||fD ƒƒ�stdƒ‚| | j|||ˆ |¡}t |t 	t | jjˆ ¡¡¡S )NFr%  zMExpected `base` to be a pointer type (but not a block pointer type or others)r
  c                 S  s"   g | ]}t |tjƒr|jn|‘qS r   r  r  r   r   r   r°   ;  s     z"make_block_ptr.<locals>.<listcomp>c                 s  s0   | ](}t |tƒo&d |  ko"dk n  V  qdS )r“  r”  N)rœ   r   r  r   r   r   rÁ   <  s     z!make_block_ptr.<locals>.<genexpr>zGExpected a list of constant integers (`int32_t` range) in `block_shape`c                 S  s"   g | ]}t |tjƒr|jn|‘qS r   r  r  r   r   r   r°   B  s     z<Expected a permutation of (0, 1, ..., len(order)-1) in orderc                 3  s   | ]}t ˆ ƒt |ƒkV  qd S rA   )r§   )r¯   r—  ©r  r   r   rÁ   F  s     zBExpected shape/strides/offsets/block_shape to have the same length)r'  rH   rB   r  r�   r$   r%   r}   rJ   r  r  r  r  rp  r¢   rÃ   rÄ   r�   r§   Zcreate_make_block_ptrrP   r&   r‘   )r˜  r�   Ústridesr(  r  Úorderr!   rP   r   r™  r   Úmake_block_ptr)  s,    


ÿ
 "ÿrœ  c                 C  s&   t ||dd�}t | | j|¡| j¡S r$  )r'  r%   r&   Zcreate_advancerP   rH   )r˜  r(  r!   r   r   r   ÚadvanceP  s    r�  )FFTF)N)T)lÚ
__future__r   Útypingr   r   r   r   r   Z_C.libtritonr	   Ú r   r%   r   r   Ú	Exceptionr   r)   r*   r1   r>   rD   rL   rT   rX   rZ   r]   r`   rc   rf   rq   rr   ru   rw   ry   rz   r{   r€   r�   r„   r…   r†   r‡   r‰   rU   rƒ   r’   r“   r”   r•   r–   r—   r˜   rž   r¦   r¥   r¬   r³   rµ   r¼   r½   rÅ   rÍ   rG   r×   r   rJ   rô   rö   rú   r  r  r	  r  r  r!  r"  r)  r+  r-  r.  r1  r4  r6  rD  rE  rF  rJ  rM  rO  rP  rT  rd  r?  rf  rt  rw  ry  r  r�  r‚  rƒ  rŒ  r’  r•  r'  rœ  r�  r   r   r   r   Ú<module>   sÌ   (    þ



:
ÿw: 	,	''		`		'