U
    ‹Ñhé3  ã                   @   s  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZ ej ej e¡¡Zej ed¡gZej ed¡Zd	gZe  ¡ d
d„ ƒZe  ¡ dd„ ƒZdd„ ZG dd„ deƒZdd„ Zdd„ ZG dd„ deƒZG dd„ deƒZ dS )é    N)ÚPath)Ú_build)Úget_cache_manager)Ú	GPUTarget)Ú	GPUDriverÚincludeÚlibÚcudac                  C   s¼   t  d¡} | r| gS t ddg¡ ¡ }dd„ | ¡ D ƒ}dd„ |D ƒ}t  d¡}|rl|sldd„ | d	¡D ƒ}d
}|rŽ|dt|ƒ 7 }|d7 }n|d7 }|d7 }tdd„ |D ƒƒs¸t	|ƒ‚|S )NZTRITON_LIBCUDA_PATHz/sbin/ldconfigz-pc                 S   s    g | ]}d |kr|  ¡ d ‘qS )úlibcuda.so.1éÿÿÿÿ)Úsplit)Ú.0Úline© r   úK/tmp/user/2202/pip-unpacked-wheel-sn6cy74c/triton/backends/nvidia/driver.pyÚ
<listcomp>   s      z libcuda_dirs.<locals>.<listcomp>c                 S   s   g | ]}t j |¡‘qS r   )ÚosÚpathÚdirname)r   Úlocr   r   r   r      s     ZLD_LIBRARY_PATHc                 S   s&   g | ]}t j t j |d ¡¡r|‘qS )r
   ©r   r   ÚexistsÚjoin)r   Údirr   r   r   r      s      ú:zlibcuda.so cannot found!
z!Possible files are located at %s.z:Please create a symlink of libcuda.so to any of the files.z<Please make sure GPU is set up and then run "/sbin/ldconfig"z- (requires sudo) to refresh the linker cache.c                 s   s$   | ]}t j t j |d ¡¡V  qdS )r
   Nr   )r   r   r   r   r   Ú	<genexpr>'   s     zlibcuda_dirs.<locals>.<genexpr>)
r   ÚgetenvÚ
subprocessÚcheck_outputÚdecodeÚ
splitlinesr   ÚstrÚanyÚAssertionError)Zenv_libcuda_pathZlibsZlocsÚdirsZenv_ld_library_pathÚmsgr   r   r   Úlibcuda_dirs   s"    


r&   c                   C   s   t ftƒ •S ©N)Úlibdevice_dirr&   r   r   r   r   Úlibrary_dirs+   s    r)   c              
   C   sì   t  |  d¡¡ ¡ }t|ƒ}| |› d�¡}|d krºt ¡ �x}tj	 
|d¡}t|dƒ�}| | ¡ W 5 Q R X t|||tƒ ttƒ}t|dƒ� }|j| ¡ |› d�dd�}W 5 Q R X W 5 Q R X dd l}	|	j ||¡}
|	j |
¡}|
j |¡ |S )	Nzutf-8z.sozmain.cÚwÚrbT)Úbinaryr   )ÚhashlibÚsha256ÚencodeÚ	hexdigestr   Zget_fileÚtempfileÚTemporaryDirectoryr   r   r   ÚopenÚwriter   r)   Úinclude_dirÚ	librariesÚputÚreadÚimportlib.utilÚutilÚspec_from_file_locationÚmodule_from_specÚloaderÚexec_module)ÚsrcÚnameÚkeyÚcacheÚ
cache_pathZtmpdirZsrc_pathÚfÚsoÚ	importlibÚspecÚmodr   r   r   Úcompile_module_from_src0   s     
.rI   c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )Ú	CudaUtilsc                    s"   t | dƒstt| ƒ | ¡| _| jS )NÚinstance)ÚhasattrÚsuperrJ   Ú__new__rK   )Úcls©Ú	__class__r   r   rN   J   s    
zCudaUtils.__new__c                 C   sP   t ttj td¡ƒ ¡ dƒ}|j| _|j| _|j	| _	|j
| _
|j| _|j| _d S )Nzdriver.cZ
cuda_utils)rI   r   r   r   r   r   Ú	read_textZload_binaryZget_device_propertiesZcuOccupancyMaxActiveClustersZset_printf_fifo_sizeZfill_1d_tma_descriptorZfill_2d_tma_descriptor)ÚselfrH   r   r   r   Ú__init__O   s    zCudaUtils.__init__)Ú__name__Ú
__module__Ú__qualname__rN   rT   Ú__classcell__r   r   rP   r   rJ   H   s   rJ   c                 C   s8   | d dkrdS ddddddd	d
ddddddddœ|  S )Nr   Ú*ZCUdeviceptrÚint32_tÚint8_tÚint16_tÚint64_tÚuint32_tÚuint8_tÚuint16_tÚuint64_tÚfloatÚdouble)Úi1Úi8Zi16Zi32Zi64Úu1Úu8Úu16Úu32Zu64Zfp16Zbf16Zfp32Zf32Zfp64r   ©Útyr   r   r   Ú	ty_to_cpp^   s(    ñðrl   c           	         s>  d  dd„ | ¡ D ƒ¡}dd„ ‰ dd„ ‰d  ‡ ‡fd	d
„| ¡ D ƒ¡}d| }t|ƒdkrvdd  dd„ | ¡ D ƒ¡ nd}‡fdd
„| ¡ D ƒ}dt|ƒdkr¦d| nd› dd  dd„ |D ƒ¡› dd  ‡ fdd
„| ¡ D ƒ¡› d|› d|› dd  dd
„ | ¡ D ƒ¡› dt|ƒdk�r0dd  dd„ | ¡ D ƒ¡ nd› d�}|S )Nú, c                 s   s$   | ]\}}t |ƒ› d |› �V  qdS )z argN©rl   ©r   Úirk   r   r   r   r   w   s     z make_launcher.<locals>.<genexpr>c                 S   s   | d dkrdS t | ƒS )Nr   rY   ú	PyObject*rn   rj   r   r   r   Ú_extracted_typey   s    z&make_launcher.<locals>._extracted_typec                 S   s"   dddddddddd	d
ddœ|  S )NÚOrD   ÚdÚlÚbÚhrp   ÚBÚHÚIÚK)rq   rb   rc   Úlongr[   r\   rZ   r]   r_   r`   r^   ra   r   rj   r   r   r   Ú	format_of~   s    ôóz make_launcher.<locals>.format_ofÚ c                    s   g | ]}ˆˆ |ƒƒ‘qS r   r   )r   rk   )rr   r}   r   r   r   Ž   s     z!make_launcher.<locals>.<listcomp>Z	iiiKKOOOOr   c                 s   s   | ]\}}d |› �V  qdS )z&_argNr   ro   r   r   r   r   �   s     c                    s   g | ]}|ˆ kr|‘qS r   r   ©r   rp   )Ú	constantsr   r   r   “   s      a	  
#include "cuda.h"
#include <stdbool.h>
#include <Python.h>
#include <dlfcn.h>

static inline void gpuAssert(CUresult code, const char *file, int line)
{
   if (code != CUDA_SUCCESS)
   {
      const char* prefix = "Triton Error [CUDA]: ";
      const char* str;
      cuGetErrorString(code, &str);
      char err[1024] = {0};
      strcat(err, prefix);
      strcat(err, str);
      PyGILState_STATE gil_state;
      gil_state = PyGILState_Ensure();
      PyErr_SetString(PyExc_RuntimeError, err);
      PyGILState_Release(gil_state);
   }
}

#define CUDA_CHECK(ans) { gpuAssert((ans), __FILE__, __LINE__); }

typedef CUresult (*cuLaunchKernelEx_t)(const CUlaunchConfig* config, CUfunction f, void** kernelParams, void** extra);

static cuLaunchKernelEx_t getLaunchKernelExHandle() {
  // Open the shared library
  void* handle = dlopen("libcuda.so.1", RTLD_LAZY);
  if (!handle) {
    PyErr_SetString(PyExc_RuntimeError, "Failed to open libcuda.so.1");
    return NULL;
  }
  // Clear any existing error
  dlerror();
  cuLaunchKernelEx_t cuLaunchKernelExHandle = (cuLaunchKernelEx_t)dlsym(handle, "cuLaunchKernelEx");
  // Check for errors
  const char *dlsym_error = dlerror();
  if (dlsym_error) {
    PyErr_SetString(PyExc_RuntimeError, "Failed to retrieve cuLaunchKernelEx from libcuda.so.1");
    return NULL;
  }
  return cuLaunchKernelExHandle;
}

static void _launch(int gridX, int gridY, int gridZ, int num_warps, int num_ctas, int clusterDimX, int clusterDimY, int clusterDimZ, int shared_memory, CUstream stream, CUfunction functionz) {
  void *params[] = { c                 s   s   | ]}d |› �V  qdS )z&argNr   r   r   r   r   r   Ã   s     a   };
  if (gridX*gridY*gridZ > 0) {
    if (num_ctas == 1) {
      CUDA_CHECK(cuLaunchKernel(function, gridX, gridY, gridZ, 32*num_warps, 1, 1, shared_memory, stream, params, 0));
    } else {
      CUlaunchAttribute launchAttr[2];
      launchAttr[0].id = CU_LAUNCH_ATTRIBUTE_CLUSTER_DIMENSION;
      launchAttr[0].value.clusterDim.x = clusterDimX;
      launchAttr[0].value.clusterDim.y = clusterDimY;
      launchAttr[0].value.clusterDim.z = clusterDimZ;
      launchAttr[1].id = CU_LAUNCH_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE;
      launchAttr[1].value.clusterSchedulingPolicyPreference = CU_CLUSTER_SCHEDULING_POLICY_SPREAD;
      CUlaunchConfig config;
      config.gridDimX = gridX * clusterDimX;
      config.gridDimY = gridY * clusterDimY;
      config.gridDimZ = gridZ * clusterDimZ;
      config.blockDimX = 32 * num_warps;
      config.blockDimY = 1;
      config.blockDimZ = 1;
      config.sharedMemBytes = shared_memory;
      config.hStream = stream;
      config.attrs = launchAttr;
      config.numAttrs = 2;
      static cuLaunchKernelEx_t cuLaunchKernelExHandle = NULL;
      if (cuLaunchKernelExHandle == NULL) {
        cuLaunchKernelExHandle = getLaunchKernelExHandle();
      }
      CUDA_CHECK(cuLaunchKernelExHandle(&config, function, params, 0));
    }
  }
}

typedef struct _DevicePtrInfo {
    CUdeviceptr dev_ptr;
    bool valid;
} DevicePtrInfo;

static inline DevicePtrInfo getPointer(PyObject *obj, int idx) {
  DevicePtrInfo ptr_info;
  ptr_info.dev_ptr = 0;
  ptr_info.valid = true;
  if (PyLong_Check(obj)) {
    ptr_info.dev_ptr = PyLong_AsUnsignedLongLong(obj);
    return ptr_info;
  }
  if (obj == Py_None) {
    // valid nullptr
    return ptr_info;
  }
  PyObject *ptr = PyObject_GetAttrString(obj, "data_ptr");
  if(ptr){
    PyObject *empty_tuple = PyTuple_New(0);
    PyObject *ret = PyObject_Call(ptr, empty_tuple, NULL);
    Py_DECREF(empty_tuple);
    Py_DECREF(ptr);
    if (!PyLong_Check(ret)) {
      PyErr_SetString(PyExc_TypeError, "data_ptr method of Pointer object must return 64-bit int");
      ptr_info.valid = false;
      return ptr_info;
    }
    ptr_info.dev_ptr = PyLong_AsUnsignedLongLong(ret);
    if(!ptr_info.dev_ptr)
      return ptr_info;
    uint64_t dev_ptr;
    int status = cuPointerGetAttribute(&dev_ptr, CU_POINTER_ATTRIBUTE_DEVICE_POINTER, ptr_info.dev_ptr);
    if (status == CUDA_ERROR_INVALID_VALUE) {
        PyErr_Format(PyExc_ValueError,
                     "Pointer argument (at %d) cannot be accessed from Triton (cpu tensor?)", idx);
        ptr_info.valid = false;
    }
    ptr_info.dev_ptr = dev_ptr;
    Py_DECREF(ret);  // Thanks ChatGPT!
    return ptr_info;
  }
  PyErr_SetString(PyExc_TypeError, "Pointer argument must be either uint64 or have data_ptr method");
  ptr_info.valid = false;
  return ptr_info;
}

static PyObject* launch(PyObject* self, PyObject* args) {
  int gridX, gridY, gridZ;
  uint64_t _stream;
  uint64_t _function;
  PyObject *launch_enter_hook = NULL;
  PyObject *launch_exit_hook = NULL;
  PyObject *kernel_metadata = NULL;
  PyObject *launch_metadata = NULL;
  ú c                    s$   g | ]\}}ˆ |ƒ› d |› d�‘qS )z _argú; r   ro   )rr   r   r   r     s     z
  if(!PyArg_ParseTuple(args, "zÑ", &gridX, &gridY, &gridZ, &_stream, &_function,
                                           &kernel_metadata, &launch_metadata,
                                           &launch_enter_hook, &launch_exit_hook ai  )) {
    return NULL;
  }

  int num_warps, num_ctas, shared_memory, clusterDimX, clusterDimY, clusterDimZ;
  if (!PyArg_ParseTuple(kernel_metadata, "iiiiii", &num_warps, &num_ctas, &shared_memory, &clusterDimX, &clusterDimY, &clusterDimZ)) {
    PyErr_SetString(PyExc_TypeError, "kernel_metadata must be a tuple");
    return NULL;
  }

  // extract launch metadata
  if (launch_enter_hook != Py_None){
    PyObject* args = Py_BuildValue("(O)", launch_metadata);
    PyObject* ret = PyObject_CallObject(launch_enter_hook, args);
    Py_DECREF(args);
    if (!ret)
      return NULL;
  }

  // raise exception asap
  r‚   c                 S   s>   g | ]6\}}|d  dkr6d|› d|› d|› d|› d�	nd‘qS )r   rY   zDevicePtrInfo ptr_infoz = getPointer(_argrm   z); if (!ptr_infoz.valid) return NULL;r~   r   ro   r   r   r   r   1  s     z®;
  Py_BEGIN_ALLOW_THREADS;
  _launch(gridX, gridY, gridZ, num_warps, num_ctas, clusterDimX, clusterDimY, clusterDimZ, shared_memory, (CUstream)_stream, (CUfunction)_functionc                 s   s4   | ],\}}|d  dkr"d|› d�nd|› �V  qdS )r   rY   Zptr_infoz.dev_ptrZ_argNr   ro   r   r   r   r   3  s     aW  );
  Py_END_ALLOW_THREADS;
  if (PyErr_Occurred()) {
    return NULL;
  }

  if(launch_exit_hook != Py_None){
    PyObject* args = Py_BuildValue("(O)", launch_metadata);
    PyObject* ret = PyObject_CallObject(launch_exit_hook, args);
    Py_DECREF(args);
    if (!ret)
      return NULL;

  }

  // return None
  Py_INCREF(Py_None);
  return Py_None;
}

static PyMethodDef ModuleMethods[] = {
  {"launch", launch, METH_VARARGS, "Entry point for all kernels with this signature"},
  {NULL, NULL, 0, NULL} // sentinel
};

static struct PyModuleDef ModuleDef = {
  PyModuleDef_HEAD_INIT,
  "__triton_launcher",
  NULL, //documentation
  -1, //size
  ModuleMethods
};

PyMODINIT_FUNC PyInit___triton_launcher(void) {
  PyObject *m = PyModule_Create(&ModuleDef);
  if(m == NULL) {
    return NULL;
  }
  PyModule_AddFunctions(m, ModuleMethods);
  return m;
}
)r   ÚitemsÚvaluesÚlenÚkeys)	r€   Ú	signatureÚidsZ	arg_declsZargs_formatÚformatZ	args_listÚparamsr?   r   )rr   r€   r}   r   Úmake_launchert   sD    ,.Ò/Ñ € ú € ù 
€ ÷ € ã  ,€ á Jr‹   c                   @   s   e Zd Zdd„ Zdd„ ZdS )ÚCudaLauncherc                    sŽ   dt ˆdƒrˆjjntƒ i}t ˆdƒr,ˆjntƒ }‡fdd„‰ ‡ fdd„| ¡ D ƒ}‡ fdd„ˆj ¡ D ƒ}t|||ƒ‰t	ˆd	ƒ}|j
| _
d S )
NZids_of_const_exprsÚfnr€   c                    s   t | tƒrˆ jj | ¡S | S r'   )Ú
isinstancer!   r�   Ú	arg_namesÚindex)rp   )r?   r   r   Ú<lambda>e  ó    z'CudaLauncher.__init__.<locals>.<lambda>c                    s   i | ]\}}ˆ |ƒ|“qS r   r   ©r   rA   Úvalue©Úcst_keyr   r   Ú
<dictcomp>f  s      z)CudaLauncher.__init__.<locals>.<dictcomp>c                    s   i | ]\}}ˆ |ƒ|“qS r   r   r“   r•   r   r   r—   g  s      Z__triton_launcher)rL   r�   Z
constexprsÚtupler€   Údictrƒ   r‡   r‹   rI   Úlaunch)rS   r?   Úmetadatarˆ   r€   r‡   rH   r   )r–   r?   r   rT   b  s    
zCudaLauncher.__init__c                 O   s   | j ||Ž d S r'   )rš   )rS   ÚargsÚkwargsr   r   r   Ú__call__l  s    zCudaLauncher.__call__N)rU   rV   rW   rT   rž   r   r   r   r   rŒ   `  s   
rŒ   c                       s0   e Zd Z‡ fdd„Zdd„ Zedd„ ƒZ‡  ZS )Ú
CudaDriverc                    s   t ƒ | _t| _tƒ  ¡  d S r'   )rJ   ÚutilsrŒ   Zlauncher_clsrM   rT   )rS   rP   r   r   rT   r  s    zCudaDriver.__init__c                 C   s6   |   ¡ }|  |¡}|d d |d  }d}td||ƒS )Nr   é
   é   é    r	   )Zget_current_deviceZget_device_capabilityr   )rS   ZdeviceZ
capabilityZ	warp_sizer   r   r   Úget_current_targetw  s
    
zCudaDriver.get_current_targetc                  C   s   dd l } | j ¡ o| jjd kS )Nr   )Útorchr	   Zis_availableÚversionZhip)r¥   r   r   r   Ú	is_active~  s    zCudaDriver.is_active)rU   rV   rW   rT   r¤   Ústaticmethodr§   rX   r   r   rP   r   rŸ   p  s   rŸ   )!Ú	functoolsr   r-   r   r1   Úpathlibr   Ztriton.runtime.buildr   Ztriton.runtime.cacher   Ztriton.backends.compilerr   Ztriton.backends.driverr   r   r   ÚrealpathÚ__file__r   r5   r(   r6   Ú	lru_cacher&   r)   rI   ÚobjectrJ   rl   r‹   rŒ   rŸ   r   r   r   r   Ú<module>   s0   

 m