U
    ‹ÑhW>  ã                   @   sä   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZ ej ej e¡¡Zej ed¡gZe  ¡ dd	„ ƒZd
d„ ZG dd„ deƒZdd„ Zdd„ ZG dd„ deƒZG dd„ deƒZdS )é    N)ÚPath)Ú_build)Úget_cache_manager)Ú	GPUTarget)Ú	GPUDriverÚincludec                     sj  d‰ t  d¡} | r@|  ˆ ¡r,t j | ¡r,| S td| › dˆ › �ƒ‚g }dd l}| ¡ D ]4}t j |ddˆ ¡}t j |¡r~|  S | 	|¡ qTt  d¡}|rÔ| 
d	¡D ]0}t j |ˆ ¡}t j |¡rÈ|  S | 	|¡ q¢t d
dg¡ ¡ }‡ fdd„| ¡ D ƒ}|D ]&}	t j |	¡�r|	  S | 	|	¡ �q t j dˆ ¡}
t j |
¡�rH|
S | 	|
¡ tdˆ › d|› �ƒ‚d S )Nzlibamdhip64.soZTRITON_LIBHIP_PATHzTRITON_LIBHIP_PATH 'z' does not point to a valid r   ÚtorchÚlibZLD_LIBRARY_PATHú:z/sbin/ldconfigz-pc                    s&   g | ]}|  ¡  ˆ ¡r| ¡ d  ‘qS )éÿÿÿÿ)ÚstripÚendswithÚsplit)Ú.0Úline©Zlib_name© úH/tmp/user/2202/pip-unpacked-wheel-sn6cy74c/triton/backends/amd/driver.pyÚ
<listcomp>5   s      z2_get_path_to_hip_runtime_dylib.<locals>.<listcomp>z/opt/rocm/lib/zcannot locate z after attempted paths )ÚosÚgetenvr   ÚpathÚexistsÚRuntimeErrorÚsiteÚgetsitepackagesÚjoinÚappendr   Ú
subprocessÚcheck_outputÚdecodeÚ
splitlines)Zenv_libhip_pathÚpathsr   r   Zenv_ld_library_pathÚdÚfZlibsZlocsÚlocZcommon_install_pathr   r   r   Ú_get_path_to_hip_runtime_dylib   s>    


r&   c              
   C   sê   t  |  d¡¡ ¡ }t|ƒ}| |› d�¡}|d kr¸t ¡ �v}tj	 
|d¡}t|dƒ�}| | ¡ W 5 Q R X t|||g tg ƒ}t|dƒ� }|j| ¡ |› d�dd�}W 5 Q R X W 5 Q R X dd l}	|	j ||¡}
|	j |
¡}|
j |¡ |S )	Nzutf-8z.sozmain.cÚwÚrbT)Úbinaryr   )ÚhashlibÚsha256ÚencodeÚ	hexdigestr   Zget_fileÚtempfileÚTemporaryDirectoryr   r   r   ÚopenÚwriter   Úinclude_dirÚputÚreadÚimportlib.utilÚutilÚspec_from_file_locationÚmodule_from_specÚloaderÚexec_module)ÚsrcÚnameÚkeyÚcacheÚ
cache_pathZtmpdirZsrc_pathr$   ÚsoÚ	importlibÚspecÚmodr   r   r   Úcompile_module_from_srcD   s     
.rD   c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )ÚHIPUtilsc                    s"   t | dƒstt| ƒ | ¡| _| jS )NÚinstance)ÚhasattrÚsuperrE   Ú__new__rF   )Úcls©Ú	__class__r   r   rI   Y   s    
zHIPUtils.__new__c                 C   sH   t ƒ }ttj td¡ƒ ¡ }| d|d¡}t|dƒ}|j	| _	|j
| _
d S )Nzdriver.cz/*py_libhip_search_path*/é   Z	hip_utils)r&   r   r   r   r   ÚdirnameÚ	read_textÚreplacerD   Zload_binaryÚget_device_properties)ÚselfÚlibhip_pathr;   rC   r   r   r   Ú__init__^   s    
zHIPUtils.__init__)Ú__name__Ú
__module__Ú__qualname__rI   rT   Ú__classcell__r   r   rK   r   rE   W   s   rE   c                 C   s8   | d dkrdS ddddddd	d
ddddddddœ|  S )Nr   Ú*ZhipDeviceptr_tÚint32_tÚint8_tÚint16_tÚint64_tÚuint32_tÚuint8_tÚuint16_tÚuint64_tÚfloatÚdouble©Úi1Úi8Zi16Zi32Zi64Úu1Úu8Úu16Úu32Zu64Zfp16Zbf16Zfp32Zf32Zfp64r   ©Útyr   r   r   Ú	ty_to_cppk   s(    ñðrm   c                    sX  t |ƒ}d dd„ | ¡ D ƒ¡}dd„ ‰ dd„ ‰d ‡ ‡fd	d
„| ¡ D ƒ¡}d| }t |ƒdkr~dd dd„ | ¡ D ƒ¡ nd}tƒ }	‡fdd
„| ¡ D ƒ}
d|	› dt |ƒdkrºd| nd› dd dd„ |
D ƒ¡› d|› dd ‡ fdd
„| ¡ D ƒ¡› d|› d|› dd dd
„ | ¡ D ƒ¡› dt |ƒdk�rJdd dd„ | ¡ D ƒ¡ nd› d�}|S )Nú, c                 s   s$   | ]\}}t |ƒ› d |› �V  qdS )z argN)rm   ©r   Úirl   r   r   r   Ú	<genexpr>„   s     z make_launcher.<locals>.<genexpr>c                 S   s8   | d dkrdS ddddddd	d
ddddddddœ|  S )Nr   rY   ú	PyObject*rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   r   rk   r   r   r   Ú_extracted_type†   s(    ñðz&make_launcher.<locals>._extracted_typec                 S   s"   dddddddddd	d
ddœ|  S )NÚOr$   r#   ÚlÚbÚhrp   ÚBÚHÚIÚK)rr   rb   rc   Úlongr[   r\   rZ   r]   r_   r`   r^   ra   r   rk   r   r   r   Ú	format_of›   s    ôóz make_launcher.<locals>.format_ofÚ c                    s   g | ]}ˆˆ |ƒƒ‘qS r   r   )r   rl   )rs   r}   r   r   r   «   s     z!make_launcher.<locals>.<listcomp>Z	iiiKKOOOOr   c                 s   s   | ]\}}d |› �V  qdS )z&_argNr   ro   r   r   r   rq   ­   s     c                    s   g | ]}|ˆ kr|‘qS r   r   ©r   rp   )Ú	constantsr   r   r   ²   s      a;  
#define __HIP_PLATFORM_AMD__
#include <hip/hip_runtime.h>
#include <Python.h>
#include <dlfcn.h>
#include <stdbool.h>
#include <dlfcn.h>

// The list of paths to search for the HIP runtime library. The caller Python
// code should substitute the search path placeholder.
static const char *hipLibSearchPaths[] = {"a  "};

// The list of HIP dynamic library symbols and their signature we are interested
// in this file.
#define HIP_SYMBOL_LIST(FOR_EACH_ERR_FN, FOR_EACH_STR_FN)                     \
  FOR_EACH_STR_FN(hipGetErrorString, hipError_t hipError)                     \
  FOR_EACH_ERR_FN(hipModuleLaunchKernel, hipFunction_t f,                     \
                  unsigned int gridDimX, unsigned int gridDimY,               \
                  unsigned int gridDimZ, unsigned int blockDimX,              \
                  unsigned int blockDimY, unsigned int blockDimZ,             \
                  unsigned int sharedMemBytes, hipStream_t stream,            \
                  void **kernelParams, void **extra)                          \
  FOR_EACH_ERR_FN(hipPointerGetAttribute, void *data,                         \
                  hipPointer_attribute attribute, hipDeviceptr_t ptr)

// The HIP symbol table for holding resolved dynamic library symbols.
struct HIPSymbolTable {
#define DEFINE_EACH_ERR_FIELD(hipSymbolName, ...)                             \
  hipError_t (*hipSymbolName)(__VA_ARGS__);
#define DEFINE_EACH_STR_FIELD(hipSymbolName, ...)                             \
  const char *(*hipSymbolName)(__VA_ARGS__);

  HIP_SYMBOL_LIST(DEFINE_EACH_ERR_FIELD, DEFINE_EACH_STR_FIELD)
};

static struct HIPSymbolTable hipSymbolTable;

bool initSymbolTable() {
  // Use the HIP runtime library loaded into the existing process if it exits.
  void *lib = dlopen("libamdhip64.so", RTLD_NOLOAD);
  if (lib) {
    // printf("[triton] chosen loaded libamdhip64.so in the process\n");
  }

  // Otherwise, go through the list of search paths to dlopen the first HIP
  // driver library.
  if (!lib) {
    int n = sizeof(hipLibSearchPaths) / sizeof(hipLibSearchPaths[0]);
    for (int i = 0; i < n; ++i) {
      void *handle = dlopen(hipLibSearchPaths[i], RTLD_LAZY | RTLD_LOCAL);
      if (handle) {
        lib = handle;
        // printf("[triton] chosen %s\n", hipLibSearchPaths[i]);
      }
    }
  }
  if (!lib) {
    PyErr_SetString(PyExc_RuntimeError, "cannot open libamdhip64.so");
    return false;
  }

  // Resolve all symbols we are interested in.
  dlerror(); // Clear existing errors
  const char *error = NULL;
#define QUERY_EACH_FN(hipSymbolName, ...)                                     \
  *(void **)&hipSymbolTable.hipSymbolName = dlsym(lib, #hipSymbolName);       \
  error = dlerror();                                                          \
  if (error) {                                                               \
    PyErr_SetString(PyExc_RuntimeError,                                       \
                    "cannot query " #hipSymbolName " from libamdhip64.so");   \
    dlclose(lib);                                                             \
    return false;                                                             \
  }

  HIP_SYMBOL_LIST(QUERY_EACH_FN, QUERY_EACH_FN)

  return true;
}

static inline void gpuAssert(hipError_t code, const char *file, int line)
{
   if (code != HIP_SUCCESS)
   {
      const char* prefix = "Triton Error [HIP]: ";
       const char* str = hipSymbolTable.hipGetErrorString(code);
      char err[1024] = {0};
      snprintf(err, 1024, "%s Code: %d, Messsage: %s", prefix, code, str );
      PyErr_SetString(PyExc_RuntimeError, err);
   }
}

#define HIP_CHECK(ans) { gpuAssert((ans), __FILE__, __LINE__); }

static void _launch(int gridX, int gridY, int gridZ, int num_warps, int num_ctas, int clusterDimX, int clusterDimY, int clusterDimZ, int shared_memory, hipStream_t stream, hipFunction_t functionz>) {
  // printf("_launch hip kernel\n");
  void *params[] = { c                 s   s   | ]}d |› �V  qdS )z&argNr   r   r   r   r   rq     s     zw };
  if (gridX*gridY*gridZ > 0) {
      HIP_CHECK(hipSymbolTable.hipModuleLaunchKernel(function, gridX, gridY, gridZ, aw  *num_warps, 1, 1, shared_memory, stream, params, 0));
    }
  }

typedef struct _DevicePtrInfo {
    hipDeviceptr_t dev_ptr;
    bool valid;
} DevicePtrInfo;

static inline DevicePtrInfo getPointer(PyObject *obj, int idx) {
  DevicePtrInfo ptr_info;
  ptr_info.dev_ptr = 0;
  ptr_info.valid = true;
  if (PyLong_Check(obj)) {
    ptr_info.dev_ptr = (hipDeviceptr_t)PyLong_AsUnsignedLongLong(obj);
    return ptr_info;
  }
  if (obj == Py_None) {
    // valid nullptr
    return ptr_info;
  }
  PyObject *ptr = PyObject_GetAttrString(obj, "data_ptr");
  if(ptr){
    PyObject *empty_tuple = PyTuple_New(0);
    PyObject *ret = PyObject_Call(ptr, empty_tuple, NULL);
    Py_DECREF(empty_tuple);
    Py_DECREF(ptr);
    if (!PyLong_Check(ret)) {
      PyErr_SetString(PyExc_TypeError, "data_ptr method of Pointer object must return 64-bit int");
      ptr_info.valid = false;
      return ptr_info;
    }
    ptr_info.dev_ptr = (hipDeviceptr_t)PyLong_AsUnsignedLongLong(ret);
    if(!ptr_info.dev_ptr)
      return ptr_info;
    uint64_t dev_ptr;
    hipError_t status = hipSymbolTable.hipPointerGetAttribute(&dev_ptr, HIP_POINTER_ATTRIBUTE_DEVICE_POINTER, ptr_info.dev_ptr);
    if (status == hipErrorInvalidValue) {
        PyErr_Format(PyExc_ValueError,
                     "Pointer argument (at %d) cannot be accessed from Triton (cpu tensor?)", idx);
        ptr_info.valid = false;
    }
    ptr_info.dev_ptr = (hipDeviceptr_t)dev_ptr;
    Py_DECREF(ret);
    return ptr_info;
  }
  PyErr_SetString(PyExc_TypeError, "Pointer argument must be either uint64 or have data_ptr method");
  return ptr_info;
}

static PyObject* launch(PyObject* self, PyObject* args) {
   // printf("launch\n");
  int gridX, gridY, gridZ;
  uint64_t _stream;
  uint64_t _function;
  PyObject *launch_enter_hook = NULL;
  PyObject *launch_exit_hook = NULL;
  PyObject *kernel_metadata = NULL;
  PyObject *launch_metadata = NULL;
  ú c                    s$   g | ]\}}ˆ |ƒ› d |› d�‘qS )z _argú; r   ro   )rs   r   r   r   O  s     z
  if(!PyArg_ParseTuple(args, "zÑ", &gridX, &gridY, &gridZ, &_stream, &_function,
                                           &kernel_metadata, &launch_metadata,
                                           &launch_enter_hook, &launch_exit_hook a=  )) {
    return NULL;
  }

  // extract kernel metadata
  int num_warps, num_ctas, shared_memory, clusterDimX, clusterDimY, clusterDimZ;
  if (!PyArg_ParseTuple(kernel_metadata, "iiiiii", &num_warps, &num_ctas, &shared_memory, &clusterDimX, &clusterDimY, &clusterDimZ)) {
    return NULL;
  }
  // extract launch metadata
  if (launch_enter_hook != Py_None){
    PyObject* args = Py_BuildValue("(O)", launch_metadata);
    PyObject* ret = PyObject_CallObject(launch_enter_hook, args);
    Py_DECREF(args);
    if (!ret)
      return NULL;
  }


  // raise exception asap
  r‚   c                 S   s>   g | ]6\}}|d  dkr6d|› d|› d|› d|› d�	nd‘qS )r   rY   zDevicePtrInfo ptr_infoz = getPointer(_argrn   z); if (!ptr_infoz.valid) return NULL;r~   r   ro   r   r   r   r   f  s     zš;
  _launch(gridX, gridY, gridZ, num_warps, num_ctas, clusterDimX, clusterDimY, clusterDimZ, shared_memory, (hipStream_t)_stream, (hipFunction_t)_functionc                 s   s4   | ],\}}|d  dkr"d|› d�nd|› �V  qdS )r   rY   Zptr_infoz.dev_ptrZ_argNr   ro   r   r   r   rq   g  s     an  );

  if(launch_exit_hook != Py_None){
    PyObject* args = Py_BuildValue("(O)", launch_metadata);
    PyObject* ret = PyObject_CallObject(launch_exit_hook, args);
    Py_DECREF(args);
    if (!ret)
      return NULL;
  }

  if(PyErr_Occurred()) {
    return NULL;
  }
  // return None
  Py_INCREF(Py_None);
  return Py_None;
}

static PyMethodDef ModuleMethods[] = {
  {"launch", launch, METH_VARARGS, "Entry point for all kernels with this signature"},
  {NULL, NULL, 0, NULL} // sentinel
};

static struct PyModuleDef ModuleDef = {
  PyModuleDef_HEAD_INIT,
  "__triton_launcher",
  NULL, //documentation
  -1, //size
  ModuleMethods
};

PyMODINIT_FUNC PyInit___triton_launcher(void) {
  if (!initSymbolTable()) {
    return NULL;
  }
  PyObject *m = PyModule_Create(&ModuleDef);
  if(m == NULL) {
    return NULL;
  }
  PyModule_AddFunctions(m, ModuleMethods);
  return m;
}
)Úlenr   ÚitemsÚvaluesr&   Úkeys)r€   Ú	signatureÚidsÚ	warp_sizeZ
start_descZ	arg_declsZargs_formatÚformatZ	args_listrS   Úparamsr;   r   )rs   r€   r}   r   Úmake_launcher�   sP    ,
ö]£_¡aŸ € ä € ã  € á 4€ Í 5,€ Ì `rŒ   c                   @   s   e Zd Zdd„ Zdd„ ZdS )ÚHIPLauncherc                    s’   dt ˆdƒrˆjjntƒ i}t ˆdƒr,ˆjntƒ }‡fdd„‰ ‡ fdd„| ¡ D ƒ}‡ fdd„ˆj ¡ D ƒ}t||||j	ƒ‰t
ˆd	ƒ}|j| _d S )
NZids_of_const_exprsÚfnr€   c                    s   t | tƒrˆ jj | ¡S | S ©N)Ú
isinstanceÚstrrŽ   Ú	arg_namesÚindex)rp   )r;   r   r   Ú<lambda>š  ó    z&HIPLauncher.__init__.<locals>.<lambda>c                    s   i | ]\}}ˆ |ƒ|“qS r   r   ©r   r=   Úvalue©Úcst_keyr   r   Ú
<dictcomp>›  s      z(HIPLauncher.__init__.<locals>.<dictcomp>c                    s   i | ]\}}ˆ |ƒ|“qS r   r   r–   r˜   r   r   rš   œ  s      Z__triton_launcher)rG   rŽ   Z
constexprsÚtupler€   Údictr„   r‡   rŒ   r‰   rD   Úlaunch)rR   r;   Úmetadatarˆ   r€   r‡   rC   r   )r™   r;   r   rT   —  s    
zHIPLauncher.__init__c                 O   s   | j ||Ž d S r�   )r�   )rR   ÚargsÚkwargsr   r   r   Ú__call__¡  s    zHIPLauncher.__call__N)rU   rV   rW   rT   r¡   r   r   r   r   r�   •  s   
r�   c                       s0   e Zd Z‡ fdd„Zedd„ ƒZdd„ Z‡  ZS )Ú	HIPDriverc                    s   t ƒ  ¡  tƒ | _t| _d S r�   )rH   rT   rE   Úutilsr�   Zlauncher_cls)rR   rK   r   r   rT   §  s    
zHIPDriver.__init__c                  C   s   dd l } | jjd k	S )Nr   )r   ÚversionÚhip)r   r   r   r   Ú	is_active¬  s    zHIPDriver.is_activec                 C   s:   |   ¡ }| j |¡}|d }|d }td| d¡d |ƒS )NÚarchZwarpSizer¥   r
   r   )Zget_current_devicer£   rQ   r   r   )rR   ZdeviceZdevice_propertiesr§   r‰   r   r   r   Úget_current_target±  s
    zHIPDriver.get_current_target)rU   rV   rW   rT   Ústaticmethodr¦   r¨   rX   r   r   rK   r   r¢   ¥  s   
r¢   )Ú	functoolsr   r*   r   r.   Úpathlibr   Ztriton.runtime.buildr   Ztriton.runtime.cacher   Ztriton.backends.compilerr   Ztriton.backends.driverr   r   rN   ÚrealpathÚ__file__r   r2   Ú	lru_cacher&   rD   ÚobjectrE   rm   rŒ   r�   r¢   r   r   r   r   Ú<module>   s*   
3  