U
    E‹ÑhÝa ã                   @   s~  U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZmZ d dlmZ d dlmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z. ddl/m0Z0 zd dl1Z2W n e3k
�rx   dZ2Y nX z¦d dl4Z5d dl6m7Z8 d dl9m:Z: d d	l4m;Z; d
dl<m=Z= e2�r e2e2j>e2j?e2j@fZAe+ejBdf eCd< e2e8e2j>e8j>e2j?e8j?e2j@e8j@iZDn
eEƒ ZAi ZDd dlFmGZGmHZHmIZI W n eJk
�r6   Y nX d dlKZKd dl5Z5d dlLZ5d dlMZ5d dlNmO  mPZQ d dl5mRZR d dlSmTZT d dl9mUZU d dlVmWZW d dlXmYZY d dlZm[Z[m\Z\ d dl]m^Z^ d dl_m`Z`maZa e bej"¡Zce#ede"ed f eCd< i Zee%edef eCd< dZfdZgdeg› d�Zhe iej¡Zki Zle%ede(em f eCd< e bdd„ ¡Zne%ede%edemf f eCd < e o¡ Zpd!d"„ Zqd ard#d$„ Zsd%d&„ Ztd aud'd(„ Zvd)d*„ Zwd+d,„ Zxd-d.„ Zy�dgd0d1„Zz�dhd4d5„Z{e j|d6d7„ ƒZ}e5j~e5je5jmfe5j€e5j�e5j‚fe5jƒe5j„e5j…fe5j†e5j‡fe5jˆe5j‰fe5jŠe5j‹fe5jŒe5j�e5jŽfe5j�e5j�e5j‘fe5j’e5j“e5j”fe5j•e5j–fi
Z—G d8d9„ d9ƒZ˜e˜ƒ Z™d:d;„ Zšd<d=„ Z›d>d?„ Zœd@dA„ Z�dBdC„ ZždDdE„ ZŸeRj dFœdGdH„Z¡dIdJ„ Z¢dKdL„ Z£dMdN„ Z¤G dOdP„ dPƒZ¥dQdR„ Z¦ej§dSk�rúej¨ej©ejªej«ej¬ej­fZ®dTdU„ Z¯dVdW„ Z°dXdY„ Z±dZd[„ Z²d\d]„ Z³d^d_„ Z´d`da„ Zµdbdc„ Z¶ddde„ Z·dfdg„ Z¸e dh¡didj„ ƒZ¹�didkdl„Zºdmdn„ Z»ej¼G dodp„ dpƒƒZ½ej¼G dqdr„ drƒƒZ¾dsZ¿ejÀe¿dt�aÁe$e-e½e¾f  eCdu< e-e½e¾f dvœdwdx„ZÂe‘ddyœdzd{„ZÃdd|œd}d~„ZÄe(e-e½e¾f  d|œdd€„ZÅej¼G d�d‚„ d‚ƒƒZÆG dƒd„„ d„e¥ƒZÇeÇƒ eÇ_Èd…d†„ ZÉdd‡œdˆd‰„ZÊdŠd‹„ ZËe5jÌdŒœd�dŽ„ZÍed�d�„ ƒZÎd‘d’„ ZÏ�djd“d”„ZÐd•d–„ ZÑd—d˜„ ZÒd™dš„ ZÓe d
¡d›dœ„ ƒZÔd�dž„ ZÕ�dkdŸd „ZÖd¡d¢„ Z×ed£ƒd¤d¥„ ƒZØe‘emeÙe–edeÚeÛdƒeÜjÝejÞe5jße5jàe5jáe5jâhZãeaƒ �r&d dläZäeã åeäjæjà¡ d¦d§„ Zçd¨d©„ Zèdªd«„ Zéd¬d­„ Zêd®d¯„ Zëd°d±„ Zìd²d³„ ZíeÛeîƒ  ï¡ ƒZðe,e'e  eCd´< eÛeîƒ  ñ¡ ƒZòe,e.e  eCdµ< eÛe ó¡  ñ¡ ƒZôe,e.e  eCd¶< eÛeõeEƒ ƒƒZöe,e&e  eCd·< eöj÷ZøeùjúZûd¸d¹„ Züdºd»„ Zýd¼d½„ Zþeÿ�Z d¾d¿„ �ZdÀdÁ„ �ZdÂdÃ„ �ZdÄdÅ„ �ZdÆdÇ„ �Z�dldÈdÉ„�ZdÊdË„ �ZdÌdÍ„ �Zedd|œdÎdÏ„�Z	edd|œdÐdÑ„�Z
dÒ�Zd dÓ�l�m�Z dÔdÕ„ �ZdÖd×„ �ZdØdÙ„ �Zdd3dÚd3d/d3d3ek�jfdÛdÜ„�ZdÝdÞ„ �Zejdßdà„ ƒ�Ze¥ƒ �Ze b�e¡�Ze#ee(e f eCdá< �eƒ �Ze(dâ eCdã< e¥ƒ �ZG dädå„ dåƒ�Ze d¡dædç„ ƒ�Zdèdé„ �Z�dmdêdë„�Zdìdí„ �Zdîdï„ �Z�dndðdñ„�Z e� !¡ �Z"dòdó„ �Z#edôdõ„ ƒ�Z$död÷„ �Z%dødù„ �Z&dúdû„ �Z'edüœdýdþ„�Z(dÿ�d „ �Z)ejB�dœ�d�d„�Z*e�dœ�d�d„�Z+e�dœ�d�d„�Z,G �d	�d
„ �d
e�j-ƒ�Z.�e.�dœ�d�d„�Z/e-e5jÌef e–�de+e–e)�e. f �dœ�d�d„�Z0�d�d„ �Z1�d�d„ �Z2�d�dg�Z3�d�dg�Z4�d�d�d�dg�Z5�e3�e4 �e5 �Z6�d�d„ �Z7�do�d �d!„�Z8�dp�d"�d#„�Z9�d$�d%„ �Z:�d&�d'„ �Z;G �d(�d)„ �d)ƒ�Z<�d*�d+„ �Z=G �d,�d-„ �d-ƒ�Z>G �d.�d/„ �d/ƒ�Z?�d0�d1„ �Z@�d2�d3„ �ZA�d4�d5„ �ZB�d6�d7„ �ZCede‘e‘�d8œ�d9�d:„�ZDej¼G �d;�d<„ �d<ƒƒ�ZEede)�eE �d=œ�d>�d?„�ZFejÞe�jGed�d@œ�dA�dB„�ZH�dC�dD„ �ZI�dE�dF„ �ZJ�dG�dH„ �ZK�dI�dJ„ �ZL�dKe–�dLœ�dM�dN„�ZM�dO�dP„ �ZN�dQ�dR„ �ZOej�dS�dT„ ƒ�ZP�dU�dV„ �ZQ�dW�dX„ �ZRG �dY�dZ„ �dZe5�jS�jTƒ�ZUe5jR�jV�d[œ�d\�d]„�ZW�d^�d_„ �ZX�d`�da„ �ZYG �db�dc„ �dcƒ�ZZ�e[ƒ �Z\e*ed eC�dd< �dq�de�df„�Z]dS (r  é    N)Úcontextmanager)Ú	lru_cacheÚwraps)ÚMethodWrapperType)ÚAnyÚCallableÚcastÚClassVarÚCounterÚDefaultDictÚDequeÚDictÚIteratorÚKeysViewÚListÚOptionalÚSetÚTupleÚTypeÚUnionÚ
ValuesViewé   )ÚRemovableHandle)Údetect_fake_mode©Ú
LazyStringé   )Úconfig.ÚNP_SUPPORTED_MODULES)Ú
FakeTensorÚis_fakeÚmaybe_get_fake_mode)Úfx)Úenable_python_dispatcher)ÚTracingContext)Úis_sparse_compressed)Úlog_compilation_event)Ú_format_graph_codeÚlazy_format_graph_code)ÚLazyModuleMixin)Ú
has_tritonÚhas_triton_packageÚcountersÚoptimus_scuba_logzAhttps://pytorch.org/docs/main/torch.compiler_troubleshooting.htmlz;https://pytorch.org/docs/main/torch.compiler_nn_module.htmlzSee z& for more information and limitations.Úcompilation_time_metricsc                   C   s
   t  t¡S ©N)ÚcollectionsÚdefaultdictÚfloat© r3   r3   úA/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/_dynamo/utils.pyÚ<lambda>w   ó    r5   Úframe_phase_timingc              
   C   sP   zdd l }|j | |d�W S  tk
rJ   d dd„ t |g| ¡D ƒ¡ Y S X d S )Nr   ©ÚheadersÚ
c                 s   s   | ]}d   tt|ƒ¡V  qdS )ú, N)ÚjoinÚmapÚstr)Ú.0Úrowr3   r3   r4   Ú	<genexpr>ƒ   s    ztabulate.<locals>.<genexpr>)ÚtabulateÚImportErrorr<   Ú	itertoolsÚchain)Úrowsr9   rB   r3   r3   r4   rB   }   s    
ÿrB   c                   C   s   t d a d S ©Nr   )Ú
curr_framer3   r3   r3   r4   Úincrement_frameŒ   s    rI   c                   C   s   t  ¡  t ¡  dad S ©Nr   )r7   Úclearr.   rH   r3   r3   r3   r4   Úreset_frame_count’   s    rL   c                 C   s   t | 7 a d S r/   )Úop_count)Zcntr3   r3   r4   Úincrement_op_countœ   s    rN   c                  C   sV   d} i }t  ¡ D ]@}| ¡ D ]2\}}| |7 } ||kr>|||< q||  |7  < qq|S )Ng        )r7   ÚvaluesÚitems)ÚtotalÚtotal_by_keyZtimingsÚkeyZtimingr3   r3   r4   Úcalculate_time_spent£   s    
rT   c                  C   sB   t ƒ } d}|  ¡ D ]"\}}|› d|› dt|dƒ› �}qt|ƒ d S )NzTIMING:ú ú:é   )rT   rP   ÚroundÚprint)rR   ÚoutrS   Úvaluer3   r3   r4   Úprint_time_report¶   s
    r\   c                 C   s   t |  |  |7  < d S r/   )r7   )rS   Ú
phase_nameÚ
time_spentr3   r3   r4   Ú_add_time_spentÀ   s    r_   Tc                    s   ‡ ‡fdd„}| r|| ƒS |S )Nc                    s   t ˆ ƒ‡ ‡‡fdd„ƒ}|S )Nc                     s¸  ˆ j }|tkrg t|< d }d }tdƒ}zŒzNtj |› d�¡�$ t ¡ }ˆ | |Ž}t ¡ | }W 5 Q R X t|  |¡ W n8 tk
r® } ztt|ƒƒ}t|ƒ}‚ W 5 d }~X Y nX W 5 ˆ�r²ttƒ}ˆrÜ|d krÜt|ˆ|ƒ nÖtjj 	¡ d k	�r²ttjj 
¡ jƒ}d|k�sd|k�r,|d k�r,t|ˆ|ƒ n†d|k�r²ttjj ¡ ƒ}|d k�r\t|ˆ|ƒ ˆdk�r²|d k�r’t|  
dd ¡}	t|  
dd ¡}
nd }	d }
t||	|
||ƒ}t|ƒ X |S )Nz-infÚforwardZ	inferenceZbackwardZinductor_compileZcode_genz (dynamo_timed))Ú__qualname__r.   r2   r>   rH   r_   ÚtorchÚ_guardsr$   Útry_getÚgetÚaot_graph_nameZCompileContextZcurrent_compile_idr7   ÚBwdCompilationMetricsÚrecord_compilation_metricsZprofilerZrecord_functionÚtimeÚappendÚ	ExceptionÚtype)ÚargsÚkwargsrS   Ú	fail_typeÚfail_reasonr^   Ú	frame_keyrf   Ú
compile_idZinductor_compile_timeZcode_gen_timeÚmetricsÚt0ÚrÚe)ÚfuncÚfwd_onlyr]   r3   r4   Útime_wrapperÛ   s|    
ÿÿþý

ÿ


ÿ þÿ þû
z>dynamo_timed.<locals>.dynamo_timed_inner.<locals>.time_wrapper)r   )rw   ry   ©rx   r]   )rw   r4   Údynamo_timed_innerÚ   s    Fz(dynamo_timed.<locals>.dynamo_timed_innerr3   )Zoriginal_functionr]   rx   r{   r3   rz   r4   Údynamo_timedÙ   s    Jr|   r>   Fc                    s|   dd„ f‡ fdd„	‰| dkrF‡fdd„t D ƒ}d}|t|d	d
�7 }|S | dkrx‡fdd„t  ¡ D ƒ}tt  ¡ ƒ}||fS dS )aÀ  
    Get metrics about torchdynamo frontend/backend compilation times.

    Accumulates information from functions tagged with `@dynamo_timed`.

    repr='str' returns a printable string for user interaction, and 'csv'
    returns headers, rows which can be logged for output

    aggregate causes values from multiple compilations (e.g. split graphs)
    to be accumulated into one value.  If false, expect more than one value
    per metric.
    c                 S   s   | S r/   r3   ©Úxr3   r3   r4   r5   7  r6   zcompile_times.<locals>.<lambda>c                    s    ˆ r|t | ƒƒS d t|| ƒ¡S )Nr;   )Úsumr<   r=   )rO   Úitem_fn)Ú	aggregater3   r4   Úfmt_fn7  s    zcompile_times.<locals>.fmt_fnr>   c                    s$   g | ]}|ˆ t | d d„ d�f‘qS )c                 S   s   | d›S )Nz.4fr3   r}   r3   r3   r4   r5   >  r6   ú*compile_times.<locals>.<listcomp>.<lambda>©r€   )r.   ©r?   Úk©r‚   r3   r4   Ú
<listcomp>=  s   ÿz!compile_times.<locals>.<listcomp>z!TorchDynamo compilation metrics:
)ÚFunctionzRuntimes (s)r8   Úcsvc                    s   g | ]}ˆ |d d„ d�‘qS )c                 S   s   | d›S )Nz.6fr3   r}   r3   r3   r4   r5   F  r6   rƒ   r„   r3   )r?   Úvr‡   r3   r4   rˆ   E  s   ÿN)r.   rB   rO   ÚlistÚkeys)Úreprr�   rF   rZ   rO   r9   r3   )r�   r‚   r4   Úcompile_times)  s    
þ
þr�   c                   C   s   t  tddd�¡ d S )Nr>   T)rŽ   r�   )ÚlogÚinfor�   r3   r3   r3   r4   Údump_compile_timesM  s    r’   c                   @   s&   e Zd Zd	dd„Zdd„ Zdd„ ZdS )
ÚDuplicateWarningCheckeré   c                 C   s   || _ |  ¡  d S r/   )ÚmaxsizeÚreset)Úselfr•   r3   r3   r4   Ú__init__a  s    z DuplicateWarningChecker.__init__c                 C   s   t  ¡ | _d S r/   )r0   ÚOrderedDictÚset©r—   r3   r3   r4   r–   e  s    zDuplicateWarningChecker.resetc                 C   sT   || j kr&| j j|dd� tjsPdS n*d | j |< t| j ƒ| jkrP| j jdd� q0dS )NT)ÚlastF)rš   Úmove_to_endr   ÚverboseÚlenr•   Úpopitem©r—   rS   r3   r3   r4   Úaddh  s    

zDuplicateWarningChecker.addN)r”   )Ú__name__Ú
__module__ra   r˜   r–   r¢   r3   r3   r3   r4   r“   `  s   
r“   c                  C   s$   t j dd¡dk} | rtƒ S t ¡ S )NZTORCH_COMPILE_DEBUGÚ0Ú1)ÚosÚenvironre   Úadd_file_handlerÚ
contextlibÚ	ExitStack)Zcompile_debugr3   r3   r4   Úsetup_compile_debugw  s    r¬   c                   C   s   t  ¡  d S r/   )Úgraph_break_dup_warning_checkerr–   r3   r3   r3   r4   Úreset_graph_break_dup_checker€  s    r®   c                     sf   t j tƒ d¡} t j| dd� t t j | d¡¡‰ t d¡‰ˆ ˆ ¡ t	 
¡ }| ‡ ‡fdd„¡ |S )NZtorchdynamoT©Úexist_okz	debug.logztorch._dynamoc                      s
   ˆ  ˆ ¡S r/   ©ÚremoveHandlerr3   ©Zlog_file_handlerÚloggerr3   r4   r5   �  r6   z"add_file_handler.<locals>.<lambda>)r§   Úpathr<   Úget_debug_dirÚmakedirsÚloggingÚFileHandlerÚ	getLoggerÚ
addHandlerrª   r«   Úcallback)Zlog_pathÚ	exitstackr3   r³   r4   r©   „  s    

r©   c                     sV   t  ¡ } tjd k	rRt tj¡‰ tjj 	¡ D ]"‰ˆ 
ˆ ¡ |  ‡ ‡fdd„¡ q*| S | S )Nc                      s
   ˆ  ˆ ¡S r/   r±   r3   r³   r3   r4   r5   —  r6   z setup_log_file.<locals>.<lambda>)rª   r«   r   Zlog_file_namer¸   r¹   rb   Ú_loggingÚ	_internalZget_loggersr»   r¼   )r½   r3   r³   r4   Úsetup_log_file‘  s    

rÀ   c                 C   s(   t ƒ › d|j› dt| ƒj› d|j› d�S )Nz/error_recordings/Ú_z.rec)r¶   Úco_namerl   r£   Úco_firstlineno)ÚexcÚcoder3   r3   r4   Úgen_record_file_name�  s    ÿÿÿrÆ   c              	   C   sz   zTt j | ¡rt d| ¡ n6t jt j | ¡dd� t| dƒ�}| |¡ W 5 Q R X W n  t	k
rt   t 
d| ¡ Y nX d S )Nz9Unable to write execution record %s; file already exists.Tr¯   Úwbz#Unable to write execution record %s)r§   rµ   Úexistsr�   Úwarningr·   ÚdirnameÚopenÚdumprk   Ú	exception)ÚfilenameZexec_recordÚfr3   r3   r4   Úwrite_record_to_file¢  s     ÿrÐ   )Úgc                 C   s&   d}| j D ]}d|jkr
|d7 }q
|S )Nr   Úcallr   )ÚnodesÚop)rÑ   ÚcÚnr3   r3   r4   Úcount_calls°  s
    


r×   c                 C   s   | S r/   r3   r}   r3   r3   r4   Úidentity¸  s    rØ   c                 C   s>   zt | ƒ W dS  tk
r$   Y dS  tk
r8   Y dS X d S )NTF)ÚhashÚ	TypeErrorÚ
ValueErrorr}   r3   r3   r4   Úhashable¼  s    rÜ   c                  O   s   d S r/   r3   ©rm   rn   r3   r3   r4   ÚnothingÇ  s    rÞ   c                   @   sJ   e Zd ZdZdd„ Zdd„ Zddd„Zd	d
„ Zdd„ Zdd„ Z	dd„ Z
dS )ÚExactWeakKeyDictionaryz\Similar to weakref.WeakKeyDictionary, but use `is`/`id` rather than `==` to compare equalityc                 C   s   t ƒ | _t ƒ | _d S r/   )ÚdictrO   Úrefsr›   r3   r3   r4   r˜   Î  s    zExactWeakKeyDictionary.__init__c                 C   s   | j t|ƒ S r/   )rO   Úidr¡   r3   r3   r4   Ú__getitem__Ò  s    z"ExactWeakKeyDictionary.__getitem__Nc                 C   s   | j  t|ƒ|¡S r/   )rO   re   râ   )r—   rS   Údefaultr3   r3   r4   re   Õ  s    zExactWeakKeyDictionary.getc                 C   s   t |ƒ| jkS r/   )râ   rO   r¡   r3   r3   r4   Ú__contains__Ø  s    z#ExactWeakKeyDictionary.__contains__c                    s<   t |ƒ‰ ˆ ˆjkr.t |‡ ‡fdd„¡ˆjˆ < |ˆjˆ < d S )Nc                    s
   ˆ  ˆ ¡S r/   )Ú
_remove_id)Úref©Úidxr—   r3   r4   r5   Þ  r6   z4ExactWeakKeyDictionary.__setitem__.<locals>.<lambda>)râ   rá   Úweakrefrç   rO   )r—   rS   r[   r3   rè   r4   Ú__setitem__Û  s    
z"ExactWeakKeyDictionary.__setitem__c                 C   s(   || j kr| j |= || jkr$| j|= d S r/   )rO   rá   )r—   ré   r3   r3   r4   ræ   á  s    

z!ExactWeakKeyDictionary._remove_idc                 C   s   | j  ¡  | j ¡  d S r/   )rá   rK   rO   r›   r3   r3   r4   rK   ç  s    
zExactWeakKeyDictionary.clear)N)r£   r¤   ra   Ú__doc__r˜   rã   re   rå   rë   ræ   rK   r3   r3   r3   r4   rß   Ë  s   
rß   c                 C   s(   t |tttfƒrt| ƒ|kS t| ƒ|kS )zisinstance() without subclasses)Ú
isinstanceÚtuplerŒ   rš   rl   )ÚobjZallowed_typesr3   r3   r4   Úistypeì  s    rð   ©é   é   c                 C   s.   t jdkrt| tƒrdS t| tjƒp,| tjkS )Nrñ   T)ÚsysÚversion_inforí   Ú_builtin_final_typing_classesÚtypingÚ_FinalÚGeneric©r[   r3   r3   r4   Ú	is_typing   s    	rû   c              
   C   s2   t sdS t| t jt jt jt jt jt jt jt j	fƒS ©NF)
Únprð   Úint8Úint16Úint32Úint64Úuint8Zuint16Zuint32Zuint64rú   r3   r3   r4   Úis_numpy_int_type  s    øþr  c                 C   s   t sdS t| t jt jt jfƒS rü   )rý   rð   Úfloat16Úfloat32Úfloat64rú   r3   r3   r4   Úis_numpy_float_type!  s    ýþr  c                 C   s:   t | ƒp8t| tjƒr$t t | d¡ƒp8t| tjjtjj	fƒS )NÚ__wrapped__)
Úis_functionrí   Ú	functoolsÚ_lru_cache_wrapperÚinspectÚgetattr_staticrb   Ú_opsZOpOverloadPacketÚ
OpOverloadrú   r3   r3   r4   Úis_function_or_wrapper/  s    þür  c                 C   s    t | tjtjtjtjtjjfƒS r/   )	rí   ÚtypesÚFunctionTypeÚBuiltinFunctionTypeÚMethodDescriptorTypeÚWrapperDescriptorTyperb   ÚjitÚScriptFunctionrú   r3   r3   r4   r	  8  s    ûþr	  c                 C   s   t | ƒd S rJ   )Ú unwrap_with_attr_name_if_wrapper)Úfnr3   r3   r4   Úunwrap_if_wrapperE  s    r  c                 C   s~   t | tjƒrt | d¡} d}nXt| ƒrHt | dd¡rHt | d| ¡} d}n.t| ƒrrt | dd¡rrt | d| ¡} d}nd }| |fS )Nr  Z_torchdynamo_inlineFZ__script_if_tracing_wrapperZ__original_fn)rí   r
  r  r  r  r	  )r  Ú	attr_namer3   r3   r4   r  I  s      ÿr  c                 C   s   t sdS t| t jƒS rü   )rý   rð   Úndarrayrú   r3   r3   r4   Úis_numpy_ndarray]  s    r  c                 C   s,   t jt jjftj˜}|t jjf }t| |ƒS )zCheck of obj is a tensor)	rb   ÚTensorÚnnÚ	Parameterr   Ztraceable_tensor_subclassesÚ_subclassesr   rð   )rï   Ztensor_listr3   r3   r4   Úistensord  s    þýr"  c                 C   s
   t | tƒS r/   )rí   r)   ©Úmodr3   r3   r4   Úis_lazy_moduleo  s    r%  r”   c                  G   s   t | Ž  d S r/   )rY   )rm   r3   r3   r4   Ú
print_onces  s    r&  c                    s6   | ‰ ‡ fdd„}|j dk	r(t|j ƒdks,t‚|j d S )zNSome black magic to create a cell object that usually only exists in a closurec                      s   ˆ S r/   r3   r3   r}   r3   r4   rÏ   |  s    zmake_cell.<locals>.fNr   r   )Ú__closure__rŸ   ÚAssertionError)ÚvalrÏ   r3   r}   r4   Ú	make_cellx  s    r*  c                 C   s–   z.t dd„ | D ƒƒ}dd„ | ¡ D ƒ}||fW S  tk
r� } zDddlm} ddlm} |d|| Ž › d	|t| ¡ ƒŽ › �|d
� W 5 d }~X Y nX d S )Nc                 s   s   | ]}|  ¡ V  qd S r/   ©Úas_proxy©r?   Úargr3   r3   r4   rA   …  s     z$proxy_args_kwargs.<locals>.<genexpr>c                 S   s   i | ]\}}||  ¡ “qS r3   r+  )r?   rS   r.  r3   r3   r4   Ú
<dictcomp>†  s      z%proxy_args_kwargs.<locals>.<dictcomp>r   ©Úunimplemented)Útypestrzcall_function args: rU   ©Zfrom_exc)	rî   rP   ÚNotImplementedErrorrÄ   r1  Zvariables.baser2  rŒ   rO   )rm   rn   Z
proxy_argsÚproxy_kwargsrv   r1  r2  r3   r3   r4   Úproxy_args_kwargsƒ  s    
þr6  c                   @   s  e Zd ZU eed< eed< eed< eed< eed< eed< eed< ee ed< ee ed	< ee ed
< ee ed< ee ed< eed< ee ed< ee ed< ee ed< ee ed< ee ed< ee ed< ee ed< ee ed< ee ed< ee ed< ee ed< eed< e	ed< dS )ÚCompilationMetricsrr   rq   rÂ   Úco_filenamerÃ   Z
cache_sizeZaccumulated_cache_sizeZguard_countZshape_env_guard_countZgraph_op_countZgraph_node_countZgraph_input_countÚ
start_timeZentire_frame_compile_time_sZbackend_compile_time_sÚinductor_compile_time_sÚcode_gen_time_sro   rp   Zfail_user_frame_filenameZfail_user_frame_linenoZnon_compliant_opsZcompliant_custom_opsZrestart_reasonsZdynamo_time_before_restart_sZhas_guarded_codeN)
r£   r¤   ra   r>   Ú__annotations__Úintr   r2   r   Úboolr3   r3   r3   r4   r7  ’  s4   
r7  c                   @   sF   e Zd ZU eed< ee ed< ee ed< ee ed< ee ed< dS )rg   rr   r:  r;  ro   rp   N)r£   r¤   ra   r>   r<  r   r2   r3   r3   r3   r4   rg   ³  s
   
rg   é@   ©ÚmaxlenÚ_compilation_metrics©Úcompilation_metricsc                    sP   t  ˆ ¡ tˆ tƒrd}nd}tˆ tƒrLtj |‡ fdd„¡ tjrLt	ˆ ƒ d S )NrD  Zbwd_compilation_metricsc                      s   dd„ t  ˆ ¡ ¡ D ƒS )Nc                 S   s(   i | ] \}}|t |tƒr t|ƒn|“qS r3   )rí   rš   rŒ   )r?   r†   r‹   r3   r3   r4   r/  Ò  s   ÿ z@record_compilation_metrics.<locals>.<lambda>.<locals>.<dictcomp>)ÚdataclassesZasdictrP   r3   rC  r3   r4   r5   Ò  s   þz,record_compilation_metrics.<locals>.<lambda>)
rB  rj   rí   r7  rb   r¾   Ztrace_structuredr   Zlog_compilation_metricsr&   )rD  Únamer3   rC  r4   rh   Ä  s    



þrh   )Únew_sizeÚreturnc                 C   s,   t tƒ| krt ¡  q tjt| d�}|ad S )Nr@  )rŸ   rB  Úpopleftr0   Údeque)rG  Z	new_dequer3   r3   r4   Úset_compilation_metrics_limitÛ  s    
rK  )rH  c                   C   s   t  ¡  d S r/   )rB  rK   r3   r3   r3   r4   Úclear_compilation_metricsã  s    rL  c                   C   s   t tƒS r/   )rŒ   rB  r3   r3   r3   r4   Úget_compilation_metricsè  s    rM  c                   @   s>   e Zd ZU dZeeef ed< eed< dd„ Ze	dd„ ƒZ
dS )	ÚCleanupHookz,Remove a global variable when hook is calledÚscoperF  c                 G   s$   t d k	rt  jd8  _| j| j= d S rG   )ÚCleanupManagerÚcountrO  rF  ©r—   rm   r3   r3   r4   Ú__call__ó  s    zCleanupHook.__call__c                 C   s,   || kst ‚t jd7  _|| |< t| |ƒS rG   )r(  rP  rQ  rN  )rO  rF  r)  r3   r3   r4   Úcreateù  s    zCleanupHook.createN)r£   r¤   ra   rì   r   r>   r   r<  rS  ÚstaticmethodrT  r3   r3   r3   r4   rN  ì  s   
rN  c                       s.   e Zd ZU dZed  ed< ‡ fdd„Z‡  ZS )rP  r   Úinstancec                    s&   | j | D ]
}|ƒ  q
tƒ  |¡ d S r/   )rO   Úsuperræ   )r—   ré   Úhook©Ú	__class__r3   r4   ræ     s    zCleanupManager._remove_id)r£   r¤   ra   rQ  r	   r<  ræ   Ú__classcell__r3   r3   rY  r4   rP    s   
rP  c                 C   s0   |   ¡  | j¡}| jr,| jdk	r,| j  ¡ |_|S )z!Clone the tensor and its gradientN)ÚcloneÚrequires_grad_Úrequires_gradÚis_leafÚgrad©r~   Úyr3   r3   r4   Úclone_tensor  s    rc  ©Údtypec             
      s(  t | ƒr| S ‡ fdd„}t ¡ ��ü | jjdkrD|| ƒW  5 Q R £ S | jtjkr„tj||  ¡ ƒ||  	¡ ƒ| j
|  ¡ d�W  5 Q R £ S t| ƒrô| jtjtjhkr°|  ¡ }|  ¡ }n|  ¡ }|  ¡ }tj||ƒ||ƒ||  ¡ ƒ| j
| jd�W  5 Q R £ S tdd„ t|  ¡ |  ¡ ƒD ƒƒ}| j�r0t |d f| ¡}ntj|d ˆ �pD| j| jd	�}|  ¡ | ¡  d |  ¡  }| |  ¡ |  ¡ |¡ zJ|  |  !¡ ¡ | j"�r¤| #| j$¡ | j"�rÈ| j%d
k	�rÈt&| j%ˆ d�|_%W n* t'k
�rô   || ƒ Y W  5 Q R £ S X t(| dƒ�r| j) *¡ |_)|W  5 Q R £ S Q R X d
S )zcopy while preserving stridesc                    sV   t  | ¡}| jr| | j¡ | jr<| jd k	r<t| jˆ d�|_t| dƒrR| j 	¡ |_|S )Nrd  Ú_dynamo_dynamic_indices)
rb   r\  r_  r]  r^  r`  Úclone_inputÚhasattrrf  Úcopyra  rd  r3   r4   Útorch_clone  s    

z clone_input.<locals>.torch_cloneZxla)Úis_coalesced)Úlayoutc                 s   s   | ]\}}|d  | V  qdS )r   Nr3   )r?   ÚshapeÚstrider3   r3   r4   rA   C  s    zclone_input.<locals>.<genexpr>é    )re  ÚdeviceNrd  rf  )+r    rb   Úno_gradrp  rl   rl  Z
sparse_cooZsparse_coo_tensorZ_indicesZ_valuesrm  rk  r%   Z
sparse_csrZ
sparse_bsrZcrow_indicesZcol_indicesZccol_indicesZrow_indicesZsparse_compressed_tensorrO   r   ÚzipÚsizern  Zis_quantizedZempty_quantizedÚemptyre  Údata_ptrZelement_sizeZas_strided_Úcopy_r\  r_  r]  r^  r`  rg  ÚRuntimeErrorrh  rf  ri  )r~   re  rj  Zcompressed_indicesZplain_indicesZneeded_sizeÚresultZcache_line_offsetr3   rd  r4   rg    sf    


ü

ûÿ 
 ÿþrg  c                 C   s¤   t | ƒtkrft| ƒ}| ¡ D ]D\}}t|tƒr<t|ƒ||< qt|tjƒsTtt |ƒƒ‚t	|ƒ||< q|S t
| ƒ}tt|ƒƒD ]$}t|| tjƒrzt	|| ƒ||< qz|S r/   )rl   rà   rP   rí   rî   Úclone_inputsrb   r  r(  rg  rŒ   ÚrangerŸ   )Úexample_inputsÚresrS   r[   Úir3   r3   r4   ry  `  s    
ry  )r)  c              
   C   sb   z|   ¡  W nP tk
r\ } z2ddlm} t ddt| ƒ¡}|d|› �ƒ|‚W 5 d }~X Y nX d S )Nr   )Ú	SkipFramez\(.*Ú z(torch.compile cannot be run in context: )ru  rw  rÄ   r~  ÚreÚsubrŽ   )r)  rv   r~  Zfunctorch_subclass_namer3   r3   r4   Úskip_frame_if_in_functorch_modes  s    ÿþr‚  c                  c   s´   t jj} t jjj}|ƒ �J | ƒ �8 t  t j ¡ ¡}t	|ƒ t j
 ¡ rTt  t j
 ¡ ¡}W 5 Q R X W 5 Q R X z
d V  W 5 t jj ¡ �( t j |¡ t j
 ¡ r¤t j
 |¡ W 5 Q R X X d S r/   )rb   Ú_CZ_DisableFuncTorchÚutilsZ_python_dispatchZ_disable_current_modesr\  ÚrandomÚget_rng_stater‚  ÚcudaÚis_availableÚset_rng_state)Zdisable_functorchZdisable_current_modesÚ	rng_stateÚcuda_rng_stater3   r3   r4   Úpreserve_rng_state€  s    

$

rŒ  c                 C   s&   t | tjjjtjjjtjjtjjfƒS r/   )	rí   rb   r  Z_traceZTopLevelTracedModuleÚ_scriptZRecursiveScriptModuler  ZScriptModule)Zmodel0r3   r3   r4   Úis_jit_model’  s    üþrŽ  c                 C   sx   t | ƒr| S ztj | |¡W S  tk
rr   ztj | ¡W  Y S  tk
rl   |r^t d¡ n
t d¡ Y nX Y nX d S )Nz	jit errorz0Both torch.jit.trace and torch.jit.script failed)	rŽ  rb   r  Útracerk   Úscriptr�   rÍ   Úerror)Úmodelr{  rž   r3   r3   r4   Útorchscriptž  s    r“  c              	   C   s,   zt  | ¡W S  ttfk
r&   Y d S X d S r/   )r  ÚgetfilerÚ   ÚOSError©rï   r3   r3   r4   r”  °  s    r”  c                 C   s   t t| ƒƒS )zLTest if an object is a namedtuple or a torch.return_types.* quasi-namedtuple)Úis_namedtuple_clsrl   r–  r3   r3   r4   Úis_namedtuple·  s    r˜  c                 C   sp   zVt | tƒrTt| dg ƒpdg}t| ddƒ}|dkpP|d tkoPt| dƒoPt| dƒW S W n tk
rj   Y nX dS )	zhTest if an object is a namedtuple or a (torch.return_types|torch.autograd.forward_ad).* quasi-namedtupleÚ	__bases__Nr¤   )útorch.return_typesztorch.autograd.forward_adr   Ú_makeÚ_fieldsF)Ú
issubclassrî   Úgetattrrh  rÚ   )ÚclsÚbasesÚmoduler3   r3   r4   r—  ¼  s    
ÿr—  c                 C   s°   | t krdddgS t| tƒs t‚t| dƒr0| jS tjG dd„ dƒƒ}| jdksRt‚| t	|t
| jƒƒƒ}dg| j }t|ƒD ]0}|d	 d
krztt||ƒ|ƒrz||t||ƒj< qz|S )zIGet the fields of a namedtuple or a torch.return_types.* quasi-namedtupleÚstartÚstopÚsteprœ  c                   @   s   e Zd ZU eed< dS )z!namedtuple_fields.<locals>.MarkerÚindexN©r£   r¤   ra   r=  r<  r3   r3   r3   r4   ÚMarkerÕ  s   
r§  rš  Nr   rÁ   )Úslicer�  rî   r(  rh  rœ  rE  Ú	dataclassr¤   r=   rz  Ún_fieldsÚdirrí   rž  r¥  )rŸ  r§  rï   ÚfieldsrF  r3   r3   r4   Únamedtuple_fieldsÊ  s    

r­  c              	      sˆ   t  ¡ �f t  t j ¡ ¡‰t j ¡ r4t  t j ¡ ¡‰ g ‰t |  	¡ |  
¡ ¡D ]}ˆ ||jt  |¡f¡ qLW 5 Q R X ‡ ‡‡fdd„}|S )Nc               	      s^   t  ¡ �L t j ˆ¡ t j ¡ r,t j ˆ ¡ ˆD ]\} }}| j|kr0|  |¡ q0W 5 Q R X d S r/   )rb   rq  r…  r‰  r‡  rˆ  Ú_versionrv  )ÚparamÚversionÚoriginal_value©r‹  rŠ  Úsaved_stater3   r4   Úrestoreì  s    


z"checkpoint_params.<locals>.restore)rb   rq  r\  r…  r†  r‡  rˆ  rD   rE   Ú
parametersÚbuffersrj   r®  )Úgmr¯  r´  r3   r²  r4   Úcheckpoint_paramsã  s    

$	r¸  c                 C   sh   t j ¡ rt jj}nt}|ƒ  t ¡  t  d¡ t 	¡ }t
|ƒD ]}| |Ž }|ƒ  q@t 	¡ }||| fS )Ni9  )rb   r‡  rˆ  ÚsynchronizerÞ   ÚgcZcollectZmanual_seedri   Úperf_counterrz  )r’  r{  Útimesr¹  rt   rÁ   rx  Út1r3   r3   r4   Útimedø  s    


r¾  c                 C   s    t dd„ t ||  d¡¡D ƒƒS )Nc                 s   s   | ]}|j V  qd S r/   )Zis_cuda©r?   r~   r3   r3   r4   rA   
  s     z check_is_cuda.<locals>.<genexpr>T)ÚallrD   rE   rµ  )r·  r{  r3   r3   r4   Úcheck_is_cuda	  s    rÁ  ro  c                 C   sn   | dkst ‚dd„ t| ƒD ƒ}t|dd … |d d…  ƒ}tdd |¡› dd |¡› d�ƒ}d	| › d
�|_|S )Nr   c                 S   s   g | ]}d |› �‘qS ©r‹   r3   )r?   r}  r3   r3   r4   rˆ     s     z rot_n_helper.<locals>.<listcomp>éÿÿÿÿzlambda ú,z: (ú)Zrot_Z_helper)r(  rz  ÚreversedÚevalr<   r£   )rÖ   ÚvarsZrotatedr  r3   r3   r4   Úrot_n_helper  s    "rÉ  c                 C   s<   t | ttfƒrttt| ƒƒS t| tjt	fƒp:t | t
thB ƒS r/   )rð   rî   Ú	frozensetrÀ  r=   Úis_safe_constantrí   ÚenumÚEnumrl   Úcommon_constant_typesr¨  rÂ  r3   r3   r4   rË  -  s    þrË  c                 C   s,   ddl m}m} t| |ƒr(| |  ¡ ¡S | S )Nr   )ÚConstantVariableÚSymNodeVariable)Ú	variablesrÏ  rÐ  rí   rT  Zevaluate_expr)r.  rÏ  rÐ  r3   r3   r4   Úspecialize_symnode6  s    
rÒ  c                 C   s*   ddl m} t| ƒ} t| |ƒr&|  ¡ S | S )Nr   ©rÏ  )rÑ  rÏ  rÒ  rí   Úas_python_constant)r.  rÏ  r3   r3   r4   Úguard_if_dyn@  s
    
rÕ  c                 C   s   t dd„ t | | ¡ ¡D ƒƒS )Nc                 s   s   | ]}|  ¡ V  qd S r/   )Úis_python_constantr¿  r3   r3   r4   rA   L  s     z&check_constant_args.<locals>.<genexpr>)rÀ  rD   rE   rO   rÝ   r3   r3   r4   Úcheck_constant_argsK  s    r×  c                 C   s^   ddl m} ddlm} d}t | | ¡ ¡D ](}t||ƒrD|d7 }q,t||ƒs, dS q,|dkS )Nr   rÓ  ©ÚUnspecializedPythonVariabler   F)Zvariables.constantrÏ  Úvariables.tensorrÙ  rD   rE   rO   rí   )rm   rn   rÏ  rÙ  Zunspec_countr~   r3   r3   r4   Úcheck_unspec_python_argsO  s    


rÛ  c                 C   s>   ddl m} t | | ¡ ¡D ]}| ¡ st||ƒs dS qdS )Nr   rØ  FT)rÚ  rÙ  rD   rE   rO   rÖ  rí   )rm   rn   rÙ  r~   r3   r3   r4   Úcheck_unspec_or_constant_args\  s
    rÜ  c                    s.   ddl m‰  t‡ fdd„t | | ¡ ¡D ƒƒS )Nr   ©ÚNumpyNdarrayVariablec                 3   s   | ]}t |ˆ ƒV  qd S r/   )rí   r¿  rÝ  r3   r4   rA   j  s   ÿz+check_numpy_ndarray_args.<locals>.<genexpr>)rÚ  rÞ  ÚanyrD   rE   rO   rÝ   r3   rÝ  r4   Úcheck_numpy_ndarray_argsg  s    þrà  Ú	dict_keysÚdict_valuesÚodict_valuesÚtuple_iteratorc                 C   s   t | ƒ}tjj |¡ |S r/   )Ú
object_newrb   r  ÚModuler˜   )rŸ  rï   r3   r3   r4   Únn_module_newx  s    rç  c                 C   s   t  tj| d¡S rG   )r
  ÚreduceÚoperatorÚmul)Úitr3   r3   r4   Úproduct~  s    rì  c                 C   s   |   ¡ \}\}}|||  S r/   )Ú
__reduce__)rë  r¥  rÁ   rï   r¢  r3   r3   r4   Útuple_iterator_getitem‚  s    rî  c                 C   s
   |   |¡S r/   )Zas_subclass)ÚtrŸ  r3   r3   r4   Úto_subclassŠ  s    rð  c                 C   s   t t t| ƒ||d ¡ƒS rG   )ÚnextrD   ÚisliceÚiter)ÚdrÖ   r3   r3   r4   Údict_keys_getitemŽ  s    rõ  c                 C   s2   | j j}| j}|rdnd}|› d|› d|› �}|S )NÚLÚGz["z"].)rZ  r£   rF  )r[   ÚlocalrF  r)  rO  Z
local_namer3   r3   r4   Ú	enum_repr’  s
    rù  c                 C   s:   || j d< t ¡ jj}tjjj 	||¡ }r6|| j d< d S )NÚexample_valueZunbacked_bindings)
Úmetar$   re   Ú	fake_modeÚ	shape_envrb   r"   ÚexperimentalÚsymbolic_shapesZcompute_unbacked_bindings)Únoderú  rý  Zsymbol_to_pathr3   r3   r4   Úset_example_valueœ  s    

 ÿr  c                 C   s2   |   ¡ jj d¡}t|ƒs.ddlm} |dƒ |S )Nrú  r   r0  z:Cannot check Tensor object identity without its fake value)r,  r   rû  re   r    rÄ   r1  )ÚvtÚfake_tensorr1  r3   r3   r4   Ú_get_fake_tensor«  s
    r  c                    sê   ddl m}m}m}m} ˆ  ¡ r@t‡ fdd„| D ƒƒ}| |¡S d}	|r^tˆ |ƒr^d}	t	ˆ ƒ‰ d }
| D ]l}|	r”t||ƒrÒˆ t	|ƒkrÒ| d¡  S qf|t
jƒ ||ˆ gi ¡}|
d krº|}
qf|t
jƒ |||
gi ¡}
qf|
d kræ| d¡}
|
S )Nr   )ÚBuiltinVariablerÏ  ÚTensorVariableÚVariableTrackerc                 3   s&   | ]}|  ¡ o| ¡ ˆ  ¡ kV  qd S r/   )rÖ  rÔ  r¿  ©Úsearchr3   r4   rA   ½  s   þz iter_contains.<locals>.<genexpr>FT)rÑ  r  rÏ  r  r  rÖ  rß  rT  rí   r  ré  ÚeqÚcall_functionÚor_)rP   r	  ÚtxZcheck_tensor_identityr  rÏ  r  r  Zfound_constZmust_check_tensor_idÚfoundr~   Úcheckr3   r  r4   Úiter_contains´  s6    ý


  ÿ
r  c                 C   s   t | tjtjjtfƒS )z4Returns whether it indexes dictionaries using its id)rí   rb   r  r  ræ  r   )r†   r3   r3   r4   Ú	key_is_idÝ  s    r  c                 C   s   dd„ |   ¡ D ƒS )Nc                 S   s    g | ]}t |ƒrt|ƒn|‘qS r3   )r  râ   r…   r3   r3   r4   rˆ   ã  s     zkey_to_id.<locals>.<listcomp>)r�   rú   r3   r3   r4   Ú	key_to_idâ  s    r  c                   sÎ   ddl m} t| ttfƒr|d ‡ fdd„| D ƒ¡}t| tƒrHd|› d�S t| tƒsVt‚t| ƒdkrnd|› d	�S d|› d
�S nNt| tj	ƒrœt
| ˆ d� dd¡S || ƒrª| jS t| tƒrÄdd„ }|| ƒS | ›S d S )Nr   )Úis_builtin_callablerÄ  c                 3   s   | ]}t |ˆ d �V  qdS ©©rø  N©Ú
const_repr©r?   Úsr  r3   r4   rA   ê  s     zconst_repr.<locals>.<genexpr>ú[ú]ú(z,)rÅ  r  ú'r  c                 S   s(   | j }|j}|dkr|jS |d |j S )NÚbuiltinsÚ.)rZ  r¤   ra   )ÚoÚklassr¡  r3   r3   r4   Úfullnameû  s
    zconst_repr.<locals>.fullname)Ztrace_rulesr  rí   rŒ   rî   r<   r(  rŸ   rÌ  rÍ  rù  Úreplacer£   rl   )r~   rø  r  Z
elems_reprr"  r3   r  r4   r  æ  s"    

r  c                   s$   d  ‡ fdd„| D ƒ¡}d| d S )NrÄ  c                 3   s   | ]}t |ˆ d �V  qdS r  r  r  r  r3   r4   rA     s     z!dict_keys_repr.<locals>.<genexpr>r  r  ©r<   )Z
const_keysrø  Zkeys_strr3   r  r4   Údict_keys_repr  s    r%  Z
__dict_key)ÚUnsupportedFakeTensorExceptionc              
   C   s`   z| ƒ W S  t k
rZ } z4ddlm} d|j› d�}t |¡ |||d� W 5 d }~X Y nX d S )Nr   r0  zUnsupported: z with fake tensor propagation.r3  )r&  rÄ   r1  Úreasonr�   rÉ   )r  rv   r1  Úmsgr3   r3   r4   Úwrap_fake_exception  s    
r)  c              
      s6   t jj |¡� t‡ fdd„ƒW  5 Q R £ S Q R X d S )Nc                      s
   t  ˆ ¡S r/   )ri  Údeepcopyr3   r–  r3   r4   r5     r6   z)deepcopy_to_fake_tensor.<locals>.<lambda>)rb   r!  r  ZFakeCopyModer)  )rï   rü  r3   r–  r4   Údeepcopy_to_fake_tensor  s    r+  c                 C   s   t  t  t  | | ¡¡¡S )z+
    Calculate root mean squared error
    )rb   ÚsqrtZmeanZsquare)rç   r|  r3   r3   r4   Úrmse"  s    r-  g-Cëâ6?c
                    sª  ˆdkrˆ‰t ˆtttjjtjfƒr¤t ˆttfƒsNtdtˆƒ› dtˆƒ› �ƒ‚t	ˆƒt	ˆƒkrjˆdƒ dS t	ˆƒt	ˆƒko¢t
‡ ‡‡‡‡‡‡
fdd„tˆˆˆƒD ƒƒS tˆƒjdkrÔtˆjˆjˆjˆ ˆ
ˆˆˆˆˆd	�
S t ˆtƒ�r~t ˆtƒsît‚tˆ ¡ ƒtˆ ¡ ƒk�s,td
tˆ ¡ ƒ› dtˆ ¡ ƒ› �ƒ‚tˆ ¡ ƒD ]@}
tˆ|
 ˆ|
 ˆ|
 ˆ ˆ
ˆˆˆˆˆd�
�s8ˆd|
ƒ  dS �q8dS t ˆtjtfƒ�rJt ˆtjjƒ�r¤t‚t ˆtjjƒ�r¸t‚dd„ ‰	‡	fdd„ˆˆˆfD ƒ\‰‰‰ˆj�rˆj�sòt‚ˆ ¡ ‰ˆ ¡ ‰t ˆtjƒ�s,tdtˆƒ› dtˆƒ› �ƒ‚ˆ�r¦ˆjˆjk�rTˆdˆjˆjƒ dS ˆjtjk�r¦ˆ�rldS tjˆjtjd�ˆjtjd�ˆ
ˆ
ˆd�}|�s¢ˆdƒ |S ˆ �r&ˆ ¡  tj¡‰ˆ ¡  tj¡‰tjˆˆˆ
ˆ
dd��rædS tjj j!ˆˆddd�}|dk �rt" #d| $¡  %¡  &¡ ¡ |dkS ˆ�s8ˆ ˆj¡‰tjˆˆˆ
ˆ
ˆd��rRdS ˆjtj'k�r.t(ˆˆƒ &¡ }t) *|¡�r„t" #d¡ t(ˆˆƒ &¡ }ˆjtj+k�r¤dnd}ˆ ,¡ dk �sòˆj-dk�rèˆj.d  ˆj.d!   k�ræd"k�sòn ˆ
d#k�röd}||| ˆ
d$  k}|�s*ˆd%||ˆ /¡ ˆj|ˆ
ƒ |S ˆ�r8dS ˆd&ˆ
ƒ dS �n\t ˆt0t1tdƒttj2fƒ�r”ˆ�rpdS ˆˆk}|�s�ˆd'tˆƒˆˆƒ |S t3ˆƒ�s¨t4ˆƒ�rúˆ�rÊt3ˆƒ�sÊt4ˆƒ�sÊˆ &¡ ‰tˆƒtˆƒk�oâˆˆk}|�söˆd(ˆˆƒ |S t5ˆƒ�r>tˆƒtˆƒk�o<tt 6ˆ¡t 6ˆ¡ˆˆ ˆ
ˆˆˆˆˆd�
S tˆƒjd)k�r’tˆƒtˆƒk�sdt‚t
‡ ‡‡‡‡‡‡‡‡‡
f
d*d„ˆj7 ¡ D ƒƒS t8d+tˆƒj› �ƒ‚dS ),z-Check correctness to see if ref and res matchNztype mismatch rU   zLength mismatchFc                 3   s0   | ](\}}}t |||ˆ ˆˆˆˆˆˆd �
V  qdS )©Ú	log_errorN)Úsame)r?   ZaiZbiZ	fp64_refi)Úcos_similarityÚ	equal_nanÚexact_dtypeÚignore_non_fpr/  Úrelax_numpy_equalityÚtolr3   r4   rA   =  s   ôözsame.<locals>.<genexpr>ZQuestionAnsweringModelOutputr.  zkeys mismatch z == ©r1  r6  r2  r3  r5  r4  r/  zAccuracy failed for key name %sTc                 S   s   t | tjƒr| S t | ¡S r/   )rí   rb   r  Útensor©rï  r3   r3   r4   Ú	to_tensorw  s    zsame.<locals>.to_tensorc                 3   s   | ]}ˆ |ƒV  qd S r/   r3   )r?   r)  )r:  r3   r4   rA   z  s     zdtype mismatch %s, %srd  )ZatolZrtolr2  z+Accuracy failed: uint8 tensor did not matchr   g�íµ ÷Æ°>)ZdimZepsg®Gáz®ï?zSimilarity score=%sz=Found nan in reference. Consider running in higher precision.g      @g       @iè  é   rÃ  éþÿÿÿr   g{®Gáz”?g      $@z\RMSE (res-fp64): %.5f, (ref-fp64): %.5f and shape=%s. res.dtype: %s, multiplier: %f, tol: %fz+Accuracy failed: allclose not within tol=%szAccuracy failed (%s): %s != %sz!Accuracy failed (numpy): %s != %s)ZMaskedLMOutputZSeq2SeqLMOutputZ!CausalLMOutputWithCrossAttentionsZLongformerMaskedLMOutputZ	InstancesZSquashedNormalZBoxesZNormalZTanhTransformZFooÚVariablec                 3   s<   | ]4}t tˆ|ƒtˆ|ƒtˆ|ƒˆ ˆ	ˆˆˆˆˆd �
V  qdS )r7  N)r0  rž  )r?   rS   )
r1  r2  r3  Úfp64_refr4  r/  rç   r5  r|  r6  r3   r4   rA     s   ôözunsupported type: )9rí   rŒ   rî   rb   r  ZParameterListÚSizer(  rl   rŸ   rÀ  rr  r£   r0  Zlossrà   rš   r�   Úsortedr  r2   r!  r   Z	is_sparseZto_densere  r>  ZallcloseÚtor  Úflattenr  Z
functionalZcosine_similarityr�   rÉ   ÚcpuÚdetachÚitemr  r-  ÚmathÚisnanÚbfloat16ZnumelÚndimrm  rs  r>   r=  rp  r  r  r  Ú	as_tensorÚ__dict__rw  )rç   r|  r>  r1  r6  r2  r3  r5  r4  r/  r†   ru   ZscoreZ	ref_errorZ	res_errorZ
multiplierZpasses_testr3   )r1  r2  r3  r>  r4  r/  rç   r5  r|  r:  r6  r4   r0  )  s@   *(
óöÿ
þöÿ

*û
ÿ
ÿþ þ
þ
ü	ù

ÿÿ
öór0  c                 C   s,   | j  d¡d }d| j› d|› d| j› d�S )Nú/rÃ  r  z' (rV   rÅ  )r8  ÚsplitrÂ   rÃ   )rÅ   Zshort_filenamer3   r3   r4   Úformat_func_info  s    rN  c                  c   s:   t j} tjt _t j}tjt _z
d V  W 5 | t _|t _X d S r/   )r   Úcache_size_limitrô   r•   Zaccumulated_cache_size_limit)ZpriorZprior_acc_limitr3   r3   r4   Údisable_cache_limit  s    
rP  Úguard_failuresz-torch._dynamo.output_graph.GraphCompileReasonÚgraph_break_reasonsc                   @   sJ   e Zd ZdZdd„ Zejjdœdd„Zdd„ Z	d	d
„ Z
dd„ Zdd„ ZdS )ÚCompileProfilerzìUtility for profiling how and what dynamo would compile.

    Can be used for
     * diagnosing recompilation issues
     * determining an appropriate compile cache limit
     * (TODO)confirming which functions got compiled/skipped
    c                 C   s   d| _ d| _t| _d S rJ   )Úframe_countrM   rP  Zbackend_ctx_ctorr›   r3   r3   r4   r˜   ?  s    zCompileProfiler.__init__©r·  c                 C   s:   |  j d7  _ |jjD ]}d|jkr|  jd7  _q|jS )Nr   rÒ   )rT  ÚgraphrÓ   rÔ   rM   r`   )r—   r·  r{  r   r3   r3   r4   rS  D  s
    
zCompileProfiler.__call__c                 C   s   | S r/   r3   r›   r3   r3   r4   Ú	__enter__L  s    zCompileProfiler.__enter__c                 C   s   d S r/   r3   )r—   Útypr)  Ú	tracebackr3   r3   r4   Ú__exit__O  s    zCompileProfiler.__exit__c                 C   s   dt iS )NrQ  )rQ  r›   r3   r3   r4   Úget_metricsR  s    zCompileProfiler.get_metricsc                    sŒ   |   ¡ }|d ‰ ‡ fdd„‰‡ fdd„‰‡‡fdd„ˆ D ƒ‰dd	„ }‡ ‡‡fd
d„}t d¡}||ƒ phd7 }|t d¡7 }||ƒ p„d7 }|S )NrQ  c                    s   t ˆ |  ƒS r/   )rŸ   ©rÅ   ©Úgfr3   r4   Únum_recompilesY  s    z.CompileProfiler.report.<locals>.num_recompilesc                    s   d  dd„ ˆ |  D ƒ¡S )Nr:   c                 S   s   g | ]}t |ƒ‘qS r3   )r>   r¿  r3   r3   r4   rˆ   ]  s     zECompileProfiler.report.<locals>.recompile_reasons.<locals>.<listcomp>r$  r\  r]  r3   r4   Úrecompile_reasons\  s    z1CompileProfiler.report.<locals>.recompile_reasonsc                    s"   g | ]}t |ƒˆ |ƒˆ|ƒg‘qS r3   )rN  ©r?   rÅ   )r_  r`  r3   r4   rˆ   _  s   ÿz*CompileProfiler.report.<locals>.<listcomp>c                      s2   dt kr.t d ‰ t‡ fdd„ˆ D ƒddgd�S d S )NZgraph_breakc                    s   g | ]}|ˆ | g‘qS r3   r3   )r?   r(  ©Zgraph_breaksr3   r4   rˆ   h  s     zFCompileProfiler.report.<locals>.graph_break_report.<locals>.<listcomp>zGraph Break ReasonZCountr8   )r,   rB   r3   r3   rb  r4   Úgraph_break_reportd  s    þz2CompileProfiler.report.<locals>.graph_break_reportc                     sJ   t ˆ ƒrFt‡fdd„ˆ D ƒƒ} tˆdddgd�}|t d| › d�¡ S d S )	Nc                 3   s   | ]}ˆ |ƒV  qd S r/   r3   ra  )r_  r3   r4   rA   n  s     zGCompileProfiler.report.<locals>.recompilation_report.<locals>.<genexpr>r‰   Z
RecompileszRecompile Reasonsr8   zC

                    Set torch._dynamo.config.cache_size_limit to z/ to avoid being cache limited.
                )rŸ   ÚmaxrB   ÚtextwrapÚdedent)Zmax_recompilesZrecomp_table)r^  r_  Úsummarized_gfr3   r4   Úrecompilation_reportl  s    þþÿz4CompileProfiler.report.<locals>.recompilation_reportaÇ  
            Torchdynamo Profiler Report
            ===========================

            Graph Breaks
            ------------
            Graph breaks happen when torchdynamo encounters code it can't safely trace.
            If you want to find out why breaks are happening, check below for each break reason
            You may gain additional insight by passing `fullgraph=True` to torch.compile,
            to stop at the first break.

        zNo graph breaks detected.a+  

            Recompilation
            -------------
            These subgraphs were recompiled more than once due to guard failures
            Guard failures indicate some condition assumed to be static by the tracer changed,
            making it unsafe to reuse the compiled program.

        zNo recompilation detected.
)r[  re  rf  )r—   rs   rc  rh  Úreportr3   )r^  r_  r`  rg  r4   ri  U  s$    þÿÿzCompileProfiler.reportN)r£   r¤   ra   rì   r˜   rb   r"   ÚGraphModulerS  rW  rZ  r[  ri  r3   r3   r3   r4   rS  6  s   rS  c                 C   s2   dt j  ¡  d¡ d tt ¡ ƒ }tj | |¡S )NZrun_z%Y_%m_%d_%H_%M_%S_%fz-pid_)ÚdatetimeÚnowÚstrftimer>   r§   Úgetpidrµ   r<   )Úroot_dirZdir_namer3   r3   r4   Ú_get_debug_dir™  s    ÿý
üÿrp  c                  C   s   t j} t| ƒS r/   )r   Zdebug_dir_rootrp  )Z
debug_rootr3   r3   r4   r¶   ¥  s    r¶   c                 C   sD   d| j kr"t| j d ƒr"| j d S |r<ddlm} |dƒ nd S d S )Nrú  r   r0  z9`FakeTensor` example value was required but not available)rû  r    Ztorch._dynamo.excr1  )r   Úrequiredr1  r3   r3   r4   Úextract_fake_example_valueª  s    

rr  c                 C   s   t | ƒ|jkst‚| S r/   )r!   rü  r(  )rv   r  r3   r3   r4   Úensure_graph_fakeµ  s    rs  c                    s(   t jjdœ‡ ‡fdd„}t jj ||¡S )N©rÖ   c                    sH   | j dkr d| jkr t| ˆˆ ƒS | jd }ˆ sDt|tjƒrDt|ˆƒS |S )Nr  rú  )rÔ   rû  Úget_fake_valuerí   rb   r  rs  )rÖ   rZ   ©Úallow_non_graph_faker  r3   r4   Úvisit»  s    

z)get_fake_values_from_nodes.<locals>.visit)rb   r"   ÚNoder   Úmap_arg)r  rÓ   rw  rx  r3   rv  r4   Úget_fake_values_from_nodesº  s    r{  c              
      s:  ddl m} ddlm}m}m}m}m} ˆj}	dˆj	krPt
ˆj	d ƒrPˆj	d S tˆˆjˆjf|ƒ\‰ ‰d‰|	dkr´tˆ ƒdkr´tˆ d tjjƒr´tˆ d ˆjƒftˆ dd… ƒ ‰ |	dkrôˆjjˆj ‰tˆƒrètˆd	ƒrèˆ ˆˆ ¡ tˆˆjƒ‰zBˆj�0 tƒ � t‡ ‡‡‡‡fd
d„ƒ}
W 5 Q R X W 5 Q R X W �nÞ |k
�rP   ‚ Y �nÆ tk
�r } �z¤|}|jdk	�rz|j}t|tj j!j"ƒ�r¢|d|j#› d�ƒ �nLt|tj j!j$ƒ�rêtj%j&j'�sÔ|d|j#› d�ƒ n|d|j#› d�ƒ �nt|tj j!j(ƒ�rp|j#}	d}t|	tj)j*ƒ�rVtj+ ,|	j-j.|	j-j/¡}|dk	�rV|\}}d|› d|› d|› d�}|d|j#› d|› d�ƒ n~t|tj0j1j2j3ƒ�rœ||j4d|› �dd�‚nRt||ƒ�r¾||j4|jd ƒ|‚n0t|t5ƒ�rîdt6|ƒk�rî|dˆj› d|› �ƒ |t6|ƒƒ 7|j8¡d‚W 5 d}~X Y nX |�s6t9 :tj;t<j=t>ˆd�|
¡}|
S ) az  
    Run the computation represented by `node` using fake tensors and return the result.

    allow_non_graph_fake: whether to allow the return result to be:
        1. non-fake or 2. fake that is not created by this instance of Dynamo.
        If `True`, you must be prepared to deal with such return values, ideally
        by further wrapping them as this graph's fakes.
    r   )ÚValueRangeErrorr   )ÚTorchRuntimeErrorr1  ÚUnsupportedÚ	UserErrorÚUserErrorTyperú  NÚcall_methodÚcall_moduleZ_initialize_hookc                      s   t ˆjˆˆ ˆˆƒS r/   )Úrun_nodeÚoutputr3   ©rm   rn   Únnmoduler   r  r3   r4   r5   ú  r6   z get_fake_value.<locals>.<lambda>zdata dependent operator: zC; to enable, set torch._dynamo.config.capture_scalar_outputs = Truezdynamic shape operator: zM; to enable, set torch._dynamo.config.capture_dynamic_output_shape_ops = Truezm; Operator does not have a meta kernel that supports dynamic output shapes, please report an issue to PyTorchr  z:It's possible that the support was implemented in module `z` and you may need to `import z`(z), otherwise zunsupported operator: z (z~see https://docs.google.com/document/d/1GgvOe7C8_NVOMLOCwDaYV1mXXyHMXY7ExoewHqooxrs/edit#heading=h.64r4npvq0w0 for how to fix)a  Tried to use data-dependent value in the subsequent computation. This can happen when we encounter unbounded dynamic value that is unknown during tracing time.  You will need to explicitly give hint to the compiler. Please take a look at torch._check OR torch._check_is_size APIs.  Zconstrain_as_size_example)Z	case_nameÚargumentz
TypeError z: )r  )?Ztorch.utils._sympy.value_rangesr|  rÄ   r}  r1  r~  r  r€  rÔ   rû  r    r{  rm   rn   rŸ   rí   rb   r  ræ  r+  rü  rî   r„  Ú
nn_modulesÚtargetr%  rh  Z_infer_parametersr#   r)  rw  Ú	__cause__r!  r  ZDataDependentOutputExceptionrw   ZDynamicOutputShapeExceptionÚ_dynamor   Z capture_dynamic_output_shape_opsZUnsupportedOperatorExceptionr  r  rƒ  Z_dispatch_pystubZ_schemarF  Zoverload_namer"   rþ  rÿ  ZGuardOnDataDependentSymNodeZCONSTRAINT_VIOLATIONrÚ   r>   Úwith_tracebackÚ__traceback__ÚpytreeZtree_map_onlyr  r
  Úpartialrs  )r   r  rw  r|  r}  r1  r~  r  r€  rÔ   Zret_valrv   ÚcauseZimport_suggestionZmaybe_pystubr¡  ÚctxrÁ   r3   r…  r4   ru  É  s¬    	
 
 ÿ&"ÿ ÿÿ ÿÿÿ ÿ ÿ
ÿÿ 
ÿú(  ÿru  c                   C   s   t tdd ƒS )Nr[   )rž  Ú_current_noder3   r3   r3   r4   Úget_current_nodeI  s    r“  c                 c   s$   t ƒ }| t_z
d V  W 5 |t_X d S r/   )r“  r’  r[   )r   Úoldr3   r3   r4   Úset_current_nodeM  s
    
r•  c                    s’  ˆj ‰tˆƒ��p ‡ ‡‡‡fdd„}zÚˆdkrHˆjˆ ˆŽW W  5 Q R £ S ˆdkr|tˆ d ˆjƒˆ dd… ˆŽW W  5 Q R £ S ˆdkr¨|dk	s�t‚|ˆ ˆŽW W  5 Q R £ S ˆd	krÌ| j ˆj¡W W  5 Q R £ S ˆd
k� rüdˆjksät‚ˆjd W W  5 Q R £ S W n| tt	fk
�rB } z ddl
m} |||ƒ|d� W 5 d}~X Y n: tk
�rz } zt||ƒƒ |j¡|‚W 5 d}~X Y nX W 5 Q R X tˆƒ‚dS )aÄ  
    Runs a given node, with the given args and kwargs.

    Behavior is dictated by a node's op.

    run_node is useful for extracting real values out of nodes.
    See get_real_value for more info on common usage.

    Note: The tracer arg is only used for 'get_attr' ops
    Note: The nnmodule arg is only used for 'call_module' ops

    Nodes that are not call_function, call_method, call_module, or get_attr will
    raise an AssertionError.
    c              	      s(   dˆ› dˆj › dˆ › dˆ› d�	t| ƒ S )NzFailed running rU   z(*z, **z):
)r‰  r>   )rv   ©rm   rn   r   rÔ   r3   r4   Úmake_error_messagej  s    z$run_node.<locals>.make_error_messager  r�  r   r   Nr‚  Zget_attrÚplaceholderrú  r0  r3  )rÔ   r•  r‰  rž  r(  Úoutput_graphZget_submodulerû  r4  r&  rÄ   r1  rk   rw  rŒ  r�  )Útracerr   rm   rn   r†  r—  rv   r1  r3   r–  r4   rƒ  W  s4    ,
 ÿþrƒ  c           
   
      sþ   ddl m} ˆ j}| |kr"||  S | j}tjj | j| j	f‡ fdd„¡\}}|dkrjd| j
krj| j
d jS |dkr ˆ jj| j }t|ƒs”t |¡}q¤|||Ž nd}ztˆ | |||ƒ}||| < W n8 tk
rø }	 z|t|	ƒƒ |	j¡d‚W 5 d}	~	X Y nX |S )	z‘
    Run the actual computation represented by `node` and return the result.
    This will execute any dependent nodes in the graph as well.
    r   )r}  c                    s
   t | ˆ ƒS r/   )Úget_real_valuert  ©rš  r3   r4   r5   –  r6   z get_real_value.<locals>.<lambda>r˜  Zgraphargr‚  N)rÄ   r}  Zreal_value_cacherÔ   rb   r"   r   rz  rm   rn   rû  Zexampler™  rˆ  r‰  r%  ri  r*  rƒ  rw  r>   rŒ  r�  )
r   rš  r}  ÚcacherÔ   rm   rn   Z	nn_moduleÚ
real_valuerv   r3   rœ  r4   r›  ˆ  s.    

þ(r›  c                    s„   ddl m‰ m} ‡ fdd„}|  ¡ D ](\}}||ƒr$td|› d||ƒ› �ƒ‚q$|  ¡ D ](\}}||ƒrVtd|› d||ƒ› �ƒ‚qVd S )Nr   )ÚFakeTensorConfigr    c                    s(   ˆ j r dd l}d| | j¡› �S dS d S )Nr   z"FAKE TENSOR CREATION TRACEBACK: 
 zNEnable TORCH_FAKE_TENSOR_DEBUG=1 to get creation stack traces on fake tensors.)ÚdebugrY  Úformat_listZ_debug_trace)rï  rY  ©rŸ  r3   r4   Ústack_or_hint²  s    z7assert_no_fake_params_or_buffers.<locals>.stack_or_hintzUnexpected fake buffer rU   zUnexpected fake param )Útorch._subclasses.fake_tensorrŸ  r    Znamed_buffersr(  Znamed_parameters)r·  r    r£  rF  Úbufferr¯  r3   r¢  r4   Ú assert_no_fake_params_or_buffers¯  s    ÿþÿþr¦  r–  c                 C   s   | j › d| j› �S )z9
    Returns the fully qualified name of the object.
    r  )r¤   ra   r–  r3   r3   r4   ÚfqnÄ  s    r§  c                 C   s   t jjjr| S |S d S r/   )rb   r‹  r   Zassume_static_by_default)Zcount1Zcount2r3   r3   r4   ÚifdynstaticdefaultË  s    
r¨  r#  c                 C   s\   t t tj tt| jƒ¡¡ƒD ]8}| d¡r|d dkrt	 
| j› d|dd… › �¡ qdS )z@
    Ensure all the files in a given submodule are imported
    z.pyr   rÁ   r  Néýÿÿÿ)r@  r§   Úlistdirrµ   rÊ   r   r>   Ú__file__ÚendswithÚ	importlibÚimport_moduler£   )r$  rÎ   r3   r3   r4   Úimport_submoduleÒ  s    "r¯  rú   c                 C   s<   z"t t t| ƒd¡tjƒr W dS W n tk
r6   Y nX dS )NÚ__getattribute__TF)rí   r  r  rl   r  r  ÚAttributeErrorrú   r3   r3   r4   Úobject_has_getattributeÛ  s    þ
r²  c                 C   sD   zt  t| ƒd¡}W n tk
r,   d }Y nX |tjjjkr@d }|S )NÚ__getattr__)r  r  rl   r±  rb   r  ræ  r³  )r[   Z
getattr_fnr3   r3   r4   Úget_custom_getattrç  s    
r´  c                   @   s   e Zd ZdZdZdZdS )ÚTensorStaticReasonr   r;  rW   N)r£   r¤   ra   Ú	PARAMETERÚ
NOT_TENSORÚNN_MODULE_PROPERTYr3   r3   r3   r4   rµ  ò  s   rµ  ©r'  c                 C   s<   | t jkrdS | t jkrdS | t jkr*dS td| › �ƒ‚d S )Nz>mark_dynamic on parameter, parameters are always static today.z2mark_dynamic on a non tensor, how did this happen?z4tensor is static because it is nn module associated.zIllegal reason )rµ  r¶  r·  r¸  r(  r¹  r3   r3   r4   Útensor_static_reason_to_messageø  s    


rº  ztorch._guards.GuardSource)r8  Ú	is_tensorÚguard_sourcerH  c                 C   sJ   |  ¡ rtjrdtjfS t| ƒtjjkr8tj	r8dtj
fS |sFdtjfS dS )a@  
    Given a tensor, source, and is_tensor flag, determine if a shape should be static.

    Args:
    tensor - the real tensor to evaluate, parameters force a static shape.
    is_tensor - internal dynamo check, essentially "is_tensor": target_cls is TensorVariable,
    tensors not in a TensorVariable for whatever reason are forced static.

    Returns a tuple, where the first element is the bool of whether or not this tensor should have a static shape.
    The second element is a TensorStaticReason, useful for passing to tensor_static_reason_to_message if needed.
    T)FN)Zis_nn_moduler   Z&force_nn_module_property_static_shapesrµ  r¸  rl   rb   r  r   Zforce_parameter_static_shapesr¶  r·  )r8  r»  r¼  r3   r3   r4   Útensor_always_has_static_shape  s    


r½  c                    s   ‡ ‡fdd„}t |ƒS )Nc                     sr   zddl m }  W n& tk
r6   dttˆ ˆƒƒ  Y S X dd„ ˆjjD ƒ}| |dddd	d
gd�}tˆ ˆjjj	|ƒS )Nr   )rB   zkTabulate module missing, please install tabulate to log the graph in tabular format, logging code instead:
c                 S   s$   g | ]}|j |j|j|j|jg‘qS r3   )rÔ   rF  r‰  rm   rn   )r?   rÖ   r3   r3   r4   rˆ   %  s    z<lazy_format_graph_tabular.<locals>.inner.<locals>.<listcomp>ÚopcoderF  r‰  rm   rn   r8   )
rB   rC   r>   r(   rV  rÓ   r'   r`   Ú__code__r8  )rB   Z
node_specsZ	graph_str©Úfn_namer·  r3   r4   Úinner  s    ÿÿÿ ÿz(lazy_format_graph_tabular.<locals>.innerr   )rÁ  r·  rÂ  r3   rÀ  r4   Úlazy_format_graph_tabular  s    rÃ  c                 C   s,   | › d|› d|› d|› dt  |¡  ¡ › d�
S )NrU   z line z 
r:   )ÚdisÚBytecode)ÚprefixrF  rÎ   Zline_norÅ   r3   r3   r4   Úformat_bytecode0  s    rÇ  Z_forward_pre_hooksZ_forward_hooksZ_backward_pre_hooksZ_backward_hooksZ_state_dict_pre_hooksZ_state_dict_hooksZ_load_state_dict_pre_hooksZ_load_state_dict_post_hooksc                   C   s    t tjjjjƒpt tjjjjƒS r/   )rŸ   rb   r  Úmodulesr¡  Z_global_backward_hooksZ_global_backward_pre_hooksr3   r3   r3   r4   Únn_module_has_global_hooks?  s    
ÿrÉ  c                 C   sŽ   t jjjj}g }| o | o | }|s*|r4| t¡ |s<|rF| t¡ |rT| t¡ g }|D ],}t	| |g ƒ}	|	D ]}
|	|
 }| 
|¡ qpq\|S r/   )rb   rƒ  r‹  Ú
eval_frameÚ
reset_codeÚextendÚforward_hook_namesÚbackward_hook_namesÚstate_dict_hook_namesrž  rj   )r$  Úcheck_forward_hooksÚcheck_backward_hooksÚcheck_state_dict_hooksrË  Zhook_dicts_to_checkZcheck_all_hooksZ	all_hooksZhook_dict_nameÚhooksÚ	hook_namerX  r3   r3   r4   Únn_module_get_all_hooksG  s(    ÿý


rÕ  c                 C   s   t | |||d�}t|ƒS )zL
    Helper function to check if a module has any hooks attached to it.
    )rÐ  rÑ  rÒ  )rÕ  r>  )r$  rÐ  rÑ  rÒ  rÓ  r3   r3   r4   Únnmodule_has_hooksi  s    	ürÖ  c                 C   sf   t | ƒr| S t| tjƒr"t| jƒS t| tjƒr:| jdd�S t| t	t
fƒr^t| ƒdd„ | D ƒƒS | S dS )z0Convert tensor and tnp.ndarray to numpy.ndarray.T)Úforcec                 s   s   | ]}t |ƒV  qd S r/   )Úto_numpy_helper©r?   rï   r3   r3   r4   rA   „  s     z"to_numpy_helper.<locals>.<genexpr>N)r    rí   Útnpr  rØ  r8  rb   r  Únumpyrî   rŒ   rl   rú   r3   r3   r4   rØ  {  s    
rØ  c                 C   s`   t dk	st‚t| t jƒr"t | ¡S t| tjƒr4| jS t| tt	fƒrXt
| ƒdd„ | D ƒƒS | S dS )zeConvert tnp.ndarray to tensor, leave other types intact. If a list/tuple, loop through it to convert.Nc                 s   s   | ]}t |ƒV  qd S r/   )Únumpy_to_tensorrÙ  r3   r3   r4   rA   ‘  s     z"numpy_to_tensor.<locals>.<genexpr>)rý   r(  rí   r  rb   rJ  rÚ  r8  rî   rŒ   rl   rú   r3   r3   r4   rÜ  ‰  s    
rÜ  c                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )Únumpy_to_tensor_wrapperc                 C   s   || _ d| j j | _d S ©NZwrapped_©rÏ   r£   )r—   rÏ   r3   r3   r4   r˜   —  s    z numpy_to_tensor_wrapper.__init__c                 C   s   d| j j› d�S )Nz<Wrapped function <original ú>>rß  r›   r3   r3   r4   Ú__repr__›  s    z numpy_to_tensor_wrapper.__repr__c                 O   s   | j ||Ž}t|ƒS r/   )rÏ   rÜ  ©r—   rm   rn   rZ   r3   r3   r4   rS  ž  s    z numpy_to_tensor_wrapper.__call__N)r£   r¤   ra   r˜   rá  rS  r3   r3   r3   r4   rÝ  –  s   rÝ  c                 C   sF   t | tjƒrt| |ƒ}t|ƒS t | tjƒrBtt | ¡|ƒ}t|ƒS d S r/   )rí   rÚ  r  rž  rÜ  rb   r  )rï   rF  rZ   r3   r3   r4   Únumpy_attr_wrapper£  s    
rã  c                   @   s.   e Zd ZdZedœdd„Zdd„ Zdd„ Zd	S )
Únumpy_method_wrapperzgConvert obj from torch.Tensor to tnp.ndarray and call method. Then convert result back to torch.Tensor.©Úmethodc                 C   s   || _ d| j  | _d S rÞ  )ræ  r£   )r—   ræ  r3   r3   r4   r˜   ¯  s    znumpy_method_wrapper.__init__c                 C   s   d| j › d�S )Nz<Wrapped method <original rà  rå  r›   r3   r3   r4   rá  ³  s    znumpy_method_wrapper.__repr__c                 O   sD   |d }t |tjƒrt |¡}t|| jƒ}||dd … |Ž}t|ƒS ©Nr   r   )rí   rb   r  rÚ  r  rž  ræ  rÜ  )r—   rm   rn   rï   Zmethod_callablerZ   r3   r3   r4   rS  ¶  s    
znumpy_method_wrapper.__call__N)r£   r¤   ra   rì   r>   r˜   rá  rS  r3   r3   r3   r4   rä  ¬  s   rä  c                   @   s6   e Zd ZdZedef dœdd„Zdd„ Zdd	„ Zd
S )Únumpy_operator_wrapperzQImplements dunder methods for tnp.ndarray via functions from the operator library.)rÔ   c                 C   s   || _ d|j› �| _d S rÞ  )rÔ   r£   )r—   rÔ   r3   r3   r4   r˜   Â  s    znumpy_operator_wrapper.__init__c                 C   s   d| j › d�S )Nz<Wrapped operator <original rà  )r£   r›   r3   r3   r4   rá  Æ  s    znumpy_operator_wrapper.__repr__c                 O   s(   |rt ‚dd„ |D ƒ}| j|Ž }t|ƒS )Nc                 s   s(   | ] }t |tjƒrt |¡n|V  qd S r/   )rí   rb   r  rÚ  r  r-  r3   r3   r4   rA   Ì  s    z2numpy_operator_wrapper.__call__.<locals>.<genexpr>)r(  rÔ   rÜ  râ  r3   r3   r4   rS  É  s    ÿ
znumpy_operator_wrapper.__call__N)	r£   r¤   ra   rì   r   r   r˜   rá  rS  r3   r3   r3   r4   rè  ¿  s   rè  c                 C   sÐ   t | tƒs| S | jršg }|  ¡ D ]4}t |tjƒrJ| |jj 	|jj
¡¡ q | |¡ q g }|  ¡ D ]4}t |tjƒrŒ| |jj 	|jj
¡¡ qb| |¡ qbn|  ¡ }|  ¡ }tj||| j| j| jd�}| ¡  |S )N)re  rp  r^  )rí   r   Z_has_symbolic_sizes_stridesrs  rb   ZSymIntrj   r   rý  Z	size_hintÚexprrn  Zempty_stridedre  rp  r^  Zzero_)r~   rs  r  rn  rb  r3   r3   r4   ÚdefakeÓ  s0    
ûrê  c                 C   s   dd l }| |jjjkS rJ   )Ztorch.utils.checkpointr„  Ú
checkpoint)rï   rb   r3   r3   r4   Úis_utils_checkpointó  s    rì  c                  K   sB   dd l m  m} ddlm} |j}tjjj	r4|j
}|j|f| ŽS )Nr   r   )Ú TorchHigherOrderOperatorVariable)Ztorch._higher_order_ops.wrapZ_higher_order_opsÚwrapZvariables.higher_order_opsrí  Ztag_activation_checkpointrb   Z
_functorchr   Zfunctionalize_rng_opsZwrap_activation_checkpointÚmake)ÚoptionsZhigher_order_opsrí  Zactivation_checkpoint_opr3   r3   r4   Úbuild_checkpoint_variableú  s    ÿ
ÿþrñ  c                 C   s8   ddl m} |ƒ }| dkrn| dkr0|r0tƒ }nd}|S )Nr   )Úis_dynamo_supportedrC  r‡  F)rÊ  rò  r*   )Zdevice_typerò  Zcompile_supportedr3   r3   r4   Úis_compile_supported	  s    ró  )r>   ÚoffsetrH  c                 C   s$   |   d¡}t|d|… jddd�ƒS )zê
    Convert byte offset `offset` of `str` into character offset.
    Byte offset is used for 3.11+ instruction column data.
    Takes things like unicode characters into consideration.

    Unchanged from CPython implementation.
    zutf-8Nr#  )Úerrors)ÚencoderŸ   Údecode)r>   rô  Zas_utf8r3   r3   r4   Ú_fix_offset	  s    
rø  c                   @   s.   e Zd ZU eed< eed< eed< eed< dS )Ú_AnchorsÚleft_end_linenoÚleft_end_offsetÚright_start_linenoÚright_start_offsetNr¦  r3   r3   r3   r4   rù  )	  s   
rù  )ÚsegmentrH  c                    s   t jdkst‚ddl}z| d|  d ¡}W n tk
rB   Y dS X t|jƒdkrVdS |  d¡‰ ‡ fdd	„}‡ fd
d„‰‡ ‡fdd„}‡ ‡fdd„}|jd }t	||j
ƒ�rœ|j}t	||jƒ�rˆtt|jjƒd }|||jjƒ}	ˆ||	ƒ\}}	ˆ | |	  }
 ¡ �s|
dk�r8|
dk�r(|||	ƒ\}}	qî|||	ƒ\}}	qî|	d }|tˆ | ƒk �rzˆ | |  }
 ¡ �sz|
dk�rz|d7 }t||	||ƒS t	||jƒ�rtt|jjƒd }|||jjƒ}ˆ||ƒ\}}ˆ | | dk�rè|||ƒ\}}�qÄtt|jƒd }|||jƒ}t||||ƒS t	||jƒ�rœtt|jjƒd }|||jjƒ}ˆ||ƒ\}}ˆ | | dk�rr|||ƒ\}}�qNtt|jƒd }|||jƒ}t||||ƒS dS )a  
    Given source code `segment` corresponding to a bytecode
    instruction, determine:
        - for binary ops, the location of the binary op
        - for indexing, the location of the brackets.
    `segment` is expected to be a valid Python expression
    )rò   é   r   Nz(
z
)r   r:   c                    s   t ˆ |  |ƒS r/   )rø  )Úlinenorô  ©Úlinesr3   r4   Ú	normalizeM	  s    z-_extract_anchors_from_expr.<locals>.normalizec                    sR   | t ˆ ƒk r*|t ˆ |  ƒkr*d}| d7 } q | t ˆ ƒk rF|t ˆ |  ƒk sJt‚| |fS rç  ©rŸ   r(  ©r   Úcolr  r3   r4   Únext_valid_charR	  s
    
 z3_extract_anchors_from_expr.<locals>.next_valid_charc                    s>   |d7 }ˆ| |ƒ\} }| t ˆ ƒk r2|t ˆ |  ƒk s6t‚| |fS rG   r  r  ©r  r  r3   r4   Ú	incrementZ	  s     z-_extract_anchors_from_expr.<locals>.incrementc                    sB   d}| d7 } ˆ| |ƒ\} }| t ˆ ƒk r6|t ˆ |  ƒk s:t‚| |fS rç  r  r  r  r3   r4   Únextlinea	  s
     z,_extract_anchors_from_expr.<locals>.nextliner   z)\#z\#r  r  )rô   rõ   r(  ÚastÚparseÚSyntaxErrorrŸ   ÚbodyrM  rí   ZExprr[   ZBinOpr   r=  ÚleftÚ
end_linenoÚend_col_offsetÚisspacerù  Z	SubscriptZCallrw   )rþ  r  Útreer  r	  r
  Z	statementré  Z
cur_linenoZcur_colÚchZ	right_colZleft_linenoZleft_colZright_linenor3   r  r4   Ú_extract_anchors_from_expr3	  sh    

 
ÿþýr  )rÅ   ÚinstrH  c                 C   s@  |j dk	st‚|j jdkrdS t | j|j j¡ ¡ }|j jdkrD|S |j jdks\|j j	dkr`|S t
||j jƒ}d}d}g }|j j|j jkrÀt
||j j	ƒ}|||… }| d| d||   ¡ nþ||d… d }| d| dt|ƒ|   ¡ t | j|j j¡ ¡ }t
||j j	ƒ}t|j jd |j jƒD ]V}t | j|¡ ¡ }	||	d 7 }t|	ƒt|	 ¡ ƒ }
| d|
 dt|	ƒ|
   ¡ �q(||d|… 7 }t|ƒt| ¡ ƒ }
| d|
 d||
   ¡ d}zt|ƒ}W n tk
�rä   Y nX |dk�r dd„ |D ƒ}nòd	d„ |D ƒ}|jd
k�r(| j|7  _|jd
k�rB| j|7  _tt|ƒƒD ]”}tt|| ƒƒD ]|}||jk �rv�qb||jk�r’||jk �r’�qb||jk�r®||jk�r®�qb||jk�r¾�qb|| | dk�rbd|| |< �qb�qNdd„ |D ƒ}d}tt|ƒƒD ]8}|t | j|j j| ¡ ¡ d 7 }||| d 7 }�q|S )a¾  
    Python 3.11+ only. Returns lines of source code (from code object `code`)
    corresponding to `inst`'s location data, and underlines relevant code to `inst`.

    Example: CALL on `g`:
    f(g(
      ^^
        h(x)))
        ^^^^^

    We need our own implementation since `format_frame_summary` in
    Python's `traceback` module doesn't handle multi-line expressions
    (and their anchor extraction code is not completely correct).
    Nr  rU   ú~r:   r   c                 S   s   g | ]}|  d d¡‘qS )r  ú^)r#  ©r?   Úmarkerr3   r3   r4   rˆ   ï	  s     z.get_instruction_source_311.<locals>.<listcomp>c                 S   s   g | ]}t |ƒ‘qS r3   )rŒ   r  r3   r3   r4   rˆ   ò	  s     r   r  c                 S   s   g | ]}d   |¡‘qS )r  r$  r  r3   r3   r4   rˆ   
  s     )Z	positionsr(  r   Ú	linecacheÚgetliner8  Úrstripr  Ú
col_offsetr  rø  rj   rŸ   rz  Úlstripr  rú  rû  rü  rý  )rÅ   r  Ú
first_lineZstart_offsetÚ
end_offsetrþ  ÚmarkersÚ	last_liner   ÚlineZ
num_spacesZanchorsZmutable_markersr  rx  r}  r3   r3   r4   Úget_instruction_source_311¬	  sˆ     ÿ"
ÿþÿÿr%  c                 C   s   t | tjƒrt| dd ƒS d S )NZ_dynamo_static_input_type)rí   rb   r  rž  r9  r3   r3   r4   Úget_static_address_type
  s    r&  c                 C   sD   t jjjt jjt jt jjf}t jjjt jjt jt jjf}| ||˜kS r/   )	rb   rƒ  Ú	GeneratorZ	get_stateZdefault_generatorr†  r‡  Z	set_stater‰  )r[   ZgettersZsettersr3   r3   r4   Úis_rng_state_getter_or_setter
  s    ûür(  c                 C   s&   t | tjƒo$| jdko$| jjtjjkS )NÚ__get__)	rí   r  r   r£   Ú__self__Ú__objclass__rb   rƒ  Z_TensorBaserú   r3   r3   r4   Úis_tensor_base_attr_getter0
  s
    ÿýr,  c                 C   s
   t | dƒS )NÚ__torch_function__)rh  rú   r3   r3   r4   Úis_torch_function_object8
  s    r.  z,torch._dynamo.variables.base.VariableTracker)r  rH  c                 C   sT   ddl m}m} ddlm} t| |ƒr*dS t| |ƒr>| | ¡ t| |ƒoRt| jdƒS )Nr   )ÚLazyVariableTrackerÚUserDefinedObjectVariable)ÚTensorWithTFOverrideVariableTr-  )	Ztorch._dynamo.variablesr/  r0  Z&torch._dynamo.variables.torch_functionr1  rí   Zrealizerh  r[   )r  r/  r0  r1  r3   r3   r4   Úhas_torch_function<
  s    


 ÿr2  c                 C   sD   d }d }t jj ¡  }r2| |jkr2|j|  }|j}|j| d||d�S )NF)Zstatic_shapesÚsymbolic_contextÚsource)rb   rc   r$   rd   Ztensor_to_contextZtensor_sourceZfrom_tensor)rï  rü  r3  r4  Ztracing_contextr3   r3   r4   Úto_fake_tensorL
  s    

   ÿr5  c                 G   s8   |D ]}t | |ƒrt| |ƒ  S qt| › d|› �ƒ‚dS )zX
    Return the first available attribute or throw an exception if none is present.
    z% does not has any of the attributes: N)rh  rž  r(  )rï   ÚattrsÚattrr3   r3   r4   Úget_first_attrY
  s    
r8  c              	   c   s8   dd„ }| r.t jj |¡�}|V  W 5 Q R X nd V  d S )Nc                 S   s   dd„ }t j| |ddd�S )Nc                 S   s(   t jjjd d  d7  < t j | |¡S )NÚcompiled_autogradZcompilesr   )rb   r‹  r„  r,   Z	_inductorÚcompile)Zgm_Zexample_inputs_r3   r3   r4   Úinner_compilerg
  s    zKmaybe_enable_compiled_autograd.<locals>.compiler_fn.<locals>.inner_compilerT)ÚbackendZ	fullgraphZdynamic)rb   r:  )r·  r;  r3   r3   r4   Úcompiler_fnf
  s    z3maybe_enable_compiled_autograd.<locals>.compiler_fn)rb   r‹  r9  Úenable)Zshould_enabler=  r‘  r3   r3   r4   Úmaybe_enable_compiled_autogradd
  s
    r?  c                  C   s   G dd„ dt ƒ} t| ƒ ƒS )Nc                   @   s   e Zd ZdS )z*invalid_removeable_handle.<locals>.InvalidN)r£   r¤   ra   r3   r3   r3   r4   ÚInvalidv
  s   r@  )rà   r   )r@  r3   r3   r4   Úinvalid_removeable_handlet
  s    rA  c                 C   s>   t | tjjƒs| S t | tjjƒr$| S | j | j¡}| j|_|S r/   )	rí   rb   r  ræ  r"   rj  rZ  Ú__new__rK  )r$  Úproxyr3   r3   r4   Únn_module_proxy€
  s    rD  c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )Ú	GmWrapperc                    s   t ƒ  ¡  || _|| _d S r/   )rW  r˜   r·  Úspec)r—   r·  rF  rY  r3   r4   r˜   Œ
  s    
zGmWrapper.__init__c                 G   s   t |ƒ}| jt || j¡Ž S r/   )rŒ   r·  rŽ  Ztree_unflattenrF  rR  r3   r3   r4   r`   ‘
  s    zGmWrapper.forward)r£   r¤   ra   r˜   r`   r[  r3   r3   rY  r4   rE  ‹
  s   rE  rU  c                    sF   t  |¡\}}|t| |ƒ|ƒ‰ dd„ t| jjƒD ƒ‰‡ ‡fdd„}|S )z—
    Mutate inputs so that they are flat and wrap gm such that it
    accepts those inputs.  This is needed for graphs that take
    bumpy inputs.
    c                 S   s,   g | ]$\}}|j d kr|j dd¡r|‘qS )r˜  Z	steal_argF)rÔ   rû  re   )r?   r}  r   r3   r3   r4   rˆ   Ÿ
  s   
 þz(flatten_graph_inputs.<locals>.<listcomp>c                     s(   t j| Ž }ˆD ]}| |  ¡  qˆ |ƒS r/   )rŽ  Zarg_tree_leavesrK   )rm   Z	flat_argsr}  ©Zcompiled_fnZidx_to_stealr3   r4   Úwrapper¥
  s    
z%flatten_graph_inputs.<locals>.wrapper)rŽ  Ztree_flattenrE  Ú	enumeraterV  rÓ   )r·  ÚinputsZ
compile_gmrF  rH  r3   rG  r4   Úflatten_graph_inputs–
  s    
þrK  c                 C   s*   t | tjjƒrt| dƒsg S | j dg ¡S )Nrû  Úlocals_to_steal)rí   rb   r"   rj  rh  rû  re   )Zmaybe_gmr3   r3   r4   Úget_locals_to_steal³
  s    rM  c                 C   s   || j d< d S )NrL  )rû  )r·  rL  r3   r3   r4   Úset_locals_to_steal¹
  s    rN  c                   @   s   e Zd Zdd„ Zdd„ ZdS )ÚLitc                 C   s
   || _ d S r/   ©r  )r—   r  r3   r3   r4   r˜   ¾
  s    zLit.__init__c                 C   s   | j S r/   rP  r›   r3   r3   r4   rá  Á
  s    zLit.__repr__N)r£   r¤   ra   r˜   rá  r3   r3   r3   r4   rO  ½
  s   rO  Úwarn_once_cachec                 C   s,   | t krd S t  | ¡ tj| |d d� d S )Nr   )Ú
stacklevel)rQ  r¢   ÚwarningsÚwarn)r(  rR  r3   r3   r4   Ú	warn_onceÈ
  s    
rU  )NNT)r>   F)N)F)r   )F)T)F)FFF)FFF)r   (^  Úatexitr0   rª   ri  rE  rk  rÄ  rÌ  r
  rº  r  rD   r  r¸   rF  ré  r§   r€  rô   re  Ú	threadingri   r  r÷   rS  rê   r   r   r   r   r   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   Zutils.hooksr   rÛ  rý   ÚModuleNotFoundErrorZtorch._loggingrb   Ztorch._numpyZ_numpyrÚ  Ztorch._guardsr   r   r  r   ZfftZlinalgr…  r   Ú
ModuleTyper<  ZNP_TO_TNP_MODULErî   r¤  r   r    r!   rC   r­  Ztorch._functorch.configZ%torch.fx.experimental.symbolic_shapesZtorch.utils._pytreer„  Z_pytreerŽ  r"   Ztorch._dispatch.pythonr#   r$   Ztorch._subclasses.meta_utilsr%   Ztorch._utils_internalr&   Ztorch.fx._utilsr'   r(   Ztorch.nn.modules.lazyr)   Ztorch.utils._tritonr*   r+   r1   r,   r>   r-   Ztroubleshooting_urlZnnmodule_doc_urlZnnmodule_doc_url_msgrº   r£   r�   r.   r2   r7   rQ  Ztimer_counterrB   rH   rI   rL   rM   rN   rT   r\   r_   r|   r�   Úregisterr’   ZFloatTensorr  ZDoubleTensorr  ÚdoubleZ
HalfTensorr  ZhalfZBFloat16TensorrH  Z
ByteTensorr  Z
CharTensorrþ   Z
LongTensorr  ÚlongZ	IntTensorr   r=  ZShortTensorrÿ   ÚshortZ
BoolTensorr>  Ztensortype_to_dtyper“   r­   r¬   r®   r©   rÀ   rÆ   rÐ   ZGraphr×   rØ   rÜ   rÞ   rß   rð   rõ   ZParamSpecArgsZParamSpecKwargsZ	ParamSpecÚTypeVarZTypeVarTupleZTypeAliasTyperö   rû   r  r  r  r	  r  r  r  r"  r%  r&  r*  r6  r©  r7  rg   Z!DEFAULT_COMPILATION_METRICS_LIMITrJ  rB  rh   rK  rL  rM  rN  rP  rV  rc  rg  ry  r  r‚  rŒ  rŽ  r“  r”  r˜  r—  r­  r¸  r¾  rÁ  rÉ  ÚcomplexÚbytesrl   ÚEllipsisrZ  ÚCodeTyperp  re  Zmemory_formatrl  rÎ  Ztritonr¢   ÚlanguagerË  rÒ  rÕ  r×  rÛ  rÜ  rà  rà   r�   rá  rO   râ  r™   rã  ró  rä  Ú__length_hint__Ztuple_iterator_lenÚobjectrB  rå  rç  rì  rî  rñ  Z	iter_nextrð  rõ  rù  r  r  r  r  r  r  r%  ZGLOBAL_KEY_PREFIXZtorch._subclassesr&  r)  r+  r-  r‘  r0  rN  rP  Zorig_code_maprŒ   rQ  rR  Zseen_code_maprS  rp  r¶   rr  rs  r{  ru  rø  r’  r“  r•  rƒ  r›  r¦  r§  r¨  r¯  r²  r´  rÍ  rµ  rº  r½  rÃ  rÇ  rÍ  rÎ  rÏ  Zall_hook_namesrÉ  rÕ  rÖ  rØ  rÜ  rÝ  rã  rä  rè  rê  rì  rñ  ró  rø  rù  r  ÚInstructionr%  r&  r(  r,  r.  r2  r5  r8  r?  rA  rD  r  ræ  rE  rj  rK  rM  rN  rO  rš   rQ  rU  r3   r3   r3   r4   Ú<module>   s¢   L
ü    ü ÿ
ÿ
P$
 
 
 
    
 
 
 ö	!ú
	
 
þ
ÿ
ÿ
J



ó	
	 





	)

!



ö m
"c


}
	
1
'
	üü
  ü$  ü	 	y l 