U
    8‹Ñh†b  ã                   @   s¢  U d dl Z d dlmZmZmZ ddlmZ d dlmZm	Z	m
Z
mZmZmZmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlZddgZe jjZdd„ Zi Ze
e	e	f ed< dd„ Z dMdd„Z!e!ej"ƒddœe#dœdd„ƒZ$e!ej%ƒdNe#dœdd„ƒZ&e!ej'ƒdOe#dœdd„ƒZ(e!ej)ƒdPe#dœdd„ƒZ*dQee# ee# ee# e+e#dœdd „Z,e!ej-ej.gƒddœe#dœd!d"„ƒZ/e!ej0ƒe#dœd#d$„ƒZ1d%d&„ Z2e!ej3ej4gƒddœe#dœd'd(„ƒZ5dd)œeeee#d*f ee#d*f ee#d*f eee#d*f  f  dœd+d,„Z6dd)œeeee#d*f ee#d*f ee#d*f eee#d*f  f  dœd-d.„Z7e!ej8d/d0�ddœe#dœd1d2„ƒZ9e!ej:d/d0�e#dœd3d4„ƒZ;d5d6„ Z<e!ej=ej>gƒddœe#dœd7d8„ƒZ?e!ej@d/d0�e#dœd9d:„ƒZAe!ejBd/d0�e#dœd;d<„ƒZCej"e$ej%e&ej'e(ej)e*ej-e/ej.e/ej0e1ej3e5ej4e5ej=e?ej>e?ej8e9ej:e;ej@eAejBeCiZd=d>„ ZDd?d@dAdBdCgZEdDdE„ ZFdFdG„ ZGdHdI„ ZHdJdK„ ZIG dLd„ deƒZJdS )Ré    N)Útree_mapÚtree_flattenÚtree_unflattené   )ÚModuleTracker)ÚListÚAnyÚDictÚOptionalÚUnionÚTupleÚIterator)Údefaultdict)ÚTorchDispatchMode)Úregister_decomposition©Úprod©ÚwrapsÚFlopCounterModeÚregister_flop_formulac                 C   s   t | tjƒr| jS | S ©N)Ú
isinstanceÚtorchZTensorÚshape)Úi© r   úF/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/utils/flop_counter.pyÚ	get_shape   s    r   Úflop_registryc                    s   t ˆ ƒd dœ‡ fdd„
ƒ}|S )N)Úout_valc                    s(   t t||| fƒ\}}}ˆ |d|i|—ŽS )NÚ	out_shape)r   r   )r    ÚargsÚkwargsr!   ©Úfr   r   Únf   s    zshape_wrapper.<locals>.nfr   ©r%   r&   r   r$   r   Úshape_wrapper   s    r(   Fc                    s   ‡ ‡fdd„}|S )Nc                    s"   ˆ st | ƒ} tˆtdd�| ƒ | S )NT)ÚregistryZunsafe)r(   r   r   )Zflop_formula©Úget_rawÚtargetsr   r   Úregister_fun"   s    z+register_flop_formula.<locals>.register_funr   )r,   r+   r-   r   r*   r   r   !   s    )r!   ©Úreturnc          	      O   s,   | \}}|\}}||kst ‚|| d | S )zCount flops for matmul.é   ©ÚAssertionError)	Úa_shapeÚb_shaper!   r"   r#   ÚmÚkÚk2Únr   r   r   Úmm_flop*   s    r9   c                 K   s
   t ||ƒS )zCount flops for addmm.)r9   ©Z
self_shaper3   r4   r!   r#   r   r   r   Ú
addmm_flop5   s    r;   c                 K   sD   | \}}}|\}}}	||ks t ‚||ks,t ‚|| |	 d | }
|
S )z"Count flops for the bmm operation.r0   r1   )r3   r4   r!   r#   Úbr5   r6   Zb2r7   r8   Úflopr   r   r   Úbmm_flop:   s    

r>   c                 K   s
   t ||ƒS )z&Count flops for the baddbmm operation.)r>   r:   r   r   r   Úbaddbmm_flopG   s    r?   )Úx_shapeÚw_shaper!   Ú
transposedr/   c           
      C   sJ   | d }|r| n|dd… }|^}}}t |ƒt |ƒ | | | d }	|	S )a  Count flops for convolution.

    Note only multiplication is
    counted. Computation for bias are ignored.
    Flops for a transposed convolution are calculated as
    flops = (x_shape[2:] * prod(w_shape) * batch_size).
    Args:
        x_shape (list(int)): The input shape before convolution.
        w_shape (list(int)): The filter shape.
        out_shape (list(int)): The output shape after convolution.
        transposed (bool): is the convolution transposed
    Returns:
        int: the number of flops
    r   r0   Nr   )
r@   rA   r!   rB   Z
batch_sizeZ
conv_shapeZc_outZc_inZfilter_sizer=   r   r   r   Úconv_flop_countO   s
    
 rC   c          
      O   s   t | |||d�S )zCount flops for convolution.©rB   )rC   )
r@   rA   Ú_biasÚ_strideÚ_paddingÚ	_dilationrB   r!   r"   r#   r   r   r   Ú	conv_flopv   s    rI   c                 C   s’   dd„ }d}|
d r4t |d ƒ}|t| ||| ƒ7 }|
d rŽt |d ƒ}|rn|t|| ƒ||ƒ||ƒdd�7 }n |t||ƒ|| ƒ||ƒdd�7 }|S )Nc                 S   s    | d | d gt | dd … ƒ S )Nr   r   r0   )Úlist)r   r   r   r   Út‹   s    zconv_backward_flop.<locals>.tr   r   FrD   )r   rC   )Úgrad_out_shaper@   rA   rE   rF   rG   rH   rB   Z_output_paddingZ_groupsZoutput_maskr!   rK   Ú
flop_countZgrad_input_shapeZgrad_weight_shaper   r   r   Úconv_backward_flop|   s    H" rN   c                 C   s¼   | \}}}}|\}}}	}
|\}}}}||  kr8|krln n0||  krP|krln n||
krl|	|krl||
kspt ‚d}|t|| ||f|| ||	fƒ7 }|t|| ||	f|| |	|fƒ7 }|S )z^
    Count flops for self-attention.

    NB: We can assume that value_shape == key_shape
    r   ©r2   r>   )Úquery_shapeÚ	key_shapeÚvalue_shaper<   ÚhÚs_qÚd_qÚ_b2Ú_h2Ús_kÚ_d2Ú_b3Ú_h3Ú_s3Úd_vÚtotal_flopsr   r   r   Úsdpa_flop_countä   s    L""r_   c                O   s   t | ||ƒS )úCount flops for self-attention.©r_   )rP   rQ   rR   r!   r"   r#   r   r   r   Ú	sdpa_flopö   s    rb   )Úgrad_out.c                 c   sN  |dk	�r&t |jƒdkst‚t |jƒdks.t‚|dksF|j| jksFt‚| j\}}	}
|j\}}}|j\}}}|dk	svt‚|dk	s‚t‚|j|jks’t‚|dd… |dd…   ¡ }|dd… |dd…   ¡ }t||ƒD ]L\}}d|	||
f}d|||f}d|||f}|dk	�r|nd}||||fV  qÔdS | j|j|j|dk	�rB|jndfV  dS )a;  
    Given inputs to a flash_attention_(forward|backward) kernel, this will handle behavior for
    NestedTensor inputs by effectively unbinding the NestedTensor and yielding the shapes for
    each batch element.

    In the case that this isn't a NestedTensor kernel, then it just yields the original shapes.
    Né   r   éÿÿÿÿ©Úlenr   r2   ÚtolistÚzip)ÚqueryÚkeyÚvaluerc   Ú	cum_seq_qÚ	cum_seq_kÚmax_qÚmax_kÚ_Úh_qrU   Úh_kÚd_kÚh_vr]   Zseq_q_lengthsZseq_k_lengthsZ	seq_q_lenZ	seq_k_lenÚnew_query_shapeÚnew_key_shapeÚnew_value_shapeÚnew_grad_out_shaper   r   r   Ú%_unpack_flash_attention_nested_shapesý   s(    
rz   c                 c   sT  |dk	�r,t |jƒdkst‚t |jƒdks.t‚|dksF|j| jksFt‚| j\}}}	}
|j\}}}}|j\}}}}|dk	s|t‚|dk	sˆt‚|j|jks˜t‚|dd… |dd…   ¡ }|dd… |dd…   ¡ }t||ƒD ]L\}}d|	||
f}d|||f}d|||f}|dk	�r|nd}||||fV  qÚdS | j|j|j|dk	�rH|jndfV  dS )a?  
    Given inputs to a efficient_attention_(forward|backward) kernel, this will handle behavior for
    NestedTensor inputs by effectively unbinding the NestedTensor and yielding the shapes for
    each batch element.

    In the case that this isn't a NestedTensor kernel, then it just yields the original shapes.
    Né   r   re   rf   )rj   rk   rl   rc   Úcu_seqlens_qÚcu_seqlens_kÚmax_seqlen_qÚmax_seqlen_krq   rr   rU   rs   rt   ru   r]   Z	seqlens_qZ	seqlens_kÚlen_qZlen_krv   rw   rx   ry   r   r   r   Ú)_unpack_efficient_attention_nested_shapes+  s(    
r�   T)r+   c             	   O   s(   t | ||||||d�}
tdd„ |
D ƒƒS )r`   )rj   rk   rl   rm   rn   ro   rp   c                 s   s"   | ]\}}}}t |||ƒV  qd S r   ra   ©Ú.0rP   rQ   rR   rq   r   r   r   Ú	<genexpr>u  s   
ÿz0_flash_attention_forward_flop.<locals>.<genexpr>©rz   Úsum)rj   rk   rl   rm   rn   ro   rp   r!   r"   r#   Úsizesr   r   r   Ú_flash_attention_forward_flop[  s    ù	þrˆ   c              	   O   s(   t | ||||||d�}
tdd„ |
D ƒƒS )r`   )rj   rk   rl   r|   r}   r~   r   c                 s   s"   | ]\}}}}t |||ƒV  qd S r   ra   r‚   r   r   r   r„   •  s   
ÿz4_efficient_attention_forward_flop.<locals>.<genexpr>©r�   r†   )rj   rk   rl   Úbiasr|   r}   r~   r   r"   r#   r‡   r   r   r   Ú!_efficient_attention_forward_flop{  s    ù	þr‹   c                 C   sR  d}|\}}}}|\}	}
}}|\}}}}| \}}}}||	  krR|  krR|kr€n n*||
  krt|  krt|kr€n n||ks„t ‚||krœ||krœ||ks t ‚d}|t|| ||f|| ||fƒ7 }|t|| ||f|| ||fƒ7 }|t|| ||f|| ||fƒ7 }|t|| ||f|| ||fƒ7 }|t|| ||f|| ||fƒ7 }|S )Nr   rO   )rL   rP   rQ   rR   r^   r<   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   Z_b4Z_h4Z_s4Z_d4r   r   r   Úsdpa_backward_flop_count›  s    P"""""rŒ   c                O   s   t | |||ƒS )z(Count flops for self-attention backward.©rŒ   )rL   rP   rQ   rR   r!   r"   r#   r   r   r   Úsdpa_backward_flop¶  s    rŽ   c
              
   O   s*   t |||| ||||	d�}tdd„ |D ƒƒS )N)rj   rk   rl   rc   rm   rn   ro   rp   c                 s   s$   | ]\}}}}t ||||ƒV  qd S r   r�   ©rƒ   rP   rQ   rR   rL   r   r   r   r„   Ö  s   
ÿz1_flash_attention_backward_flop.<locals>.<genexpr>r…   )rc   rj   rk   rl   ÚoutZ	logsumexprm   rn   ro   rp   r"   r#   Úshapesr   r   r   Ú_flash_attention_backward_flop»  s    ø
þr’   c
              
   O   s*   t |||| ||||	d�}tdd„ |D ƒƒS )N)rj   rk   rl   rc   r|   r}   r~   r   c                 s   s$   | ]\}}}}t ||||ƒV  qd S r   r�   r�   r   r   r   r„   ÷  s   
ÿz5_efficient_attention_backward_flop.<locals>.<genexpr>r‰   )rc   rj   rk   rl   rŠ   r�   r|   r}   r~   r   r"   r#   r‘   r   r   r   Ú"_efficient_attention_backward_flopÜ  s    ø
þr“   c                 C   s   t | tƒs| fS | S r   )r   Útuple)Úxr   r   r   Únormalize_tuple  s    
r–   Ú ÚKÚMÚBÚTc                 C   s0   t dtttƒd tt| ƒƒd d ƒƒ}t| S )Nr   r   r0   rd   )ÚmaxÚminrg   ÚsuffixesÚstr)ÚnumberÚindexr   r   r   Úget_suffix_str  s    (r¢   c                 C   s&   t  |¡}| d|  d›}|t |  S )Niè  z.3f)rž   r¡   )r    Úsuffixr¡   rl   r   r   r   Úconvert_num_with_suffix  s    
r¤   c                 C   s   |dkrdS | | d›S )Nr   ú0%z.2%r   )ÚnumZdenomr   r   r   Úconvert_to_percent_str&  s    r§   c                    s   t ˆ ƒ‡ fdd„ƒ}|S )Nc                    s   t | ƒ\}}ˆ |Ž }t||ƒS r   )r   r   )r"   Z	flat_argsÚspecr�   r$   r   r   r&   ,  s    z)_pytreeify_preserve_structure.<locals>.nfr   r'   r   r$   r   Ú_pytreeify_preserve_structure+  s    r©   c                       s®   e Zd ZdZdeeejje	ejj f  e
eeeeef  dœdd„Ze
dœd	d
„Zeeeee
f f dœdd„Zddd„Z‡ fdd„Z‡ fdd„Zddd„Zdd„ Z‡  ZS )r   aþ  
    ``FlopCounterMode`` is a context manager that counts the number of flops within its context.

    It does this using a ``TorchDispatchMode``.

    It also supports hierarchical output by passing a module (or list of
    modules) to FlopCounterMode on construction. If you do not need hierarchical
    output, you do not need to use it with a module.

    Example usage

    .. code-block:: python

        mod = ...
        with FlopCounterMode(mod) as flop_counter:
            mod.sum().backward()

    Nr0   T)ÚmodsÚdepthÚdisplayÚcustom_mappingc                 C   s`   t dd„ ƒ| _|| _|| _|d kr&i }|d k	r<tjddd� tdd„ | ¡ D ƒ–| _tƒ | _	d S )Nc                   S   s   t tƒS r   )r   Úintr   r   r   r   Ú<lambda>O  ó    z*FlopCounterMode.__init__.<locals>.<lambda>z<mods argument is not needed anymore, you can stop passing itr0   )Ú
stacklevelc                 S   s*   i | ]"\}}|t |d dƒr|nt|ƒ“qS )Z_get_rawF)Úgetattrr(   ©rƒ   r6   Úvr   r   r   Ú
<dictcomp>X  s      z,FlopCounterMode.__init__.<locals>.<dictcomp>)
r   Úflop_countsr«   r¬   ÚwarningsÚwarnr   Úitemsr   Úmod_tracker)Úselfrª   r«   r¬   r­   r   r   r   Ú__init__I  s    þzFlopCounterMode.__init__r.   c                 C   s   t | jd  ¡ ƒS )NÚGlobal)r†   r¶   Úvalues©r»   r   r   r   Úget_total_flops\  s    zFlopCounterMode.get_total_flopsc                 C   s   dd„ | j  ¡ D ƒS )a  Return the flop counts as a dictionary of dictionaries.

        The outer
        dictionary is keyed by module name, and the inner dictionary is keyed by
        operation name.

        Returns:
            Dict[str, Dict[Any, int]]: The flop counts as a dictionary.
        c                 S   s   i | ]\}}|t |ƒ“qS r   )Údictr³   r   r   r   rµ   i  s      z3FlopCounterMode.get_flop_counts.<locals>.<dictcomp>)r¶   r¹   r¿   r   r   r   Úget_flop_counts_  s    
zFlopCounterMode.get_flop_countsc                    s  |d krˆj }|d krd}dd l}d|_dddg}g }ˆ ¡ ‰ tˆ ƒ‰d‰‡ ‡‡‡fdd	„}tˆj ¡ ƒD ]>}|d
krxqj| d¡d }||kr�qj|||d ƒ}| 	|¡ qjd
ˆjkrðˆsðt
|ƒD ] \}	}
d||	 d  ||	 d< qÀ|d
dƒ| }t|ƒdk�r
d
ddgg}|j||dd�S )Ni?B r   TÚModuleZFLOPz% TotalFc                    sŽ   t ˆj|   ¡ ƒ}ˆ|ˆ kO ‰d| }g }| ||  t|ˆƒt|ˆ ƒg¡ ˆj|   ¡ D ]0\}}| |d t|ƒ t|ˆƒt|ˆ ƒg¡ qX|S )Nú z - )r†   r¶   r¾   Úappendr¤   r§   r¹   rŸ   )Úmod_namer«   r^   Úpaddingr¾   r6   r´   ©Zglobal_flopsZglobal_suffixZis_global_subsumedr»   r   r   Úprocess_mody  s     ýýz.FlopCounterMode.get_table.<locals>.process_modr½   Ú.r   rÄ   Ú0r¥   )ÚleftÚrightrÍ   )ÚheadersZcolalign)r«   ÚtabulateZPRESERVE_WHITESPACErÀ   r¢   Úsortedr¶   ÚkeysÚcountÚextendÚ	enumeraterg   )r»   r«   rÏ   Úheaderr¾   rÉ   ÚmodZ	mod_depthZ
cur_valuesÚidxrl   r   rÈ   r   Ú	get_tablek  s6    
zFlopCounterMode.get_tablec                    s"   | j  ¡  | j ¡  tƒ  ¡  | S r   )r¶   Úclearrº   Ú	__enter__Úsuperr¿   ©Ú	__class__r   r   rÚ   §  s    


zFlopCounterMode.__enter__c                    s0   t ƒ j|Ž  | j ¡  | jr,t|  | j¡ƒ d S r   )rÛ   Ú__exit__rº   r¬   ÚprintrØ   r«   )r»   r"   rÜ   r   r   rÞ   ­  s    
zFlopCounterMode.__exit__r   c                 C   s(   |r|ni }|||Ž}|   |j|||¡S r   )Ú_count_flopsZ_overloadpacket)r»   ÚfuncÚtypesr"   r#   r�   r   r   r   Ú__torch_dispatch__³  s    
z"FlopCounterMode.__torch_dispatch__c                 C   sR   || j krN| j | }|||d|i—Ž}t| jjƒD ]}| j| |  |7  < q2|S )Nr    )r   Úsetrº   Úparentsr¶   )r»   Zfunc_packetr�   r"   r#   Zflop_count_funcrM   Úparr   r   r   rà   ¸  s    

zFlopCounterMode._count_flops)Nr0   TN)N)r   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r
   r   r   ÚnnrÃ   r   r®   Úboolr	   r   r¼   rÀ   rŸ   rÂ   rØ   rÚ   rÞ   rã   rà   Ú__classcell__r   r   rÜ   r   r   5  s$       ûû
<
)F)N)N)N)F)Kr   Ztorch.utils._pytreer   r   r   Zmodule_trackerr   Útypingr   r   r	   r
   r   r   r   Úcollectionsr   Ztorch.utils._python_dispatchr   Ztorch._decompr   Úmathr   Ú	functoolsr   r·   Ú__all__ÚopsZatenr   r   Ú__annotations__r(   r   Úmmr®   r9   Zaddmmr;   Zbmmr>   Zbaddbmmr?   rì   rC   ZconvolutionZ_convolutionrI   Zconvolution_backwardrN   r_   Z'_scaled_dot_product_efficient_attentionZ#_scaled_dot_product_flash_attentionrb   rz   r�   Z_flash_attention_forwardrˆ   Z_efficient_attention_forwardr‹   rŒ   Z0_scaled_dot_product_efficient_attention_backwardZ,_scaled_dot_product_flash_attention_backwardrŽ   Z_flash_attention_backwardr’   Z_efficient_attention_backwardr“   r–   rž   r¢   r¤   r§   r©   r   r   r   r   r   Ú<module>   sÎ    $
	
 üû'ôgû
6ö3û
6ö0
÷õõó ó!               ñ
