U
    D‹ÑhA?  ã                
   @   s&  d dl Z d dlmZmZmZmZmZ d dlZd dlm	Z	 d dl
mZmZ dddgZe  e¡ZdZG d	d
„ d
ƒZG dd„ deƒZee d¡dd„ ƒZd ZG dd„ dƒZG dd„ dƒZdd„ Zdeedf eeeef  eeeedf  eeeef  eee ee f dœdd„Zee dœdd„ZdS )é    N)ÚAnyÚDictÚListÚOptionalÚTuple©Úmap_aggregate)Útree_flattenÚtree_unflattenÚTensorChunkSpecÚsplit_args_kwargs_into_chunksÚmerge_chunksFc                   @   s   e Zd ZdZdd„ ZdS )Ú_CustomReducera$  
    Custom reducer class that can be used to specify a custom operation that
    reduces losses of multiple microbatches into one value.

    Example:
    >>> # xdoctest: +SKIP
    >>> sum_reducer = _CustomReducer(
    >>>     torch.tensor(0.0),
    >>>     lambda a, b: a + b
    >>> )
    c                 C   s   || _ || _d S ©N)Ú
init_valueÚ	reduce_fn)Úselfr   r   © r   úU/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/distributed/pipelining/microbatch.pyÚ__init__(   s    z_CustomReducer.__init__N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r   r      s   r   c                   @   s   e Zd ZdS )Ú_LossReducerN©r   r   r   r   r   r   r   r   -   s   r   g        c                 C   s   | | S r   r   )ÚaÚbr   r   r   Ú<lambda>1   ó    r   c                   @   sf   e Zd ZU dZdd„ Zeed< dd„ Zdd„ Ze	e
ed	f d
œdd„ƒZe	eeef d
œdd„ƒZdS )r   z2
    Class used to specify chunking of inputs
    c                 C   s
   || _ d S r   ©Ú	split_dim)r   r!   r   r   r   r   =   s    zTensorChunkSpec.__init__r!   c                 C   s    | j j› d| j j› d| j› d�S )NÚ.ú(ú))Ú	__class__r   r   r!   ©r   r   r   r   Ú__repr__B   s    ÿzTensorChunkSpec.__repr__c                 C   s   d| j › d�S )NzTensorChunkSpec(r$   r    r&   r   r   r   Ú__str__G   s    zTensorChunkSpec.__str__.)Ú
chunk_dimsc                 C   s   t | dd„ ƒ}|S )aŠ  
        A helper for creating a tuple of `TensorChunkSpec` from a tuple of chunk
        dimensions (int's).
        Example:
            >>> # xdoctest: +SKIP
            >>> # There are three positional arguments to the model, and
            >>> # we are chunking them along dimension 0, 0 and 1, respectively
            >>> args_chunk_spec = TensorChunkSpec.from_tuple((0, 0, 1))
        c                 S   s   t | ƒS r   ©r   ©Údimr   r   r   r   Y   r   z,TensorChunkSpec.from_tuple.<locals>.<lambda>r   )r)   Úargs_chunk_specr   r   r   Ú
from_tupleJ   s
    þzTensorChunkSpec.from_tuplec                 C   s   t | dd„ ƒ}|S )a\  
        A helper for creating a dictionary of `TensorChunkSpec` from a
        dictionary of chunk dimensions (int's).
        Example:
            >>> # xdoctest: +SKIP
            >>> # Chunk dimension 0 for the "id" argument, 1 for the "mask" argument
            >>> kwargs_chunk_spec = TensorChunkSpec.from_dict({"id": 0, "mask": 1})
        c                 S   s   t | ƒS r   r*   r+   r   r   r   r   k   r   z+TensorChunkSpec.from_dict.<locals>.<lambda>r   )r)   Úkwargs_chunk_specr   r   r   Ú	from_dict]   s
    þzTensorChunkSpec.from_dictN)r   r   r   r   r   ÚintÚ__annotations__r'   r(   Ústaticmethodr   r.   r   Ústrr0   r   r   r   r   r   8   s   

ÿ
ÿc                   @   s   e Zd ZdS )Ú
_ReplicateNr   r   r   r   r   r5   q   s   r5   c           #      C   sò  i }g }|}d}t | ƒt |ƒksDtdt|  ¡ ƒ› dt| ¡ ƒ› �ƒ‚|  ¡ D �]â\}}t|ƒ\}	}
| |
¡ || }|dk	s€t‚t|ƒ\}}t |	ƒt |ƒkr°td|› d|› �ƒ‚g }t|	|ƒD �]f\}}|t	ksÜt
|tjƒsî| |g| ¡ q¾t
|tƒ�rt
|tjƒ�st|› d�ƒ‚| |j¡}||k �rr|�rVt d|› d	|› d
|› d�¡ |}ntd|› d|› d|› d�ƒ‚t |||j¡}t�rg }d}|D ]f}t |¡}|| |j¡ }tdddƒg|j }t||ƒ||j< |||< | |¡ || |j¡7 }�q”| |¡ n
| |¡ d}q¾td|› �ƒ‚q¾|||< qLg }t|ƒD ]N}i }| ¡ D ]0\}}g }|D ]}| || ¡ �q^|||< �qN| |¡ �q>g }|D ]V} i }!t |ƒt | ƒk�s´t‚t|  ¡ |ƒD ]\\}}}"t||"ƒ|!|< �qÂ| |!¡ �q–|S )aW  
    Given a dictionary of args, and a dictionary of chunking specs, shard the
    args according to the chunking specs.

    Args:
        args_dict: Dictionary of args
        args_chunk_spec: Dictionary of chunking specs
        num_chunks: Number of chunks to shard the args into

    Returns:
        args_split: List of sharded args
    Tzargs_dict.keys() = z args_chunk_spec.keys() = NzArgument value z9 did not have the same number of values as as chunk spec z is not a tensorz%Tensor size on chunking dimension is z', downsizing the number of chunks from z to r"   zArg z% on chunking dimension has a size of z$, smaller than the number of chunks zŒ. PiPPy cannot reduce the number of chunks because other arguments have bigger chunk-dimension sizes. Please adjust your num_chunks setting.r   FzUnrecognized chunk spec: )ÚlenÚAssertionErrorÚlistÚkeysÚitemsr	   ÚappendÚ
ValueErrorÚzipr5   Ú
isinstanceÚtorchZTensorr   Úsizer!   ÚloggerÚwarningÚRuntimeErrorÚtensor_splitÚ_debug_mask_minibatchesZ
zeros_likeÚsliceÚndimÚ	TypeErrorÚranger
   )#Z	args_dictr-   Z
num_chunksZargs_sharded_replicatedZ	arg_specsÚreal_num_chunksZfirst_tensorZarg_keyÚargZflatÚspecÚ
chunk_specZchunk_spec_flatÚ_Zsharded_arg_flatÚvZchunk_vZv_split_dim_sizeZchunk_tensorsZexpanded_chunksZsplit_dim_idxZchunk_tensorÚnew_valZ	upper_idxÚslice_indicesZchunks_flatÚ	chunk_idxÚ
chunk_argsÚkeyZarg_single_chunkZv_flatÚ
args_splitÚchunkZper_chunk_argsZarg_specr   r   r   Ú_shard_dict_of_argsu   s˜    ÿþ
ÿ
ÿÿ  ÿ
 ÿ



rW   .)ÚargsÚkwargsÚchunksr-   r/   Úreturnc           	         s  |dkri }|dkr&t tƒft| ƒ }|dkr>t |t tƒ¡}ttt| ƒƒtt|ƒƒ|ƒ}t|ƒ}t|||ƒ}t|ƒ|k ržt|ƒ}ttt| ƒƒtt|ƒƒ|ƒ}t|ƒt|ƒkrÊtdt|ƒ› dt|ƒ› �ƒ‚g }|D ](‰ | t	‡ fdd„t
tˆ ƒƒD ƒƒ¡ qÒ||fS )a  
    Given a sequence of args and kwargs, split them into a number of chunks
    according to  their respective chunking specs.

    Args:
        args: Tuple of args
        kwargs: Dict of kwargs
        chunks: Number of chunks to split the args and kwargs into
        args_chunk_spec: chunking specs for args, in same shape as args
        kwargs_chunk_spec: chunking specs for kwargs, in same shape as kwargs

    Returns:
        args_split: List of sharded args
        kwargs_split: List of sharded kwargs
    Nz;args and kwargs are split into different number of chunks: z, c                 3   s   | ]}ˆ | V  qd S r   r   )Ú.0Úi©rS   r   r   Ú	<genexpr>Y  s     z0split_args_kwargs_into_chunks.<locals>.<genexpr>)r   ÚDEFAULT_CHUNK_DIMr6   ÚdictÚfromkeysrW   Ú	enumeraterC   r;   ÚtuplerI   )	rX   rY   rZ   r-   r/   Zargs_split_dictrJ   Zkwargs_splitrU   r   r^   r   r   ö   s@    8

ýý

ýÿ&)rZ   c                    s@  |dk	rt |ƒ\}}n"t | d ƒ\}}ttƒgt|ƒ }g ‰| D ]>}t |ƒ\}}t|ƒt|ƒkrttd|› d|› �ƒ‚ˆ |¡ q@g }t|ƒD �]¦\‰ }	t|	tƒ�r¬‡ ‡fdd„ttˆƒƒD ƒ}
t	�r�|
d j
}|
dd… D ]}|j
|ksÚt‚qÚtjtj|dd	iŽt|
ƒ|	jd
�}g }d}t|
ƒt|ƒk�s.t‚t|
|ƒD ]T\}}|| |	j¡ }tdddƒg|j }t||ƒ||	j< || }| |¡ |}�q8n|
}| tj||	jd�¡ qŒt|	tƒ�rò|	j}ttˆƒƒD ]}|	 |ˆ| ˆ  ¡}�qÊ| |¡ qŒˆd ˆ  }tdtˆƒƒD ]}ˆ| ˆ  |k�st‚�q| |¡ qŒt||ƒS )zæ
    Given a list of chunks, merge them into a single value according to
    the chunk spec.

    Args:
        chunks: list of chunks
        chunk_spec: Chunking spec for the chunks

    Returns:
        value: Merged value
    Nr   zChunk z did not match chunk spec c                    s   g | ]}ˆ| ˆ  ‘qS r   r   )r\   rR   ©Zarg_idxZchunks_flattenedr   r   Ú
<listcomp>¤  s   ÿz merge_chunks.<locals>.<listcomp>é   ZdeviceÚmeta)Úsectionsr,   r+   )r	   r   r`   r6   r<   r;   rc   r>   rI   rE   Úshaper7   r?   rD   Úemptyr!   r=   r@   rF   rG   Úcatr   r   r   r
   )rZ   rM   Zspec_flattenedZflatten_specZchunk0_flatrV   Zchunk_flattenedrN   Zargs_flattenedrK   Zpartial_valuesZoverall_shapeÚvalZmeta_chunksZvalues_to_catZchunk_start_idxZpartial_valueZ
meta_chunkZchunk_end_idxrQ   ZslicedZreduced_valrR   Úvaluer   re   r   r   ^  sd    -
þ
ý

 
ÿ)NN)ÚloggingÚtypingr   r   r   r   r   r?   Ztorch.fx.noder   Ztorch.utils._pytreer	   r
   Ú__all__Ú	getLoggerr   rA   rE   r   r   ZtensorZsum_reducerr`   r   r5   rW   r4   r1   r   r   r   r   r   r   Ú<module>   s<   ý
9   û
úiÿ