U
    D‹ÑhgX  ã                   @   sÌ  d dl Z d dlZd dlmZmZ d dlmZmZmZm	Z	m
Z
mZmZmZmZmZmZmZ d dlmZ dddddd	d
dgZejeejgeej f dœdd„Zeje	eje	eef f eeejgeej f dœdd„Zejeeej  eej e	eef e	eje	eef f dœdd„Zedœdd„ZG dd„ deƒZejeeeeej  edœdd„ZG dd„ deƒZG dd
„ d
eƒZ ejeeeedœdd„Z!ejeeeeej  ed œd!d„Z"ejee# ed"œd#d$„Z$ejeeed%œd&d'„Z%ed(ƒddfejeeeeeeej   eeeej   ed)œd*d„Z&ej'ej(he&_)ej*he&_+e j,eee
d+ d,œd-d„ƒZ-ejeejd.œd/d	„Z.ejeejd0œd1d2„Z/d9ejeeeej eej0 eeeejef d4œd5d6„Z1G d7d8„ d8ƒZ2dS ):é    N)ÚABCÚabstractmethod)ÚAnyÚCallableÚcastÚDictÚ	GeneratorÚIterableÚOptionalÚSequenceÚSetÚTupleÚTypeÚUnionÚalways_wrap_policyÚlambda_auto_wrap_policyÚtransformer_auto_wrap_policyÚsize_based_auto_wrap_policyÚenable_wrapÚwrapÚCustomPolicyÚModuleWrapPolicy©Úroot_moduleÚfnc                    s8   | h‰t jttt j dœ‡ ‡‡fdd„‰ ˆ | ddƒ dS )aQ  
    This applies ``fn`` to every module in the module tree of ``root_module``
    following a post-order traversal. If ``fn`` returns an :class:`nn.Module`,
    then this replaces the original module with the newly returned one in the
    tree. Otherwise, ``fn`` should return ``None``, in which case the module is
    not changed.
    )ÚmoduleÚmodule_nameÚparent_modulec                    sœ   |   ¡ D ]&\}}|ˆkrˆ |¡ ˆ ||| ƒ qˆ| ƒ}|d k	r˜t|tjƒs`td|› d| › �ƒ‚|srtd| › �ƒ‚t|tjƒsŒtd|› �ƒ‚t|||ƒ d S )Nz=Non-root modules should have their parent module set but got z for zTNon-root modules should have their module name set but got an empty module name for z.fn should return None or an nn.Module but got )Únamed_childrenÚaddÚ
isinstanceÚnnÚModuleÚAssertionErrorÚsetattr)r   r   r   Zchild_module_nameZchild_moduleZoptional_module©Ú_post_order_apply_innerr   Zvisited_modules© úI/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/distributed/fsdp/wrap.pyr&   8   s&    
ÿÿ ÿþz2_post_order_apply.<locals>._post_order_apply_innerÚ N)r!   r"   Ústrr
   r   r'   r%   r(   Ú_post_order_apply*   s    ýr+   )r   Útarget_module_to_kwargsÚfsdp_fnÚreturnc                    s$   t jtt j dœ‡ ‡‡fdd„}|S )z¶
    This constructs the "wrap" function to pass to :func:`_post_order_apply`
    based on ``target_module_to_kwargs``, which should be constructed from the
    wrapping policy.
    )r   r.   c                    s(   | ˆkr$| ˆk	r$ˆ|  }ˆ | f|ŽS d S ©Nr'   )r   Úkwargs©r-   r   r,   r'   r(   r   ^   s    z_construct_wrap_fn.<locals>.fn)r!   r"   r
   )r   r,   r-   r   r'   r1   r(   Ú_construct_wrap_fnS   s     r2   )r   Úmodule_classesÚignored_modulesÚroot_kwargsr,   c                 C   sP   t t|ƒƒ}|  ¡ D ]6}||kr$qqt||ƒr||kr>|||< d || d< q|S )NZmixed_precision)ÚtupleÚsetÚmodulesr    )r   r3   r4   r5   r,   Zmodule_classes_tupler   r'   r'   r(   Ú$_run_mixed_precision_override_policyi   s    
r9   ©r.   c                  O   s   dS )zª
    A simple recursive wrap policy that always returns ``True``. This means
    that every submodule is wrapped by the wrapper class in
    :func:`_recursive_wrap`.
    Tr'   )Úargsr0   r'   r'   r(   r   }   s    c                	   @   sJ   e Zd ZdZeejeej ee	e
f eejee	e
f f dœdd„ƒZdS )Ú_Policyzk
    This defines an abstract base class that represents a policy for applying
    a module-level API.
    ©r   r4   r5   r.   c                 C   s   dS )zˆ
        This should return a dict ``target_module_to_kwargs`` that maps from
        each target module to wrap to its kwargs.
        Nr'   )Úselfr   r4   r5   r'   r'   r(   Ú_run_policyŒ   s    z_Policy._run_policyN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r!   r"   r   r   r*   r   r?   r'   r'   r'   r(   r<   †   s   
ûr<   )r   ÚrecurseÚnonwrapped_numelr3   r.   c                 C   s   |rdS t | t|ƒƒS )a   
    This auto wrap policy wraps every module that is an instance of any type in
    ``module_classes`` as its own FSDP instance. The root module given by
    ``module`` is always wrapped as an FSDP instance regardless. Since the
    wrapping proceeds bottom up, each FSDP instance manages the parameters in
    its subtree excluding any already managed by a child FSDP instance.

    Args:
        module (nn.Module): Current module being considered.
        recurse (bool): If ``False``, then this function must decide whether
            ``module`` should be wrapped as an FSDP instance or not. If
            ``True``, then the function is still recursing down the module
            tree as a part of the DFS.
        nonwrapped_numel (int): Parameter numel not yet wrapped.
        module_classes (Set[Type[nn.Module]]): Set of module classes that are
            wrapped as FSDP instances.

    Returns:
        ``True`` if ``recurse=True``, and whether ``module`` should be wrapped
        if ``recurse=False``.
    T©r    r6   )r   rD   rE   r3   r'   r'   r(   Ú_module_wrap_policyš   s    rG   c                       s|   e Zd ZdZeeej  dœdd„Zeje	ej e
eef e
eje
eef f dœdd„Zdd	„ Zed
œ‡ fdd„Z‡  ZS )r   z{
    This policy applies to every module of the specified module classes,
    passing in the kwargs given to the root.
    )r3   c                 C   s   t |ƒ}|| _t|ƒ| _d S r/   )r7   Ú_module_classesr*   Ú_module_classes_str)r>   r3   Zmodule_classes_setr'   r'   r(   Ú__init__À   s    zModuleWrapPolicy.__init__r=   c                 C   sD   t | jƒ}i }| ¡ D ](}||kr&qqt||ƒrt |¡||< q|S r/   )r6   rH   r8   r    Úcopy)r>   r   r4   r5   r3   r,   r   r'   r'   r(   r?   Å   s    

zModuleWrapPolicy._run_policyc                 O   s   t ||d| jd�S )Néÿÿÿÿ)rE   r3   )rG   rH   )r>   r   rD   r;   r0   r'   r'   r(   Ú__call__Õ   s       ÿzModuleWrapPolicy.__call__r:   c                    s   t ƒ  ¡ d| j› d� S )Nú(ú))ÚsuperÚ__repr__rI   ©r>   ©Ú	__class__r'   r(   rQ   Û   s    zModuleWrapPolicy.__repr__)r@   rA   rB   rC   r	   r   r!   r"   rJ   r   r   r*   r   r?   rM   rQ   Ú__classcell__r'   r'   rS   r(   r   º   s   
ûc                   @   sp   e Zd ZdZeejgeee	e
ef f f dœdd„Zejeej e	e
ef e	eje	e
ef f dœdd„ZdS )	r   aý  
    This policy takes in a lambda function that maps a given ``nn.Module`` to
    either ``False``, ``True``, or a kwarg dictionary.
    - If the function returns ``False`` or an empty dictionary, then the module
      does not have the API applied.
    - If the function returns ``True``, then the module has the API applied
      with the root's kwargs.
    - If the function returns a non-empty dictionary, then the module has the
      API applied, and the dictionary overrides the root's kwargs.

    Example::

        >>> # xdoctest: +SKIP("undefined variables")
        >>> model = init_transformer_model(...)
        >>> def lambda_fn(module: nn.Module):
        >>>     if module is model.lm_head:
        >>>         return {"sharding_strategy": ShardingStrategy.SHARD_GRAD_OP}
        >>>     elif isinstance(module, TransformerBlock):
        >>>         return True
        >>>     return False
        >>> policy = CustomPolicy(lambda_fn)
        >>> fsdp_model = FSDP(model, auto_wrap_policy=policy)
    )Ú	lambda_fnc                 C   s
   || _ d S r/   )Ú
_lambda_fn)r>   rV   r'   r'   r(   rJ   ø   s    zCustomPolicy.__init__r=   c                 C   sr   i }|  ¡ D ]`}||krq|  |¡}t|ttfƒs@td|› �ƒ‚|sFqt |¡}t|tƒrd| |¡ |||< q|S )Nz_The lambda_fn passed to CustomPolicy should return False/True or a kwarg dict, but it returned )r8   rW   r    ÚdictÚboolÚ
ValueErrorrK   Úupdate)r>   r   r4   r5   r,   r   Úresr0   r'   r'   r(   r?   û   s     
ÿ



zCustomPolicy._run_policyN)r@   rA   rB   rC   r   r!   r"   r   rY   r   r*   r   rJ   r   r?   r'   r'   r'   r(   r   ß   s   *
û)r   rD   rE   rV   r.   c                 C   s   |rdS || ƒS )aU  
    A convenient auto wrap policy to wrap submodules based on an arbitrary user
    function. If `lambda_fn(submodule) == True``, the submodule will be wrapped as
    a `wrapper_cls` unit.

    Return if a module should be wrapped during auto wrapping.

    The first three parameters are required by :func:`_recursive_wrap`.

    Args:
        module (nn.Module): Current module being considered.
        recurse (bool): If ``False``, then this function must decide whether
            ``module`` should be wrapped as an FSDP instance or not. If
            ``True``, then the function is still recursing down the module
            tree as a part of the DFS.
        nonwrapped_numel (int): Parameter numel not yet wrapped.

        lambda_fn (Callable[[nn.Module], bool]): If this returns ``True``, then
            this module will be wrapped.
    Tr'   )r   rD   rE   rV   r'   r'   r(   r     s    )r   rD   rE   Útransformer_layer_clsr.   c                 C   s   t | |||ƒS )a-  
    See :func:`_module_wrap_policy`, where ``transformer_layer_cls`` is the
    same as ``module_classes``. Note that shared parameters must be wrapped in
    the same FSDP instance, so this auto wrap policy can help wrap shared
    embeddings into the same FSDP instance for transformer models.
    )rG   )r   rD   rE   r]   r'   r'   r(   r   2  s    )r   r3   rD   c                 O   s   |rdS t | t|ƒƒS d S )NTrF   )r   r3   rD   r;   r0   r'   r'   r(   Ú_wrap_module_cls_individuallyA  s    r^   )r   rD   rE   r.   c                    s   t ‡ ‡‡fdd„|D ƒƒS )zv
    A policy that wraps ``module`` if any policy in the passed in iterable of
    ``policies`` returns ``True``.
    c                 3   s   | ]}|ˆ ˆˆd �V  qdS )©r   rD   rE   Nr'   )Ú.0Úpolicy©r   rE   rD   r'   r(   Ú	<genexpr>W  s   ÿz_or_policy.<locals>.<genexpr>)Úany)r   rD   rE   Zpoliciesr'   rb   r(   Ú
_or_policyM  s    
þre   g    „×—A)r   rD   rE   Úmin_num_paramsÚforce_leaf_modulesÚexclude_wrap_modulesr.   c                 C   s`   |dkrt jn|}|dkr t jn|}|}||k}|rH|oFt| t|ƒƒ S |oZt| t|ƒƒ S dS )a¯  
    A size-based auto wrap policy.

    Args:
        module (nn.Module): Current module being considered.
        recurse (bool): If ``False``, then this function must decide whether
            ``module`` should be wrapped as an FSDP instance or not. If
            ``True``, then the function is still recursing down the module
            tree as a part of the DFS.
        nonwrapped_numel (int): Parameter numel not yet wrapped.

        min_num_params (int): Customizable policy input that controls the size
            threshold over which a module is ready to be wrapped. This is in
            units of numel.
        force_leaf_modules (Set[Type[nn.Module]]): Set of module types to keep
            as leaves, i.e. their children will never be wrapped.
        exclude_wrap_modules (Set[Type[nn.Module]]): Set of module types to be
            excluded in wrapping.

    Returns:
        Whether ``module`` should be wrapped.
    N)r   ÚFORCE_LEAF_MODULESÚEXCLUDE_WRAP_MODULESr    r6   )r   rD   rE   rf   rg   rh   Zmin_nonwrapped_numelZis_larger'   r'   r(   r   ]  s    !ÿýÿý)NNN)Úwrapper_clsÚwrapper_kwargsr.   c              	   k   s,   d| i|–}t f |Ž� dV  W 5 Q R X dS )aÑ  
    Context manager to wrap modules using a wrapper.

    Useful for when you'd like to apply the same configuration arguments to all
    child modules that you wrap. A particularly important use case is wrapping
    large layers so that they get sharded (in-place) during initialization, to
    avoid running out of system memory. Large layers can indicate that they
    should be sharded via the ``wrap`` annotation and this context manager can
    provide the exact configuration for these nested instances.

    Usage::

        with enable_wrap(wrapper_cls, **params):
            # Wraps layer in FSDP by default if within context
            self.l1 = wrap(torch.nn.Linear(5, 5))

    Args:
        wrapper_cls:
            Class that `wrap` annotation will `wrap` modules with, such as
            `FullyShardedDataParallel`.
        **wrapper_kwargs:
            Configuration settings that will be passed to all ``wrap``
            instances inside the context
    rk   N)Ú_ConfigAutoWrap)rk   rl   r0   r'   r'   r(   r   ˜  s     ÿþ)r   Úwrap_overridesr.   c                 K   s2   t jr.t jdk	st‚t j|–}t| t jf|ŽS | S )a  
    Annotate that a module should be wrapped. Annotated modules will only be
    wrapped if inside of an :func:`enable_wrap` context manager. This allows
    a module to be initialized both with and without a wrapper without code
    change.

    The class that this function wraps the passed in ``nn.Module`` with is the
    passed in ``wrapper_cls`` argument into ``enable_wrap``. Both
    ``enable_wrap`` and ``wrap`` can take in kwargs specifying how to construct
    the ``wrapper_cls`` instance. In the case of duplicate kwargs in
    ``enable_wrap`` and ``wrap``, the argument passed into ``wrap`` will be
    respected.

    Usage::

        with enable_wrap(wrapper_cls=FSDP, **fsdp_config):
            # Wraps layer in FSDP by default if within context
            self.l1 = wrap(torch.nn.Linear(5, 5))

    Args:
        module (nn.Module): module to wrap (if in :func:`enable_wrap` context)
        **wrap_overrides: configuration overrides that will take priority over
            the values provided by the :func:`enable_wrap` context
    N)rm   Úin_autowrap_contextrk   r#   r0   Ú_wrap)r   rn   r'   r'   r(   r   ¼  s    
þý)r   rk   r.   c                 K   s8   |d k	st ‚t| dƒr,|| j–}|| f|ŽS || f|ŽS )NÚ_wrap_overrides)r#   Úhasattrrq   )r   rk   r0   Z	overridesr'   r'   r(   rp   á  s
    

rp   F)r   Úauto_wrap_policyrk   r4   Úignored_paramsÚonly_wrap_childrenr0   r.   c              	      s2  |dk	st dƒ‚|dk	s t dƒ‚|  ¡ D ]@\}}||kr:q(zt|tt|ƒƒrPt ‚W q( tk
rf   Y q(X q(t‡ fdd„|  ¡ D ƒƒ}	|dk	s�t ‚|| d|	d��r*d}
|  ¡ D ]D\}}||kr¾q¬t	f ||||ˆ d	œ|—Ž\}}t
| ||ƒ |
|7 }
q¬|	|
 }|�s"|| d
|d��r"t| |f|Ž|	fS | |
fS | dfS )aÚ  
    Wraps submodules of ``module`` for which ``auto_wrap_policy`` returns
    ``True`` with ``wrapper_cls``.

    Args:
        module (nn.Module): Module to recursively wrap.
        auto_wrap_policy (Callable): A callable representing a policy that
            determines which modules to recursively wrap with ``wrapper_cls``.
        ignored_modules (Set[torch.nn.Module]): Modules to ignore when
            wrapping.
        ignored_params (Set[torch.nn.Parameter]): Parameters to ignore when
            wrapping; these should be the parameters contained in the modules
            in ``ignored_modules``.
    Returns:
        (nn.Module, int):
            ``module`` after wrapping and the numel recursively wrapped.
    NzMust specify auto_wrap_policy.zMust specify wrapper_clsc                 3   s   | ]}|ˆ kr|  ¡ V  qd S r/   )Znumel)r`   Úp©rt   r'   r(   rc     s     z"_recursive_wrap.<locals>.<genexpr>Tr_   r   )r   rs   rk   r4   rt   F)r#   Znamed_modulesr    r   ÚtypeÚ	TypeErrorÚsumÚ
parametersr   Ú_recursive_wrapr$   rp   )r   rs   rk   r4   rt   ru   r0   Ú_ÚchildrE   Ztotal_wrapped_numelÚnameZwrapped_childZnum_wrapped_paramsÚ	remainderr'   rw   r(   r|   î  sL    ÿûú

  ÿr|   c                   @   s    e Zd ZU dZdZeed< dZee	 ed< i Z
eeef ed< eeef dœdd	„Zeedd
œdd„ƒZeddœdd„ƒZddœdd„Zeeeddœdd„ZdS )rm   zŒ
    Helper class to wrap modules based on default config args via a context manager.
    See :func:`enable_wrap` for more information.
    Fro   Nrk   r0   ©r0   c                 K   s
   || _ d S r/   r�   )r>   r0   r'   r'   r(   rJ   B  s    z_ConfigAutoWrap.__init__)r0   r.   c                 C   sH   t jrtdƒ‚dt _d|  ¡ ks(tdƒ‚tt| d ƒt _| d= | t _d S )Nz]You are already within an autowrap context and we currently do not supported nested autowrap.Trk   z9Expected to pass in wrapper_cls arg into _ConfigAutoWrap.)	rm   ro   ÚNotImplementedErrorÚkeysr#   r   r   rk   r0   r�   r'   r'   r(   Úenable_autowrap_contextE  s    ÿ
ÿþz'_ConfigAutoWrap.enable_autowrap_contextr:   c                   C   s   dt _d t _i t _d S )NF)rm   ro   rk   r0   r'   r'   r'   r(   Údisable_autowrap_contextU  s    z(_ConfigAutoWrap.disable_autowrap_contextc                 C   s   |   | j¡ d S r/   )r„   r0   rR   r'   r'   r(   Ú	__enter__[  s    z_ConfigAutoWrap.__enter__)Úexc_typeÚexc_valÚexc_tbr.   c                 C   s   |   ¡  d S r/   )r…   )r>   r‡   rˆ   r‰   r'   r'   r(   Ú__exit__^  s    z_ConfigAutoWrap.__exit__)r@   rA   rB   rC   ro   rY   Ú__annotations__rk   r
   r   r0   r   r*   r   rJ   Ústaticmethodr„   r…   r†   rŠ   r'   r'   r'   r(   rm   8  s   
rm   )F)3Ú
contextlibrK   Úabcr   r   Útypingr   r   r   r   r   r	   r
   r   r   r   r   r   Ztorch.nnr!   Ú__all__r"   r+   r*   r2   r9   rY   r   r<   ÚintrG   r   r   r   r   rx   r^   re   r   Z
ModuleListZ
ModuleDictrj   ZMultiheadAttentionri   Úcontextmanagerr   r   rp   Ú	Parameterr|   rm   r'   r'   r'   r(   Ú<module>   s¶   8øþ*ü
û	û %8   þû  ÿûùø7
 þ#% úøJ