U
    D‹ÑhÂ}  ã                   @   s‚  U d dl Z d dlZd dlZd dlmZmZmZmZmZm	Z	 d dl
Z
d dlmZ ddlmZ ddgZeƒ s d dlZG dd	„ d	ƒZd
d„ Zeejd _eejd _nÞd dlmZmZmZmZmZmZmZmZmZm Z  e  !e"¡Z#e�rzd dl$m%Z% W n  e&k
�r   e# 'd¡ Y nX G dd„ dej(ƒZ)e)ƒ Z*e)e+d< de,dœdd„Z-G dd„ dƒZddœe,ee.df eee,df  edœdd„ZdS )é    N)ÚDictÚListÚOptionalÚTupleÚTYPE_CHECKINGÚUnion)Úis_availableé   )Únot_noneÚinit_device_meshÚ
DeviceMeshc                   @   s   e Zd ZdS )Ú_DeviceMeshStubN)Ú__name__Ú
__module__Ú__qualname__© r   r   úK/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/distributed/device_mesh.pyr      s   r   c                   C   s   d S ©Nr   r   r   r   r   Ú_init_device_mesh_stub   s    r   ztorch.distributed.device_mesh)
Ú_find_pg_by_ranks_and_tagÚ_get_default_groupÚ_get_group_tagÚget_process_group_ranksÚget_rankÚget_world_sizeÚinit_process_groupÚis_initializedÚ	new_groupÚProcessGroup)Ú	ArrayLikezCDeviceMesh requires numpy >= 1.21 to be installed for type checkingc                   @   sÀ   e Zd Zddœdd„Zddœdd„Zdeedf dd	œd
d„Zded dœdd„Z	dee
 dœdd„Zeee
dœdd„ƒZeee
dœdd„ƒZdee
dœdd„Zde
eeej ddœdd„ZdS )Ú_MeshEnvN©Úreturnc                 C   s   g | _ i | _i | _d S r   )Ú
mesh_stackÚchild_to_parent_mappingÚmesh_dim_group_options©Úselfr   r   r   Ú__init__?   s    þz_MeshEnv.__init__r   c                 C   s    t | jƒdkrtdƒ‚| jd S )Nr   z#No device mesh is currently active!éÿÿÿÿ)Úlenr#   ÚRuntimeErrorr&   r   r   r   Úget_current_meshF   s    z_MeshEnv.get_current_mesh.)Úparent_meshÚsubmesh_dim_namesr"   c                    sº   ‡ fdd„|D ƒ}‡ fdd„|D ƒ}t tˆ jjƒƒ}|D ]}| |¡ q8ˆ jj||žŽ jd|žŽ }ˆ  ¡ }|D ]"}	tˆ j	|	|dd�}
||	krn|
}qnˆ |_
‡ fdd„|D ƒ|_ˆ | j|< |S )	Nc                    s   g | ]}t ˆ jƒ |¡‘qS r   )r
   Úmesh_dim_namesÚindex©Ú.0Úmesh_dim_name©r-   r   r   Ú
<listcomp>O   s   ÿz._MeshEnv.create_child_mesh.<locals>.<listcomp>c                    s   g | ]}ˆ j  |¡‘qS r   )ÚmeshÚsize©r2   Úmesh_dimr4   r   r   r5   S   s    r)   F©r/   Ú_init_backendc                    s   g | ]}ˆ j | ‘qS r   )Ú_dim_group_infosr8   r4   r   r   r5   n   s    )r)   )ÚlistÚranger6   ÚndimÚremoveZpermuteÚreshaper   r   Údevice_typeÚ_parent_meshr<   r$   )r'   r-   r.   Zsubmesh_dimsZsubmesh_dim_sizesZmesh_dims_remainedZsubmesh_dimÚpg_ranks_by_dimZcur_rankZmesh_ndÚsubmeshZres_submeshr   r4   r   Úcreate_child_meshK   sB    
þ
ÿ ÿ þþü
ÿ
z_MeshEnv.create_child_mesh)Údevice_meshr"   c                 C   s   | j  |d ¡S r   )r$   Úget)r'   rG   r   r   r   Úget_parent_meshu   s    z_MeshEnv.get_parent_meshc                 C   sD   |   |¡}|j}|r@|r@t|ƒdks,tdƒ‚|d }|  ||¡S dS )zŸ
            Return the index of the mesh dim in the parent mesh.
            The device_mesh passed in needs to be sliced out from a parent mesh.
            é   z%The child mesh can only be a 1D mesh.r   N)rI   r/   r*   ÚAssertionErrorÚget_mesh_dim_by_name)r'   rG   r-   Zchild_mesh_dim_namesZchild_mesh_dim_namer   r   r   Úget_parent_mesh_dimx   s    

ÿþz_MeshEnv.get_parent_mesh_dim)rB   r"   c                 C   s   t | ƒ ¡ S r   )Ú_get_device_handleÚdevice_count©rB   r   r   r   Únum_devices_per_host‡   s    z_MeshEnv.num_devices_per_hostc                 C   s   t ƒ t | ¡ S r   )r   r    rQ   rP   r   r   r   Ú	num_hosts‹   s    z_MeshEnv.num_hosts)rG   r3   r"   c                 C   sT   |j d kst|j ƒdkr tdƒ‚||j krDtd|› d�d|j › �ƒ‚t|j  |¡ƒS )Nr   zNo `mesh_dim_names` found.zMesh dimension 'z' does not exist.z.Available mesh dimensions are: mesh_dim_names=)r/   r*   ÚKeyErrorr
   r0   )r'   rG   r3   r   r   r   rL   ‘   s    ÿþÿ


þz_MeshEnv.get_mesh_dim_by_name)ÚdimÚbackendÚ
pg_optionsr"   c                 C   s   ||f| j |< d S r   )r%   )r'   rT   rU   rV   r   r   r   Ú_set_mesh_dim_group_options¢   s    z$_MeshEnv._set_mesh_dim_group_options)N)r   r   r   r(   r,   r   ÚstrrF   r   rI   ÚintrM   ÚstaticmethodrQ   rR   rL   r   ÚOptionsrW   r   r   r   r   r    >   s.    
þ* þ üûr    Ú_mesh_resourcesÚcudarP   c                 C   s   t t| dƒS )a:  
        Get the module corresponding to the device_type which is cuda or cuda-like device.
        For example, when the device_type is cuda, the module `torch.cuda` is returned.
        Return None when there is no corresponding module for device_type, otherwise
        return the corresponding module.
        N)ÚgetattrÚtorchrP   r   r   r   rN   ¬   s    rN   c                
   @   sÚ  e Zd ZU dZeed< ejed< ee	edf  ed< dddœee
ejd	f ee	edf  edd
œdd„Zdd„ Zdd„ Zd dœdd„Zddœdd„Zedœdd„Zdd„ Zeedœdd„Ze
ee	edf f d dœdd„Zd5ee
eef  ed œd!d"„Zee dœd#d$„Zed6dd%œe
eee f eee
ejd	f  ee	edf  d d&œd'd(„ƒZd7ee ed œd)d*„Zeedœd+d,„ƒZee	edf dœd-d.„ƒZ edœd/d0„Z!d8ee
eef  ed œd1d2„Z"eee  dœd3d4„Z#dS )9r   aÈ  
        DeviceMesh represents a mesh of devices, where layout of devices could be
        represented as a n-d dimension array, and each value of the n-d dimensional
        array is the global id of the default process group ranks.

        DeviceMesh could be used to describe the layout of devices across the cluster,
        and serves as a proxy for communication among the device lists within the cluster.

        DeviceMesh can be used as a context manager.

        .. note::
            DeviceMesh follows SPMD programming model, which means the same PyTorch Python program
            is running on all processes/ranks in the cluster. Therefore, users need to make sure the
            `mesh` array (which describes the layout of devices) should be identical across all ranks.
            Inconsistent `mesh` will lead to silent hang.

        Args:
            device_type (str): The device type of the mesh. Currently supports: "cpu", "cuda/cuda-like".
            mesh (ndarray): A multi-dimensional array or an integer tensor describing the layout
                of devices, where the IDs are global IDs of the default process group.

        Returns:
            DeviceMesh: A :class:`DeviceMesh` object representing the device layout.

        The following program runs on each process/rank in an SPMD manner. In this example, we have 2
        hosts with 4 GPUs each.
        A reduction over the first dimension of mesh will reduce across
        columns (0, 4), .. and (3, 7), a reduction over the second dimension
        of mesh reduces across rows (0, 1, 2, 3) and (4, 5, 6, 7).

        Example::
            >>> # xdoctest: +SKIP("no rank")
            >>> from torch.distributed.device_mesh import DeviceMesh
            >>>
            >>> # Initialize device mesh as (2, 4) to represent the topology
            >>> # of cross-host(dim 0), and within-host (dim 1).
            >>> mesh = DeviceMesh(device_type="cuda", mesh=[[0, 1, 2, 3],[4, 5, 6, 7]])
        rB   r6   .r/   NTr:   r   )rB   r6   r/   r;   r"   c                C   sö   || _ t|tjƒr,|jjdkr,td|› �ƒ‚t|tjƒrJ| ¡ jtj	d�ntj
|dtj	d�| _|rjt|ƒnd | _t| j ¡  ¡ ƒ| _d | _t ¡ | _|dkrò|r°|  ¡  |  ¡  | jtƒ k ¡ }| d¡dksÒt‚| d¡dkrì|d  ¡ nd | _d S )NÚcpuz!`mesh` must be a CPU tensor, got ©Údtype©Údevicerb   Zxlar   )r   rJ   )rB   Ú
isinstancer_   ÚTensorrd   ÚtypeÚ
ValueErrorÚdetachÚtorY   Útensorr6   Útupler/   ÚflattenÚtolistÚ_flatten_mesh_listrC   Ú	threadingÚ	get_identÚ
_thread_idÚ_get_or_create_default_groupÚ_init_process_groupsr   Znonzeror7   rK   Ú_coordinate_on_dim)r'   rB   r6   r/   r;   Zrank_coordsr   r   r   r(   á   s&    
ÿý
ÿzDeviceMesh.__init__c                 C   sœ   t ƒ }|stƒ  tƒ }| j ¡ |kr:td| j ¡ › d�ƒ‚t| jƒ}|s–|r–| ¡ }||kr†|| dkr†td|› d|› d| j› d�ƒ‚| 	t
ƒ | ¡ tƒ S )Nz=Mesh should not be bigger than default world size, but found z ranks!r   z8DeviceMesh only support homogeneous hardware, but found z ranks and ú z	 devices!)r   r   r   r6   Únumelr+   rN   rB   rO   Z
set_devicer   r   )r'   Zdefault_initializedZ
world_sizeZdevice_handlerQ   r   r   r   rs   	  s(    ÿ
ÿ
þÿz'DeviceMesh._get_or_create_default_groupc           	      C   s  g }| j jdkrF| j  ¡ tƒ krF| ttƒ ƒtttƒ ƒƒtƒ j	f¡ n¼t| j jƒD ]®}| j  
d|¡ d| j  |¡¡}|D ]†}| ¡ }|tjkržtj| \}}nd\}}t|||d�}|  ¡ |krxt|ƒ|krätd| j› d|› d�ƒ‚| tt|ƒƒ||j	f¡ qxqR|| _d S )NrJ   r)   )NN)ZranksrU   rV   zFEach device mesh dimension should get only one process group, but got z in ú!)r6   r?   rw   r   Úappendr   r   r=   r>   Ú
group_nameZswapdimsrA   r7   rn   r\   r%   r   r   r*   r+   r
   r<   )	r'   Zdim_group_infosrT   rD   Zdim_meshZsubgroup_ranksrU   rV   Z	dim_groupr   r   r   rt   &  sN    ýÿ	 
ÿ
ýþýÿ
ýÿzDeviceMesh._init_process_groupsr!   c                 C   s   t j | ¡ | S r   )r\   r#   ry   r&   r   r   r   Ú	__enter__j  s    zDeviceMesh.__enter__c                 C   s   t j ¡  d S r   )r\   r#   Úpop)r'   Úexc_typeÚ	exc_valueÚexc_tracebackr   r   r   Ú__exit__p  s    zDeviceMesh.__exit__c                 C   s6   | j sd| j ¡ › d�nd| j ¡ › d| j › d�}|S )NzDeviceMesh(ú)z, mesh_dim_names=)r/   r6   rn   )r'   Zdevice_mesh_reprr   r   r   Ú__repr__t  s
    ÿýzDeviceMesh.__repr__c                 C   s>   t | dd ƒ| _| js8t| j| jj| j| j| j| j	fƒ| _| jS )NÚ_hash)
r^   rƒ   Úhashro   r6   ÚshaperB   r/   rC   rr   r&   r   r   r   Ú__hash__|  s    úÿ
zDeviceMesh.__hash__)Úotherr"   c                 C   sr   t |tƒsdS t| ƒt|ƒkr"dS | j|jkol| jj|jjkol| j|jkol| j|jkol| j|jkol| j	|j	kS d S )NFT)
re   r   Úidro   r6   r…   rB   r/   rC   rr   )r'   r‡   r   r   r   Ú__eq__Œ  s    
ÿ
þ
ý
ü
úzDeviceMesh.__eq__)r/   r"   c                    sÌ   ˆ j stdƒ‚t|tƒr|fn|}d|› dˆ j › d�}|ˆ j krDˆ S t|ƒtˆ j ƒkslt‡ fdd„|D ƒƒsvt|ƒ‚nF|d }ˆ j  |¡}t|ˆ j |t|ƒ… ƒD ]\}}||kr¢t|ƒ‚q¢t	 
ˆ |¡}|S )aä  
            Slice the current DeviceMesh based on the mesh_dim_name given to create a child
            DeviceMesh.

            Args:
                mesh_dim_name (Union[str, Tuple[str]]): the name or the tuple of names of the
                mesh dimension of the parent DeviceMesh to create the child DeviceMesh for.
            Returns:
                A :class:`DeviceMesh` object

            The following program runs on each process/rank in an SPMD manner. In this example, we have 2
            hosts with 4 GPUs each.
            Calling mesh["tp"] on rank 0, 1, 2, 3 would return a 1D child DeviceMesh:([0, 1, 2, 3]).
            Calling mesh["tp"] on rank 4, 5, 6, 7 would return a 1D child DeviceMesh:([4, 5, 6, 7]).
            Calling mesh["dp"] on rank 0, 4 would return a 1D child DeviceMesh:([0, 4]).
            Calling mesh["dp"] on rank 1, 5 would return a 1D child DeviceMesh:([1, 5]).
            Calling mesh["dp"] on rank 2, 6 would return a 1D child DeviceMesh:([2, 6]).
            Calling mesh["dp"] on rank 3, 7 would return a 1D child DeviceMesh:([3, 7]).

            Example::
                >>> # xdoctest: +SKIP("no rank")
                >>> from torch.distributed.device_mesh import DeviceMesh
                >>>
                >>> # Initialize device mesh as (2, 4) to represent the topology
                >>> # of cross-host(dim 0), and within-host (dim 1).
                >>> mesh = DeviceMesh(device_type="cuda", mesh=[[0, 1, 2, 3],[4, 5, 6, 7]])
            z1Cannot slice a DeviceMesh without mesh_dim_names!zInvalid mesh_dim_name zG specified. Valid mesh_dim_names should be a contiguous subsequence of Ú.c                 3   s   | ]}|ˆ j kV  qd S r   ©r/   r1   r&   r   r   Ú	<genexpr>Ç  s    z)DeviceMesh.__getitem__.<locals>.<genexpr>r   )r/   r+   re   rX   r*   ÚallrS   r0   Úzipr\   rF   )r'   r/   Ú	error_msgZoutermost_dim_nameZoutermost_dim_idxÚiÚjrE   r   r&   r   Ú__getitem__›  s,    ÿÿ
ÿ
þ
zDeviceMesh.__getitem__)r9   r"   c                 C   sŒ   t | dƒstdƒ‚| jjdkr>|dkr>td| jj› d�ddƒ‚| jjdkrX|dkrXd	}nt|tƒrnt | |¡n|}tt	| j
| dd
… Ž ƒS )aÂ  
            Returns the single ProcessGroup specified by mesh_dim, or, if mesh_dim is not specified and the
            DeviceMesh is 1-dimensional, returns the only ProcessGroup in the mesh.

            Args:
                mesh_dim (str/int, optional): it can be the name of the mesh dimension or the index
                of the mesh dimension. Default is None.

            Returns:
                A :class:`ProcessGroup` object.
            r<   z*DeviceMesh process groups not initialized!rJ   NúFound the DeviceMesh have ú dimensionsúJOptional kwarg `mesh_dim` needs to be specified when device_mesh.ndim > 1.zmIf you want to get the list of all the ProcessGroups in the DeviceMesh,please use `get_all_groups()` instead.r   r	   )Úhasattrr+   r6   r?   re   rX   r\   rL   r
   r   r<   ©r'   r9   r   r   r   Ú	get_groupÚ  s"    
ýÿýÿzDeviceMesh.get_groupc                    s   ‡ fdd„t ˆ jjƒD ƒS )z 
            Returns a list of ProcessGroups for all mesh dimensions.

            Returns:
                A list of :class:`ProcessGroup` object.
            c                    s   g | ]}ˆ   |¡‘qS r   )r˜   )r2   r�   r&   r   r   r5     s     z-DeviceMesh.get_all_groups.<locals>.<listcomp>)r>   r6   r?   r&   r   r&   r   Úget_all_groupsþ  s    zDeviceMesh.get_all_groupsr‹   )ÚgrouprB   r6   r/   r"   c                C   s<  t | tƒrŒt| ƒ}t |tjƒr*| ¡ |ks:|dk	rR||krRtdt|ƒ› d|› �ƒ‚tj|dtj	d�}t
|||dd�}t| ƒ|| jfg|_|S t| ƒ}t|ƒdkr¨td	ƒ‚|dkr¸td
ƒ‚t |tjƒrØ| ¡ jtj	dd�ntj|dtj	d�}|jt|ƒk�rtd| ¡ › dt|ƒ› d�ƒ‚t
|||dd�}dd„ |D ƒ|_|S )aN  
            Contstructs a :class:`DeviceMesh` with ``device_type`` from an
            existing :class:`ProcessGroup`.

            The constructed device mesh has number of dimensions equal to the
            number of groups passed. If more than one group is passed, then the
            ``mesh`` argument is required.
            NzInvalid mesh z for ProcessGroup with ranks r`   rc   Fr:   r   z.Expects at least one ProcessGroup to be passedz0Must pass mesh if passing multiple ProcessGroups)rb   rd   zEExpects mesh with ndim equal to number of ProcessGroups but got mesh z and z ProcessGroupsc                 S   s    g | ]}t |ƒt|ƒ|jf‘qS r   )r   r   rz   )r2   rš   r   r   r   r5   <  s
   üýz)DeviceMesh.from_group.<locals>.<listcomp>)re   r   r   r_   rf   rn   rh   rX   rk   rY   r   r   rz   r<   r=   r*   ri   rj   r?   )rš   rB   r6   r/   Zgroup_ranksrG   Úgroupsr   r   r   Ú
from_group  s\    

ÿ
ÿþþÿüÿ
ÿýÿ   ÿúzDeviceMesh.from_groupc                 C   s   |d kr| j  ¡ S | j  |¡S r   )r6   rw   r7   r—   r   r   r   r7   F  s    zDeviceMesh.sizec                 C   s   | j jS r   )r6   r?   r&   r   r   r   r?   I  s    zDeviceMesh.ndimc                 C   s   t | jjƒS r   )rl   r6   r…   r&   r   r   r   r…   M  s    zDeviceMesh.shapec                 C   s   t ƒ S )z:
            Returns the current global rank.
            )r   r&   r   r   r   r   Q  s    zDeviceMesh.get_rankc                 C   sb   | j dkr*|dkr*td| jj › d�dƒ‚n|dkr6d}t|  |¡ƒ}t|tƒsVtdƒ‚tt|ƒƒS )a{  
            Returns the local rank of the given mesh_dim of the DeviceMesh.

            Args:
                mesh_dim (str/int, optional): it can be the name of the mesh dimension or the index
                of the mesh dimension. Default is None.

            Returns:
                An integer denotes the local rank.

            The following program runs on each process/rank in an SPMD manner. In this example, we have 2
            hosts with 4 GPUs each.
            Calling mesh_2d.get_local_rank(mesh_dim=0) on rank 0, 1, 2, 3 would return 0.
            Calling mesh_2d.get_local_rank(mesh_dim=0) on rank 4, 5, 6, 7 would return 1.
            Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 0, 4 would return 0.
            Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 1, 5 would return 1.
            Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 2, 6 would return 2.
            Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 3, 7 would return 3.

            Example::
                >>> # xdoctest: +SKIP("no rank")
                >>> from torch.distributed.device_mesh import DeviceMesh
                >>>
                >>> # Initialize device mesh as (2, 4) to represent the topology
                >>> # of cross-host(dim 0), and within-host (dim 1).
                >>> mesh = DeviceMesh(device_type="cuda", mesh=[[0, 1, 2, 3],[4, 5, 6, 7]])
            rJ   Nr“   r”   r•   r   z1We expect ProcessGroup before calling `get_rank`!)	r?   r+   r6   r
   r˜   re   r   rK   r   )r'   r9   Zmesh_dim_groupr   r   r   Úget_local_rankW  s    þ ÿþzDeviceMesh.get_local_rankc                 C   s   | j r| j S dS )z©
            Return the relative indices of this rank relative to all
            dimensions of the mesh. If this rank is not part of the mesh, return None.
            N)ru   r&   r   r   r   Úget_coordinate�  s    zDeviceMesh.get_coordinate)N)N)N)N)$r   r   r   Ú__doc__rX   Ú__annotations__r_   rf   r   r   r   Úboolr(   rs   rt   r{   r€   r‚   r†   Úobjectr‰   r’   rY   r   r˜   r   r™   rZ   rœ   r7   Úpropertyr?   r…   r   r�   rž   r   r   r   r   r   µ   sX   
'
úù(Dþ?$	 ýûú>*r‹   .)rB   Ú
mesh_shaper/   r"   c             	   C   s¾   |dk	r\t t|ƒƒt |ƒkr,tdd|› �ƒ‚t |ƒt |ƒkr\tddt |ƒ› dt |ƒ› d�ƒ‚| rz|  ¡ sztd| › d	�d
ƒ‚t d¡�" tjt |¡tj	d� 
|¡}W 5 Q R X t| ||d�}|S )a¤  
        Initializes a `DeviceMesh` based on `device_type`, `mesh_shape`, and `mesh_dim_names` parameters.

        This creates a DeviceMesh with an n-dimensional array layout, where `n` is the length of `mesh_shape`.
        If `mesh_dim_names` is provided, each dimension is labeled as `mesh_dim_names[i]`.

        .. note::
            `init_device_mesh` follows SPMD programming model, meaning the same PyTorch Python program
            runs on all processes/ranks in the cluster. Ensure `mesh_shape` (the dimensions of the nD array
            describing device layout) is identical across all ranks. Inconsistent `mesh_shape` may lead to hanging.

        .. note::
            If no process group is found, init_device_mesh will initialize distributed process group/groups
            required for distributed communications behind the scene.

        Args:
            device_type (str): The device type of the mesh. Currently supports: "cpu", "cuda/cuda-like".
                Passing in a device type with a GPU index, such as "cuda:0", is not allowed.
            mesh_shape (Tuple[int]): A tuple defining the dimensions of the multi-dimensional array
                describing the layout of devices.
            mesh_dim_names (Tuple[str], optional): A tuple of mesh dimension names to assign to each dimension
                of the multi-dimensional array describing the layout of devices. Its length must match the length
                of `mesh_shape`. Each string in `mesh_dim_names` must be unique.

        Returns:
            DeviceMesh: A :class:`DeviceMesh` object representing the device layout.

        Example::
            >>> # xdoctest: +SKIP("no rank")
            >>> from torch.distributed.device_mesh import init_device_mesh
            >>>
            >>> mesh_1d = init_device_mesh("cuda", mesh_shape=(8,))
            >>> mesh_2d = init_device_mesh("cuda", mesh_shape=(2, 8), mesh_dim_names=("dp", "tp"))

        Nz"Each mesh_dim_name must be unique.z/Found repeated mesh_dim_name in mesh_dim_names z6mesh_shape and mesh_dim_names should have same length!zFound len(mesh_dim_names): z and len(mesh_shape):rŠ   z4Device type with GPU index is not supported but got z. zUIf you maintained a 'torch.device' object, it's recommended to pass in 'device.type'.r`   ra   )rB   r6   r/   )r*   Úsetr+   Úisalphar_   rd   ZarangeÚmathÚprodrY   Úviewr   )rB   r¤   r/   r6   rG   r   r   r   r   ˆ  s0    )þþ
þ&ý)r]   )/Úloggingr§   rp   Útypingr   r   r   r   r   r   r_   Ztorch.distributedr   Zutils._typing_utilsr
   Ú__all__Úsysr   r   Úmodulesr   r   Z"torch.distributed.distributed_c10dr   r   r   r   r   r   r   r   r   r   Ú	getLoggerr   ÚloggerZnumpy.typingr   ÚImportErrorÚwarningÚlocalr    r\   r    rX   rN   rY   r   r   r   r   Ú<module>   sP     þÿ0
ÿ
l	   Zü
û