U
    8‹Ñhn  ã                   @   sN  d dl Z ddlmZ ddlmZ d dlmZmZmZm	Z	m
Z
mZmZmZmZmZ d dlmZ d dlmZ er„d d	lmZ d d
lmZ dgZeedœdd„Zeedœdd„Zddœdd„Zddœdd„Zd#ee
ee  edœdd„Zd$ee j eee e j!f  ee	e	e j  dœdd„Z"eded �Z#d%e#eee e j!f  ee	e# d!œd"d„Z$dS )&é    Né   )ÚModuleé   )Úcomm)
ÚTYPE_CHECKINGÚDictÚIteratorÚListÚOptionalÚSequenceÚSetÚTypeVarÚUnionÚcast©Ú_get_device_index)ÚOrderedDict)ÚScriptModule)ÚEnabledProxyÚ	replicate©ÚmoduleÚreturnc                 C   s   dd l }t| |jjƒS ©Nr   )Ú	torch.jitÚ
isinstanceÚjitr   ©r   Útorch© r   úI/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/nn/parallel/replicate.pyÚ_is_script_module   s    r!   c                 C   s   dd l }t| |jjƒS r   )r   r   Z_CZScriptMethodr   r   r   r    Ú_is_script_method   s    r"   r   )r   c                  C   s   dd l } | j ¡ S r   )r   r   r   ©r   r   r   r    Ú_init_script_module   s    r$   r   c                  C   s   dd l } | jjjS r   )Útorch.jit._stater   Ú_stateZ_enabledr#   r   r   r    Ú_is_jit_enabled   s    r'   )r   Úmemor   c                 C   sŽ   t tt  dœdd„}tƒ sdS |d kr,tƒ }| | ¡ t| ƒrb| || ƒ¡ tdd„ || ƒD ƒƒS |  ¡ D ]}||krxqjt	||ƒsj dS qjdS )Nr   c                 S   s   |   ¡ }t|ƒ |S ©N)ÚmodulesÚnext)r   Úgenr   r   r    Údescendant_modules-   s    z0_replicatable_module.<locals>.descendant_modulesTc                 s   s   | ]}t |ƒV  qd S r)   )r!   )Ú.0Z
descendantr   r   r    Ú	<genexpr>;   s   ÿz'_replicatable_module.<locals>.<genexpr>F)
r   r   r'   ÚsetÚaddr!   ÚupdateÚallÚchildrenÚ_replicatable_module)r   r(   r-   Úchildr   r   r    r5   *   s"    
ÿ
r5   F)ÚtensorsÚdevicesÚdetachr   c                    sd   ddl m} |rt ˆ|¡S tˆƒdkr\|j|fˆžŽ ‰ ‡ ‡fdd„tdtˆ ƒtˆƒƒD ƒS g S d S )Nr   )Ú	Broadcastr   c                    s    g | ]}ˆ ||t ˆƒ … ‘qS r   )Úlen)r.   Úi©Ztensor_copiesr7   r   r    Ú
<listcomp>T   s   ÿz0_broadcast_coalesced_reshape.<locals>.<listcomp>)Z
_functionsr:   r   Zbroadcast_coalescedr;   ÚapplyÚrange)r7   r8   r9   r:   r   r=   r    Ú_broadcast_coalesced_reshapeH   s    ÿrA   ÚT)Úbound)Únetworkr8   r9   r   c                    s  t | ƒstdƒ‚|sg S dd„ |D ƒ}t|ƒ}t|  ¡ ƒ}dd„ t|ƒD ƒ}t|||ƒ}t|  ¡ ƒ}g }g }	|D ]$}
|
jrŠ|sŠ| 	|
¡ qp|	 	|
¡ qpdd„ t|ƒD ƒ}dd„ t|	ƒD ƒ}t|||d�}t|	|d	d�}t|  
¡ ƒ}d
d„ |D ƒ‰ i }t|ƒD ]>\}}|||< t|ƒD ]$}| ¡ }tƒ |_ˆ |  	|¡ �qqüt|ƒD �]¸\}}|j ¡ D ]t\}}|d k�r’t|ƒD ]}ˆ | | }d |j|< �qrn8|| }t|ƒD ]&}ˆ | | }t||ˆ | | ƒ �q¢�qX|j ¡ D ]‚\}}|d k�rt|ƒD ]}ˆ | | }d |j|< �qònF|| }t|ƒD ]4}ˆ | | }|| | }t|||ƒ ||j|< �q"�qØ|j ¡ D ]”\}}
|
d k�r t|ƒD ]}ˆ | | }d |j|< �q€nX|
j�r¼|�s¼|}||
 }n|}||
 }t|ƒD ]&}ˆ | | }t|||| | ƒ �qÐ�qf�qD‡ fdd„t|ƒD ƒS )NzKCannot replicate network where python modules are childrens of ScriptModulec                 S   s   g | ]}t |d ƒ‘qS )Tr   )r.   Úxr   r   r    r>   i   s     zreplicate.<locals>.<listcomp>c                 S   s   i | ]\}}||“qS r   r   )r.   ÚidxÚparamr   r   r    Ú
<dictcomp>m   s      zreplicate.<locals>.<dictcomp>c                 S   s   i | ]\}}||“qS r   r   ©r.   rF   Úbufr   r   r    rH   y   s      c                 S   s   i | ]\}}||“qS r   r   rI   r   r   r    rH   z   s      )r9   Tc                 S   s   g | ]}g ‘qS r   r   )r.   Ú_r   r   r    r>   €   s     c                    s   g | ]}t tˆ | d  ƒ‘qS )r   )r   rB   )r.   Új©Zmodule_copiesr   r    r>   º   s     )r5   ÚRuntimeErrorr;   ÚlistÚ
parametersÚ	enumeraterA   ÚbuffersZrequires_gradÚappendr*   r@   Z_replicate_for_data_parallelr   Z_former_parametersZ_modulesÚitemsÚsetattrÚ_parametersÚ_buffers)rD   r8   r9   Znum_replicasÚparamsZparam_indicesZparam_copiesrR   Z
buffers_rgZbuffers_not_rgrJ   Zbuffer_indices_rgZbuffer_indices_not_rgZbuffer_copies_rgZbuffer_copies_not_rgr*   Zmodule_indicesr<   r   rL   ZreplicaÚkeyr6   Z
module_idxrG   Z	param_idxZ
param_copyZbuffer_copiesZ
buffer_idxr   rM   r    r   ]   s€    




 )N)F)F)%r   r*   r   Ú r   Útypingr   r   r   r	   r
   r   r   r   r   r   Ztorch._utilsr   Úcollectionsr   r   r   r%   r   Ú__all__Úboolr!   r"   r$   r'   r5   ZTensorÚintZdevicerA   rB   r   r   r   r   r    Ú<module>   s:   0! ýü ýü