U
    D‹ÑhÓ*  ã                   @   sn  d dl Z d dlZd dlmZmZ d dlmZmZmZm	Z	m
Z
mZmZ d dlm  m  m  mZ d dlmZmZ d dlmZ d dlmZ d dlmZmZmZ d dlmZ d d	l m!Z! d d
l"m#Z# d dl$m%Z% dddgZ&e%e'ƒZ(eG dd„ dƒƒZ)G dd„ dƒZ*eee+df e	e e+dœdd„Z,e!ee
e+ e
e- f dœdd„Z.e)eee+df e	e ee-ef dœdd„Z/dS )é    N)Ú	dataclassÚfield)ÚAnyÚCallableÚDictÚListÚOptionalÚTupleÚUnion)ÚeventsÚmetrics)Ú
WorkerSpec)ÚLocalElasticAgent)ÚDefaultLogsSpecsÚ	LogsSpecsÚSignalException)ÚChildFailedError)ÚRendezvousParameters)Úparse_rendezvous_endpoint)Ú
get_loggerÚLaunchConfigÚelastic_launchÚlaunch_agentc                   @   sö   e Zd ZU dZeed< eed< eed< dZee ed< dZ	e
ed< d	Ze
ed
< dZe
ed< dZe
ed< eed�Zee
ef ed< dZeed< dZeed< dZeed< dZe
ed< dZee
 ed< eed�Zee
e
f ed< dZee
 ed< dd„ ZdS )r   aM  
    Creates a rendezvous config.

    Args:
        min_nodes: Minimum amount of nodes that the user function will
                        be launched on. Elastic agent ensures that the user
                        function start only when the min_nodes amount enters
                        the rendezvous.
        max_nodes: Maximum amount of nodes that the user function
                        will be launched on.
        nproc_per_node: On each node the elastic agent will launch
                            this amount of workers that will execute user
                            defined function.
        rdzv_backend: rdzv_backend to use in the rendezvous (zeus-adapter, etcd).
        rdzv_endpoint: The endpoint of the rdzv sync. storage.
        rdzv_configs: Key, value pair that specifies rendezvous specific configuration.
        rdzv_timeout: Legacy argument that specifies timeout for the rendezvous. It is going
            to be removed in future versions, see the note below. The default timeout is 900 seconds.
        run_id: The unique run id of the job (if not passed a unique one will be
                deduced from run environment - flow workflow id in flow - or auto generated).
        role: User defined role of the worker (defaults to "trainer").
        max_restarts: The maximum amount of restarts that elastic agent will conduct
                    on workers before failure.
        monitor_interval: The interval in seconds that is used by the elastic_agent
                        as a period of monitoring workers.
        start_method: The method is used by the elastic agent to start the
                    workers (spawn, fork, forkserver).
        metrics_cfg: configuration to initialize metrics.
        local_addr: address of the local node if any. If not set, a lookup on the local
                machine's FQDN will be performed.
        local_ranks_filter: ranks for which to show logs in console. If not set, show from all.
    ..note:
        `rdzv_timeout` is a legacy argument that will be removed in future.
        Set the timeout via `rdzv_configs['timeout']`

    Ú	min_nodesÚ	max_nodesÚnproc_per_nodeNÚ
logs_specsÚ Úrun_idZdefault_roleÚroleÚrdzv_endpointZetcdÚrdzv_backend)Údefault_factoryÚrdzv_configséÿÿÿÿÚrdzv_timeouté   Úmax_restartsgš™™™™™¹?Úmonitor_intervalÚspawnÚstart_methodÚlog_line_prefix_templateÚmetrics_cfgÚ
local_addrc                 C   sF   d}| j dkr| j | jd< nd| jkr0|| jd< | jd krBtƒ | _d S )Ni„  r$   Útimeout)r%   r#   r   r   )ÚselfZdefault_timeout© r0   úL/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/distributed/launcher/api.pyÚ__post_init__U   s    



zLaunchConfig.__post_init__)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚintÚ__annotations__r   r   r   r   Ústrr   r    r!   r   Údictr#   r   r   r%   r'   r(   Úfloatr*   r+   r,   r-   r2   r0   r0   r0   r1   r      s$   
%c                   @   s2   e Zd ZdZeeeedf dœdd„Zdd„ Z	dS )r   a  
    Launches an torchelastic agent on the container that invoked the entrypoint.

        1. Pass the ``entrypoint`` arguments as non ``kwargs`` (e.g. no named parameters)/
           ``entrypoint`` can be a function or a command.
        2. The return value is a map of each worker's output mapped
           by their respective global rank.

    Usage

    ::

    def worker_fn(foo):
        # ...

    def main():
        # entrypoint is a function.
        outputs = elastic_launch(LaunchConfig, worker_fn)(foo)
        # return rank 0's output
        return outputs[0]

        # entrypoint is a command and ``script.py`` is the python module.
        outputs = elastic_launch(LaunchConfig, "script.py")(args)
        outputs = elastic_launch(LaunchConfig, "python")("script.py")
    N)ÚconfigÚ
entrypointc                 C   s   || _ || _d S ©N)Ú_configÚ_entrypoint)r/   r<   r=   r0   r0   r1   Ú__init__|   s    zelastic_launch.__init__c                 G   s   t | j| jt|ƒƒS r>   )r   r?   r@   Úlist)r/   Úargsr0   r0   r1   Ú__call__„   s    zelastic_launch.__call__)
r3   r4   r5   r6   r   r
   r   r9   rA   rD   r0   r0   r0   r1   r   a   s
   ý)r=   rC   Úreturnc                 C   sF   t | tƒr| jS t | tƒr>| tjkr8tdd„ |D ƒdƒS | S ndS dS )aÀ  Retrieve entrypoint name with the rule:
    1. If entrypoint is a function, use ``entrypoint.__qualname__``.
    2. If entrypoint is a string, check its value:
        2.1 if entrypoint equals to ``sys.executable`` (like "python"), use the first element from ``args``
            which does not start with hifen letter (for example, "-u" will be skipped).
        2.2 otherwise, use ``entrypoint`` value.
    3. Otherwise, return empty string.
    c                 s   s   | ]}|d  dkr|V  qdS )r   ú-Nr0   )Ú.0Úargr0   r0   r1   Ú	<genexpr>—   s      z'_get_entrypoint_name.<locals>.<genexpr>r   N)Ú
isinstancer   r3   r9   ÚsysÚ
executableÚnext)r=   rC   r0   r0   r1   Ú_get_entrypoint_nameˆ   s    


rN   )Úrdzv_parametersrE   c                 C   sX   | j dkrdS | j}| ¡ }|s(tdƒ‚t|dd�\}}|dkrPtd|› d�ƒ‚||fS )NZstatic)NNzKEndpoint is missing in endpoint. Try to add --master-addr and --master-portr$   )Údefault_portzport is missing in endpoint: z. Try to specify --master-port)ÚbackendÚendpointÚstripÚ
ValueErrorr   )rO   rR   Úmaster_addrÚmaster_portr0   r0   r1   Ú_get_addr_and_portž   s    
ÿ
ÿrW   )r<   r=   rC   rE   c                 C   s¼  | j s&tt ¡ jƒ}t d|¡ || _ t||ƒ}t d|| j	| j
| j| j | j| j| j| j| j| jj| jdœ¡ tf | j| j| j | j	| j
| jdœ| j—Ž}t|ƒ\}}t| j| j|t|ƒt |¡| j| j||| jd�
}t|| j| j| jd�}	d}
z¶zLt! "t! #| j¡¡ |	 $¡ }t% &|	 '¡ ¡ | (¡ �r2t)||j*d�‚|j+W W ¢jS  t)k
�rT   ‚ Y nN t,k
�r|   d	}
t% &|	 -¡ ¡ ‚ Y n& t.k
�r    t% &|	 -¡ ¡ ‚ Y nX W 5 |
�r¶|j  ¡  X d S )
Nz3config has no run_id, generated a random run_id: %saê  Starting elastic_operator with launch configs:
  entrypoint       : %(entrypoint)s
  min_nodes        : %(min_nodes)s
  max_nodes        : %(max_nodes)s
  nproc_per_node   : %(nproc_per_node)s
  run_id           : %(run_id)s
  rdzv_backend     : %(rdzv_backend)s
  rdzv_endpoint    : %(rdzv_endpoint)s
  rdzv_configs     : %(rdzv_configs)s
  max_restarts     : %(max_restarts)s
  monitor_interval : %(monitor_interval)s
  log_dir          : %(log_dir)s
  metrics_cfg      : %(metrics_cfg)s
)r=   r   r   r   r   r!   r    r#   r'   r(   Zlog_dirr,   )rQ   rR   r   r   r   r-   )
r   Zlocal_world_sizer=   rC   Úrdzv_handlerr'   r(   rU   rV   r-   )Úspecr   r*   r+   T)ÚnameÚfailuresF)/r   r9   ÚuuidZuuid4r7   ÚloggerÚwarningrN   Úinfor   r   r   r!   r    r#   r'   r(   r   Zroot_log_dirr,   r   r-   rW   r   r   ÚtupleÚrdzv_registryZget_rendezvous_handlerr   r*   r+   rX   Úshutdownr   Zinitialize_metricsZMetricsConfigÚrunr   ÚrecordZget_event_succeededZ	is_failedr   r[   Zreturn_valuesr   Zget_event_failedÚ	Exception)r<   r=   rC   r   Zentrypoint_namerO   rU   rV   rY   ZagentZshutdown_rdzvÚresultr0   r0   r1   r   ±   s�    
ôòúù
öü
þ)0rK   r\   Zdataclassesr   r   Útypingr   r   r   r   r   r	   r
   Z-torch.distributed.elastic.rendezvous.registryZdistributedZelasticZ
rendezvousÚregistryra   Ztorch.distributed.elasticr   r   Z*torch.distributed.elastic.agent.server.apir   Z:torch.distributed.elastic.agent.server.local_elastic_agentr   Z)torch.distributed.elastic.multiprocessingr   r   r   Z0torch.distributed.elastic.multiprocessing.errorsr   Z$torch.distributed.elastic.rendezvousr   Z*torch.distributed.elastic.rendezvous.utilsr   Z'torch.distributed.elastic.utils.loggingr   Ú__all__r3   r]   r   r   r9   rN   r7   rW   r   r0   r0   r0   r1   Ú<module>	   s:   $
C( þþ
ü