U
    D‹Ñh¥2  ã                   @   s  d dl Z d dlmZmZmZmZmZmZmZ d dl	Z	d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZm Z m!Z!m"Z"m#Z#mZm$Z$ d dl%m&Z&m'Z' d dl(m)Z)m*Z* d dl+m,Z, d dl-m.Z. d dl/m0Z0m1Z1m2Z2 d dl3m4Z4 d dl5m6Z6 d dl7m8Z8 ee9eeee:  ee: f f Z;dgZ<d)e:e9e9dœdd„Z=d*eej> edœdd„Z?e	j@eAdœdd„ZBd+eee: e9e	j@dœd d!„ZCe#ee;eej> f d"œd#d$„ZDG d%d&„ d&eƒZEd,e#e9e.ee' e#d'œd(d„ZFdS )-é    N)ÚcastÚDictÚListÚOptionalÚSequenceÚTupleÚUnion)Ú_get_device_module)ÚShardedTensor)ÚTensorProperties)ÚShard)ÚChunkShardingSpec)ÚDTensor)Úunflatten_state_dict)ÚDefaultLoadPlanner)ÚBytesStorageMetadataÚChunkStorageMetadataÚMetadataÚMetadataIndexÚSTATE_DICT_TYPEr   ÚTensorStorageMetadata)ÚLoadPlanÚLoadPlanner)Ú_create_read_itemsÚ create_read_items_for_chunk_list)Úload_state_dict)ÚStorageReader)Ú_element_wise_addÚ_element_wise_subÚ_normalize_device_info)Ú_get_default_group)Ú_create_chunk_sharded_tensor)Ú_remote_deviceÚ!load_sharded_optimizer_state_dictÚcuda)Úglobal_rankÚdevice_typeÚreturnc                 C   s2   |dkrdS t |ƒ}| ¡ r.t|| | ¡  ƒS dS )NÚcpu)r	   Zis_availabler   Údevice_count)r%   r&   Údevice_module© r+   úT/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/distributed/checkpoint/optimizer.pyÚ_gen_rank_device4   s     
ÿr-   )Úpgr'   c                    sl   t j ˆ ¡j‰ˆ d kr2‡fdd„tt  ¡ ƒD ƒ}n‡ ‡fdd„tˆ  ¡ ƒD ƒ}tdtt	t
ttf  |ƒd�S )Nc                    s"   g | ]}d |› dt |ˆ ƒ› �‘qS ©úrank:ú/)r-   ©Ú.0Úidx)Úpg_device_typer+   r,   Ú
<listcomp>D   s   ÿz(_create_colwise_spec.<locals>.<listcomp>c              
      s*   g | ]"}d |› dt t ˆ |¡ˆƒ› �‘qS r/   )r-   ÚdistZget_global_rankr2   ©r.   r5   r+   r,   r6   I   s   ÿr   ©ZdimÚ
placements)r7   Údistributed_c10dÚ_get_pg_default_deviceÚtypeÚrangeÚget_world_sizeÚsizer   r   r   r   r"   Ústr)r.   r:   r+   r8   r,   Ú_create_colwise_spec?   s    

þ
þþrB   )Úvalr'   c                 C   sŽ   t | ƒtkrZt|  ¡ ƒdkr dS t |  ¡ d jƒtkr:dS t |  ¡ d jƒtkrŠtdƒ‚n0t | ƒtkrŠt | jƒtks‚t | jƒtkrŠtdƒ‚dS )Nr   FTz2Cannot handle DTensor nested insided ShardedTensorzCannot handle nested DTensor)r=   r
   ÚlenÚlocal_shardsÚtensorr   Ú
ValueErrorZ_local_tensor)rC   r+   r+   r,   Ú_is_nested_tensorS   s    
ÿÿrH   )Úpropsr@   r&   r'   c              
   C   s.   t j|| j| j| j| jtt jt|ƒ 	¡ ƒd�S )N)r@   ÚdtypeÚlayoutÚrequires_gradÚ
pin_memoryÚdevice)
ÚtorchÚemptyrJ   rK   rL   rM   r   rN   r	   Zcurrent_device)rI   r@   r&   r+   r+   r,   Ú_alloc_tensorb   s    úrQ   )Ú
state_dictr'   c                 C   sŒ   i }d}|   ¡ D ]r\}}d| ¡ f||< t|ƒrt| ¡ ƒdksHtdƒ‚t|tƒsZtdƒ‚| ¡ d }|jj	|jj
f||< |jj}q||fS )a+  
    Load the right TP slice of the optimizer state.

    This is not easy since the per-tensor slicing can't be inferred from checkpoint metadata.
    We take advantage of the model state_dict producing a sliced ST to figure out what we need to load.
    This is pretty fragile and it might be easier for FSDP to compute this info for us.
    Returns a dictionary where keys are the same of the state_dict and the value is a tuple of
    (offset, size) for the current rank TP slice.
    N.B. The state_dict *MUST* come from FSDP.sharded_state_dict.
    Né   z%Cannot handle ST with multiple shardsz$Can only handle nested ShardedTensorr   )Úitemsr@   rH   rD   rE   ÚAssertionErrorÚ
isinstancer
   ÚmetadataÚshard_offsetsÚshard_sizesrF   Z_process_group)rR   ÚspecsÚdp_pgÚkeyÚvalueZshardr+   r+   r,   Ú_get_state_dict_2d_layouto   s.    ÿþ ÿþþ
þr^   c                       sv   e Zd ZU eeef ed< eed< eed< eee	e
 f ddœ‡ fdd„Zedœd	d
„Zeejdœ‡ fdd„Z‡  ZS )Ú_ReaderWithOffsetÚtranslationrR   rW   N)Úfqn_to_offsetr'   c                    s*   t ƒ  ¡  || _ti ƒ| _i | _i | _d S ©N)ÚsuperÚ__init__ra   r   rW   rR   r`   )Úselfra   ©Ú	__class__r+   r,   rd   ™   s
    

z_ReaderWithOffset.__init__)r'   c                 C   s"  g }i | _ | j ¡ D �]\}}| jj| }t|tƒsF|t|||ƒ7 }q|| jkrb|t|||ƒ7 }q| j| }t	| 
¡ ƒdks€t‚| 
¡ d }tt t|jj|ƒ¡t |jj¡d�g}t|tt|ƒ|ƒ}|D ]D}	|	jjd k	sÞt‚t|	jj|ƒ}
tj|	jt |
¡d�}|| j |	j< qÊ||7 }qt|ƒS )NrS   r   )ÚoffsetsZsizes)Úoffset)r`   rR   rT   rW   Ústate_dict_metadatarV   r
   r   ra   rD   rE   rU   r   rO   ÚSizer   rX   rY   r   r   r   Z
dest_indexri   r   ÚdataclassesÚreplacer   )re   ÚrequestsZfqnÚobjZmdri   Zoriginal_shardZlocal_chunksÚreqsÚriZoriginal_offsetZoriginal_indexr+   r+   r,   Úcreate_local_plan    sF    


ÿüÿ	  ÿ ÿ
z#_ReaderWithOffset.create_local_plan)Úindexr'   c                    s   t ƒ  | j ||¡¡S rb   )rc   Úlookup_tensorr`   Úget)re   rs   rf   r+   r,   rt   Ê   s    z_ReaderWithOffset.lookup_tensor)Ú__name__Ú
__module__Ú__qualname__r   r   Ú__annotations__r   r   rA   r   Úintrd   r   rr   rO   ÚTensorrt   Ú__classcell__r+   r+   rf   r,   r_   ”   s   
 *r_   )Úmodel_state_dictÚoptimizer_keyÚstorage_readerÚplannerr'   c              	   C   sX  |  ¡ }t| ƒ\}}tj |¡j}t|ƒ}|dkr~g }	tt ¡ ƒD ],}
t	||
| 
¡  ƒ}|	 d|
› d|› �¡ qBtd|	d�}nt|ƒ}i }i }|j ¡ D �]Œ\}}|j| }|d |krºq˜t|tƒrÎd||< q˜|j ¡ dkròt|j|j|ƒ||< q˜|dk�r.tt|j|j|ƒt ¡ t ¡ | 
¡ tƒ d�||< q˜|d	 }| |d|jf¡d }t|jj|jj|jj|jj|jj d
�}| !t" #|¡|¡}g }t |¡}|j$D ]>}t%t&|j'ƒ (¡ |k�r¶�q˜| t)t|j|j*|ƒ|d�¡ �q˜t+j,|||d�}||k�r|| d dk	�rt%t-t. || d ƒ||< |||< q˜t/|||dk	�r@t0|ƒn|d� t1||jƒ}|S )aç  
    Load a state_dict in conjunction with FSDP sharded optimizer state.

    This is the current recommended way to checkpoint FSDP.
    >>> # xdoctest: +SKIP
    >>> import torch.distributed.checkpoint as dist_cp
    >>> # Save
    >>> model: torch.nn.Model
    >>> optim_params = model.parameters()
    >>> optim = torch.optim.SGD(optim_params, lr=0.01)
    >>> # Save
    >>> with FSDP.state_dict_type(model, StateDictType.SHARDED_STATE_DICT):
    >>>     state_dict = {
    >>>         "optimizer": FSDP.optim_state_dict(model, optim),
    >>>         "model": model.state_dict()
    >>>     }
    >>>     dist_cp.save_state_dict(
    >>>         state_dict=optim_state,
    >>>         storage_writer=dist_cp.FileSystemWriter("checkpoint"),
    >>>         planner=dist_cp.DefaultSavePlanner(),
    >>>     )
    >>>
    >>> # Load
    >>> with FSDP.state_dict_type(model_tp, StateDictType.SHARDED_STATE_DICT):
    >>>     model_state_dict = model_tp.state_dict()
    >>>     checkpoint = {
    >>>         "model": model_state_dict
    >>>     }
    >>>     dist_cp.load_state_dict(
    >>>         state_dict=checkpoint,
    >>>         storage_reader=dist_cp.FileSystemReader(checkpoint_file),
    >>>         planner=dist_cp.DefaultLoadPlanner(),
    >>>     )
    >>>     model.load_state_dict(checkpoint["model_state"])
    >>>
    >>>     optim_state = dist_cp.load_sharded_optimizer_state_dict(
    >>>         model_state_dict,
    >>>         optimizer_key="optimizer",
    >>>         storage_reader=dist_cp.FileSystemReader("checkpoint"),
    >>>     )
    >>>
    >>>     flattened_osd = FSDP.optim_state_dict_to_load(
    >>>        model, optim, optim_state["optimizer"]
    >>>     )
    >>>
    >>>     optim.load_state_dict(flattened_osd)
    Nr0   r1   r   r9   z
<bytes_io>rS   )ÚrankZ
world_sizeZnum_devices_per_noder.   é   )rJ   rK   rL   Úmemory_formatrM   )rF   rW   )Zprocess_group)rR   r   r€   )2Zread_metadatar^   r7   r;   r<   r=   r	   r>   r?   r   r)   Úappendr   rB   rj   rT   Zplanner_datarV   r   r@   ZnumelrQ   Ú
propertiesr!   Zget_rankr    ru   ÚShardTensorPropertiesrJ   rK   rL   rƒ   rM   Zbuild_metadatarO   rk   Zshards_metadatar   r"   Z	placementr�   r   rY   r
   Z+_init_from_local_shards_and_global_metadatar   rz   r   r_   r   )r}   r~   r   r€   rW   Zlayout_specsr[   Zdp_pg_device_typer*   r:   ÚiZdevice_infoZsharding_specrR   ra   r\   r]   Zkey_pathZspec_keyZ
alloc_sizer…   Zst_mdrE   Zcurrent_rankZshard_mdÚstr+   r+   r,   r#   Î   sš    5 
ÿ

  ÿ

ûû

  ÿüÿ	  ÿ
ü)r$   )N)r$   )N)Grl   Útypingr   r   r   r   r   r   r   rO   Ztorch.distributedZdistributedr7   Ztorch._utilsr	   Z+torch.distributed._shard.sharded_tensor.apir
   Z0torch.distributed._shard.sharded_tensor.metadatar   r†   Z-torch.distributed._shard.sharded_tensor.shardr   Z:torch.distributed._shard.sharding_spec.chunk_sharding_specr   Ztorch.distributed._tensorr   Z)torch.distributed.checkpoint._nested_dictr   Z,torch.distributed.checkpoint.default_plannerr   Z%torch.distributed.checkpoint.metadatar   r   r   r   r   r   Z$torch.distributed.checkpoint.plannerr   r   Z,torch.distributed.checkpoint.planner_helpersr   r   Z.torch.distributed.checkpoint.state_dict_loaderr   Z$torch.distributed.checkpoint.storager   Z"torch.distributed.checkpoint.utilsr   r   r   Z"torch.distributed.distributed_c10dr    Z#torch.distributed.fsdp._shard_utilsr!   Ztorch.distributed.remote_devicer"   rA   rz   ZSTATE_DICT_2D_LAYOUTÚ__all__r-   ZProcessGrouprB   r{   ÚboolrH   rQ   r^   r_   r#   r+   r+   r+   r,   Ú<module>   sb   $$	 ÿ ÿþ ÿ  þþ%> üû