U
    6‹Ñh¯!  ã                   @   sÒ   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	 d dl
mZ d dlmZ d dlmZmZmZmZ dd„ ZdaG d	d
„ d
eƒZeddddddddddg	ƒG dd„ dejƒƒZG dd„ deƒZdS )é    N)Úsuppress)Úcached_propertyÚwraps)Úparse_qs)ÚAbstractFileSystem)Ú"get_package_version_without_importÚinfer_storage_optionsÚmirror_fromÚtokenizec                    s   t ˆ ƒ‡ fdd„ƒ}|S )Nc               
      sj   zˆ | |ŽW S  t k
rd } z:|js&‚ |j^}} t|tƒrRd|krRttj|ƒ|‚n‚ W 5 d }~X Y nX d S )Nzdoes not exist)ÚOSErrorÚargsÚ
isinstanceÚstrÚFileNotFoundErrorÚerrnoÚENOENT)r   ÚkwargsÚ	exceptionÚmessage©Úfunc© úJ/tmp/user/2202/pip-unpacked-wheel-org7786_/fsspec/implementations/arrow.pyÚwrapper   s    
z wrap_exceptions.<locals>.wrapper)r   )r   r   r   r   r   Úwrap_exceptions   s    r   c                       sø   e Zd ZdZdZ‡ fdd„Zedd„ ƒZedd„ ƒZ	e
d	d
„ ƒZd-dd„Zdd„ Zdd„ Zdd„ Zedd„ ƒZedd„ ƒZedd„ ƒZed.dd„ƒZed/dd „ƒZed0d!d"„ƒZed1d#d$„ƒZed%d&„ ƒZed'd(„ ƒZd2‡ fd)d*„	Z‡ fd+d,„Z‡  ZS )3ÚArrowFSWrapperzwFSSpec-compatible wrapper of pyarrow.fs.FileSystem.

    Parameters
    ----------
    fs : pyarrow.fs.FileSystem

    ú/c                    s    t dƒa|| _tƒ jf |Ž d S )NZpyarrow)r   ÚPYARROW_VERSIONÚfsÚsuperÚ__init__)Úselfr   r   ©Ú	__class__r   r   r    3   s    zArrowFSWrapper.__init__c                 C   s   | j jS ©N)r   Ú	type_name©r!   r   r   r   Úprotocol9   s    zArrowFSWrapper.protocolc                 C   s   dt | jj| jjƒ S )NZhdfs_)r
   r   ÚhostÚportr&   r   r   r   Úfsid=   s    zArrowFSWrapper.fsidc                 C   s*   t |ƒ}|d }| d¡r&|dd … }|S )NÚpathz//é   )r   Ú
startswith)Úclsr+   Úopsr   r   r   Ú_strip_protocolA   s
    
zArrowFSWrapper._strip_protocolFc                    sN   ˆ   |¡}ddlm} ‡ fdd„ˆ j ||ƒ¡D ƒ}|r<|S dd„ |D ƒS d S )Nr   )ÚFileSelectorc                    s   g | ]}ˆ   |¡‘qS r   )Ú_make_entry©Ú.0Úentryr&   r   r   Ú
<listcomp>N   s   ÿz%ArrowFSWrapper.ls.<locals>.<listcomp>c                 S   s   g | ]}|d  ‘qS )Únamer   r3   r   r   r   r6   U   s     )r0   Ú
pyarrow.fsr1   r   Úget_file_info)r!   r+   Zdetailr   r1   Úentriesr   r&   r   ÚlsJ   s    

þzArrowFSWrapper.lsc                 K   s$   |   |¡}| j |g¡\}|  |¡S r$   )r0   r   r9   r2   )r!   r+   r   Úinfor   r   r   r<   W   s    
zArrowFSWrapper.infoc                 C   s8   |   |¡}z|  |¡ W n tk
r.   Y dS X dS d S )NFT)r0   r<   r   ©r!   r+   r   r   r   Úexists\   s    
zArrowFSWrapper.existsc                 C   sn   ddl m} |j|jkrd}n<|j|jkr0d}n*|j|jkrVttjt	 
tj¡|jƒ‚nd}|j|j||jdœS )Nr   )ÚFileTypeÚ	directoryÚfileÚother)r7   ÚsizeÚtypeÚmtime)r8   r?   rD   Z	DirectoryZFileZNotFoundr   r   r   ÚosÚstrerrorr+   rC   rE   )r!   r<   r?   Úkindr   r   r   r2   e   s    üzArrowFSWrapper._make_entryc                 K   s¾   |   |¡ d¡}|   |¡ d¡}|  |d¡�ˆ}|› dt d¡› �}z6|  |d¡�}t ||¡ W 5 Q R X | j 	||¡ W n6 t
k
r®   ttƒ� | j |¡ W 5 Q R X ‚ Y nX W 5 Q R X d S )Nr   Úrbz.tmp.é   Úwb)r0   ÚrstripÚ_openÚsecretsZ	token_hexÚopenÚshutilÚcopyfileobjr   ÚmoveÚBaseExceptionr   r   Údelete_file)r!   Úpath1Úpath2r   ZlstreamZ	tmp_fnameZrstreamr   r   r   Úcp_filex   s    
zArrowFSWrapper.cp_filec                 K   s2   |   |¡ d¡}|   |¡ d¡}| j ||¡ d S )Nr   )r0   rL   r   rR   )r!   rU   rV   r   r   r   r   Úmvˆ   s    zArrowFSWrapper.mvc                 C   s   |   |¡}| j |¡ d S r$   )r0   r   rT   r=   r   r   r   Úrm_fileŽ   s    
zArrowFSWrapper.rm_fileNc                 C   sF   |   |¡ d¡}|  |¡r6|r,| j |¡ qBtdƒ‚n| j |¡ d S )Nr   z0Can't delete directories without recursive=False)r0   rL   Úisdirr   Ú
delete_dirÚ
ValueErrorrT   )r!   r+   Ú	recursiveZmaxdepthr   r   r   Úrm“   s    

zArrowFSWrapper.rmrI   Tc           	      K   s    |dkr |r| j j}qR| j j}n2|dkr2| j j}n |dkrD| j j}ntd|›�ƒ‚i }|dksb|s€tt d¡d ƒdkr€d |d< ||f|Ž}t	| ||||f|ŽS )	NrI   rK   Úabz'unsupported mode for Arrow filesystem: Ú.r   é   Úcompression)
r   Zopen_input_fileZopen_input_streamZopen_output_streamZopen_append_streamr\   Úintr   ÚsplitÚ	ArrowFile)	r!   r+   ÚmodeÚ
block_sizeÚseekabler   ÚmethodÚ_kwargsÚstreamr   r   r   rM   ž   s    



zArrowFSWrapper._openc                 K   s2   |   |¡}|r| j|dd� n| jj|dd� d S )NT)Úexist_okF©r]   )r0   Úmakedirsr   Ú
create_dir)r!   r+   Zcreate_parentsr   r   r   r   Úmkdirµ   s    
zArrowFSWrapper.mkdirc                 C   s   |   |¡}| jj|dd� d S )NTrm   )r0   r   ro   )r!   r+   rl   r   r   r   rn   ½   s    
zArrowFSWrapper.makedirsc                 C   s   |   |¡}| j |¡ d S r$   )r0   r   r[   r=   r   r   r   ÚrmdirÂ   s    
zArrowFSWrapper.rmdirc                 C   s   |   |¡}| j |¡jS r$   )r0   r   r9   rE   r=   r   r   r   ÚmodifiedÇ   s    
zArrowFSWrapper.modifiedc                    s&   |dk|d< t ƒ j|fd d dœ|—ŽS )N)Nr   rh   )ÚstartÚend)r   Úcat_file)r!   r+   rs   rt   r   r"   r   r   ru   Ì   s    zArrowFSWrapper.cat_filec                    s   d|d< t ƒ j||f|Ž d S )NFrh   )r   Úget_file)r!   ZrpathZlpathr   r"   r   r   rv   Ð   s    zArrowFSWrapper.get_file)F)FN)rI   NT)T)F)NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Zroot_markerr    Úpropertyr'   r   r*   Úclassmethodr0   r;   r<   r>   r2   r   rW   rX   rY   r^   rM   rp   rn   rq   rr   ru   rv   Ú__classcell__r   r   r"   r   r   (   sB   



	





r   rk   ÚreadÚseekÚtellÚwriteÚreadableÚwritableÚcloserC   rh   c                   @   s&   e Zd Zddd„Zdd„ Zdd„ ZdS )	re   Nc                 K   s.   || _ || _|| _|| _| | _| _|| _d S r$   )r+   rf   r   rk   Ú	blocksizerg   r   )r!   r   rk   r+   rf   rg   r   r   r   r   r    ä   s    zArrowFile.__init__c                 C   s   | S r$   r   r&   r   r   r   Ú	__enter__î   s    zArrowFile.__enter__c                 G   s   |   ¡ S r$   )r„   )r!   r   r   r   r   Ú__exit__ñ   s    zArrowFile.__exit__)N)rw   rx   ry   r    r†   r‡   r   r   r   r   re   Õ   s   

re   c                       s2   e Zd ZdZdZd‡ fdd„	Zed	d
„ ƒZ‡  ZS )ÚHadoopFileSystemz]A wrapper on top of the pyarrow.fs.HadoopFileSystem
    to connect it's interface with fsspecZhdfsÚdefaultr   Né   c           
         s:   ddl m} |||||||d�}	tƒ jf d|	i|—Ž dS )a;  

        Parameters
        ----------
        host: str
            Hostname, IP or "default" to try to read from Hadoop config
        port: int
            Port to connect on, or default from Hadoop config if 0
        user: str or None
            If given, connect as this username
        kerb_ticket: str or None
            If given, use this ticket for authentication
        replication: int
            set replication factor of file for write operations. default value is 3.
        extra_conf: None or dict
            Passed on to HadoopFileSystem
        r   )rˆ   )r(   r)   ÚuserÚkerb_ticketÚreplicationÚ
extra_confr   N)r8   rˆ   r   r    )
r!   r(   r)   r‹   rŒ   r�   rŽ   r   rˆ   r   r"   r   r   r    û   s    úzHadoopFileSystem.__init__c                 C   s�   t | ƒ}i }| dd ¡r$|d |d< | dd ¡r<|d |d< | dd ¡rT|d |d< | dd ¡rŒt|d ƒ}| dd ¡rŒt|d d ƒ|d< |S )Nr(   Úusernamer‹   r)   Z	url_queryr�   r   )r   Úgetr   rc   )r+   r/   ÚoutZqueriesr   r   r   Ú_get_kwargs_from_urls"  s    z&HadoopFileSystem._get_kwargs_from_urls)r‰   r   NNrŠ   N)	rw   rx   ry   rz   r'   r    Ústaticmethodr’   r}   r   r   r"   r   rˆ   õ   s         ù'rˆ   )r   ÚiorF   rN   rP   Ú
contextlibr   Ú	functoolsr   r   Úurllib.parser   Zfsspec.specr   Zfsspec.utilsr   r   r	   r
   r   r   r   ÚIOBasere   rˆ   r   r   r   r   Ú<module>   s8    .÷þ