
    g                     f   d dl m Z mZmZ d dlZd dlZd dlZd dlmZ d dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlmZ d dlmZmZmZ d dlmZmZmZmZmZmZmZmZmZmZm Z  d dl!m"Z" ejF                  jI                  ejF                  jK                  e&            Z' G d d	e      Z(ejR                  d
        Z*ejR                  d        Z+ejR                  d        Z,ejR                  d        Z-ejR                  d        Z.ejR                  d        Z/ejR                  d        Z0ejR                  d        Z1ejR                  d        Z2dZ3ejR                  d        Z4ejR                  d        Z5ejR                  d        Z6ejR                  d        Z7ejR                  d        Z8 ejR                   ejr                  dd       ejr                  dd       ejr                  dd       ejr                  d d!ejt                  jv                  "       ejr                  d#d$ejt                  jx                  "       ejr                  d%d&ejt                  jz                  "       ejr                  d'd(ejt                  jj                  "       ejr                  d)d*       ejr                  d+d,       ejr                  d-d.       ejr                  d/d0       ejr                  d1d2       ejr                  d3d4ejt                  jv                  "      g5      d6        Z>ejR                  d7        Z?ejR                  d8        Z@ejR                  d9        ZAejR                  d:        ZBd; ZCd< ZDd= ZEd> ZFd? ZGejt                  jv                  d@        ZHdA ZIdB ZJdC ZKdD ZLdE ZMdF ZNdG ZOdH ZPdI ZQdJ ZRdK ZSdL ZTdM ZUdN ZVdO ZWdP ZXdQ ZYdR ZZdS Z[dT Z\dU Z]dV Z^dW Z_ejt                  j                  ejt                  j                  dXdde_fddYe_fdZdej                  fdZd[ej                  fg      d\               Zbd] Zcd^ Zdejt                  j                  ejt                  j                  d_dde_fddYe_fdZdej                  fdZd[ej                  fg      d`               Zfejt                  j                  ejt                  j                  dadde_e_fddYe_e_fdZdej                  ej                  fdZd[ej                  ej                  fg      db               Zgdc Zhdd Zide Zjdf Zkdg Zlejt                  jx                  dh        Zmejt                  jv                  di        Znejt                  jv                  dj        Zoejt                  jv                  dk        Zpejt                  jz                  dl        Zqejt                  jj                  dm        Zrejt                  j                  dndoedpfdqedrfdsedrfdtedrfduedvfdwedxfdyedxfdzed{fd|ed{fd}ed~fdxedxfd{ed{fdedfg      d        Zsejt                  j                  ddpdxdrd{e&g      d        Ztejt                  jv                  d        Zuejt                  jx                  d        Zvd Zwd Zxd Zyd Zzd Z{d Z|d Z}d Z~d Zd Zd Zejt                  jv                  d        Zejt                  jv                  d        Zejt                  jv                  d        Zejt                  jv                  d        Zd Zejt                  jv                  d        Zejt                  jv                  d        Zejt                  j                  ejt                  j                  ejt                  jv                  d                      Zejt                  jv                  d        Zy)    )datetimetimezone	timedeltaN)urlopen)assert_file_not_found)_filesystem_uriProxyHandler_configure_s3_limited_user)FileTypeFileInfoFileSelector
FileSystemLocalFileSystemSubTreeFileSystem_MockFileSystemFileSystemHandlerPyFileSystemFSSpecHandler
copy_files)find_free_portc                   z    e Zd ZddZd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zd Zy)DummyHandlerc                     || _         y N)_value)selfvalues     J/var/www/openai/venv/lib/python3.12/site-packages/pyarrow/tests/test_fs.py__init__zDummyHandler.__init__.   s	        c                 `    t        |t              r| j                  |j                  k(  S t        S r   
isinstancer   r   NotImplementedr   others     r   __eq__zDummyHandler.__eq__1   &    e./;;%,,..r    c                 `    t        |t              r| j                  |j                  k7  S t        S r   r"   r%   s     r   __ne__zDummyHandler.__ne__6   r(   r    c                      y)Ndummy r   s    r   get_type_namezDummyHandler.get_type_name;   s    r    c                     |S r   r-   r   paths     r   normalize_pathzDummyHandler.normalize_path>   s    r    c                 r   g }|D ]  }d|v r*|j                  t        |t        j                               1d|v r*|j                  t        |t        j                               _d|v r*|j                  t        |t        j
                               d|v r|j                  t                      t         |S )Nfiledirnotfoundbadtype)appendr   r   File	DirectoryNotFoundobjectIOError)r   pathsinfor2   s       r   get_file_infozDummyHandler.get_file_infoA   s    D~HT8==9:$HT8+=+=>?t#HT8+<+<=>d"FH%  r    c                 .   |j                   dk7  r#|j                  rg S t        |j                         t        dt        j
                  d      t        dt        j                        g}|j                  r |t        dt        j
                  d      gz  }|S )Nsomedirsomedir/file1{   )sizesomedir/subdir1somedir/subdir1/file2i  )base_dirallow_not_foundFileNotFoundErrorr   r   r:   r;   	recursive)r   selectorinfoss      r   get_file_info_selectorz#DummyHandler.get_file_info_selectorQ   s    	)''	'(9(9::_hmm#>&(:(:;
 0(--cJ E r    c                 >    |dk(  r|du sJ y |dk(  r|du sJ y t         )NrL   Tnon-recursiveF)r>   )r   r2   rL   s      r   
create_dirzDummyHandler.create_dira   s3    ;$$$_$%%%Mr    c                     |dk(  sJ y )N
delete_dirr-   r1   s     r   rT   zDummyHandler.delete_diri   s    |###r    c                 @    |j                  d      st        |dk(  sJ y )N/delete_dir_contents)strip
ValueError)r   r2   missing_dir_oks      r   rW   z DummyHandler.delete_dir_contentsl   s"    zz#,,,,r    c                      y r   r-   r.   s    r   delete_root_dir_contentsz%DummyHandler.delete_root_dir_contentsq   s    r    c                     |dk(  sJ y )Ndelete_filer-   r1   s     r   r^   zDummyHandler.delete_filet   s    }$$$r    c                      |dk(  sJ |dk(  sJ y )N	move_frommove_tor-   r   srcdests      r   movezDummyHandler.movew   s    k!!!y   r    c                      |dk(  sJ |dk(  sJ y )Ncopy_file_fromcopy_file_tor-   rb   s      r   	copy_filezDummyHandler.copy_file{   s    &&&&~%%%r    c                     d|v rt        |      dj                  |      j                  d      }t        j                  |      S )Nr7   z{0}:input_streamutf8rK   formatencodepaBufferReaderr   r2   datas      r   open_input_streamzDummyHandler.open_input_stream   s?    #D))!((.55f=t$$r    c                     d|v rt        |      dj                  |      j                  d      }t        j                  |      S )Nr7   z{0}:input_filerk   rl   rq   s      r   open_input_filezDummyHandler.open_input_file   s?    #D))&&t,33F;t$$r    c                 H    d|v rt        |      t        j                         S Nr7   rK   ro   BufferOutputStreamr   r2   metadatas      r   open_output_streamzDummyHandler.open_output_stream   $    #D))$$&&r    c                 H    d|v rt        |      t        j                         S rw   rx   rz   s      r   open_append_streamzDummyHandler.open_append_stream   r}   r    N)*   )__name__
__module____qualname__r   r'   r*   r/   r3   rA   rO   rR   rT   rW   r\   r^   re   ri   rs   ru   r|   r   r-   r    r   r   r   -   s\    

  $-
%!&%%'
'r    r   c                 8    t        t               fddd      S )Nc                 *    | z  j                         S r   as_posixptempdirs    r   <lambda>zlocalfs.<locals>.<lambda>       'A+//1r    Tfspathfnallow_move_dirallow_append_to_filedictr   requestr   s    `r   localfsr      s    1!	 r    c                 \    t        t        t        t                           fddd      S )Nc                 *    | z  j                         S r   r   r   s    r   r   zpy_localfs.<locals>.<lambda>   r   r    Tr   )r   r   r	   r   r   s    `r   
py_localfsr      s)    _%6781!	 r    c                 2    t        t               d dd      S )Nc                     | S r   r-   r   s    r   r   zmockfs.<locals>.<lambda>       r    Tr   )r   r   r   s    r   mockfsr      s    !	 r    c                 V    t        t        t        t                           d dd      S )Nc                     | S r   r-   r   s    r   r   zpy_mockfs.<locals>.<lambda>   r   r    Tr   )r   r   r	   r   r   s    r   	py_mockfsr      s(    _%678!	 r    c                 <    t        t        d      fddd      S )NTuse_mmapc                 *    | z  j                         S r   r   r   s    r   r   z#localfs_with_mmap.<locals>.<lambda>   r   r    r   r   r   s    `r   localfs_with_mmapr      s!    D)1!	 r    c                 N    t        t        t        |      |d         d dd      S )Nr   c                     | S r   r-   r   s    r   r   z!subtree_localfs.<locals>.<lambda>   r   r    Tr   )r   r   str)r   r   r   s      r   subtree_localfsr      s*    S\74=9!	 r    c              #     K   | j                   j                  j                  d       ddlm} |d   \  }}d} || d| ddt        d	
      d      }	 |j                  |       t        ||j                  dd       |j                  |       y # t        $ r%}t        j                  d| d|        Y d }~Vd }~ww xY ww)Ngcsr   GcsFileSystem
connectionpyarrow-filesystem/:httpT-   )secondstest-project-id)endpoint_overridescheme	anonymousretry_time_limit
project_idzCould not create directory in : Fr   )configpyarrowrequires
pyarrow.fsr   r   rR   OSErrorpytestskipr   __add__rT   )r   
gcs_serverr   hostportbucketr   es           r   gcsfsr      s     NN##E*(L)JD$"F	!F!D6*"2.$
B@
f ~~"	  MM&  @4RD1#>??@s0   ACB &-C	CB<7C<CCc              #   0  K   | j                   j                  j                  d       ddlm} |d   \  }}}}d} |||dj                  ||      ddd	      }|j                  |       t        ||j                  d
d
       |j                  |       y w)Ns3r   S3FileSystemr   r   {}:{}r   T)
access_key
secret_keyr   r   allow_bucket_creationallow_bucket_deletionFr   )
r   r   r   r   r   rm   rR   r   r   rT   )	r   	s3_serverr   r   r   r   r   r   r   s	            r   s3fsr      s     NN##D)')2<)@&D$
J"F	!..t4""
B MM&
~~"	  MM&s   BBc                 R    d}t        t        ||d         |j                  dd      S )Nzpyarrow-filesystem/prefix/r   Fr   )r   r   r   )r   r   prefixs      r   subtree_s3fsr     s/    )FVT$Z0~~"	 r    a  {
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:ListAllMyBuckets",
                "s3:PutObject",
                "s3:GetObject",
                "s3:ListBucket",
                "s3:PutObjectTagging",
                "s3:DeleteObject",
                "s3:GetObjectVersion"
            ],
            "Resource": [
                "arn:aws:s3:::*"
            ]
        }
    ]
}c              #   "  K   | j                   j                  j                  d       ddlm} |d   \  }}}}| d| }d}d}	 |||||||      }
|
j                  |	       t        |
|	j                  d	d	
       |
j                  |	       y w)Nazurer   AzureFileSystemr   r   r   r   account_nameaccount_keyblob_storage_authoritydfs_storage_authorityblob_storage_schemedfs_storage_schemeTr   )	r   r   r   r   r   rR   r   r   rT   )r   azure_serverr   r   r   r   r   azurite_authorityazurite_scheme	containerr   s              r   azurefsr   ,  s     NN##G,*,8,F)D$k&$(N%I	l%00A/@-;,:
<B MM)
  !	  MM)s   BBc                     | j                   j                  j                  d       t        j                         st        j                  d       ddlm} |\  }}} ||||      }t        |d dd      S )	NhdfsCannot locate libhdfsr   HadoopFileSystem)r   userc                     | S r   r-   r   s    r   r   zhdfs.<locals>.<lambda>V  r   r    Tr   )
r   r   r   ro   have_libhdfsr   r   r   r   r   )r   hdfs_connectionr   r   r   r   r   s          r   r   r   I  sg    NN##F+??+,+&D$	$T	5B!	 r    c                     t        j                  d      }|j                  d      }t        t	        t        |            fddd      S )Nfsspecr5   c                 *    | z  j                         S r   r   r   s    r   r   z#py_fsspec_localfs.<locals>.<lambda>b  r   r    Tr   )r   importorskip
filesystemr   r   r   r   r   r   r   s    `  r   py_fsspec_localfsr   \  sF      *F			6	"Bb)*1!	 r    c                     t        j                  dd      }|j                  dk(  rt        j                  d       |j	                  d      }t        t        t        |            d dd	      S )
Nr   z0.7.5)
minversionz0.8.5z,Bug in fsspec 0.8.5 for in-memory filesystemmemoryc                     | S r   r-   r   s    r   r   z$py_fsspec_memoryfs.<locals>.<lambda>q  r   r    Tr   )r   r   __version__r   r   r   r   r   r   s       r   py_fsspec_memoryfsr   h  sb      g>FW$BC			8	$Bb)*!	 r    c           
   #   L  K   t        j                  d      }|d   \  }}}}d}|j                  ||t        dj	                  ||                  }t        t        |            }|j                  |       t        ||j                  dd	       |j                  |       y w)
Nr   r   r   zhttp://{}:{})endpoint_url)keysecretclient_kwargsFTr   )
r   r   r   r   rm   r   r   rR   r   rT   )	r   r   r   r   r   r   r   r   r   s	            r   py_fsspec_s3fsr  w  s     v&D)2<)@&D$
J"F			(=(=dD(IJ 
 
B
 
mB'	(BMM&
~~!	  MM&s   B"B$r   zLocalFileSystem())idr   zLocalFileSystem(use_mmap=True)r   z$SubTreeFileSystem(LocalFileSystem())r   r   )r  marksr   r   r   r   r   r   r   z_MockFileSystem()r   z-PyFileSystem(ProxyHandler(LocalFileSystem()))r   z-PyFileSystem(ProxyHandler(_MockFileSystem()))r   z5PyFileSystem(FSSpecHandler(fsspec.LocalFileSystem()))r   z8PyFileSystem(FSSpecHandler(fsspec.filesystem("memory")))r  z0PyFileSystem(FSSpecHandler(s3fs.S3FileSystem())))paramsc                 8    | j                  | j                        S r   )getfixturevalueparamr   s    r   filesystem_configr    s    x ""7==11r    c                     | d   S Nr   r-   r  s    r   r   r     s    T""r    c                     | d   S )Nr   r-   r  s    r   r   r     s    X&&r    c                     | d   S )Nr   r-   r  s    r   r   r     s    -..r    c                     | d   S )Nr   r-   r  s    r   r   r     s    344r    c                    t        | j                  t              sJ t        | j                  t              sJ | j                  dk\  sJ | j                  t        j                  | j                  j                         dz        k(  sJ | j                  j                  }|J |j                  d       t        d      k(  sJ y )Nr   g    eA)r#   mtimer   mtime_nsintr   approx	timestamptzinfo	utcoffsetr   )	file_infor  s     r   check_mtimer    s    ioox000i((#..."""!!#c)"+ + + + __##FD!Yq\111r    c                 <    | j                   J | j                  J y r   )r  r  r  s    r   check_mtime_absentr    s$    ??"""%%%r    c                 J    | j                   t        |        y t        |        y r   )r  r  r  r  s    r   check_mtime_or_absentr     s    9%Ir    c                 P    | j                   dk(  rt        j                  d       y y )Npy::fsspec+('s3', 's3a')zNot working with fsspec's s3fsreason)	type_namer   xfailr   s    r   skip_fsspec_s3fsr(    s!    	||11<= 2r    c                 P    | j                   dk(  rt        j                  |       y y )Nabfsr#  )r%  r   r   )r   r$  s     r   
skip_azurer+    s    	||v6" r    c                    ddl m} t        | t        dd       | d   \  }}}} |dddj	                  ||      d      }|j                  d	       t        j                  t        j                  d
      5  |j                  d       d d d        t        j                  t        j                  d      5  |j                  d       d d d        y # 1 sw Y   IxY w# 1 sw Y   y xY w)Nr   r   test_fs_limited_user
limited123r   r   r   )r   r   r   r   zexisting-bucket/testzBucket 'new-bucket' not foundmatchz
new-bucketzWould delete bucketzexisting-bucket)r   r   r
   _minio_limited_policyrm   rR   r   raisesro   ArrowIOErrorrT   )r   r   r   r   _r   s         r   +test_s3fs_limited_permissions_create_bucketr5    s    'y*?5|E .D$1	)!..t4	
B MM()	r.M	N
l# 
O 
r.C	D
'( 
E	D 
O	N 
E	Ds   5C4CCC$c                  \   t        j                  dt        j                        } t	        d      }|j
                  dk(  sJ |j                  dk(  sJ |j                  t        j                  k(  sJ |j                  J t        |       t	        dt        j                  dd      }|j
                  dk(  sJ |j                  dk(  sJ |j                  t        j                  k(  sJ |j                  dk(  sJ |j                  d	k(  sJ t        |       t	        d
t        j                  |       }|j
                  d
k(  sJ |j                  d
k(  sJ |j                  t        j                  k(  sJ |j                  J |j                   | k(  sJ |j                  dk(  sJ t        |       y )NR]foo/barbarzfoo/baz.txtrE   g  `A)typerF   r  zbaz.txtl    t<|+. foo)r:  r  l    4 |+. )r   fromtimestampr   utcr   r2   	base_namer:  r   UnknownrF   r  r:   r  r  r;   r  )dtr@   s     r   test_file_info_constructorrA    s   			
HLL	9BID99	!!!>>U"""99(((((99tMC&(D99%%%>>Y&&&99%%%99==////E 2 2"=D99>>U"""99*****99::==////r    c                  t    t        j                  t              5  t                d d d        y # 1 sw Y   y xY wr   )r   r2  	TypeErrorr   r-   r    r   'test_cannot_instantiate_base_filesystemrD  6  s    	y	! 
"	!	!s   .7c                     t               } t               }t               }| j                  |       sJ | j                  |      sJ t        j                  t
              5  | j                  d       d d d        | | cxk(  r|k(  sJ  J | dk7  sJ ||k(  sJ |t               k7  sJ t        d|       t        d|       k(  sJ t        d|       t        d|      k7  sJ t        d|       t        d|       k7  sJ y # 1 sw Y   xY w)Nstring   /basez/other)r   r   equalsr   r2  rC  r   )fs0fs1fs2s      r   test_filesystem_equalsrM  ;  s    

C

C

C::c??::c??	y	!

8 
"#!8O8#::/####Wc*.?.MMMMWc*.?.MMMMWc*.?#.NNNN 
"	!s   C77D c                     t        j                  t        d      5  | j                  d        d d d        | J y # 1 sw Y   xY w)Nzgot NoneTyper/  )r   r2  rC  rI  r'  s    r   test_filesystem_equals_nonerO  O  s5    	y	7
		$ 
8 >> 
8	7s	   ;Ac                  P   t               } t        d|       }|j                  dk(  sJ |j                  | k(  sJ t	        |      j                  d      sJ t        dt                     }|j                  dk(  sJ |j                  | k(  sJ t	        |      j                  d      sJ y )NrH  z/base/zHSubTreeFileSystem(base_path=/base/, base_fs=<pyarrow._fs.LocalFileSystemz/another/base/zPSubTreeFileSystem(base_path=/another/base/, base_fs=<pyarrow._fs.LocalFileSystem)r   r   	base_pathbase_fsrepr
startswith)r   subfss     r   test_subtree_filesystemrV  V  s    Ggw/E??h&&&==G###;!! #I J J J .0ABE??....==G###;!! #J K K Kr    c                     | j                   j                  d      d   dk(  rt        j                  d       |j	                  |       }|j                  |      }t        |t              sJ |j                  |       sJ y )N::mock"MockFileSystem is not serializabler#  )	r%  splitr   r&  dumpsloadsr#   r   rI  )r   pickle_module
serializedrestoreds       r   test_filesystem_picklingrb  f  sm    	||$#v-@A$$R(J"":.Hh
+++??2r    c                    | j                   j                  d      d   dk(  rt        j                  d       t	        |         |d      } |d      } |d      }| j                  |       | j                  |      5  	 d d d        | j                  |      5 }|j                  d	       d d d        |j                  |j                  |             }|j                  |||g      \  }}	}
|j                  t        j                  k(  sJ |	j                  t        j                  k(  sJ |
j                  t        j                  k(  sJ y # 1 sw Y   xY w# 1 sw Y   xY w)
NrX  rY  rZ  r[  r#  	a/aa/aaa/a/bbc.txt   test)r%  r\  r   r&  r(  rR   r|   writer^  r]  rA   r:  r   r;   r:   )r   r   r_  aaabbcfpra  aaa_infobb_infoc_infos              r   ,test_filesystem_is_functional_after_picklingrp  p  s    	||$#v-@AR

C	BwAMM#			r	" 
#			q	!R
 
" ""=#6#6r#:;H ( 6 6R| DHgv==H.....<<8==(((;;(--''' 
#	"	!	!s   =EEEEc                  p    t               } | j                  dk(  sJ t               } | j                  dk(  sJ y )NlocalrZ  )r   r%  r   r'  s    r   test_type_namers    s6    		B<<7"""		B<<6!!!r    c                 0    | j                  d      dk(  sJ y )Nr;  )r3   r'  s    r   test_normalize_pathru    s     U#u,,,r    c           	           G d d      }dd |       t               i g d t        j                         g}|D ]5  }t        j                  t
              5  | j                  |       d d d        7 y # 1 sw Y   BxY w)Nc                       e Zd Zy)-test_non_path_like_input_raises.<locals>.PathN)r   r   r   r-   r    r   Pathrx    s    r    ry     g?c                       y)Nrz  r-   r-   r    r   r   z1test_non_path_like_input_raises.<locals>.<lambda>  s    ar    )tuplepathlibry  r   r2  rC  rR   )r   ry  invalid_pathsr2   s       r   test_non_path_like_input_raisesr    sb      TVUWb"i\\^%M]]9%MM$ &% %%s   A00A9	c                     |d      } |d      } |d      } |d      }| j                  |       | j                  |      5  	 d d d        | j                  |      5 }|j                  d       d d d        | j                  ||||g      \  }}}	}
|j                  |k(  sJ dt        |      v sJ |j                  dk(  sJ | j                  dk(  r|j                  t        j                  k(  s0J |j                  t        j                  k(  sJ d	t        |      v sJ |j                  J t        |       |j                  t        |      k(  sJ |j                  d
k(  sJ |j                  dk(  sJ |j                  t        j                   k(  sJ dt        |      v sJ |j                  dk(  sJ | j                  dvrt#        |       |	j                  t        |      k(  sJ |	j                  dk(  sJ |	j                  dk(  sJ |	j                  t        j                   k(  sJ dt        |	      v sJ |	j                  dk(  sJ | j                  dvrt#        |	       |
j                  t        |      k(  sJ |
j                  dk(  sJ |
j                  dk(  sJ |
j                  t        j                  k(  sJ |
j                  J |
j$                  J dt        |
      v sJ t'        |
       | j                  |      }|j                  |j                  k(  sJ |j                  |j                  k(  sJ y # 1 sw Y   xY w# 1 sw Y   xY w)Nrd  re  rf  zzzrg  ri   r"  zFileType.Directoryrj  zFileType.Filer   )zpy::fsspec+memoryr"  txtrG  zFileType.NotFound)rR   r|   rh  rA   r2   rS  	extensionr%  r:  r   r<   r;   rF   r   r   r>  r:   r  r  r  )r   r   ri  rj  rk  r  rl  rm  rn  ro  zzz_info	aaa_info2s               r   test_get_file_infor    s&   

C	BwA
-CMM#			r	" 
#			q	!R
 
" +-*:*:CQ;L*M'Hgvx==CDN"""###	||11}} 1 1111}} 2 2222#tH~555==   (#<<3r7"""$$$"""<<8==(((d7m+++<<1	||LLG;;#a&   w&&&u$$$;;(--'''d6l***;;!	||LLF==CH$$$&&&###==H-----==   >>!!!$x.000x    %I==INN***==INN***c 
#	"	!	!s   L9M9MMc                     |d      } |d      } |d      } |d      } |d      } |d      }	 | j                  |       | j                  |      5  	 d d d        | j                  |      5  	 d d d        | j                  |       | j                  |      5  	 d d d        | j                  |       t        |dd	      }|j                  |k(  sJ | j	                  |      }	| j
                  d
k(  rt        |	      dk(   nt        |	      dk(  sJ |	D ]  }
|
j                  j                  |      s6|
j                  j                  |      s|
j                  j                  |      r|
j                  t        j                  k(  sJ |
j                  j                  d      j                  |      s*|
j                  j                  d      j                  |      r|
j                  t        j                  k(  s&J t        dj                  |
j                              t!        |
        t        |d      }| j	                  |      }	| j
                  d
k(  rt        |	      dk(  sJ t        |	      dk(  sJ | j#                  |       y # 1 sw Y   :xY w# 1 sw Y   ,xY w# 1 sw Y   xY w# | j#                  |       w xY w)Nzselector-dir/zselector-dir/test_file_azselector-dir/test_file_bzselector-dir/test_dir_az#selector-dir/test_dir_a/test_file_czselector-dir/test_dir_bFT)rJ   rL   r"  rG     rV   zunexpected path {}rL      )rR   r|   r   rI   rA   r%  lenr2   endswithr:  r   r:   rstripr;   rY   rm   r   rT   )r   r   rI   file_afile_bdir_afile_cdir_brM   rN   r@   s              r    test_get_file_info_with_selectorr    sZ   o&H./F./F,-E9:F,-E- 
h""6* +""6* +
e""6* +
e  %*.0  H,,,  *<<55J!Ou:?"?D		""6*dii.@.@.HII&&v.yyHMM111))""3'007))""3'007yyH$6$6666 !5!<!<TYY!GHH!$'   E:  *<<55u:?"?u:?"? 	hW +*** +*L 	hsT   "J5 JJ5 /J1*J5 J(GJ5 JJ5 J% J5 (J2-J5 5Kc                 X   t        |         |d      }t        j                  t        j                        5  | j                  |       d d d        | j                  |       | j                  |        |d      }| j                  |d       | j                  |       y # 1 sw Y   XxY w)Nztest-directory/zdeeply/nested/test-directory/Tr  )r(  r   r2  ro   r3  rT   rR   )r   r   ds      r   test_create_dirr    s     R !A	r	'
a 
( MM!MM!./AMM!tM$MM! 
(	's   B  B)c                    t        |         |d      } |d      }| j                  |       | j                  |       t        j                  t
        j                        5  | j                  |       d d d        t        j                  t
        j                        5  | j                  |       d d d        y # 1 sw Y   GxY w# 1 sw Y   y xY w)N
directory/directory/nested/)r(  rR   rT   r   r2  ro   r3  r   r   r  nds       r   test_delete_dirr  "  s    R|A	#	$BMM"MM!	r	'
b 
(	r	'
a 
(	' 
(	'	'	's   !B9C9CCc                 d   t        |         |d      } |d      }| j                  |       | j                  |       | j                  |       | j                  |      }|j                  t
        j                  k(  sJ  |d      } |d      } |d      }| j                  |       | j                  |       | j                  |      5 }|j                  d       d d d        | j                  |       | j                  |      }|j                  t
        j                  k(  sJ y # 1 sw Y   KxY w)Nr  r  
directory2zdirectory2/nestedzdirectory2/nested/target-file   data)	r(  rR   rT   rA   r:  r   r<   r|   rh  )r   r   r  r  dir_infofss          r   $test_delete_dir_with_explicit_subdirr  0  s	   
 R|A	#	$B MM!MM"MM!"H==H----- 	|A	#	$B./AMM!MM"			q	!Q	 
" MM!"H==H----- 
"	!s   
D&&D/c                 |   t        |         |d      } |d      }| j                  |       | j                  |       t        j                  t
        j                        5  | j                  |       d d d        | j                  |d       t        j                  t
        j                        5  | j                  |       d d d        | j                  |       t        j                  t
        j                        5  | j                  |       d d d        y # 1 sw Y   xY w# 1 sw Y   dxY w# 1 sw Y   y xY w)Nr  r  T)rZ   )r(  rR   rW   r   r2  ro   r3  rT   r  s       r   test_delete_dir_contentsr  P  s    R|A	#	$BMM"1	r	'
b 
(2d3	r	'
r" 
(MM!	r	'
a 
(	' 
(	' 
(	' 
(	's$   !D1D&?D2D#&D/2D;c                    | d   }| d   } |d      } |d      }|j                  |       t        j                  t        j                        5  |j                  d       d d d        t        j                  t        j                        5  |j                  d       d d d        t        j                  t        j                        5  |j                  d       d d d        |j                  dd	       |j                  dd	       |j                  dd	       t        j                  t        j                        5  |j                  |       d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)
Nr   r   r  r  r  rV   //Taccept_root_dir)rR   r   r2  ro   ArrowInvalidrW   r3  rT   )r   r   r   r  r  s        r   _check_root_dir_contentsr  b  s   	BHF|A	#	$BMM"	r	'
r" 
(	r	'
s# 
(	r	'
t$ 
( 2t43546	r	'
a 
(	' 
(	'	'	'	'	' 
(	's0   EE&	E2?E>E#&E/2E;>Fc                 0    t        |        t        |       y r   )r  )r   r   s     r   test_delete_root_dir_contentsr  x  s    V$Y'r    c                      |d      } |d      }| j                  |      5  	 d d d        | j                  ||       | j                  |       | j                  |       y # 1 sw Y   >xY w)Nztest-copy-source-fileztest-copy-target-file)r|   ri   r^   r   r   r  ts       r   test_copy_filer  }  s_    &'A&'A			q	! 
" LLANN1NN1 
"	!s   A!!A*c                    t        | d        |d      } |d      }| j                  |       |ra| j                  ||       t        j                  t
        j                        5  | j                  |       d d d        | j                  |       y t        j                  t
        j                        5  | j                  ||       d d d        y # 1 sw Y   ZxY w# 1 sw Y   y xY w)N.Not implemented yet in for Azure. See GH-40025zsource-dir/ztarget-dir/)r+  rR   re   r   r2  ro   r3  rT   )r   r   r   r  r  s        r   test_move_directoryr    s    rCD 	}A}AMM!
1]]2??+MM! ,
a]]2??+GGAqM ,+	 ,+ ,+s   %C4CCC%c                 x   t        |        t        | d        |d      } |d      }| j                  |      5  	 d d d        | j                  ||       t	        j
                  t        j                        5  | j                  |       d d d        | j                  |       y # 1 sw Y   jxY w# 1 sw Y   'xY w)Nr  ztest-move-source-fileztest-move-target-file)	r(  r+  r|   re   r   r2  ro   r3  r^   r  s       r   test_move_filer    s     R rCD&'A&'A			q	! 
" GGAqM	r	'
q 
(NN1 
"	! 
(	's   B$8B0$B-0B9c                     |d      }| j                  |      5  	 d d d        | j                  |       t        j                  t        j
                        5  | j                  |       d d d         |d      }| j                  |        |d      }| j                  |      5 }|j                  d       d d d        | j                  |       y # 1 sw Y   xY w# 1 sw Y   sxY w# 1 sw Y   3xY w)Nztest-delete-target-fileztest-delete-nestedztest-delete-nested/target-filer  )	r|   r^   r   r2  ro   r3  rR   rh  rT   )r   r   r   r  r  r  s         r   test_delete_filer    s    ()A			q	! 
" NN1	r	'
q 
( 	#$AMM!/0A			q	!Q	 
" MM! 
"	! 
(	' 
"	!s#   CC$C(CC%(C1c                     | S r   r-   )vs    r   identityr    s    Hr    )compressionbuffer_size
compressor@   gzip   c                     |d      }ddz  }| j                  |      5 }|j                   ||             d d d        | j                  |||      5 }|j                         }d d d        |k(  sJ y # 1 sw Y   =xY w# 1 sw Y   xY w)Nzopen-input-streams   some data for reading
   )r|   rh  rs   read)	r   r   r  r  r  r   rr   r  results	            r   test_open_input_streamr    s     	"#A%+D			q	!Q	
4 ! 
" 
		ak	:a 
; T>> 
"	! 
;	:s   A3A?3A<?Bc                     |d      }ddz  }| j                  |      5 }|j                  |       d d d        t        d      dz  }| j                  |      5 }|j	                         }d d d        |k(  sJ | j                  |      5 }|j                  |       |j	                         }d d d        |||d  k(  sJ y # 1 sw Y   xY w# 1 sw Y   bxY w# 1 sw Y   ,xY w)Nzopen-input-file	   some data   r  )r|   rh  r  ru   r  seek)r   r   r   rr   r  	read_fromr  r  s           r   test_open_input_filer    s     !A$D			q	!Q	 
" L!C'I			A	! 
T>>			A	!	y 
 T)*%%%% 
"	! 
	 
	s#   B>C
	"C>C
CCc                      |d      }t        j                  t              5  | j                  |       d d d        y # 1 sw Y   y xY w)Nzopen-input-stream-not-found)r   r2  rK   rs   )r   r   r   s      r    test_open_input_stream_not_foundr    s4    ,-A	(	)
Q 
*	)	)s	   =A)r  r  decompressorc                      |d      }ddz  }| j                  |||      5 }|j                  |       d d d        | j                  |||      5 }|j                  t	        |            |k(  sJ 	 d d d        y # 1 sw Y   FxY w# 1 sw Y   y xY w)Nzopen-output-streams   some data for writingr  )r|   rh  rs   r  r  )r   r   r  r  r  r   rr   r  s           r   test_open_output_streamr    s     	#$A#d*D			q+{	;q	 
< 
		ak	:avvc$i D((( 
;	: 
<	; 
;	:s   A8 B8BB)r  r  r  r  c                 .    |d      } |d      }| j                  |      5 }	|	j                  |       d d d        |rh| j                  |||      5 }
|
j                  d       d d d        | j                  |      5 }
|
j	                         }d d d         |      }|dk(  sJ y t        j                  t        j                        5  | j                  |||       d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   rxY w# 1 sw Y   y xY w)Nzopen-append-streams   already existing)r  r  s   
newly addeds   already existing
newly added)	r|   rh  r   rs   r  r   r2  ro   ArrowNotImplementedError)r   r   r  r  r  r  r   r   initialr  r  r  s               r   test_open_append_streamr    s    	#$A,-G			q	!Q	 
" ""1+/: # <?@GG$%< !!!$VVXF % f%9999]]2667!!!.9 " ; 87 
"	!< < %$ 87s/   C'C3=C?	D'C03C<?DDc                     |d      }ddi}d}| j                  ||      5 }|j                  |       d d d        | j                  |      5 }|j                         |k(  sJ |j	                         }d d d        | j
                  dv sd| j
                  v r=t        | d       |j                         D ]  \  }}|   |j                         k(  rJ  y i k(  sJ y # 1 sw Y   xY w# 1 sw Y   vxY w)	Nzopen-output-stream-metadataContent-Typezx-pyarrow/testr  )r{   )r   r   r*  rZ  zWAzure filesystem currently only returns system metadata not user metadata. See GH-40026)	r|   rh  rs   r  r{   r%  r+  itemsrn   )	r   r   r   r{   rr   r  got_metadatakr  s	            r    test_open_output_stream_metadatar  0  s    ,-A 01HD			q8		4	 
5 
		a	 Avvx4zz| 
! 
||,,",,0F %	& NN$DAq?ahhj000 % r!!! 
5	4 
!	 s   C&C(C%(C1c                      t        d       t        j                  t              5  t        d       d d d        y # 1 sw Y   y xY w)NFr   )xxx)r   r   r2  rC  r-   r    r   test_localfs_optionsr  G  s*    U#	y	!E" 
"	!	!s	   <Ac                    | d   }t               5  |j                  d       d d d        t               5  |j                  d       d d d        t               5  |j                  dd       d d d        t               5  |j	                  d       d d d        t               5  |j                  d       d d d        t               5  |j                  dd       d d d        t               5  |j                  dd       d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   kxY w# 1 sw Y   y xY w)Nr   z/non/existent/filez/non/existent/dirFr  z/non/existentz/xxx)r   rs   r|   rR   rT   r^   re   ri   )r   r   s     r   test_localfs_errorsr  O  s     
B		 
12 
!		 
23 
!		 
)U; 
!		 
)* 
!		 
*+ 
!		 
( 
!		 
_f- 
!	  
!	 	 	 	 	 	 	 	 	 	 	 	 	 sR   DDD>D+"D7E+EDDD(+D47E EEc                    | d   }t        j                  t              }|j                  }|j	                  |j                         |j                         g      \  }}|j                  |j                         j                  k(  sJ |j                  |j                         j                  k(  sJ t        |       |j                  |j                         j                  k(  sJ t        |       y r  )r}  ry  __file__parentrA   r   rF   statst_sizer  st_mtime_nsr  )r   r   	file_pathdir_pathr  r  s         r   test_localfs_file_infor  c  s    	BX&IH,,i.@.@.B.6.?.?.A.C DY>>Y^^-55555!1!=!====	 ; ;;;;r    c                     t        j                  dt        j                        }t	        |      }|j                  d      5  	 d d d        |j                  dg      \  }|j                  |k(  sJ y # 1 sw Y   /xY w)Nr7  r;  )r   r<  r   r=  r   r|   rA   r  )r   r@  r   r@   s       r   test_mockfs_mtime_roundtripr  q  sg    			
HLL	9B		B			u	% 
&ug&FT:: 
&	%s   A11A:c           	      J   ddl m} t        j                         } |dd|dddd	      }t	        ||      sJ |j
                  dk(  sJ |j                  dk(  sJ | j                  | j                  |            |k(  sJ  |       }t	        ||      sJ | j                  | j                  |            |k(  sJ  |d
      }t	        ||      sJ | j                  | j                  |            |k(  sJ  |ddd      }t	        ||      sJ | j                  | j                  |            |k(  sJ t        j                  t              5   |d       d d d        t        j                  t              5   |d
d       d d d        t        j                  t              5   |d
d       d d d        t        j                  t              5   |t        j                                d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   _xY w# 1 sw Y   y xY w)Nr   r   abczservice_account@apachezus-west2httpslocalhost:8999r   )access_tokentarget_service_accountcredential_token_expirationdefault_bucket_locationr   r   r   Tr   authenticated-read
text/plainACLr  default_metadataaccess)r  r  )r   r  acct)r   r  )r  )r   r   r   nowr#   r  r   r^  r]  r   r2  rY   )r_  r   r@  r   s       r   test_gcs_optionsr  {  s   (	B	E.F35/9%9I"3
5B b-(((%%333==----}222672===	Bb-(((}222672===		&Bb-(((}222672===	0D9E)G 
HBb-(((}222672===	z	"8, 
#	z	"8< 
#	z	"VD 
#	z	"(,,.A 
#	" 
#	"	"	"	"	"	"	"s0   
G58H$HH5G>H
HH"c                 .
   ddl m}m}m}m}  |dddddd	      }t        ||      sJ |j                  dk(  sJ | j                  | j                  |            |k(  sJ  |d
ddd      }t        ||      sJ | j                  | j                  |            |k(  sJ  | |d            }t        ||      sJ  | |d            }t        ||      sJ  |d
      }t        ||      sJ | j                  | j                  |            |k(  sJ ||k7  sJ  |d      }t        ||      sJ | j                  | j                  |            |k(  sJ  |d      }t        ||      sJ | j                  | j                  |            |k(  sJ  |dddd      }t        ||      sJ | j                  | j                  |            |k(  sJ ||k7  sJ  |dd      }t        ||      sJ | j                  | j                  |            |k(  sJ  |ddd      }t        ||      sJ | j                  | j                  |            |k(  sJ  |dd      }t        ||      sJ | j                  | j                  |            |k(  sJ  |dd      }t        ||      sJ | j                  | j                  |            |k(  sJ ||k7  sJ  |dd      }t        ||      sJ | j                  | j                  |            |k(  sJ t        j                  t              5   |d        d d d        t        j                  t              5   |d!       d d d        t        j                  t              5   |dd"       d d d        t        j                  t              5   |dd#       d d d        t        j                  t              5   |ddd$%       d d d        t        j                  t              5   |ddd&       d d d        t        j                  t              5   |d$d'       d d d        t        j                  t              5   |d(d)g*       d d d        t        j                  t              5   | |              d d d        y # 1 sw Y   qxY w# 1 sw Y   SxY w# 1 sw Y   4xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)+Nr   )AwsDefaultS3RetryStrategyAwsStandardS3RetryStrategyr   S3RetryStrategyr  r  token	us-east-2r  r  )r   r   session_tokenregionr   r   rolesessionr  d   )role_arnsession_nameexternal_idload_frequencyr  )max_attempts)retry_strategy)r  Tr  )background_writesr  r  r  )r  r  )r   r   )r   r   )check_directory_existence_before_creationg      ?g      ?)request_timeoutconnect_timeout)r   force_virtual_addressing)r   )r   )r   r  )r   r  arn)r   r   r  )r   r   r   )r  r   r;  r9  r  )r   r  r  r   r  r#   r  r^  r]  r   r2  rY   )r_  r  r  r   r  r   rL  s          r   test_s3_optionsr    s   - - 
h$+K$8H
JB b,'''99###}222672===	vI"&s
<Bb,'''}222672=== 
1qA
CBb,'''	0a@
BBb,'''

'Cc<(((}22378C???"99		%Bb,'''}222672===		-Bb,'''}222672===
0D9E)GHC c<(((}22378C???"99	D	MBb,'''}222672===	D@D
FBb,'''}222672===	c4	@Bb,'''}222672===
tS
ACc<(((}22378C???"99	(8SW	XBb,'''}222672===	z	") 
#	z	") 
#	z	"@ 
#	z	"@ 
#	z	"Hu	
 
# 
z	"H	
 
# 
z	"et4 
#	z	"uen5 
#	z	"O$56 
#	") 
#	"	"	"	"	"	"	"	"	" 
#	" 
#	"	"	"	"	"sl   .
R'
R4S0SS	S'6S3"S?T'R14R>SSS$'S03S<?TTc                    ddl m} dddd}d}ddd	d}d
} ||      }t        ||      sJ |j                  |j	                  |            |k(  sJ  ||      }t        ||      sJ |j                  |j	                  |            |k(  sJ  ||      }t        ||      sJ |j                  |j	                  |            |k(  sJ  ||      }t        ||      sJ |j                  |j	                  |            |k(  sJ  ||      } ||      }	||	k(  sJ |j                  |j	                  |            |	k(  sJ |j                  |j	                  |	            |k(  sJ  ||      } ||      }	||	k(  sJ |j                  |j	                  |            |	k(  sJ |j                  |j	                  |	            |k(  sJ  ||      } ||      }	||	k(  sJ |j                  |j	                  |            |	k(  sJ |j                  |j	                  |	            |k(  sJ  ||      } ||      }	||	k(  sJ |j                  |j	                  |            |	k(  sJ |j                  |j	                  |	            |k(  sJ  ||      } ||      }	||	k(  sJ |j                  |j	                  |            |	k(  sJ |j                  |j	                  |	            |k(  sJ  ||      } ||      }	||	k(  sJ |j                  |j	                  |            |	k(  sJ |j                  |j	                  |	            |k(  sJ  ||      } ||      }	||	k7  sJ |j                  |j	                  |            |	k7  sJ |j                  |j	                  |	            |k7  sJ  ||      } ||      }	||	k7  sJ |j                  |j	                  |            |	k7  sJ |j                  |j	                  |	            |k7  sJ  ||      } ||      }	||	k7  sJ |j                  |j	                  |            |	k7  sJ |j                  |j	                  |	            |k7  sJ  ||      } ||      }	||	k7  sJ |j                  |j	                  |            |	k7  sJ |j                  |j	                  |	            |k7  sJ  ||      } |       }	||	k7  sJ |j                  |j	                  |            |	k7  sJ |j                  |j	                  |	            |k7  sJ  ||      } |       }	||	k7  sJ |j                  |j	                  |            |	k7  sJ |j                  |j	                  |	            |k7  sJ  ||      } |       }	||	k7  sJ |j                  |j	                  |            |	k7  sJ |j                  |j	                  |	            |k7  sJ  ||      } |       }	||	k7  sJ |j                  |j	                  |            |	k7  sJ |j                  |j	                  |	            |k7  sJ t        j                  t              5   |d       d d d        t        j                  t              5   |ddd       d d d        t        j                  t              5   |ddd       d d d        t        j                  t              5   |ddd       d d d        t        j                  t        j                        5   |d       d d d        t        j                  t        j                        5   |dddd       d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   txY w# 1 sw Y   y xY w)Nr   r   r   	localhosti'#  )r   r   r   zhttp://localhost:8999r  i  zhttps://localhost:8080)proxy_options)r   r  #  r  )r   r   )r   r   )r   r   zhttpsB://localhost:9000httpA)r   r   r#   r^  r]  r   r2  rC  KeyErrorro   r  )
monkeypatchr_  r   proxy_opts_1_dictproxy_opts_1_strproxy_opts_2_dictproxy_opts_2_strr   rK  rL  s
             r   test_s3_proxy_optionsr#    s   ' $*;M.#*KN/ 
$5	6Bb,'''}222672===	$5	6Bb,'''}222672=== 
$4	5Bb,'''}222672===	$4	5Bb,'''}222672=== %6
7C
%6
7C#::}22378C???}22378C???
%6
7C
%6
7C#::}22378C???}22378C??? %5
6C
%5
6C#::}22378C???}22378C???
%5
6C
%5
6C#::}22378C???}22378C??? %6
7C
%5
6C#::}22378C???}22378C???
%6
7C
%5
6C#::}22378C???}22378C??? %6
7C
%6
7C#::}22378C???}22378C???
%6
7C
%5
6C#::}22378C???}22378C???
%5
6C
%6
7C#::}22378C???}22378C???
%5
6C
%5
6C#::}22378C???}22378C??? %6
7C
.C#::}22378C???}22378C???
%5
6C
.C#::}22378C???}22378C???
%6
7C
.C#::}22378C???}22378C???
%5
6C
.C#::}22378C???}22378C??? 
y	!#>? 
" 
x	 K#FG 
! 
x	 gt#DE 
! 
x	 fk#JK 
! 
r	'#<= 
( 
r	'g{,0$2 	3 
(	' 
"	! 
!	  
!	  
!	  
(	' 
(	'sH   
^-^^&	^2
^>6_
^^#&^/2^;>_
_c                     ddl m}   | dd      }d}t        j                  t        |      5 }|j                  d       d d d        d	t        j                        v ry  | d
d      }|j                  d       y # 1 sw Y   =xY w)Nr   r   z
eu-north-1T)r  r   a  When getting information for bucket 'voltrondata-labs-datasets': AWS Error UNKNOWN \(HTTP status 301\) during HeadBucket operation: No response body. Looks like the configured region is 'eu-north-1' while the bucket is located in 'us-east-2'.|NETWORK_CONNECTIONr/  voltrondata-labs-datasetsNETWORK_CONNECTIONr  )r   r   r   r2  r   rA   r   r   )r   r   msgexcs       r   test_s3fs_wrong_regionr)    sy    ' 
\T	:B!C
 
wc	*c
45 
+ s399~-	[D	9B01 
+	*s   A<<Bc                 
   ddl m}  |d      }t        ||      sJ | j                  | j	                  |            |k(  sJ  |dd      }t        ||      sJ | j                  | j	                  |            |k(  sJ ||k7  sJ  |ddd	d
dd      }t        ||      sJ | j                  | j	                  |            |k(  sJ ||k7  sJ t        j                  t              5   |        d d d        y # 1 sw Y   y xY w)Nr   r   zfake-account-name)r   fakeaccountkey)r   r   zfake-accountfakeaccountzfake-blob-authorityzfake-dfs-authorityr  r   )r   r   r#   r^  r]  r   r2  rC  )r_  r   rK  rL  fs3s        r   test_azurefs_optionsr.    s   *
':
;Cc?+++}22378C???
':&68Cc?+++}22378C???#::
~=1F0D.5-4	6C
 c?+++}22378C???#::	y	! 
"	!	!s   (C99Dc                 H   ddl m} t        j                         st	        j
                  d       | \  }}}d}d}d}d}	 |||d|||	      }
|j                  |	j                  ||d|||            }|j                  |	j                  ||d
|||            }|j                  |	j                  ||d
|dz   ||            } |||      }|j                  dj                  ||            } |||d      } |||dd      } |||dt        j                  d            } |||dd      } |||ddddi      }|
|k(  sJ ||k(  sJ ||k7  sJ ||k7  sJ ||k7  sJ ||k7  sJ ||k7  sJ ||k7  sJ ||k7  sJ ||k(  sJ ||k7  sJ ||k7  sJ t	        j                  t              5   |        d d d        t	        j                  t              5  |j                  d       d d d        |
||||||||||fD ]'  }|j                  |j                  |            |k(  r'J  | \  }}} ||||      }|j                  t        d            sJ |j                  dj                  |||            }|j                  t        d            sJ y # 1 sw Y   xY w# 1 sw Y   xY w)Nr   r   r      i   i   zIhdfs://{}:{}/?user={}&replication={}&buffer_size={}&default_block_size={}libhdfs)r   replicationr  default_block_sizemerz  zhdfs://{}:{}	localuser)r   
cache_path)r   kerb_ticketcache_path2
hdfs_tokenabcd)r   r7  
extra_confr  rV   zhdfs://{}:{}/?user={})r   r   ro   r   r   r   from_urirm   r}  ry  r2  rC  r^  r]  rA   r   )r   r_  r   r   r   r   r2  r  r3  urihdfs1hdfs2hdfs3hdfs4hdfs5hdfs6hdfs7hdfs8hdfs9hdfs10hdfs11r   r   s                          r   test_hdfs_optionsrI    s   +??+,&D$KK$$C T4i)4+0BDE %%cjjdI{K9K' E %%cjjdD+{4F' E %%cjjdD+/;8J' E T4(E%%n&;&;D$&GHET4k:ET4k)57ET4k)0l)CEEdD{*79FdD{*6*6)?AF E>>E>>E>>E>>E>>E>>E>>E>>E>>E>>U??U??	y	! 
"	y	!!!!$ 
" eUE5%ff&""=#6#6r#:;rAAA& 'D$D$T2Dl3/000$$&&tT48D l3/000# 
"	!	!	!s   &JJJJ!)r=  expected_klassexpected_pathzmock:r  zmock:foo/barr8  zmock:/foo/barzmock:///foo/barzmock:///some%20path/%C3%A9u   some path/ézfile:/rV   zfile:///zfile:/foo/barz/foo/barzfile:///foo/barzfile:///some%20path/%C3%A9u   /some path/éu   /some path/%20éc                 ^    t        j                  |       \  }}t        ||      sJ ||k(  sJ y r   )r   r<  r#   )r=  rJ  rK  r   r2   s        r   test_filesystem_from_urirM    s6    & ""3'HBb.)))=   r    r2   c                     t        j                  |       }t        j                  |      \  }} t	        |t
              sJ | |j                         j                         j                         k(  sJ y r   )	r}  ry  r   r<  r#   r   resolveabsoluter   )r2   r   r   s      r    test_filesystem_from_path_objectrQ    s\    
 	TA""1%HBb/***199;'')224444r    c                 P   ddl m} | d   \  }}}}dj                  ||||      }t        j                  |      \  }}t        ||      sJ |dk(  sJ |j                  |       |j                  |g      \  }	|	j                  |k(  sJ |	j                  t        j                  k(  sJ y )Nr   r   r   zZs3://{}:{}@mybucket/foo/bar?scheme=http&endpoint_override={}:{}&allow_bucket_creation=Truemybucket/foo/bar)r   r   rm   r   r<  r#   rR   rA   r2   r:  r   r;   )
r   r   r   r   r   r   r=  r   r2   r@   s
             r   test_filesystem_from_uri_s3rT  )  s    ')2<)@&D$
J(6*j$5  ""3'HBb,'''%%%%MM$tf%FT9999*****r    c                 B   ddl m} | d   \  }}dd| d| dz   dz   }t        j                  |      \  }}t	        ||      sJ |d	k(  sJ |j                  |       |j                  |g      \  }|j                  |k(  sJ |j                  t        j                  k(  sJ y )
Nr   r   r   zgs://anonymous@z/mybucket/foo/bar?scheme=http&endpoint_override=r   &z0retry_limit_seconds=5&project_id=test-project-idrS  )r   r   r   r<  r#   rR   rA   r2   r:  r   r;   )r   r   r   r   r=  r   r2   r@   s           r   test_filesystem_from_uri_gcsrW  =  s    (L)JD$<TF!D6KL=>C ""3'HBb-(((%%%%MM$tf%FT9999*****r    c                     t               } t        |       }t        |t              sJ |j                  dk(  sJ |j                  | u sJ t        j                  t              5  t        d        d d d        y # 1 sw Y   y xY wNz	py::dummy)r   r   r#   r%  handlerr   r2  rC  )rZ  r   s     r   test_py_filesystemr[  Q  sg    nG	g	Bb,'''<<;&&&::   	y	!T 
"	!	!s   "A77B c                  (   t        d      } t        d      }t        d      }t        |       }t        |       }t        |      }t        |      }||usJ ||usJ ||usJ ||k(  sJ ||k7  sJ ||k(  sJ |t               k7  sJ |t               k7  sJ y )Nrz  r0  )r   r   r   r=   )handler1handler2handler3rK  rL  r-  fs4s          r   test_py_filesystem_equalityra  \  s    AHAHAH
x
 C
x
 C
x
 C
x
 Cc>>c>>c>>#::#::#::/####&(??r    c                     t               }t        |      }| j                  |      }| j                  |      }t	        |t
              sJ ||k(  sJ |j                  |k(  sJ |j                  dk(  sJ y rY  )r   r   r]  r^  r#   r   rZ  r%  )r_  rZ  r   r`  ra  s        r   test_py_filesystem_picklingrc  p  sw    nG	g	B$$R(J"":.Hh
+++r>>w&&&,,,r    c                  R   t               } t        |       }t        |t              sJ t        j                  |       }d }  |       J d } |       J t               } t        |       }t        j                  |       }d }  |       |j
                  u sJ  |       J d } |       J y r   )r   r   r#   weakrefrefrZ  )rZ  r   wrs      r   test_py_filesystem_lifetimerh  |  s    nG	g	Bb,'''	W	BG4	B4<< nG	g	B	W	BG42::4	B4<<r    c                     t               } t        |       }|j                  dg      \  }|j                  dk(  sJ |j                  t
        j                  k(  sJ |j                  dg      \  }|j                  dk(  sJ |j                  t
        j                  k(  sJ |j                  dg      \  }|j                  dk(  sJ |j                  t
        j                  k(  sJ t        j                  t              5  |j                  dg       d d d        t        j                  t              5  |j                  dg       d d d        y # 1 sw Y   >xY w# 1 sw Y   y xY w)Nzsome/dirz	some/filer7   r8   r  )r   r   rA   r2   r:  r   r;   r:   r<   r   r2  rC  r>   )rZ  r   r@   s      r    test_py_filesystem_get_file_inforj    s$   nG	g	Bzl+FT99
"""99*****{m,FT99###99%%%zl+FT99
"""99)))))	y	!
)% 
" 
w	
%! 
 	 
"	! 
 	s   ;E/EEE c                  Z   t               } t        |       }t        d      }|j                  |      }t	        |      dk(  sJ |d   j
                  dk(  sJ |d   j                  t        j                  k(  sJ |d   j                  dk(  sJ |d   j
                  dk(  sJ |d   j                  t        j                  k(  sJ |d   j                  J t        dd	
      }|j                  |      }t	        |      dk(  sJ |d   j
                  dk(  sJ |d   j
                  dk(  sJ |d   j
                  dk(  sJ t        d      }t        j                  t              5  |j                  |       d d d        t        dd	      }|j                  |      g k(  sJ y # 1 sw Y   -xY w)NrC   )rI   r0  r   rD   rE   rz  rG   T)rI   rL   r  rH   r7   )rI   rJ   )r   r   r   rA   r  r2   r:  r   r:   rF   r;   r   r2  rK   )rZ  r   rM   rN   s       r   )test_py_filesystem_get_file_info_selectorrl    s   nG	g	BY/HX&Eu:??8==O+++8==HMM)))8==C8==----8==H.....8==   Y$?HX&Eu:??8==O+++8==----8==3333Z0H	(	)
" 
* ZFHH%+++	 
*	)s   #F!!F*c                  X   t               } t        |       }|j                  dd       |j                  dd       t        j                  t
              5  |j                  d       d d d        |j                  d       |j                  d       d	D ]H  }t        j                  t              5  |j                  |       d d d        |j                  |d
       J |j                  d       |j                  dd       |j                  dd       y # 1 sw Y   xY w# 1 sw Y   `xY w)NrL   Tr  rQ   FfoobarrT   rW   )r  rV   r  r  r^   r`   ra   rg   rh   )r   r   rR   r   r2  r>   rT   rW   rY   r^   re   ri   )rZ  r   r2   s      r   test_py_filesystem_opsro    s    nG	g	BMM+M.MM/UM3	w	
h 
  MM,01]]:&""4( '
tT:   NN=!GGK#LL!>2 
 	 '&s   D/D D D)	c                      t        t                     } | j                  d      5 }|j                         dk(  sJ 	 d d d        t	        j
                  t              5  | j                  d       d d d        y # 1 sw Y   =xY w# 1 sw Y   y xY w)Nsomefiles   somefile:input_streamr7   )r   r   rs   r  r   r2  rK   r   r  s     r   test_py_open_input_streamrs    sk    	ln	%B			j	)Qvvx3333 
*	(	)
Z( 
*	) 
*	)	)	)   A8B8BBc                      t        t                     } | j                  d      5 }|j                         dk(  sJ 	 d d d        t	        j
                  t              5  | j                  d       d d d        y # 1 sw Y   =xY w# 1 sw Y   y xY w)Nrq  s   somefile:input_filer7   )r   r   ru   r  r   r2  rK   rr  s     r   test_py_open_input_filerv    sk    	ln	%B			J	'1vvx1111 
(	(	)
:& 
*	) 
(	'	)	)rt  c                      t        t                     } | j                  d      5 }|j                  d       d d d        y # 1 sw Y   y xY wNrq  r  )r   r   r|   rh  rr  s     r   test_py_open_output_streamry    6    	ln	%B			z	*a	 
+	*	*   A  A	c                      t        t                     } | j                  d      5 }|j                  d       d d d        y # 1 sw Y   y xY wrx  )r   r   r   rh  rr  s     r   test_py_open_append_streamr}    rz  r{  c                     ddl m}  t        j                  j	                  d      xs d} | d      }|j
                  |k(  sJ  | dd      }|j                  t        d	            }t        |      dkD  sJ d
}|j                  |      5 }|j                         }d|v sJ |d   dk(  sJ |d   dk(  sJ 	 d d d        y # 1 sw Y   y xY w)Nr   r   PYARROW_TEST_S3_REGIONz	us-east-1Tr  r  )r   r  z"voltrondata-labs-datasets/nyc-taxizCvoltrondata-labs-datasets/nyc-taxi/year=2019/month=6/part-0.parquetr  zLast-Modifieds   2022-07-12T23:32:00ZETags%   "4c6a76826a695c6ac61592bc30cda3df-16")r   r   osenvirongetr  rA   r   r  rs   r{   )r   default_regionr   entriesr  r  mds          r   test_s3_real_awsr    s     (jjnn%=> "! 		%B99&&&	[	9B|, . /Gw<!
OC			c	"aZZ\###/"&==== &zEEEE 
#	"	"s   +C  C	c                     t        j                  d      \  } }| j                  dk(  sJ | j                  |      5 }d|j	                  d      v sJ 	 d d d        t        j                  d      \  } }| j                  dk(  sJ t        j                  t        d      5  t        j                  d	       d d d        t        j                  d
      \  } }| j                  dk(  sJ y # 1 sw Y   xY w# 1 sw Y   ?xY w)Nzs3://mf-nwp-models/README.txt	eu-west-1s&   Meteo-France Atmospheric models on AWS2   z.s3://mf-nwp-models/README.txt?region=us-east-2r  zBucket '.*' not foundr/  zs3://x-arrow-nonexistent-bucketz0s3://x-arrow-nonexistent-bucket?region=us-east-3z	us-east-3)r   r<  r  rs   r  r   r2  r>   )r   r2   r  s      r   !test_s3_real_aws_region_selectionr    s     ""#BCHB99###			d	#q8AFF2JFFF 
$ ""8:HB99### 
w&=	>=> 
?"":<HB99### 
$	# 
?	>s   C&C2&C/2C;c                  $   ddl m}   | d      dk(  sJ  | d      dk(  sJ t        j                  t        d      5   | d	       d d d        t        j                  t        d      5   | d
       d d d        y # 1 sw Y   6xY w# 1 sw Y   y xY w)Nr   resolve_s3_regionr%  r  zmf-nwp-modelsr  zNot a valid bucket namer/  r8  z	s3:bucket)r   r  r   r2  rY   r  s    r   test_resolve_s3_regionr  (  sz    ,89[HHH_-<<<	z)B	C)$ 
D	z)B	C+& 
D	C 
D	C	C	Cs   	A:(	B:BBc           
      `   |d   }|d   } |d      }|j                  |      5 }|j                  d       d d d        | \  }}}	}
d|	 d|
 d| d| d| 
}t        |d	z        }t        ||       t	               }|j                  |      5 }|j                         dk(  sJ 	 d d d        t        |d
z        }t        |||       |j                  |      5 }|j                         dk(  sJ 	 d d d        t        |dz        }t        |      }t        ||       |j                  |      5 }|j                         dk(  sJ 	 d d d        t        |dz        }t        |||       |j                  |      5 }|j                         dk(  sJ 	 d d d        t        |dz        }t        ||dd       |j                  |      5 }|j                         dk(  sJ 	 d d d        y # 1 sw Y   xY w# 1 sw Y   RxY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)Nr   r   rf  rg  zs3://r   @z?scheme=http&endpoint_override=zc_copied1.txtzc_copied2.txt)source_filesystemzc_copied3.txtzc_copied4.txt)destination_filesystemzc_copied5.txtrz  F
chunk_sizeuse_threads)r|   rh  r   r   r   rs   r  r   )s3_connectionr   r   r   r   r2   r  r   r   r   r   
source_urilocal_path1r   local_path2local_path3destination_urilocal_path4local_path5s                      r   test_copy_filesr  4  s"   	dB(^F '?D			t	$	 
% *7&D$
J

|1ZL$
)$q	8 
 g/0Kz;'G		"	";	/1vvx7""" 
0 g/0Kt[B7		"	";	/1vvx7""" 
0 g/0K%k2Oz?+		"	";	/1vvx7""" 
0 g/0Kz;wG		"	";	/1vvx7""" 
0 g/0Kz;1%H		"	";	/1vvx7""" 
0	/U 
%	$ 
0	/ 
0	/ 
0	/ 
0	/ 
0	/sG   G%G2G?,H8HH$%G/2G<?H	HH!$H-c                    t               | dz  }|j                          j                  t        |dz              5 }|j	                  d       d d d        j                  t        |dz              5 }|j	                  d       d d d        fd}| dz  }|j                          t        t        |      t        |              ||       | dz  }|j                          t        t        |      t        |      	        ||       | d
z  }|j                          t        t        |            }t        t        |            }t        ||        ||       | dz  }	|	j                          t        ||	        ||	       | dz  }
|
j                          t        ||
dd        ||
       y # 1 sw Y   fxY w# 1 sw Y   <xY w)Nsourcefile1   test1file2   test2c                 &   j                  t        | dz              5 }|j                         dk(  sJ 	 d d d        j                  t        | dz              5 }|j                         dk(  sJ 	 d d d        y # 1 sw Y   FxY w# 1 sw Y   y xY w)Nr  r  r  r  )rs   r   r  )destination_dirr  r   s     r   check_copied_filesz5test_copy_files_directory.<locals>.check_copied_filest  s    &&s?W+D'EF!668x''' G&&s?W+D'EF!668x''' GF GFFFs   A;B;BBdestination1destination2)r  r  destination3destination4destination5rz  Fr  )r   mkdirr|   r   rh  r   r   )r   
source_dirr  r  destination_dir1destination_dir2destination_dir3r  r  destination_dir4destination_dir5r   s              @r   test_copy_files_directoryr  i  s   G 8#J		#	#C
W(<$=	>!	 
?		#	#C
W(<$=	>!	 
?( /s:$4 56'( /s:$4 5!(J'( / Z1J%c*:&;<Oz?+'( /z+,'( /z+uM'(W 
?	>	>	>s   F45G4F>Gc                  T    d} t        j                  t        j                  d| g       y )Na  if 1:
        import pytest
        from pyarrow.fs import (FileSystem, S3FileSystem,
                                ensure_s3_initialized, finalize_s3)

        fs, path = FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        assert fs.region == 'eu-west-1'
        f = fs.open_input_stream(path)
        f.read(50)

        finalize_s3()

        with pytest.raises(ValueError, match="S3 .* finalized"):
            f.read(50)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            fs.open_input_stream(path)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            S3FileSystem(anonymous=True)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        -c
subprocess
check_callsys
executablecodes    r   test_s3_finalizer    s$    D* 3>>467r    c                  T    d} t        j                  t        j                  d| g       y )Na  if 1:
        import pytest
        from pyarrow.fs import resolve_s3_region, ensure_s3_initialized, finalize_s3

        resolve_s3_region('mf-nwp-models')

        finalize_s3()

        # Testing both cached and uncached accesses
        with pytest.raises(ValueError, match="S3 .* finalized"):
            resolve_s3_region('mf-nwp-models')
        with pytest.raises(ValueError, match="S3 .* finalized"):
            resolve_s3_region('voltrondata-labs-datasets')
        r  r  r  s    r    test_s3_finalize_region_resolverr    s$    D 3>>467r    c                  T    d} t        j                  t        j                  d| g       y )Na  if 1:
        import threading
        import pytest
        from pyarrow.fs import (FileSystem, S3FileSystem,
                                ensure_s3_initialized, finalize_s3)
        threads = []
        fn = lambda: FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        for i in range(4):
            thread = threading.Thread(target = fn)
            threads.append(thread)
            thread.start()

        for thread in threads:
            thread.join()

        finalize_s3()
        r  r  r  s    r   test_concurrent_s3fs_initr    s$    
D" 3>>467r    c            	      H   	 t        j                  ddg       t               } dddd|  dt        j                  j                  t        d      g}t        j                  |t         j                  	      }	 d
|  d}t        j                         }d }t        j                         |dz   k  r(	 t        |      5 }|j                  dk(  sJ 	 d d d        nt        j"                  d|d|        |j%                          |j'                         dk(  sJ y # t        $ r t        j                  d       Y w xY w# 1 sw Y   pxY w# t        $ r!}|}t        j                   d       Y d }~nd }~ww xY wt        j                         |dz   k  r⌻# |j%                          w xY w)Nuwsgiz	--versionz"uwsgi not installed on this Pythonz-iz--httpz
127.0.0.1:z--wsgi-filezwsgi_examples.py)stdinzhttp://127.0.0.1:z/s3/r     g?zCould not fetch r      )r  r  rK   r   r   r   r  r2   joinherePopenDEVNULLtimer   statusr   sleepfail	terminatewait)r   argsprocurl
start_timeerrorrespr   s           r   test_uwsgi_integrationr    sk   :w45 DT8z$%8277<<.@ACDD
(:(:;D!$t,YY[
iikJN* S\T;;#--- "
 KK*3'E7;<99;"3  :89: "\   

3  iikJN* 	s_   D 66F -E 8D:E F D76D7:E?E 	E0E+&F +E00F F!)r   r   r   r  r  r}  urllib.requestr   r  r  r  r   re  r   ro   pyarrow.tests.test_ior   pyarrow.tests.utilr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   pyarrow.utilr   r2   dirnameabspathr  r  r   fixturer   r   r   r   r   r   r   r   r   r1  r   r   r   r   r  r
  markr   r   r   r  r   r   r   r   r  r  r   r(  r+  r5  rA  rD  rM  rO  rV  rb  rp  rs  ru  r  r  r  r  r  r  r  r  r  r  r  r  r  r  parametrizecompressr  r  r  
decompressr  r  r  r  r  r  r  r  r  r#  r)  r.  rI  rM  rQ  rT  rW  r[  ra  rc  rh  rj  rl  ro  rs  rv  ry  r}  r  r  r  r  r  r  r  	processes	threadingr  r  r-   r    r   <module>r     s  $ 3 2  	  "  
     7< <$ $ $ $ ( 
wwrwwx01f'$ f'R              :  4   ,  8  $      , FLL FLL+ FLL1 FLLkknn
 FLLkkoo
 FLLkk
 FLLkk
 FLL FLL: FLL: FLLB FLLE FLL=kknnk: :v2w:v2 # # ' ' / / 5 5	2&
>
#
 ) )*:
O(K (,"- 8+v5 p".@$,(
	(($ 0	tX	r8	t}}%	dmm$	
 
&&  2	tX	r8	t'	doo&		) 	) @	tXx(	r8X&	t}}doo6	dmmT__5	; ;0".#.( "B "BJ Y7 Y7x I3 I3X 2 2.  4 D1 D1N C or"_i0oy13!?NC$#&oz24!?OD/3*-*<=!F $!%$! 
iX.5	5 + +& + +&(	-*".,:3()' F F. $ $, ' ' 1# 1#h1)h 8 86 8 8& 8   8,  r    