
    g&                        d dl Z d dlZd dlZd dlZd dlZ	 d dlmZ d dl	m
Z
 	 d dlZd dlmZ d dl	mZ ej$                  j                  Zej$                  j                  d        Zej$                  j                  d        Zej$                  j                  d        Zej$                  j                  d        Zej$                  j                  d        Zej$                  j                  d	        Zd
 Zd Zej$                  j                  ej$                  j9                  dg d      d               Zej$                  j                  ej$                  j9                  ddg      d               Zej$                  j                  ej$                  j9                  dddg      d               Zd Z ej$                  jB                  ej$                  j9                  dd      d               Z"y# e$ r dZY w xY w# e$ r dxZZY w xY w)    N)_write_table)alltypes_samplec                     t        d      } t        j                  j                  |       }t	        j
                         }t        ||dd       |j                  d       t        j                  |      }|j                  d       t        j                  ||      }t        j                  | |j                         j                                y )N'  sizesnappy2.6)compressionversionr   )metadata)r   paTablefrom_pandasioBytesIOr   seekpqread_metadataParquetFiletmassert_frame_equalread	to_pandas)dfa_tablebufr   filehs        \/var/www/openai/venv/lib/python3.12/site-packages/pyarrow/tests/parquet/test_parquet_file.pytest_pass_separate_metadatar    .   s     
e	$Bhh""2&G
**,C#8UCHHQK$HHHQKNN32E"ejjl4467    c                     d\  } }t        |       }t        j                  j                  |      }t	        j
                         }t        ||| |z  dd       |j                  d       t        j                  |      }|j                  |k(  sJ t        |      D cg c]  }|j                  |       }}t        j                  |      }t        j                  ||j!                                y c c}w Nr      r   r	   r
   row_group_sizer   r   r   )r   r   r   r   r   r   r   r   r   r   num_row_groupsrangeread_row_groupconcat_tablesr   r   r   )	NKr   r   r   pfi
row_groupsresults	            r   test_read_single_row_groupr2   B   s     DAq	a	 Bhh""2&G
**,C#a!e%u6 HHQK		B!!!05a91"##A&J9j)F"f..01 :s   C0c                     d\  } }t        |       }t        j                  j                  |      }t	        j
                         }t        ||| |z  dd       |j                  d       t        j                  |      }t        |j                  d d       }t        |      D cg c]  }|j                  ||       }}t        j                  |      }	t        j                   ||   |	j#                                t        |      D cg c]  }|j                  |||z          }}t        j                  |      }	t        j                   ||   |	j#                                y c c}w c c}w 	Nr$   r   r	   r
   r&   r      )columns)r   r   r   r   r   r   r   r   r   r   listr6   r)   r*   r+   r   r   r   )
r,   r-   r   r   r   r.   colsr/   r0   r1   s
             r   -test_read_single_row_group_with_column_subsetr9   Y   s8   DAq	a	 Bhh""2&G
**,C#a!e%u6 HHQK		B

2AD>CAhGh"##At#4hJGj)F"T(F$4$4$67 FK1XNX"##Atd{#;XJNj)F"T(F$4$4$67 H Os   #E$E)c                     d\  } }t        |       }t        j                  j                  |      }t	        j
                         }t        ||| |z  dd       |j                  d       t        j                  |      }|j                  |k(  sJ |j                  t        |            }t        j                  ||j                                y r#   )r   r   r   r   r   r   r   r   r   r   r(   read_row_groupsr)   r   r   r   )r,   r-   r   r   r   r.   r1   s          r   test_read_multiple_row_groupsr<   r   s    DAq	a	 Bhh""2&G
**,C#a!e%u6 HHQK		B!!!a)F"f..01r!   c                  @   d\  } }t        |       }t        j                  j                  |      }t	        j
                         }t        ||| |z  dd       |j                  d       t        j                  |      }t        |j                  d d       }|j                  t        |      |      }t        j                  ||   |j!                                |j                  t        |      ||z         }t        j                  ||   |j!                                y r4   )r   r   r   r   r   r   r   r   r   r   r7   r6   r;   r)   r   r   r   )r,   r-   r   r   r   r.   r8   r1   s           r   0test_read_multiple_row_groups_with_column_subsetr>      s    DAq	a	 Bhh""2&G
**,C#a!e%u6 HHQK		B

2ADa$7F"T(F$4$4$67 a$+>F"T(F$4$4$67r!   c                  n   d\  } }t        |       }t        j                  j                  |      }t	        j
                         }t        ||| |z  dd       |j                  d       t        j                  |      }|j                         dk(  sJ |j                  |j                  d d       dk(  sJ y )	Nr$   r   r	   r
   r&   r   r   r%   )r   r   r   r   r   r   r   r   r   r   scan_contentsr6   )r,   r-   r   r   r   r.   s         r   test_scan_contentsrA      s    DAq	a	 Bhh""2&G
**,C#a!e%u6 HHQK		B&&&BJJrN+u444r!   c                 b   | dz  }t        j                  t        |             dt        |       d}t        j                  t
              5 }t        j                  |       d d d        j                  t              rt        j                  dk(  ry |j                  |       y # 1 sw Y   DxY w)N	directoryzCannot open for reading: path 'z' is a directorywin32)osmkdirstrpytestraisesIOErrorr   r   errisinstancePermissionErrorsysplatformmatch)tempdirpathmsgexcs       r   0test_parquet_file_pass_directory_instead_of_filerT      s~    [ DHHSY+CI;6F
GC	w	3
t 
 
)cllg.EIIcN	 
 	s   B%%B.c                     t        j                  t        j                  ddg      t        j                  ddg      gddg      } t        j                         }t	        j
                  | |       t	        j                  |j                               }|j                  j                  d      j                         ddgk(  sJ |j                  j                  d	      j                         ddgk(  sJ d
D ]E  }t        j                  t        t        f      5  |j                  j                  |       d d d        G y # 1 sw Y   RxY w)Nr%      foobarintsstrs)namesr      )r5   )r   tablearrayBufferOutputStreamr   write_tabler   getvaluereaderread_column	to_pylistrH   rI   
ValueError
IndexError)r^   biofindexs       r   test_read_column_invalid_indexrk      s    HHbhh1v&%(@A"F+-E



!CNN5#
s||~&A88",,.1a&88888",,.5%.@@@]]J
34HH  ' 54 44s   D>>E	
batch_size)i,    i  c           	      6   d}d}t        |      }| dz  }t        j                  j                  |      }t	        ||d|       t        j                  |      }|j                  d d |j                  dd  fD ]  }|j                  ||      }	t        d	||z   |      }
t        |	|
      D ]l  \  }}t        |||z         }t        j                  |j                         |j                  ||d d f   j                   d d |f   j#                  d
             n  y )Ni  rm   r   pandas_roundtrip.parquetr
   r   
chunk_size
   )rl   r6   r   Tdrop)r   r   r   r   r   r   r   r6   iter_batchesr)   zipminr   r   r   iloclocreset_index)rP   rl   
total_sizerq   r   filenamearrow_tablefile_r6   batchesbatch_startsbatchstartends                 r    test_iter_batches_columns_readerr      s    JJ	j	)B33H((&&r*Kh&( NN8$EJJsORZZ_5$$
G$LQ
: 5zB6LE5j%*"45C!!!c	1%))!W*5AAtAL 7 6r!   rq   rm   c                    t        dd      }| dz  }t        j                  j                  |      }|j                  j
                  J t        ||d|       t        j                  |      }d }t         ||            }d}t        |j                        D ]  }	t        j                  ||   j                         |j                  |	g      j                         j!                  d	             |d
z  }t        j                  ||   j                         j#                  d      |j                  |	g      j                         j$                  d	d  j#                  d             |d
z  } y )Nr   T)r   categoricalro   r
   rp   c              3   ~   K   t        | j                        D ]!  }| j                  d|g      }|D ]  }|  # y w)N  )rl   r0   )r)   r(   ru   )ri   	row_groupr   r   s       r   get_all_batchesz1test_iter_batches_reader.<locals>.get_all_batches   sG     q//0Inn%; % G
 ! ! 1s   ;=r   r   r\   rs   )r   r   r   r   schemapandas_metadatar   r   r   r7   r)   r(   r   r   r   r;   headrz   rx   )
rP   rq   r   r|   r}   r~   r   r   batch_nor/   s
             r   test_iter_batches_readerr      sX    
e	6B33H((&&r*K--999h&( NN8$E ?5)*GH5''(
H'')!!1#&00277<	

 	A
H'')5545@!!1#&00277=II J 	
 	A )r!   
pre_bufferFTc                 @   d\  }}t        |      }t        j                  j                  |      }t	        j
                         }t        ||||z  dd       |j                  d       t        j                  ||       }|j                         j                  |k(  sJ y )Nr$   r   r	   r
   r&   r   )r   )r   r   r   r   r   r   r   r   r   r   r   num_rows)r   r,   r-   r   r   r   r.   s          r   test_pre_bufferr     s     DAq	a	 Bhh""2&G
**,C#a!e%u6 HHQK	
	3B779"""r!   c                    | j                  d      }t        j                  ddgddgd      }t        j                  ||       t        |d      5 }t        j                  |      5 }|j                          |j                  rJ |j                  rJ 	 ddd       |j                  rJ j                  rJ 	 ddd       j                  sJ j                  sJ t        j                  |      5 }|j                          |j                  rJ 	 ddd       |j                  sJ y# 1 sw Y   xY w# 1 sw Y   }xY w# 1 sw Y   0xY w)z
    Unopened files should be closed explicitly after use,
    and previously opened files should be left open.
    Applies to read_table, ParquetDataset, and ParquetFile
    zfile.parquetr   r\   )col1col2rbN)	joinpathr   r^   r   ra   openr   r   closed)rP   fnr^   ri   ps        r   #test_parquet_file_explicitly_closedr     s    
		.	)BHHq!fq!f56ENN5" 
b$1^^A!FFHxx<xx<x  88|88|8 
 88O888O8 
	q	88|8 
 88O8  
 
	s0   D5&-D)$D52E)D2	.D55D>E
use_uri)TFc                    | \  }}}|r|n|f}|ri nt        |      }t        j                  dt        d      i      }t	        j
                  |||       t	        j                  |i |}|j                         |k(  sJ |j                  rJ |j                          |j                  sJ t	        j                  |i |5 }	|	j                         |k(  sJ |	j                  rJ 	 d d d        	j                  sJ y # 1 sw Y   xY w)N)
filesystemarr   )
dictr   r^   r)   r   ra   r   r   r   close)
s3_example_fsr   s3_fss3_uris3_pathargskwargsr^   parquet_fileri   s
             r   !test_parquet_file_with_filesystemr   <  s     +E67F7,DR 6FHHc59%&ENN5'e4>>4262L%'''""""		(	(Avvx5   88|8 
) 88O8 
)	(s   $C==D)#r   rE   rM   rH   pyarrowr   pyarrow.parquetparquetr   pyarrow.tests.parquet.commonr   ImportErrorpandaspdpandas.testingtestingr   r   mark
pytestmarkr    r2   r9   r<   r>   rA   rT   rk   parametrizer   r   r   r   s3r    r!   r   <module>r      s  $ 
 	 
   9< [[  
 8 8& 2 2, 8 80 2 2( 8 8, 5 5 

( '89 : . v.( / (V t}5# 6 #: M2 3 C	  	B  NBs"   G, G: ,G76G7:	HH