
    g-                        d dl Z d dlZd dlmZ 	 d dlmZ d dlmZm	Z	m
Z
 	 d dlZd dlmZ e j"                  j
                  Ze j"                  j                  d        Zd Zd Ze j"                  j                  d        Ze j"                  j                  d        Ze j"                  j                  e j"                  j1                  d	d ej2                         g      d
               Ze j"                  j6                  e j"                  j                  d               Ze j"                  j                  e j"                  j1                  d	d ej2                         g      d               Ze j"                  j                  e j"                  j<                  d               Ze j"                  j                  e j"                  j<                  d               Z e j"                  j                  e j"                  j<                  d               Z!e j"                  j                  d        Z"d Z#d Z$y# e$ r dZY w xY w# e$ r dxZZY w xY w)    N)fs)_read_table_test_dataframe_range_integersc                    t        d      }d|d<   t        j                  j                  |d      }t        j                         }t        j                  ||j                  d      }g }t        d      D ]X  }||d<   t        j                  j                  |d      }|j                  |       |j                  |j                                Z |j                          |j                         }t        t        j                  |            }t!        j"                  |d	
      }	t%        j&                  |j)                         |	       y Nd   r   	unique_idFpreserve_index2.6version
   Tignore_index)r   paTablefrom_pandasBufferOutputStreampqParquetWriterschemarangewrite_tableappendcopyclosegetvaluer   BufferReaderpdconcattmassert_frame_equal	to_pandas
tempdirdfarrow_tableoutwriterframesibufresultexpecteds
             ^/var/www/openai/venv/lib/python3.12/site-packages/pyarrow/tests/parquet/test_parquet_writer.py#test_parquet_incremental_file_buildr2   ,   s    		BB{O((&&r%&@K



!Cc;#5#5uEFF2Y;hh**2e*D;'bggi   LLN
,,.C-.Fyyd3H&**,h7    c                 P   t        j                  dt        j                               t        j                  dt        j                               g}t        j                  |      }t        j
                  dg      t        j
                  dg      g}t         j                  j                  |ddg      }| dz  }t        j                  ||ddd	      5 }t        j                  t              5  |j                  |       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)
NPOSdesc   blazsimple_validate_schema.parquetr   snappyspark)r   compressionflavor)r   fielduint32stringr   arrayr   from_arraysr   r   pytestraises
ValueErrorr   )r'   simple_fieldssimple_schemasimple_from_arraysimple_tablepathws          r1    test_validate_schema_write_tablerK   G   s     			$
%M
 IIm,M 1#%(9:88''(9E6?KL55D			$"'&.w
@CD]]:&MM,' '
@ 
@ '&
@ 
@s$   D-D?DD	DD%c                    t        j                  t              5  t        j                  t        j
                  dt        j                               g      }t        j                  d |       d d d        t        j                  t              5  t        j                  | dz  d        d d d        y # 1 sw Y   ExY w# 1 sw Y   y xY w)Nx	some_path)	rB   rC   	TypeErrorr   r   r=   int32r   r   )r'   some_schemas     r1   test_parquet_invalid_writerrR   ]   s    	y	!ii#rxxz!: ;<
{+ 
" 
y	!
;.5 
"	!	 
"	! 
"	!s   AB1B=1B:=Cc                    t        d      }d|d<   t        j                  j                  |d      }t        j                         }t        j                  ||j                  d      5 }g }t        d      D ]X  }||d<   t        j                  j                  |d      }|j                  |       |j                  |j                                Z 	 d d d        |j                         }t        t        j                  |            }t        j                   d	
      }	t#        j$                  |j'                         |	       y # 1 sw Y   sxY wr   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   s
             r1   test_parquet_writer_context_objrT   g   s   		BB{O((&&r%&@K



!C			#{115	AVrAB{O((..r%.HK{+MM"'')$  
B ,,.C-.Fyyd3H&**,h7 
B	As   (A)EEc                    t        d      }d|d<   t        j                  j                  |d      }t        j                         }d}	 t        j                  ||j                  d      5 }g }t        d	      D ]h  }||d<   t        j                  j                  |d      }|j                  |       |j                  |j                                |d
k(  s_t        |       	 d d d        |j                         }	t!        t        j"                  |	            }
t%        j&                  d      }t)        j*                  |
j-                         |       y # 1 sw Y   sxY w# t        $ r}t        |      |k(  sJ Y d }~d }~ww xY w)Nr	   r   r
   Fr   zArtificial Errorr   r   r      Tr   )r   r   r   r   r   r   r   r   r   r   r   r   rD   	Exceptionstrr   r   r    r!   r"   r#   r$   r%   )r'   r(   r)   r*   
error_textr+   r,   r-   er.   r/   r0   s               r1   .test_parquet_writer_context_obj_with_exceptionr[      sH   		BB{O((&&r%&@K



!C#J$c)00&+-06F2Y"#; hh222e2L"";/bggi(6$Z00 - ,,.C-.Fyyd3H&**,h7'- -  $1v###$s7   	"E# +A,EE%E# E E# #	F,FF
filesystemc                    t        d      }t        j                  j                  |d      }t        j                  j                  |d      }t        | dz        }t        | dz        }t        j                  ||j                  |d      5 }|j                  |       d d d        t        |      j                         }t        j                  ||       t        j                  ||j                  |d      5 }|j                  |       d d d        t        |      j                         }t        j                  ||       t        j                  ||j                  |d      5 }|j                  |       d d d        t        |      j                         }t        j                  ||       t        j                  ||j                  |d      5 }|j                  |       d d d        t        |      j                         }t        j                  ||       y # 1 sw Y   ~xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   _xY w)Nr	   Fr   zdata_table.parquetzdata_batch.parquetr   r\   r   )r   r   r   r   RecordBatchrX   r   r   r   r   r   r%   r#   r$   write_batchwrite)	r'   r\   r(   tablebatch
path_table
path_batchr+   r/   s	            r1   "test_parquet_writer_write_wrappersrf      s    
	BHH  E :ENN&&r%&@EW334JW334J			ELLZ
	5!

 $..0F&"%			ELLZ
	5!

 $..0F&"%			ELLZ
	U

 $..0F&"%			ELLZ
	U

 $..0F&"%=
 

 

 

 
s0   H9H(%H5IH%(H25H>I
c                      dd fd} |dz  dz
  d        |dz  d        |dz  dz   d        |dz   dz   d        |dz  dz  d        |d d        |dz   d d       y )Ni   i   c                 X   t         j                  j                  t        | d      gdg      }|t	        j
                  |dz         nt	        j
                  |dz  |       t	        j                  dz        }|
n|}|j                  |k(  sJ t        |	      }t        |dz
        D ]"  }|j                  |      j                  |k(  r"J  | ||dz
  z  z
  }|dk(  r$|j                  |dz
        j                  |k(  sJ y |j                  |dz
        j                  |k(  sJ y )NbrM   )namesztest.parquet)row_group_sizer7   r   )r   r   rA   r   r   r   read_metadatanum_row_groupsminr   	row_groupnum_rows)	data_size
chunk_sizeexpect_num_chunksrb   metadataexpected_chunk_sizelatched_chunk_size	chunk_idx	remainderabs_max_chunk_sizedefault_chunk_sizer'   s            r1   check_chunk_sizez8test_parquet_writer_chunk_size.<locals>.check_chunk_size   sR   $$Is+&
 %  NN5'N":;NN5'N"::V##Gn$<=4>4F0J&&*;;;; !46HI0145I%%i099=OOOO 6 !48IA8M!NO	>%%!A%''/x3EF F F %%&7!&;<EERRRr3      r	      r7    )r'   r{   ry   rz   s   ` @@r1   test_parquet_writer_chunk_sizer      s     %)S. '!+-?#-EqI'!+-?C'!+-?#-EqI'#-/AC/GK '!+-?!-CQG 'q1'!+T15r3   c                 h   t        d      }t        j                  j                  |d      }t	        | dz        }t        j                  ||j                  |d      5 }|j                  |       d d d        t        |      j                         }t        j                  ||       y # 1 sw Y   9xY w)Nr	   Fr   zdata.parquetr   r^   )r   r   r   r   rX   r   r   r   r   r   r%   r#   r$   )r'   r\   r(   rb   rI   r+   r/   s          r1   $test_parquet_writer_filesystem_localr      s     
	BHH  E :Ew'(D			ellz5
	5!

 ((*F&"%
 
s   B((B1c                 X   t        d      }t        j                  j                  |d      }| \  }}}t	        j
                  ||j                  |d      5 }|j                  |       d d d        t        |      j                         }t        j                  ||       y # 1 sw Y   9xY w)Nr	   Fr   r   r^   r   r   r   r   r   r   r   r   r   r%   r#   r$   s3_example_fsr(   rb   r   urirI   r+   r/   s           r1   !test_parquet_writer_filesystem_s3r     s     
	BHH  E :E!MBT			ellr5
	5!

 '')F&"%
 
s   B  B)c                 Z   t        d      }t        j                  j                  |d      }| \  }}}t	        j
                  ||j                  d      5 }|j                  |       d d d        t        ||      j                         }t        j                  ||       y # 1 sw Y   ;xY w)Nr	   Fr   r   r   r\   r   r   s           r1   %test_parquet_writer_filesystem_s3_urir     s     
	BHH  E :E!MBT			#u||U	;v5! 
< "-779F&"%	 
<	;s   B!!B*c                 d   t        d      }t        j                  j                  |d      }| \  }}|dz   }t	        j
                  ||j                  |d      5 }|j                  |       d d d        t        ||      j                         }t        j                  ||       y # 1 sw Y   ;xY w)Nr	   Fr   z/test.parquetr   r^   r   r   )s3_example_s3fsr(   rb   r   	directoryrI   r+   r/   s           r1   #test_parquet_writer_filesystem_s3fsr   +  s     
	BHH  E :E#MB	&D			ellr5
	5!

 "-779F&"%
 
s   B&&B/c                  L   t        d      } t        j                  j                  | d      }t	        j
                         }t        j                  t        d      5  t        j                  t        j                         |j                  |       d d d        y # 1 sw Y   y xY w)Nr	   Fr   zspecified path is file-like)matchr   )r   r   r   r   r   LocalFileSystemrB   rC   rD   r   r   r   r   )r(   rb   r\   s      r1   ,test_parquet_writer_filesystem_buffer_raisesr   =  ss    		BHH  E :E##%J 
z)F	G
!!#U\\j	
 
H	G	Gs   5BB#c                    t        j                  dg di      }| dz  }t        j                  ||j                        5 }|j                  |       d d d        t        j                  |      }d|j                  v sJ |j                  d   sJ | dz  }t        j                  ||j                  d      5 }|j                  |       d d d        t        j                  |      }|j                  J y # 1 sw Y   xY w# 1 sw Y   9xY w)Na)r7   r|   r}   ztest_with_schema.parquets   ARROW:schemaztest_without_schema.parquetF)store_schema)r   rb   r   r   r   r   rl   rt   )r'   rb   path1r+   metapath2s         r1    test_parquet_writer_store_schemar   J  s    HHc9%&E 00E			%	.&5! 
/ E"Ddmm+++==))) 33E			%E	Bf5! 
C E"D==    
/	. 
C	Bs   C46D 4C= D	c                    t         j                  j                  t        j                  g d      gdg      }| dz  }t	        j
                  ||j                        5 }|j                  |       |j                  ddd       |j                  dd	d
       d d d        t	        j                  |      }|j                  j                  }|d   dk(  sJ |d   dk(  sJ |d   dk(  sJ y # 1 sw Y   SxY w)NrP   )typef0zmetadata.parquet1rM   )key1key223)r   key3s   key1   1s   key2   2s   key3   3)r   r   rA   r@   r   r   r   r   add_key_value_metadataParquetFilert   )r'   rb   rI   r+   readerrt   s         r1   -test_parquet_writer_append_key_value_metadatar   _  s    HH  "((2G"<!=vFE''D			$	-5!%%sC&@A%%sC&@A 
. ^^D!F''HG$$$G$$$G$$$ 
.	-s   :C))C2)%rB   pyarrowr   r   pyarrow.parquetparquetr   pyarrow.tests.parquet.commonr   r   r   ImportErrorpandasr!   pandas.testingtestingr#   mark
pytestmarkr2   rK   rR   rT   r[   parametrizer   rf   large_memoryr   r   s3r   r   r   r   r   r   r~   r3   r1   <module>r      sP  $    ? ? [[  
 8 84(,6 8 80 8 8> B( %&	 
%&P &6  &6R B( &	 
& &  & 
&  
& &  &  	
 	
!*%I
  	B  NBs"   H* 
H8 *H54H58	II