
    g                        d dl Z d dlmZ d dlmZ d dlZd dlmZ d dlZd dl	m
Z
 d dlZd dlZd dlZd dlZ	 d dlZd dlZd dlmZmZmZ i ai ai ad Zd Zd	 Zd
 Zd Zd Z d Z!d Z"d Z#d Z$d Z%d Z&d Z'd Z(d Z)d Z*d Z+d+dZ,	 	 d,dZ-d Z.d-dZ/d Z0	 d.dZ1h dZ2d+dZ3d  Z4d! Z5d+d"Z6	 d+d#Z7d$ Z8d% Z9d& Z:d' Z;d( Z<d) Z=d* Z>y# e$ r dZY w xY w)/    N)Sequence)futures)deepcopy)zip_longest)_pandas_api	frombytesis_threading_enabledc            	      |   t         s0t         j                  i t        j                  j                  dt        j                  j
                  dt        j                  j                  dt        j                  j                  dt        j                  j                  dt        j                  j                  dt        j                  j                  dt        j                  j                  dt        j                  j                  d	t        j                  j                  d
t        j                  j                  dt        j                  j                  dt        j                  j                   dt        j                  j"                  dt        j                  j$                  dt        j                  j&                  dt        j                  j(                  dt        j                  j*                  dt        j                  j,                  dt        j                  j.                  di       t         S )Nemptyboolint8int16int32int64uint8uint16uint32uint64float16float32float64datetimebytesunicode)_logical_type_mapupdatepalibType_NA	Type_BOOL	Type_INT8
Type_INT16
Type_INT32
Type_INT64
Type_UINT8Type_UINT16Type_UINT32Type_UINT64Type_HALF_FLOAT
Type_FLOATType_DOUBLEType_DATE32Type_DATE64Type_TIME32Type_TIME64Type_BINARYType_FIXED_SIZE_BINARYType_STRING     J/var/www/openai/venv/lib/python3.12/site-packages/pyarrow/pandas_compat.pyget_logical_type_mapr7   .   s       "
FFNNG"
FFf"
 FFf"
 FFw	"

 FFw"
 FFw"
 FFw"
 FF"
 FF"
 FF"
 FF""I"
 FFy"
 FF	"
 FF"
 FF"
  FF!"
" FF#"
$ FFFF))7FF	)"
 	, r5   c                    t               }	 || j                     S # t        $ r t        | t        j
                  j                        rY yt        | t        j
                  j                        r&dj                  t        | j                              cY S t        | t        j
                  j                        r| j                  dcY S dcY S t        | t        j
                  j                        rY yY yw xY w)Ncategoricalzlist[{}]
datetimetzdatetimedecimalobject)r7   idKeyError
isinstancer   r   DictionaryTypeListTypeformatget_logical_type
value_typeTimestampTypetzDecimal128Type)
arrow_typelogical_type_maps     r6   rD   rD   K   s    +-
.. 	j"&&"7"78 
BFFOO4$$%5j6K6K%LMM
BFF$8$89#-==#<<L*L
BFF$9$9:	s(    -C5
AC52C5C5$C54C5c                     t         st         j                  t        j                  dt        j                  dt        j
                  dt        j                  dt        j                  dt        j                  dt        j                  dt        j                  dt        j                  d	t        j                  d
t        j                  dddt        j                  dt        j                  di       t         S )Nr   r   r   r   r   r   r   r   r   r   r   datetime64[D]r   stringr   )_numpy_logical_type_mapr   npbool_r   r   r   r   r   r   r   r   r   r   str_bytes_r4   r5   r6   get_numpy_logical_type_maprS   \   s    "&&HHfGGVHHgHHgHHgHHgIIxIIxIIxJJ	JJ	VGGXIIw(
 	  #"r5   c                 J   t               }	 || j                  j                     S # t        $ rt t	        | j                  d      rY yt        | j                        j                  d      rt        | j                        cY S t        j                  |       }|dk(  rY y|cY S w xY w)NrG   r:   
datetime64rM   r   )	rS   dtypetyper?   hasattrstr
startswithr   infer_dtype)pandas_collectionnumpy_logical_type_mapresults      r6   get_logical_type_from_numpyr_   r   s    79%&7&=&=&B&BCC 
$**D1  &&'22<@(..//(():;X
s!   % B"9B"B"B"!B"c                 ~   | j                   }t        |      dk(  rVt        | d|       }|J t        |j                        |j
                  d}t        |j                  j                         }||fS t        |d      r1dt        j                  j                  |j                        i}d}||fS d }t        |      }||fS )Ncategorycat)num_categoriesorderedrG   timezonedatetime64[ns])rV   rY   getattrlen
categoriesrd   codesrX   r   r   tzinfo_to_stringrG   )columnrV   catsmetadataphysical_dtypes        r6   get_extension_dtype_inforp      s    LLE
5zZvuf-!$//2||
 TZZ--. 8## 
	 7 7 AB) 8## U8##r5   c                 b   t        |      }t        |       \  }}|dk(  r|j                  |j                  d}d}|>t	        |t
              s.t        dj                  |t        |      j                              |)t	        |t
              sJ t        t        |                   ||dn||||dS )a  Construct the metadata for a given column

    Parameters
    ----------
    column : pandas.Series or pandas.Index
    name : str
    arrow_type : pyarrow.DataType
    field_name : str
        Equivalent to `name` when `column` is a `Series`, otherwise if `column`
        is a pandas Index then `field_name` will not be the same as `name`.
        This is the name of the field in the arrow Table's schema.

    Returns
    -------
    dict
    r<   )	precisionscaler=   z6Column name must be a string. Got column {} of type {}Nonename
field_namepandas_type
numpy_typern   )
rD   rp   rr   rs   r@   rY   	TypeErrorrC   rW   __name__)rl   rv   rI   rw   logical_typestring_dtypeextra_metadatas          r6   get_column_metadatar      s    " $J/L#;F#C L.y #--%%
  
4 5DKKd4j))
 	
 J!< D<  * 2f
#"" r5   c           	      T   t        |D cg c]  }t        |t              s| c}      }t        |      }	|d|	|z
   }
||	|z
  d }g }t        | ||
      D ]&  \  }}}t	        ||||      }|j                  |       ( g }|durMg }t        |||      D ]  \  }}}t        |t              r|j                  5t        |j                  t              s|j                  |j                         t	        |t        |j                        ||      }|j                  |        t        |      dkD  r t        j                  d| dt        d       g }t        |j                  d	|j                  g      }t        |j                  d
|j                  j                  g      }t        ||      D ]"  \  }}t        ||      }|j                  |       $ ng x}x}}dt        j                   ||||z   dt"        j$                  dt&        j(                  d      j+                  d      iS c c}w )a  Returns a dictionary containing enough metadata to reconstruct a pandas
    DataFrame as an Arrow Table, including index columns.

    Parameters
    ----------
    columns_to_convert : list[pd.Series]
    df : pandas.DataFrame
    index_levels : List[pd.Index]
    index_descriptors : List[Dict]
    preserve_index : bool
    types : List[pyarrow.DataType]

    Returns
    -------
    dict
    N)rv   rI   rw   Fr   z&The DataFrame has non-str index name `z@` which will be converted to string and not roundtrip correctly.   
stacklevellevelsnamess   pandaspyarrow)libraryversion)index_columnscolumn_indexescolumnscreatorpandas_versionutf8)rh   r@   dictzipr   appendrv   rY   _column_name_to_stringswarningswarnUserWarningrg   r   _get_simple_index_descriptorjsondumpsr   __version__r   r   encode)columns_to_convertdfcolumn_namesindex_levelsindex_descriptorspreserve_indextypesdescrnum_serialized_index_levelsntypesdf_typesindex_typescolumn_metadatacolsanitized_namerI   rn   index_column_metadatanon_str_index_nameslevel
descriptorr   r   r   rv   s                            r6   construct_metadatar      s_   $ #&:K 'G:K.8.E (-:K 'G #H ZF:f::;H!<<=>KO+./A/;X,G'^Z&s2<2@B 	x(,G U" -0{1B.D)E:z*d+ zz%jS.I#**5::6*,UZZ8%%	H "((2!.D$ "#a'MM89L8M N0 0 	+ X

|<

Gbjjoo->?vu-KE43E4@H!!(+ . FHGG1N 	4::.,&)>>$>> *11	
 	 6&> m'Gs   H%c                     t        |       \  }}t        |       }d|v rt        j                  dt        d       |dk(  r|rJ ddi}|||||dS )	NmixedzlThe DataFrame has column names of mixed type. They will be converted to strings and not roundtrip correctly.r   r   r   encodingUTF-8ru   )rp   r_   r   r   r   )r   rv   r}   r~   rx   s        r6   r   r     sp    #;E#B L.-e4K+@A	' i!!!$g.""" r5   c                    t        | t              r| S t        | t              r| j                  d      S t        | t              r"t        t	        t        t        |                   S t        | t              rt        d      | yt        |       S )a!  Convert a column name (or level) to either a string or a recursive
    collection of strings.

    Parameters
    ----------
    name : str or tuple

    Returns
    -------
    value : str or tuple

    Examples
    --------
    >>> name = 'foo'
    >>> _column_name_to_strings(name)
    'foo'
    >>> name = ('foo', 'bar')
    >>> _column_name_to_strings(name)
    "('foo', 'bar')"
    >>> import pandas as pd
    >>> name = (1, pd.Timestamp('2017-02-01 00:00:00'))
    >>> _column_name_to_strings(name)
    "('1', '2017-02-01 00:00:00')"
    r   z%Unsupported type for MultiIndex levelN)	r@   rY   r   decodetuplemapr   r   rz   rv   s    r6   r   r   -  sv    2 $	D%	 {{6""	D%	 54d;<==	D(	#?@@	t9r5   c                     | j                   #| j                   |vrt        | j                         S dj                  |      S )zReturn the name of an index level or a default name if `index.name` is
    None or is already a column name.

    Parameters
    ----------
    index : pandas.Index
    i : int

    Returns
    -------
    name : str
    z__index_level_{:d}__)rv   r   rC   )indexir   s      r6   _index_level_namer   T  s9     zz%**L"@&uzz22%,,Q//r5   c                 H   t        | ||      }| j                  j                  s-t        dj	                  t        | j                                    |t        | ||      S g }|durt        | j                        ng }g }g }|D ]t  }| |   }	t        |      }t        j                  |	      rt        dj	                  |            |j                  |	       |j                  d        |j                  |       v g }
g }t        |      D ]  \  }}t        |||      }t!        |t        j"                  j$                        r|t'        |      }n5|j                  |       |j                  d        |}|j                  |       |
j                  |        ||z   }||||
|||fS )Nz Duplicate column names found: {}F-Sparse pandas data (column {}) not supported.)_resolve_columns_of_interestr   	is_unique
ValueErrorrC   list$_get_columns_to_convert_given_schema_get_index_level_valuesr   r   r   	is_sparserz   r   	enumerater   r@   pd
RangeIndex_get_range_index_descriptor)r   schemar   r   r   r   r   convert_fieldsrv   r   r   index_column_namesr   index_levelr   	all_namess                   r6   _get_columns_to_convertr   g  s   *2vw?G::.55d2::6FG
 	
 3BOOL .<5-H) 
 Nh&t,  %?FFtLN N 	!!#&d#D!  #L1; a>{KNN$=$=>&/<E%%k2!!$'E%%d+  ' 2 11I |%79J,n> >r5   c                 &   g }g }g }g }g }g }|j                   D ]  }		 | |	   }
d}t        |	      }	t        j                  |
      rt        dj	                  |	            |j                  |	      }|j                  |
       |j                  |       |j                  |	       |s|j                  |	       |j                  |	       |j                  |
        ||z   }|||||||fS # t        $ r 	 t        | |	      }
n-# t        t        f$ r t        dj	                  |	            w xY w|du rt        dj	                  |	            |>t        |
t        j                  j                        rt        dj	                  |	            d}Y kw xY w)z
    Specialized version of _get_columns_to_convert in case a Schema is
    specified.
    In that case, the Schema is used as the single point of truth for the
    table structure (types, which columns are included, order of columns, ...).
    FzNname '{}' present in the specified schema is not found in the columns or indexzlname '{}' present in the specified schema corresponds to the index, but 'preserve_index=False' was specifieda  name '{}' is present in the schema, but it is a RangeIndex which will not be converted as a column in the Table, but saved as metadata-only not in columns. Specify 'preserve_index=True' to force it being added as a column, or remove it from the specified schemaTr   )r   r?   _get_index_level
IndexErrorrC   r   r@   r   r   r   r   r   rz   fieldr   )r   r   r   r   r   r   r   r   r   rv   r   is_indexr   r   s                 r6   r   r     s    LNL	T(CH2 't,  %?FFtLN N T"!!#&e$D!%%d+$$T*$U X 11I|%79J,n> >U  	<&r40j) <..4fTl< <<
 &   &t. . !(sKNN$=$=> '
 (.vd|5 5 H-	s*   C((	F2C?>F?*D))A#FFc                     |}|| j                   j                  vr"t        |      rt        |t	        d      d       }| j                   j                  |      S )z_
    Get the index level of a DataFrame given 'name' (column name in an arrow
    Schema).
    __index_level_)r   r   _is_generated_index_nameintrh   get_level_values)r   rv   keys      r6   r   r     sO    
 C288>>!&>t&D $s+,R0188$$S))r5   c                 f    	 t        j                  |        | S # t        $ r t        |       cY S w xY wN)r   r   rz   rY   r   s    r6   _level_namer     s1    

4 4ys    00c                     dt        | j                        t        j                  | d      t        j                  | d      t        j                  | d      dS )Nrangestartstopstep)kindrv   r   r   r   )r   rv   r   get_rangeindex_attribute)r   s    r6   r   r     sM     EJJ'55eWE44UFC44UFC r5   c                     t        t        | d| g            }t        |      D cg c]  }| j                  |       c}S c c}w )Nr   )rh   rg   r   r   )r   nr   s      r6   r   r     s>    GE8eW-.A/4Qx8x!E""1%x888s   A c                     ||t        d      ||j                  }|S | |D cg c]  }|| j                  v s| }}|S | j                  }|S c c}w )NzJSchema and columns arguments are mutually exclusive, pass only one of them)r   r   r   )r   r   r   cs       r6   r   r     sw    g1 < = 	=		,, N 
	%9gbjj1g9 N **N	 :s
   AAc           	         t        | d ||      \  }}}}}}}g }	|D ]%  }
|
j                  }t        j                  |      r"t	        j
                  |
d      j                  }nt        j                  |      r\t        |
t        j                  j                        r|
j                  d      n|
d d }t	        j
                  |d      j                  }n]t        ||
j                  d       \  }}t        j                  j                  ||      }|!t	        j
                  |
d      j                  }|	j!                  |       ( t#        || |||||	      }||	|fS )NT)from_pandasr   )r   valuesr   is_categoricalr   arrayrW   is_extension_array_dtyper@   r   Seriesheadget_datetimetz_typerV   r   _ndarray_to_arrow_typer   r   )r   r   r   r   r   _r   r   r   r   r   r   type_r   rn   s                  r6   dataframe_to_typesr     s8    ""dNG
DYE%%f-HHQD166E11&9!+;>>(("*AFF1I/0!u HHU5::E/FMFEFF11&%@E}5::U   "Bm>5H
 eX%%r5   c           	      N   t        | |||      \  }}}}	}
}}|Dt        |       t        | j                        }}||dz  kD  r|dkD  rt        j                         }nd}t               sd}fd}d }|dk(  r&t        ||      D cg c]  \  }} |||       }}}ng }t        j                  |      5 }t        ||      D ]R  \  }} ||j                        r|j                   |||             1|j                  |j                  |||             T 	 d d d        t        |      D ]3  \  }}t        |t        j                        s!|j                         ||<   5 |D cg c]  }|j                    }}|Vg }t        ||      D ]0  \  }}||nd}|j                  t        j"                  ||             2 t        j$                  |      }t'        || ||
|	||      }|j(                  rt+        |j(                        n	t-               }|j/                  |       |j1                  |      }d }t        |      dk(  r<	 |	d   d   }|dk(  r.|	d   d	   } |	d   d
   }!|	d   d   }"t        t3        | |!|"            }|||fS c c}}w # 1 sw Y   qxY wc c}w # t4        $ r Y (w xY w)Nd      c                    |d}d }n|j                   }|j                  }	 t        j                  | |d      }|s=|j                  dkD  r.t        dj                  t        |      |j                              |S # t        j                  t        j
                  t        j                  f$ rA}|xj                  dj                  | j                  | j                        fz  c_        |d }~ww xY w)NT)rW   r   safez0Conversion failed for column {!s} with type {!s}r   z>Field {} was non-nullable but pandas column had {} null values)nullablerW   r   r   ArrowInvalidArrowNotImplementedErrorArrowTypeErrorargsrC   rv   rV   
null_countr   rY   )r   r   field_nullabler   r^   er   s         r6   convert_columnz+dataframe_to_arrays.<locals>.convert_columnU  s    =!NE"^^NJJE	XXc4dKF &"3"3a"7 228&U9?9J9J3LM M  ++!!# 	 FFIsxx36 6FG	s   A< <2C/.<C**C/c                     t        | t        j                        xrF | j                  j                  xr. t        | j                  j                  t        j                        S r   )	r@   rO   ndarrayflags
contiguous
issubclassrV   rW   integer)arrs    r6   _can_definitely_zero_copyz6dataframe_to_arrays.<locals>._can_definitely_zero_copyk  sB    3

+ 7		$$7399>>2::6	8r5   rt   r   r   r   r   r   r   )r   rh   r   r   	cpu_countr	   r   r   ThreadPoolExecutorr   r   submitr   r@   Futurer^   rW   r   r   r   rn   r   r   r   with_metadatar   r   )#r   r   r   nthreadsr   r   r   r   r   r   r   r   r   nrowsncolsr  r  r   farraysexecutorr   	maybe_futxr   fieldsrv   r   pandas_metadatarn   n_rowsr   r   r   r   s#        `                             r6   dataframe_to_arraysr  =  s    /r6>/68Y 2wBJJu53;519||~HH!,8
 1}!"4nEGEda !A&E 	 G ''1X.?1,QXX6MM.A"67MM(//.!Q"GH	 @ 2 &f-LAy)W^^4%,,.q	 . $$VQVVVE$~y%0KD%+4DMM"((4/0 1 6"(Bm>5O -3OOx(HOOO$!!(+F F
6{a	$Q'/Dw)!,W5(+F3(+F3U5$56 66!!]G 21 %8  		s+   J 4A"J$J ;J J	J$#J$c                 4   | j                   j                  t        j                  k7  r| |fS t	        j
                  |      r4|2|j                  }|j                  }t        j                  ||      }| |fS |t        j                  | j                         }| |fS r   )rV   rW   rO   rU   r   is_datetimetzrG   unitr   	timestampfrom_numpy_dtype)r   rV   r   rG   r  s        r6   r   r     s    ||BMM)u}  'EMXXzzT2&
 5=	 
##FLL15=r5   c                    ddl mc m} | j                  dd      }| d   }d| v r)t        j
                  j                  || d   | d         }nd| v rt        j                  |j                        \  }}	t        || d         }
t	        j                         r2t        j                  j                  |j                  d	      |
d
      }nv|}|rr|j                  |||j                   |
      }|S d| v rK| d   }t#        |      dk(  sJ ||d      }||   }t%        |d      st'        d      |j)                  |      }n|}|r|j                  ||      S ||fS )a  
    Construct a pandas Block from the `item` dictionary coming from pyarrow's
    serialization or returned by arrow::python::ConvertTableToPandas.

    This function takes care of converting dictionary types to pandas
    categorical, Timestamp-with-timezones to the proper pandas Block, and
    conversion to pandas ExtensionBlock

    Parameters
    ----------
    item : dict
        For basic types, this is a dictionary in the form of
        {'block': np.ndarray of values, 'placement': pandas block placement}.
        Additional keys are present for other types (dictionary, timezone,
        object).
    columns :
        Column names of the table being constructed, used for extension types
    extension_columns : dict
        Dictionary of {column_name: pandas_dtype} that includes all columns
        and corresponding dtypes that will be converted to a pandas
        ExtensionBlock.

    Returns
    -------
    pandas Block

    r   Nblock	placement
dictionaryrd   )ri   rd   re   r   F)rV   copy)r$  klassrV   py_arrayr   __from_arrow__zGThis column does not support to be converted to a pandas ExtensionArray)r$  )pandas.core.internalscore	internalsgetr   categorical_type
from_codesrO   datetime_datarV   make_datetimetz	is_ge_v21r   r   view
make_blockDatetimeTZBlockrh   rX   r   r)  )itemr   extension_columnsreturn_block_int	block_arrr$  r
  r  r   rV   r#  rv   pandas_dtypes                 r6   _reconstruct_blockr<    s   8 )($'I[!It**55$|"4O 6 % 
t	""9??3ad:&67  "..&&w'u5 ' C C	Y.2.B.B.3 ( 5 	t	:9~"""y|$(.|%56 : ; ;))#.si88I~r5   c                     t        j                         rd} t        j                  j	                  |      }t        j
                  | |      S )NnsrG   )r   is_v1r   r   string_to_tzinfodatetimetz_type)r  rG   s     r6   r1  r1    s:    		 	 	$B&&t33r5   c           	      j   g }g }|j                   j                  }|sI|G|d   }|j                  dg       }|d   }t        ||      }t	        ||||      \  }}	t        |||      }
n6t        j                  j                  |j                        }	t        |g |      }
t        |       t        |||      }|j                  }t        j                  j                  | ||t!        |
j#                                     }t        j$                         r/ddlm} |D cg c]  }t+        |||
d       }} |||	|      }|S dd	lm} dd
lm} |D cg c]  }t+        |||
       }}||	g} |||      }t        j4                         r|j7                  ||j8                        }|S  ||      }|S c c}w c c}w )Nr   r   r   r   )create_dataframe_from_blocksF)r8  )r   r   )BlockManager)	DataFrame)r   r  r-  _add_any_metadata_reconstruct_index_get_extension_dtypesr   r   r   num_rows'_check_data_column_metadata_consistency_deserialize_column_indexr   r   r   table_to_blocksr   keysis_ge_v3pandas.api.internalsrD  r<  r*  rE  pandasrF  r2  	_from_mgraxes)optionstableri   ignore_metadatatypes_mapperall_columnsr   r  r   r   ext_columns_dtypesr   r   r^   rD  r6  blocksr   rE  rF  rS  mgrs                         r6   table_to_dataframer\    s    KNll22O:%i0(,,-=rB+O<!%9)%1B*5|Eu2;. ))%..925"lK+K8'{NKG%%LVV##GUJ$();)@)@)B$CEFE
 
  l$6UL 	 

 *&wO	6$ 
 t\3EF 	 
 64(  "$$S#((3B 	 3B	+

s   F+F0>   r   r   r   r   r   r   r=   r   r   r   r   r   r   c                 P   i }t         j                  |S |D ]X  }	 |d   }|d   }|t        vst        j                  |      }t        |t         j                        sGt        |d      sT|||<   Z | j                  D ]I  }|j                  }	t        |	t        j                        s*	 |	j                         }|||j                  <   K |r7| j                  D ](  }|j                  }	 ||	      }||||j                  <   * |S # t        $ r |d   }Y w xY w# t        $ r Y w xY w)a  
    Based on the stored column pandas metadata and the extension types
    in the arrow schema, infer which columns should be converted to a
    pandas extension dtype.

    The 'numpy_type' field in the column metadata stores the string
    representation of the original pandas dtype (and, despite its name,
    not the 'pandas_type' field).
    Based on this string representation, a pandas/numpy dtype is constructed
    and then we can check if this dtype supports conversion from arrow.

    rw   rv   ry   r)  )r   extension_dtyper?   _pandas_supported_numpy_typesr;  r@   rX   r   rW   r   BaseExtensionTypeto_pandas_dtyperv   NotImplementedError)
rU  columns_metadatarW  ext_columnscol_metarv   rV   r;  r   typs
             r6   rI  rI  @  s@    K ""* %	$L)D &55 '33E:L,(C(CD<)9:(4K% %  jjc2//07"224 +7EJJ'  \\E**C',L'*6EJJ'	 " ?  	$F#D	$$ ' s#   D)DDD	D%$D%c                 ,    t        d | D              sJ y )Nc              3   H   K   | ]  }|d    du xr d|v xs |d    du  yw)rv   Nrw   r4   ).0r   s     r6   	<genexpr>z:_check_data_column_metadata_consistency.<locals>.<genexpr>~  s<      A 
6d		0|q0JQvYd5JJs    ")all)rX  s    r6   rK  rK  y  s#    
     r5   c           
      T   |rY|D ci c]$  }|j                  dt        |d               |d   & }}| j                  D cg c]  }|j                  ||       }}n| j                  }t        |      dkD  r^t        j
                  j                  j                  t        t        t        j                  |            |D cg c]  }|d   	 c}      }n+t        j
                  j                  ||r|d   d   nd       }t        |      dkD  rt        ||      }|S c c}w c c}w c c}w )Nrw   rv   r   r   r   r   )r-  r   r   rh   r   r   
MultiIndexfrom_tuplesr   r   astliteral_evalIndex"_reconstruct_columns_from_metadata)	block_tablerX  r   r   columns_name_dictrv   columns_values	col_indexr   s	            r6   rL  rL    sA    !
  EE, 7&	 BCQvYN  	 

 ;F:R:R
:R$!!$-:R 	 
 %11 >Q ..++77S%%~676DEn9V$nE 8 

 ..&&n!26!:RV ' 

 >Q4WnMN9

 Fs   )D D ?D%
c                    |D ci c]  }|j                  d|d         | }}g }g }| }|D ]  }	t        |	t              rt        | ||	||      \  }}
}|
o)|	d   dk(  rI|	d   }t        j
                  j                  |	d   |	d   |	d   |      }
t        |
      t        |       k7  rzt        d	j                  |	d               |j                  |
       |j                  |        t        j
                  }t        |      d
kD  r!|j                  j                  ||      }||fS t        |      d
k(  r5|d   }t        ||j                        s|j                  ||d         }||fS |j                  | j                        }||fS c c}w )Nrw   rv   r   r   r   r   r   )r   rv   zUnrecognized index kind: {}r   rm  r   r   )r-  r@   rY   _extract_index_levelr   r   r   rh   r   rC   r   rn  from_arraysrr  rJ  )rU  r   rX  rW  r   field_name_to_metadataindex_arraysindex_namesresult_tabler   r   
index_namer   r   s                 r6   rH  rH    s    A 	
lAfI&)   LKL"eS!4H|U,BL5R1L+z"6]g%vJ%..33E'N49&M9>v9C 4 EK ;3u:-:$fU6]35 5K(:&) #, 
B <1)),k)J  
\	a	Q%*HHUQH8E  enn-[s   Fc                 &   ||   d   }t        ||      }| j                  j                  |      }|dk(  r|d d fS | j                  |      }|j	                  |      }	d |	_        |j                  |j                  j                  |            }||	|fS )Nrv   )rW  ) _backwards_compatible_index_namer   get_field_indexrl   	to_pandasrv   remove_column)
rU  r~  rw   r{  rW  logical_namer  r   r   r   s
             r6   ry  ry    s    )*5f=L1*lKJ$$Z0ABwT4''
,,q/C--\-:KK--++J7L j00r5   c                 (    | |k(  rt        |       ry|S )a1  Compute the name of an index column that is compatible with older
    versions of :mod:`pyarrow`.

    Parameters
    ----------
    raw_name : str
    logical_name : str

    Returns
    -------
    result : str

    Notes
    -----
    * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager`
    N)r   )raw_namer  s     r6   r  r    s    $ <$<X$Fr5   c                 6    d}t        j                  ||       d uS )Nz^__index_level_\d+__$)rematch)rv   patterns     r6   r   r     s    &G88GT"$..r5   c                     t         st         j                  dddt        j                  t        j                  t        j                  t        j
                  t        j                  t        j                  t        j                  d
       t         S )NrL   rf   )
r   r;   r:   r   r   rM   r	  floatingr<   r   )_pandas_logical_type_mapr   rO   rQ   rR   r   r   object_r4   r5   r6   get_pandas_logical_type_mapr    s\     $ ''#(*wwYYggxx

zzZZ)
 	 $#r5   c                     t               }	 ||    S # t        $ r. d| v rt        j                  cY S t        j                  |       cY S w xY w)a  Get the numpy dtype that corresponds to a pandas type.

    Parameters
    ----------
    pandas_type : str
        The result of a call to pandas.lib.infer_dtype.

    Returns
    -------
    dtype : np.dtype
        The dtype that corresponds to `pandas_type`.
    r   )r  r?   rO   r  rV   )rx   pandas_logical_type_maps     r6   _pandas_type_to_numpy_typer    sM     :;%&{33 %k!::xx$$	%s    AAAc                    t         j                  }t        | dd      xs | g}t        | dd      xs dg}t        ||i       D cg c]=  \  }}||j	                  dt        |j                              |j	                  dd      f? }}}g }t        j                  dd      }	|D ]]  \  }}
}t        |
      }|t        j                  k(  r|j                  |	      }|
d	k(  rt        j                  j                  |d
   d   d         }|j!                  |d      j#                  |      }t        j$                         r|j'                  t        j(                  |      d
         }nf|
dk(  rAt         j                  j+                  |D cg c]  }t-        j.                  |       c}      }n |j                  |k7  r|j1                  |      }|j                  |k7  r|
d	k7  r|j1                  |      }|j3                  |       ` t5        |      dkD  r|j7                  ||| j8                        S |j+                  |d
   |d
   j                  | j:                        S c c}}w c c}w )a_  Construct a pandas MultiIndex from `columns` and column index metadata
    in `column_indexes`.

    Parameters
    ----------
    columns : List[pd.Index]
        The columns coming from a pyarrow.Table
    column_indexes : List[Dict[str, str]]
        The column index metadata deserialized from the JSON schema metadata
        in a :class:`~pyarrow.Table`.

    Returns
    -------
    result : MultiIndex
        The index reconstructed using `column_indexes` metadata with levels of
        the correct type.

    Notes
    -----
    * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager`
    r   Nrj   )	fillvaluerx   ry   r   r   r:   r   rn   re   T)utcr<   r   rm  )rV   rv   )r   r   rg   r   r-  rY   rV   operatormethodcallerr  rO   rR   r   r   r   rA  to_datetime
tz_convertrO  as_unitr0  rr  r<   Decimalastyper   rh   rn  r   rv   )r   r   r   r   labelsr   rw  levels_dtypes
new_levelsencoderr;  numpy_dtyperV   rG   r   s                  r6   rs  rs  5  s7   , 
B Wh-:'FWgt,6F !,Nb!
!
E9 
	mS-=>	|T	*	,!
   J##Hg6G,9(|[*<8 BIIIIg&E<'((q!*-j9;BNN5dN3>>rBE##% b&6&6u&=a&@AY&NN((e)Le'//!*<e)LME[[E!LL'E;;+%,,*FLL-E% 3 -:6 :}}Zw}}}EExx
1Z]-@-@w||xTTS: *Ms   AI 	I&
c                    i }i }| j                   }|d   }|D cg c]  }t        |t              r| }}t        |      }t        |d         |z
  }t	        |d         D ],  \  }	}
|
j                  d      }|s|
d   }|	|k\  r||	|z
     }|d}|j                  |      }|dk7  sG|
d   dk(  sP| |   }t        |j                  t        j                  j                        s|
d	   }|s|j                  d
      }|s||j                  j                  k7  s|j                         }t        j                  d|      }t        j                  j                  ||      }t        j                   ||   j"                  |      ||<   |||<   / t        |      dkD  rg }g }t%        t        | j                               D ]a  }	|	|v r)|j'                  ||	          |j'                  ||	          0|j'                  | |	          |j'                  | j                   |	          c t        j(                  j+                  |t        j                   |            S | S c c}w )Nr   r   rw   rv   rt   r  rx   r:   rn   re   r>  r?  )rW   r   )r   )r   r@   rY   rh   r   r-  r  rW   r   r   rF   rG   r  r   Arrayr   r   rv   r   r   Tablerz  )rU  r  modified_columnsmodified_fieldsr   r   idx_coln_index_levels	n_columnsr   re  r  idxr   rn   metadata_tz	convertedtz_aware_typer  r   r  s                        r6   rG  rG  ~  sM   O\\F#O4M,9 2M"7C0 MM 2'NOI./.@I !!;<8<<-'HI~(Y7!$$X."9&,6Cj!#((BFF,@,@A#J/&ll:6;#((++#= #I$&LL+$FM$&HH$8$8>K %9 %MM ,.88F3K4D4D4A,COC(,9$S)= =@ q s5<<()A$$/23oa01uQx(ell1o. * xx##GBIIf4E#FFe2s   Ic                     t         j                  j                  |      }| j                  j	                  d      j                  j                  |      } | S )zB
    Make a datetime64 Series timezone-aware for the given tz
    r  )r   r   rA  dttz_localizer  )seriesrG   s     r6   make_tz_awarer    sA     
	 	 	$Bii##E*R

2 Mr5   r   )r   NT)NNT)NFN)?rp  collections.abcr   
concurrentr   concurrent.futures.threadr&  r   r<   	itertoolsr   r   r  r  r   numpyrO   ImportErrorr   r   pyarrow.libr   r   r	   r   rN   r  r7   rD   rS   r_   rp   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r   r<  r1  r\  r_  rI  rK  rL  rH  ry  r  r   r  r  rs  rG  r  r4   r5   r6   <module>r     sI  &  $  !   !   	   D D    :"#,"$&*ZSl($N0&=>@=>@
*9
&D IM!b"J&BJ4 JN2n! 6rB3n ?C1&0/
$&%.FUR:By$  	Bs   C CC