gw NddlZddlZddlZddlZddlmZddlm Z ddlm Z m Z  ddl m Z ejjZd'dZdZejj'dddgd Zejj'd ej*d d d fdZdZejj'dddgdZdZdZejj'dddgdZdZdZejj'dddgdZdZejj'dddgdZ ejjBdZ"ejjBdZ#ejj'dejHejJddejHejJdejJdejJddkDgd Z&d!Z'd"Z(d#Z)d$Z*d%Z+d&Z,y#e$rdZ YwxYw)(N)tobytes) ArrowInvalidArrowNotImplementedErrorcDddlm}|tj|S)Nr)_get_udf_context)pyarrow._computerpadefault_memory_pool) batch_lengthrs Q/var/www/openai/venv/lib/python3.12/site-packages/pyarrow/tests/test_substrait.pymock_udf_contextr &s1 B224l CCctjjt||}tj j ||j5}|j|ddd|S#1swY|SxYw)N)schema) ospathjoinstrr ipcRecordBatchFileWriterr write_table)tmpdir file_nametablerwriters r _write_dummy_data_to_diskr+s\ 77<<F Y /D  % %d5<< % @F5! A K A Ks A11A; use_threadsTFcd}d}tjgdgdg}t|||}t|j dt j |j}tjj|}tj||}|j} |jdg| jdgk(sJy)Nax { "version": { "major": 9999 }, "relations": [ {"rel": { "read": { "base_schema": { "struct": { "types": [ {"i64": {}} ] }, "names": [ "foo" ] }, "local_files": { "items": [ { "uri_file": "FILENAME_PLACEHOLDER", "arrow": {} } ] } } }} ] } zread_data.arrowfoonamesFILENAME_PLACEHOLDERrr rrrreplacepathlibPathas_uri _substrait_parse_json_plan substrait run_queryread_allselect rrsubstrait_queryrrrquerybufreaderres_tbs r test_run_serialized_queryr;2sO<"I HHo&ug 6E $VY >D O++ T 2 9 9 ;= >E -- ( ( /C  + >F __ F << FMM5'$: :: :rr7sbuffersbytesr c|t|tjtfsIdt |d}t j t|5tj|dddyd}t j t|5tj|dddy#1swYyxYw#1swYyxYw)Nz)Expected 'pyarrow.Buffer' or bytes, got ''matchz1ParseFromZeroCopyStream failed for substrait.Plan) isinstancer Bufferbytestypepytestraises TypeErrorr1r2OSError)rr7msgs r test_run_query_input_typesrI`s ebii/ 09$u+aH ]]9C 0    &1 >C wc *E" + * 1 + *s B&B2&B/2B;cd}tjjt|}d}t j t |5tj|dddy#1swYyxYw)Nz2 { "relations": [ ] } zPlan has no relationsr>) r r/r0rrDrErr1r2)r7r8 exec_messages r test_invalid_planrLpsP E -- ( ( 8C*L |< 8C  9 8 8s A''A0cd}d}tjgdgdg}t|||}t|j dt j |j}tjjt|}tj||}|j} |jdg| jdgk(sJy)Na { "version": { "major": 9999 }, "relations": [ {"rel": { "read": { "base_schema": { "struct": { "types": [ {"i64": {}} ] }, "names": [ "bar" ] }, "local_files": { "items": [ { "uri_file": "FILENAME_PLACEHOLDER", "arrow": {}, "metadata" : { "created_by" : {}, } } ] } } }} ] } zbinary_json_data.arrowrbarr&r(r)r*r5s r (test_binary_conversion_with_json_optionsrO}sOB)I HHo&ug 6E $VY >D O++ T 2 9 9 ;= >E -- ( ( 8C  + >F __ F << FMM5'$: :: :rcF|d|}|D]}|j|syy)N#TF)endswith)fnsext_filefn_namesuffixfns r has_functionrXs0z7) $F ;;v  rc|tjj}t|ddsJt|ddsJy)Nzfunctions_arithmetic.yamladdsum)r r/get_supported_functionsrX)supported_functionss r test_get_supported_functionsr^sK--??A +3U << < +3U << .table_providersP/0 0 1X X% %%  1X  56 6ra { "version": { "major": 9999 }, "relations": [ {"rel": { "read": { "base_schema": { "struct": { "types": [ {"i64": {}} ] }, "names": [ "x" ] }, "namedTable": { "names": ["t1"] } } }} ] } rmr) r Table from_pydictrfieldint64r/r0rr1r2r3) rrmr6r8r9r:rjrkrls @@@r test_named_tablerss88''i(89L88''i(89Lyy"((3 345H 7O2 -- ( ()A BC \\ # # N $EF __ F \ !! !rc<tjjdgdifd}d}tjj t |}d}t jt|5tj||dddy#1swYyxYw)Nr`racH|s td|ddk(rStdNrdrrergrhr'_rks r rmz;test_named_table_invalid_table_name..table_provider0/0 0 1X  56 6ra { "version": { "major": 9999 }, "relations": [ {"rel": { "read": { "base_schema": { "struct": { "types": [ {"i64": {}} ] }, "names": [ "x" ] }, "namedTable": { "names": ["t3"] } } }} ] } zInvalid NamedTable Sourcer>rm) r rorpr/r0rrDrErr1r2)rmr6r8rKrks @r #test_named_table_invalid_table_namer{sr88''i(89L7O2 -- ( ()A BC.L |< 8C? 9 8 8s 1BBcRtjjdgdifd}d}t|}tjj t|}d}t jt|5tj||dddy#1swYyxYw)Nr`racH|s td|ddk(rStdrvrhrws r rmz4test_named_table_empty_names..table_providerryra { "version": { "major": 9999 }, "relations": [ {"rel": { "read": { "base_schema": { "struct": { "types": [ {"i64": {}} ] }, "names": [ "x" ] }, "namedTable": { "names": [] } } }} ] } z!names for NamedTable not providedr>rz) r rorprr/r0rDrErr1r2)rmr6r7r8rKrks @r test_named_table_empty_namesr~s{88''i(89L7O0 O $E -- ( ( 8C6L |< 8C? 9 8 8s <BB&c ` tjjdgdi fd}d}tjj |}tj j |||}|j}|\}} jdd|td d} || k(sJy) Nr`racH|s td|ddk(rStdrvrhr'rx test_tables r rmz.test_udf_via_substrait..table_providerJ0/0 0 1X  56 6rs { "extensionUris": [ { "extensionUriAnchor": 1 }, { "extensionUriAnchor": 2, "uri": "urn:arrow:substrait_simple_extension_function" } ], "extensions": [ { "extensionFunction": { "extensionUriReference": 2, "functionAnchor": 1, "name": "y=x+1" } } ], "relations": [ { "root": { "input": { "project": { "common": { "emit": { "outputMapping": [ 1, 2, ] } }, "input": { "read": { "baseSchema": { "names": [ "t", ], "struct": { "types": [ { "i64": { "nullability": "NULLABILITY_REQUIRED" } }, ], "nullability": "NULLABILITY_REQUIRED" } }, "namedTable": { "names": [ "t1" ] } } }, "expressions": [ { "selection": { "directReference": { "structField": {} }, "rootReference": {} } }, { "scalarFunction": { "functionReference": 1, "outputType": { "i64": { "nullability": "NULLABILITY_NULLABLE" } }, "arguments": [ { "value": { "selection": { "directReference": { "structField": {} }, "rootReference": {} } } } ] } } ] } }, "names": [ "x", "y", ] } } ] } rnr y ) r rorpr/r0r1r2r3 add_columnr ) unary_func_fixturerrmr6r8r9r:functionname expected_tbrs @r test_udf_via_substraitrFs%%sI&67J7cOJ -- ( ( 9C \\ # # N $EF __ F'NHd''3jo1/0K [  rc|tjjdgdifd}d}tjj |}t j tj5}tjj||ddddtjvsJy#1swY#xYw)Nr`racH|s td|ddk(rStdrvrhrs r rmz=test_udf_via_substrait_wrong_udf_name..table_providerrrs { "extensionUris": [ { "extensionUriAnchor": 1 }, { "extensionUriAnchor": 2, "uri": "urn:arrow:substrait_simple_extension_function" } ], "extensions": [ { "extensionFunction": { "extensionUriReference": 2, "functionAnchor": 1, "name": "wrong_udf_name" } } ], "relations": [ { "root": { "input": { "project": { "common": { "emit": { "outputMapping": [ 1, 2, ] } }, "input": { "read": { "baseSchema": { "names": [ "t", ], "struct": { "types": [ { "i64": { "nullability": "NULLABILITY_REQUIRED" } }, ], "nullability": "NULLABILITY_REQUIRED" } }, "namedTable": { "names": [ "t1" ] } } }, "expressions": [ { "selection": { "directReference": { "structField": {} }, "rootReference": {} } }, { "scalarFunction": { "functionReference": 1, "outputType": { "i64": { "nullability": "NULLABILITY_NULLABLE" } }, "arguments": [ { "value": { "selection": { "directReference": { "structField": {} }, "rootReference": {} } } } ] } } ] } }, "names": [ "x", "y", ] } } ] } rzzNo function registered) r rorpr/r0rDrE ArrowKeyErrorr1r2rvalue)rmr6r8excinfors @r %test_udf_via_substrait_wrong_udf_namers%%sI&67J7cOJ -- ( ( 9C r'' (G s>B ) #s7=='9 99 9 ) (s ."B22B;cbtjjdgdifd}d}tjj t |}tj j|||}|j}tjjdgdi}||k(sJy)Nr`racSN)r'rin_tables r rmz/test_output_field_names..table_provider<sra { "version": { "major": 9999 }, "relations": [ { "root": { "input": { "read": { "base_schema": { "struct": { "types": [{"i64": {}}] }, "names": ["x"] }, "namedTable": { "names": ["t1"] } } }, "names": ["out"] } } ] } rnout) r rorpr/r0rr1r2r3)rrmr6r8r9r:expectedrs @r test_output_field_namesr8sxx##S)$45HO4 -- ( ()A BC \\ # # N $EF __ Fxx##UI$67H X  rcbtjjgdgdgddfd}d}tjj |}tj j ||d}|j}tjjd d gd d gd }||k(sJy)Nr r r!r!)r r!r"r#)?rrr)kv1v2cSrrrs r rmz7test_scalar_aggregate_udf_basic..table_providerk rs { "extensionUris": [ { "extensionUriAnchor": 1, "uri": "urn:arrow:substrait_simple_extension_function" }, ], "extensions": [ { "extensionFunction": { "extensionUriReference": 1, "functionAnchor": 1, "name": "sum_mean" } } ], "relations": [ { "root": { "input": { "extensionSingle": { "common": { "emit": { "outputMapping": [ 0, 1 ] } }, "input": { "read": { "baseSchema": { "names": [ "k", "v1", "v2", ], "struct": { "types": [ { "i64": { "nullability": "NULLABILITY_REQUIRED" } }, { "i64": { "nullability": "NULLABILITY_NULLABLE" } }, { "fp64": { "nullability": "NULLABILITY_NULLABLE" } } ], "nullability": "NULLABILITY_REQUIRED" } }, "namedTable": { "names": ["t1"] } } }, "detail": { "@type": "/arrow.substrait_ext.SegmentedAggregateRel", "segmentKeys": [ { "directReference": { "structField": {} }, "rootReference": {} } ], "measures": [ { "measure": { "functionReference": 1, "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", "outputType": { "fp64": { "nullability": "NULLABILITY_NULLABLE" } }, "arguments": [ { "value": { "selection": { "directReference": { "structField": { "field": 1 } }, "rootReference": {} } } }, { "value": { "selection": { "directReference": { "structField": { "field": 2 } }, "rootReference": {} } } } ] } } ] } } }, "names": [ "k", "v_avg" ] } } ], } Frnr r!g@g@)rv_avgr rorpr/r0r1r2r3varargs_agg_func_fixturermr6r8r9r:rrs @r test_scalar_aggregate_udf_basicrcs%%,# %J |Oz -- ( ( 9C \\ # # N$?F __ F((&&Vs(K [  rcntjjgdgdgdgddfd}d}tjj |}tj j ||d }|j}tjjgd gd gd d }||k(sJy)N)r r r r r!r!r!r!)r rrr rr rr )r r!r"r#r$rb)rrrrg@g@g@g@)trrrcSrrrs r rmz5test_hash_aggregate_udf_basic..table_providerrrs  { "extensionUris": [ { "extensionUriAnchor": 1, "uri": "urn:arrow:substrait_simple_extension_function" }, ], "extensions": [ { "extensionFunction": { "extensionUriReference": 1, "functionAnchor": 1, "name": "sum_mean" } } ], "relations": [ { "root": { "input": { "extensionSingle": { "common": { "emit": { "outputMapping": [ 0, 1, 2 ] } }, "input": { "read": { "baseSchema": { "names": [ "t", "k", "v1", "v2", ], "struct": { "types": [ { "i64": { "nullability": "NULLABILITY_REQUIRED" } }, { "i64": { "nullability": "NULLABILITY_REQUIRED" } }, { "i64": { "nullability": "NULLABILITY_NULLABLE" } }, { "fp64": { "nullability": "NULLABILITY_NULLABLE" } } ], "nullability": "NULLABILITY_REQUIRED" } }, "namedTable": { "names": ["t1"] } } }, "detail": { "@type": "/arrow.substrait_ext.SegmentedAggregateRel", "groupingKeys": [ { "directReference": { "structField": { "field": 1 } }, "rootReference": {} } ], "segmentKeys": [ { "directReference": { "structField": {} }, "rootReference": {} } ], "measures": [ { "measure": { "functionReference": 1, "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", "outputType": { "fp64": { "nullability": "NULLABILITY_NULLABLE" } }, "arguments": [ { "value": { "selection": { "directReference": { "structField": { "field": 2 } }, "rootReference": {} } } }, { "value": { "selection": { "directReference": { "structField": { "field": 3 } }, "rootReference": {} } } } ] } } ] } } }, "names": [ "t", "k", "v_avg" ] } } ], } Frnr)r rrr ) @rg"@g&@)rrrrrs @r test_hash_aggregate_udf_basicrs%%&&'7 9JNO^ -- ( ( 9C \\ # # N$?F __ F((&&  &(K [  rexprr`rr2ctjtjdtjtjdtjg}tjj |gdg|}tjj |}||jk(sJt|jdk(sJd|jvsJy)Nr`r test_exprr ) r rrqint32r1serialize_expressionsdeserialize_expressionslen expressions)rrr8returneds r test_serializing_expressionsrs YY bhhj! bhhj!F ,, , ,dVk]F KC||33C8H X__ $$ $ x## $ )) ) (.. .. .rc `tjddftjddftjddftjdftjdftj dfd}tj |jDcgc]\}\}}tj|| c}}}fd}|jD];\}\}}|tj|tj|| k(=ycc}}}w) Nsrmsns test_strings test_string) time_seconds time_millis time_nanos date_millis large_string large_binaryctjj|gdg}tjj|}|jk(sJyNr)r r1rrr)rr8rrs r check_round_tripz3test_arrow_specific_types..check_round_tripsEll00$+O<<77<(((r)rC) r time32time64date64rrritemsrqpcscalar)fieldsrtyprxrvalrs @r test_arrow_specific_typesrs3+ $+yy* Q'*M:*N; FYY&,,.Q.hsAs+.Q RF) #LLNjsC$299Ss+CCD+Rs)#D) c tjtjdtjtjdtjtjdtj tj tjtjdtjtj tjgtjtjdtjtjdtjtjdtjtjdtjgfd}D] }|| y)N binary_view string_view dictionaryreectjtj|j}tj j |gdg}tj j|}|jk(sJyr) ris_nullrqrr r1rrr)rqrr8r alt_schemars r check_one_wayz/test_arrow_one_way_types..check_one_ways`zz"((5::./ll00$+O<<77<X__,,,r) r rrqrrrrstringrun_end_encodedbinary)rrqrrs @@r test_arrow_one_way_typesrs  YY HH]BNN$4 5 HH]BNN$4 5 HH\2==RYY[#I J HHUB..rxxz299;G H  F HH]BIIK 0 HH]BIIK 0 HH\299; / HHUBIIK (  J- erctjtjdtjtjdtjg}t j t jdd}t j t jdd}t jt5}tjj|gg|ddddtjvsJt jt5}tjj|gddg|ddddt|jvsJt jt5}tjj|gdg|dddd t|jvsJy#1swYxYw#1swYxYw#1swY;xYw) Nr`rrzzneed to have the same lengthr%rNrzNo match for FieldRef) r rrqrrequalrDrE ValueErrorr1rrr)rrbad_exprrs r test_invalid_expression_ser_desrsV YY bhhj! bhhj!F 88BHHSM1 %Dxx q)H z "g **D62v> # )S-? ?? ? z "g **D6E5>6J # )S-? ?? ? z "g **H:xH # "c'--&8 88 8 # " # " # "s$#G,%G $G)GG&)G2ctjtjdtjtjdtjg}t j t jddt j t jdt jdg}tj j|ddg|}tj j|}||jk(sJt|jdk(sJt j t jddt j t jdt jdg}t|jdt|dk(sJt|jdt|dk(sJy) Nr`rrfirstsecondr!rr ) r rrqrrrr1rrrrr)rexprsr8r norm_exprss r %test_serializing_multiple_expressionsrsY YY bhhj! bhhj!FXXbhhsmQ '"((3-#)O PE ,, , ,UWh4G PC||33C8H X__ $$ $ x## $ )) )((288A;*BHHRXXa["((1+,NOJ x##G, -Z]1C CC C x##H- .#jm2D DD Drc<tjtjdtjtjdtjg}t j t jdd}t j t jdd}|j |}tjj|}||jk(sJt|jdk(sJt|jdt|k(sJtjj||gddg|}tjt5}tj j#|dddd tj$vsJtjj|gd g|}tj j#|}t|t|k(sJy#1swY~xYw) Nr`rrrr expressionrrzcontained multiple expressions weirdname)r rrqrrr to_substraitr1rrrrrrDrEr Expressionfrom_substraitr)rr expr_normr8rrexpr2s r test_serializing_with_computer s YY bhhj! bhhj!F 88BHHSM1 %D!a(I   F #C||33C8H X__ $$ $ x## $ )) ) x##L1 2c)n DD D ,, , ,dD\GX;NPV WC z "g $$S) # +s7==/A AA A ,, , ,dVk]F KC MM ( ( -E u:Y '' ' # "s 5 HHctjtjdtjg}t j d}t j d}t j ||g}tjt5tjj|dg|dddtjj|dg|d}tjj|}||jk(sJt|jdk(sJt|jdt|dk(sJy#1swYxYw) Nr`rr#rT)allow_arrow_extensionsr r)r rrquint32rr shift_leftrDrErr1rrrrr)rabrr8rs r test_serializing_udfsr$sYY biik"F " A ! A ]]1a !E / 0 **56(FC 1 ,, , , x - >C||33C8H X__ $$ $ x## $ )) ) x##F+ ,E!H == = 1 0s #EE#)r)-rr,rDpyarrowr pyarrow.computecomputer pyarrow.librrrpyarrow.substraitr1 ImportErrormark pytestmarkr r parametrizer; py_bufferrIrLrOrXr^rsr{r~rrrnumpyrrrrqrrrrrrrrrr rsc$  >) [[ " " D u 6*;7*;Z<2<< #:Ha"HI #J # !u 6,;7,;b<u 6-"7-"`'@T'@Tu 6x!7x!vs:lu 6'7'TQ!Q!hg!g!T BHHXRXXc]A BHHXRXXc]HBHHSM* BHHSMB" /  /E(89( E (6>M IsHH$#H$