g#ddlZddlmZddlZddlmZddlmZddlZddl m Z m Z m Z m Z mZddlmZmZmZddlmZmZmZdd lmZdd lmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#ejHd Z%e&Z'd e ed e e(e(ffdZ)d e ed e*fdZ+d e ed e*fdZ,d ed e-fdZ.d ed e e-fdZ/d ed e-fdZ0d ede-d e e-e-ffdZ1d ed efdZ2 dd7Z?d8e-d9e-d e-fd:Z@d;ZAy)>N)StringIO)path)deepcopy)TupleSetOptionalListAny) DictSchemaSchema NamedSchemas)FlatDictRepositorySchemaRepositoryErrorAbstractSchemaRepository) AVRO_TYPES) PRIMITIVES UnknownTypeSchemaParseExceptionRESERVED_PROPERTIESOPTIONAL_FIELD_PROPERTIESRESERVED_FIELD_PROPERTIESJAVA_FINGERPRINT_MAPPINGFINGERPRINT_ALGORITHMSRABIN_64rabin_fingerprintz[A-Za-z_][A-Za-z0-9_]*schemareturncd}d}|D]@}t|}|dk(r |dz }|dz }|dk(s|dk(r|dz }.|tvs7|dz }|dz }B||fS)Nrrecordr enumfixed)extract_record_typer)rrecord_type_countnamed_type_countsextracted_types H/var/www/openai/venv/lib/python3.12/site-packages/fastavro/_schema_py.py&_get_name_and_record_counts_from_unionr)!s ,Q/ X %  "   !  v %7)B  !  : -  !   "  . ..c$t|ddk(S)Nr r)rs r(is_single_record_unionr.5 1& 9! < AAr*c$t|ddk(S)Nrr r,r-s r(is_single_name_unionr19r/r*cRt|tr|dSt|try|S)Ntypeunion) isinstancedictlistr-s r(r#r#=s(&$f~&$ Mr*cht|tsy|}|d}|jd}|r|d|Sy)Nr3 logicalType-)r5r6get)rd_schemartlts r(extract_logical_typer?GsB fd #H & B m $B Qrd| r*c t|ddS)aQReturns the fullname of a schema Parameters ---------- schema Input schema Example:: from fastavro.schema import fullname schema = { 'doc': 'A weather reading.', 'name': 'Weather', 'namespace': 'test', 'type': 'record', 'fields': [ {'name': 'station', 'type': 'string'}, {'name': 'time', 'type': 'long'}, {'name': 'temp', 'type': 'int'}, ], } fname = fullname(schema) assert fname == "test.Weather" r ) schema_namer-s r(fullnamerCSs8 vr "1 %%r* parent_nsc |d}|jd|}d|vr|jddd|fS|r ||d|fSd|fS#t$rtd|wxYw)Nnamez4"name" is a required field missing from the schema: namespace.r rrA)KeyErrorrr;rsplit)rrDrFrGs r(rBrBrs f~  ; 2I d{{{3"1%t++ YKq///4x  "B6( K   s AAct|ddS)aReturns a schema where all named types are expanded to their real schema NOTE: The output of this function produces a schema that can include multiple definitions of the same named type (as per design) which are not valid per the avro specification. Therefore, the output of this should not be passed to the normal `writer`/`reader` functions as it will likely result in an error. Parameters ---------- schema: dict Input schema Example:: from fastavro.schema import expand_schema original_schema = { "name": "MasterSchema", "namespace": "com.namespace.master", "type": "record", "fields": [{ "name": "field_1", "type": { "name": "Dependency", "namespace": "com.namespace.dependencies", "type": "record", "fields": [ {"name": "sub_field_1", "type": "string"} ] } }, { "name": "field_2", "type": "com.namespace.dependencies.Dependency" }] } expanded_schema = expand_schema(original_schema) assert expanded_schema == { "name": "com.namespace.master.MasterSchema", "type": "record", "fields": [{ "name": "field_1", "type": { "name": "com.namespace.dependencies.Dependency", "type": "record", "fields": [ {"name": "sub_field_1", "type": "string"} ] } }, { "name": "field_2", "type": { "name": "com.namespace.dependencies.Dependency", "type": "record", "fields": [ {"name": "sub_field_1", "type": "string"} ] } }] } TF)expand _write_hint) parse_schemar-s r( expand_schemarOsB t ??r*FTrLrM_force_ignore_default_error named_schemasrLrMrQrRc |i}t|trGd|vrCd|vr!|djD] \}}|||< nt|d||t |t |S|s|rt|d||t |t |St|trd|vr|St|t r|Dcgc]}t||||||c}St|d||t |t |Scc}w)a|Returns a parsed avro schema It is not necessary to call parse_schema but doing so and saving the parsed schema for use later will make future operations faster as the schema will not need to be reparsed. Parameters ---------- schema Input schema named_schemas Dictionary of named schemas to their schema definition expand If true, named schemas will be fully expanded to their true schemas rather than being represented as just the name. This format should be considered an output only and not passed in to other reader/writer functions as it does not conform to the avro specification and will likely cause an exception _write_hint Internal API argument specifying whether or not the __fastavro_parsed marker should be added to the schema _force Internal API argument. If True, the schema will always be parsed even if it has been parsed and has the __fastavro_parsed marker _ignore_default_error Internal API argument. If True, when a union has the wrong default value, an error will not be raised. Example:: from fastavro import parse_schema from fastavro import writer parsed_schema = parse_schema(original_schema) with open('weather.avro', 'wb') as out: writer(out, parsed_schema, records) Sometimes you might have two schemas where one schema references another. For the sake of example, let's assume you have a `Parent` schema that references a `Child` schema`. If you were to try to parse the parent schema on its own, you would get an exception because the child schema isn't defined. To accommodate this, we can use the `named_schemas` argument to pass a shared dictionary when parsing both of the schemas. The dictionary will get populated with the necessary schema references to make parsing possible. For example:: from fastavro import parse_schema named_schemas = {} parsed_child = parse_schema(child_schema, named_schemas) parsed_parent = parse_schema(parent_schema, named_schemas) __fastavro_parsed__named_schemasrArP)r5r6items _parse_schemaset NO_DEFAULTr7rN) rrSrLrMrQrRkeyvaluer&s r(rNrNs7~ &$$76$A  &$%67==? U%* c"@ !%       E   !   FD !&9V&C FD !   '&;         E   !    s(C!default schema_typeignore_default_errorc`|ryt|trd|}nd|}td|d|)Nza schema in union with type: z schema type: zDefault value must match )r5r7r)r]r^r_texts r(_raise_default_value_errorrbDsD K &.{m<{m,  tfM NNr*r\cH t|S#ttf$r|cYSwxYwN)float TypeError ValueError)r\s r( _maybe_floatrhQs*U| z " s !!c^|dk(r||dk(rt|tr|dk(rt|tr{|dk(rt|trf|dk(rtt|trH|dk(rtt|tr*|dk(rt|t r|dk(rt|t sy y ) NnullbooleanstringbytesdoublereintlongFT)r5boolstrrhrero)r]rs r(_default_matches_schemarsXs 6 g1 i  7D(A h z'3'? g j#&> h z,w2G'O g jg1F&N eOJw$< f Z%= r*rGnamesc r t|trP|Dcgc]}t|||d||t|} }|tur#| D]}t ||s| St |||| St|t sX|tvr#|turt ||s t ||||Sd|vr |r|dz|z}||vr t||r d||vr||S|S|d} |jD cic]\} } | tvr| | } } } | | d<d|vr|d| d<| jd}|dk(r| jd}|r#t|tr|d krtd || jd }|rt|tr|d krtd d |z| dk(rY|d}ttjtj dd|zdz z}||kDrtd|dd|z|r|r||krtdd|z| dk(rCt|d||d||t|| d<|turt|ts t || || S| dk(rCt|d||d||t|| d<|turt|t s t || || S| dk(rvt#||\}}||vrtd||j%|t'||turt|t(s t || || ||<|| d<|d| d<| S| dk(rkt#||\}}||vrtd||j%||turt|t(s t || || ||<|| d<|d| d<| S| dk(s| d k(rt#||\}}||vrtd||j%||turt|t s t || || ||<g}|jd!gD]!}|j+t-||||||#|| d<|| d!<|r1| jDcic]\}}|| c}}||<d"| d#<|| d$<| S| tvr| | d<|tur| d%k(r|| d&k(rt|t.r~| d'k(rt|t(ri| d(k(rt|t(rT| d)k(rt|t0r?| d*k(rt|t0r*| d+k(rt|tr| d,k(rt|ts t || || St|cc}wcc} } wcc}}w)-NFrHrFr3docr9decimalscalerz.decimal scale must be a positive integer, not precisionz.decimal precision must be a positive integer, znot r"sizer zdecimal precision of z doesn't fit zinto array of length z,decimal scale must be less than or equal to zthe precision of arrayrWmapvaluesr!zredefined named type: symbolsr errorfieldsTrUrVrjrkrlrmrnrerorp)r5r7rXrZrsrbr6rrrWrr;rormathfloorlog10rBadd_validate_enum_symbolsrrappend parse_fieldrqre)rrGrLrMrtrSr]r_r&parsed_schemasr^r[r\ parsed_schema logical_typerxryrz max_precision_rCrfieldkvs r(rXrXhsv&$   $     * $#*7A6 $+7FD$' 4::a=AHqL3Q(R$SM =023I;mL 5dV<= y5'8*B))56 ' !%2w$ &M' "j(GT1J*7KAUVPME !&3x $ 'M( #j(GT1J*7KAUVtqF "%fi8KAx5 *-CH:+NOO IIh  "6 *j(GS1I*7KAUV&3M( #$,M& !'-i'8M) $TQG #%fi8KAx5 *-CH:+NOO IIh j(GS1I*7KAUV&3M( #$,M& !$*6NM& !xuH $ w(>"-fi"@ Ix5 *-CH:+NOO IIh j(GT1J*7KAUV&3M( #FHb1 !%,  2%-M& !&,M( #=J.avsc`. Parameters ---------- schema_path Full schema name, or path to schema file if default repo is used. repo: Schema repository instance. named_schemas Dictionary of named schemas to their schema definition _write_hint Internal API argument specifying whether or not the __fastavro_parsed marker should be added to the schema _injected_schemas Internal API argument. Set of names that have been injected Consider the following example with default FlatDictRepository... namespace.Parent.avsc:: { "type": "record", "name": "Parent", "namespace": "namespace", "fields": [ { "name": "child", "type": "Child" } ] } namespace.Child.avsc:: { "type": "record", "namespace": "namespace", "name": "Child", "fields": [] } Code:: from fastavro.schema import load_schema parsed_schema = load_schema("namespace.Parent.avsc") )rsplitsplitextrrY _load_schema) rrrSrMrrBfile_dir file_name _file_exts r( load_schemarfsqDK |"jj5)!%y!9 Y!(+  E T=+7H r*cj |j|}t|||||S#t$r}|d}~wwxYwrd)load_parse_schema_with_repor)rBrrS write_hintinjected_schemasrrs r(rrsH ;'&         ! s" 2-2cr t|}t|||S#t$r}|j} t ||d|}n#t $r|wxYw|d|vrEt ||} t|tst|tr| d}|j|dt|||||cYd}~Sd}~wwxYw)NrSrMF)rSrrrFr) rrNrrFrr_inject_schemar5rrr7rr) rrrSrr schema_copyrmissing_subject sub_schemainjected_schemas r(rrs }-  '"   ** %) !1 J% K  f %5 5,VZ@O&#&*VT*B(+  F!3 4& D+z3C  % s2 B6 B1AB1 AAB1+B61B6c&|}|dur||fSt|trJg}|D]?}|r|j|t||||\}}|j||dus>|}A||fSt|ts*|t vr||fSd|vr |r|dz|z}||dk(r|dfS||fS|d} | dk(rt|d|||\}}||d<||fS| dk(rt|d|||\}}||d<||fS| d k(r||fS| d k(r||fS| d k(s| d k(rxt ||\}} g} |jd gD]G} |r| j| t| d|||\}}|| d<| j| |dusF|}I| r| |d <||fS| t vr||fStd)NTrHrFr3r}rWr~rr!r"r rrzKInternal error; You should raise an issue in the fastavro github repository) r5r7rrr6rrBr; Exception) outer_schema inner_schemans is_injectedrGr4 each_schema return_schemainjectedr^rrrs r(rrsXId[((,%'K [)*8y++' x ]+t#"*K(k!! d + : %, , l "y$s?\9L </ /% % , ,#6* ' !&4W%|Y ' #M8%2L !) ) E !&4X& i' #M8&3L ") ) F ", , G #, , H $ w(>&|Y?LIqF%))(B7MM%(.<f |Y /+M8%2E&MMM%(4'&. 8)/ X&, , J &, ,P r*)rMordered_schemascg}i}t|D]9\}}|dzt|k(r|nd}t|||}|j|;|ddd}|j d} |r |j d} t | | |r | S)aReturns a schema loaded from a list of schemas. The list of schemas should be ordered such that any dependencies are listed before any other schemas that use those dependencies. For example, if schema `A` depends on schema `B` and schema B depends on schema `C`, then the list of schemas should be [C, B, A]. Parameters ---------- ordered_schemas List of paths to schemas _write_hint Internal API argument specifying whether or not the __fastavro_parsed marker should be added to the schema Consider the following example... Parent.avsc:: { "type": "record", "name": "Parent", "namespace": "namespace", "fields": [ { "name": "child", "type": "Child" } ] } namespace.Child.avsc:: { "type": "record", "namespace": "namespace", "name": "Child", "fields": [] } Code:: from fastavro.schema import load_schema_ordered parsed_schema = load_schema_ordered( ["path/to/namespace.Child.avsc", "path/to/Parent.avsc"] ) r FrNr) enumeratelenrrpopr) rrMloaded_schemasrSidxr_lastrtop_first_orderrrs r(load_schema_orderedrGsnN"$M%o6["Qw#o*>> E }%  f% 7%TrT*O"&&q)L $((+ |Z0  r*c`t}tt|||jS)a'Returns a string represening the parsing canonical form of the schema. For more details on the parsing canonical form, see here: https://avro.apache.org/docs/current/spec.html#Parsing+Canonical+Form+for+Schemas Parameters ---------- schema Schema to transform )r_to_parsing_canonical_formrNgetvalue)rfos r(to_parsing_canonical_formrs& B|F3R8 ;;=r*c t|trX|jdt|D]'\}}|dk7r|jdt ||)|jdyt|t s|jd|dy|d}|dk(r6|jd|d t |d ||jd y|d k(r6|jd|d t |d||jd y|dk(rp|d}|jd|d|dt|dD]0\}}|dk7r|jd|jd|d2|jdy|dk(r&|d}|d}|jd|d|d|d y|dk(s|dk(r|d}|jd|dt|dD]U\}}|dk7r|jd|d}|jd|dt |d||jd W|jdy|t vr|jd|dyy)N[r,]"r3r}z {"type":"z ","items":rW}r~z ","values":rr!rFz {"name":"z ","type":"z ","symbols":[rz]}r"rzz ","size":r rz","type":"record","fields":[rz ","type":)r5r7writerrr6r) rrrr&r^rFsymbolrzrs r(rrs[&$  'FCax &q" -(   % 1VHAVn ' ! HHz+j9 : &vg ; HHSM E ! HHz+k: ; &vh'7 < HHSM F "&>D HHz$z+mL M( ):; V!8HHSM1VHA' < HHTN G #&>D&>D HHz$z+ivRP Q H $ w(>&>D HHz$'CD E'x(89 U!8HHSMV}:dV956*5="= : HHTN J & HHq Q' ('r*parsing_canonical_form algorithmc|tvrtd|ddtztj||}|tk(rt |j Stj||j }|jS)atReturns a string represening a fingerprint/hash of the parsing canonical form of a schema. For more details on the fingerprint, see here: https://avro.apache.org/docs/current/spec.html#schema_fingerprints Parameters ---------- parsing_canonical_form The parsing canonical form of a schema algorithm The hashing algorithm z%Unknown schema fingerprint algorithm z. zValid values include: ) rrgrr;rrencodehashlibnew hexdigest)rrhs r( fingerprintrs..3I;b A&'=&>? @  ),,Y BIH !7!>!>!@AA I5<<>?A ;;=r*c|d}|D]2}t|trtj|r)t dt |t t |k7r t dd|vr|d|vr t dyy)NrzEEvery symbol must match the regular expression [A-Za-z_][A-Za-z0-9_]*z%All symbols in an enum must be uniquer]z.Default value for enum must be in symbols list)r5rr SYMBOL_REGEX fullmatchrrrY)rrrs r(rrsYG&#&l.D.DV.L&W   7|s3w<(("#JKKFvi0?"#STT @r*rd)rAF)Briorrosrcopyrretypingrrrr r typesr r r repositoryrrrconstr_schema_commonrrrrrrrrrrcompilerobjectrZror)rqr.r1rrr#r?rCrBrOrNrbrhrsrXrrrrrrrrrrr*r(rs2  2233    rzz34 X /4</E#s(O/(B4<BDBBfB$B3  HSM &Z&C&> suS#X"A@&A@VA@L-1z "'z z L)z   z  z  z  z  z z O  O" O:> O S & T  Z ZZ Z Z s8 Z  ZZZ ZzH04,0,0 PP + ,PL) P  P  C) P Pf " JXx8<H#YH04H HVf"9)x> Ur*