g6EndZddlmZddlmZmZmZmZmZm Z m Z m Z m Z m Z mZmZmZddlmZmZmZmZmZmZmZmZGddeZGddeeZGd d eZGd d eZGd deZGddeZGdde Z GddeZ!GddeZ"GddeZ#GddeZ$eeeeegZ%eeeegZ&ee eegZ'ee eegZ(ee e gZ)GddeZ*Gdde*Z+Gdd e*Z,Gd!d"e*Z-Gd#d$e*Z.Gd%d&e*Z/eee"e#e$gZ0eeee!gZ1eeee!gZ2eeee!gZ3Gd'd(e*eZ4Gd)d*e4Z5Gd+d,e4Z6Gd-d.e4Z7Gd/d0e4Z8 d4d1Z9e:d2k(re9y3y3)5a Data classes and parser implementations for *incremental* chart parsers, which use dynamic programming to efficiently parse a text. A "chart parser" derives parse trees for a text by iteratively adding "edges" to a "chart". Each "edge" represents a hypothesis about the tree structure for a subsequence of the text. The "chart" is a "blackboard" for composing and combining these hypotheses. A parser is "incremental", if it guarantees that for all i, j where i < j, all edges ending at i are built before any edges ending at j. This is appealing for, say, speech recognizer hypothesis filtering. The main parser class is ``EarleyChartParser``, which is a top-down algorithm, originally formulated by Jay Earley (1970). ) perf_counter) BottomUpPredictCombineRuleBottomUpPredictRuleCachedTopDownPredictRuleChart ChartParserEdgeIEmptyPredictRule"FilteredBottomUpPredictCombineRule!FilteredSingleEdgeFundamentalRuleLeafEdge LeafInitRuleSingleEdgeFundamentalRuleTopDownInitRule)!FeatureBottomUpPredictCombineRuleFeatureBottomUpPredictRule FeatureChartFeatureChartParserFeatureEmptyPredictRule FeatureSingleEdgeFundamentalRuleFeatureTopDownInitRuleFeatureTopDownPredictRulec<eZdZdZdZdZdZdZdZdZ dZ y ) IncrementalChartcjtd|jD|_i|_i|_y)Nc3 K|]}gywN.0xs K/var/www/openai/venv/lib/python3.12/site-packages/nltk/parse/earleychart.py z.IncrementalChart.initialize..@s>,=q,= )tuple _positions _edgelists _edge_to_cpls_indexesselfs r" initializezIncrementalChart.initialize>s.>DOO,=>>  c4t|jSr)list iteredgesr*s r"edgeszIncrementalChart.edgesIsDNN$%%r-c(d|jDS)Nc3.K|] }|D]}|ywrr)r edgelistedges r"r#z-IncrementalChart.iteredges..MsJ__s)r'r*s r"r0zIncrementalChart.iteredgesLsJT__JJr-c F|j|}ik(r t|Stj}t |}||j vr|j |t fd|D}t|j ||j|gS)Nc3(K|] }| ywrr)r key restrictionss r"r#z*IncrementalChart.select..^s=*3\#&*sr'itersortedkeysr%r) _add_indexgetr+endr9r4 restr_keysvalss ` r"selectzIncrementalChart.selectOs??3' 2 > !L--/0 :&  T]] * OOJ '=*==DMM*-c266tR@AAr-cj|D] }tt|rtd|ztd|j Dx}|j |<t |jD]F\}}||}|D]7tfd|D}|j|gj9Hy)NBad restriction: %sc3 K|]}iywrrrs r"r#z.IncrementalChart._add_index..h1P>O">Or$c3@K|]}t|ywrgetattrr r8r5s r"r#z.IncrementalChart._add_index..nsHZc/WT3/1Z hasattrr ValueErrorr%r&r) enumerater' setdefaultappend r+rBr8indexrAr4 this_indexrCr5s @r"r>zIncrementalChart._add_indexasC5#& !6!<== -21Pdoo>O1P,PP j)'t7MCsJ HZHH%%dB/66t<!8r-cj}|jjD]=\}}tfd|D}||j |gj ?y)Nc3@K|]}t|ywrrJrLs r"r#z:IncrementalChart._register_with_indexes..tsD#+s+-rMrAr)itemsr%rRrSr+r5rArBrUrCs ` r"_register_with_indexesz'IncrementalChart._register_with_indexesqsYhhj!%!4!4!6 JDDDD #J ! !$ + 2 24 8"7r-c\|j|jj|yr)r'rArS)r+r5s r" _append_edgezIncrementalChart._append_edgews  #**40r-c:t|jdzS)N)range num_leavesr*s r"r&zIncrementalChart._positionszsT__&*++r-N) __name__ __module__ __qualname__r,r1r0rDr>r\r^r&rr-r"rr=s, &KB$= 9 1,r-rceZdZdZdZdZy)FeatureIncrementalChartc Jj|}ik(r t|Stj}t |}|j vrj |t fd|D}tj ||j|gS)Nc3FK|]}j|ywr)_get_type_if_possible)r r8r9r+s r"r#z1FeatureIncrementalChart.select..s% EOcD & &|C'8 9Zs!r:r@s` ` r"rDzFeatureIncrementalChart.selects??3' 2 > !L--/0 :&  T]] * OOJ ' EO  DMM*-c266tR@AAr-cn|D] }tt|rtd|ztdj Dx}j |<t jD]G\}}||}|D]8tfd|D}|j|gj:Iy)NrFc3 K|]}iywrrrs r"r#z5FeatureIncrementalChart._add_index..rHr$c3^K|]$}jt|&ywrrjrKr r8r5r+s r"r#z5FeatureIncrementalChart._add_index..s1)../AwtS/A/CD)*-rNrTs ` @r"r>z"FeatureIncrementalChart._add_indexsC5#& !6!<== -21Pdoo>O1P,PP j)'t7MCsJ )%%dB/66t< !8r-cj}jjD]>\}}tfd|D}||j |gj @y)Nc3^K|]$}jt|&ywrrnros r"r#zAFeatureIncrementalChart._register_with_indexes..s-LVS**+=74+=+?@JrprYr[s`` r"r\z.FeatureIncrementalChart._register_with_indexessbhhj!%!4!4!6 JLVD #J ! !$ + 2 24 8 "7r-N)rcrdrerDr>r\rr-r"rgrg~sB(=&9r-rgceZdZdZy)CompleteFundamentalRulec#K|j}|j||d|jD]9}|j|j}|j |||s6|;ywNT)startrA is_completelhs)rArDnextsymmove_dot_forwardinsert_with_backpointer)r+chartgrammar left_edgerA right_edgenew_edges r"_apply_incompletez)CompleteFundamentalRule._apply_incompletesmmmo ,,3Di6G6G6I' J!11*..2BCH,,Xy*M  s A)A3,A3Nrcrdrerrr-r"rtrts r-rtc eZdZeZdZy) CompleterRulec#|Kt|ts&|jj|||Ed{yy7wr isinstancer _fundamental_ruleapplyr+r}r~r5s r"rzCompleterRule.applys6$)--33E7DI I I* I 1<:<Nrcrdrertrrrr-r"rr/1Jr-rc eZdZeZdZy) ScannerRulec#|Kt|tr&|jj|||Ed{yy7wrrrs r"rzScannerRule.applys6 dH %--33E7DI I I & IrNrrr-r"rrrr-rc eZdZy) PredictorRuleNrcrdrerr-r"rrr-rceZdZdZy)FilteredCompleteFundamentalRulec#hK|jr|j|||Ed{yy7wr)rx_apply_completers r"rz%FilteredCompleteFundamentalRule.applys4    ++E7DA A A  As '202N)rcrdrerrr-r"rrsBr-rceZdZdZy)FeatureCompleteFundamentalRulec#K|j}|j}|j||d|jD]}|j ||||Ed{ y7wrv)rrArDrzr)r+r}r~rfrrArs r"rz0FeatureCompleteFundamentalRule._apply_incompletesh  # #mmo ,,3Di6G6G6I' Jxxw :F F F  GsAA&A$A&Nrrr-r"rrsGr-rceZdZeZy)FeatureCompleterRuleNrcrdrerrrr-r"rr 68r-rceZdZeZy)FeatureScannerRuleNrrr-r"rrrr-rc eZdZy)FeaturePredictorRuleNrrr-r"rrrr-rc(eZdZdZeddefdZddZy)IncrementalChartParsera An *incremental* chart parser implementing Jay Earley's parsing algorithm: | For each index end in [0, 1, ..., N]: | For each edge such that edge.end = end: | If edge is incomplete and edge.next is not a part of speech: | Apply PredictorRule to edge | If edge is incomplete and edge.next is a part of speech: | Apply ScannerRule to edge | If edge is complete: | Apply CompleterRule to edge | Return any complete parses in the chart r2c&||_||_||_||_g|_g|_|D]b}|j dk(r|jj|.|j dk(r|j j|Ytdy)a Create a new Earley chart parser, that uses ``grammar`` to parse texts. :type grammar: CFG :param grammar: The grammar used to parse texts. :type trace: int :param trace: The level of tracing that should be used when parsing a text. ``0`` will generate no tracing output; and higher numbers will produce more verbose tracing output. :type trace_chart_width: int :param trace_chart_width: The default total width reserved for the chart in trace output. The remainder of each line will be used to display edges. :param chart_class: The class that should be used to create the charts used by this parser. rr`z9Incremental inference rules must have NUM_EDGES == 0 or 1N) _grammar_trace_trace_chart_width _chart_class_axioms_inference_rules NUM_EDGESrSrP)r+r~strategytracetrace_chart_width chart_classrules r"__init__zIncrementalChartParser.__init__/s4   "3' "D~~" ##D)1$%%,,T2 R r-Nc >| |j}|j}t|}|jj ||j |}|j}|j |jdzz}|rt|j||jD])}t|j||}||||||+|j} t|jdzD]} |dkDr td| dt|j| } | s3| j} | D]V} t| j||| }||| ||||D]'}|j!| k(s| j#|)X| rn|S)Nr`z * Processing queue: )rA)r_trace_new_edgesr/rcheck_coveragerrrbprintpretty_format_leavesrrrrarDpoprArS)r+tokensrtrace_new_edgesr}r~trace_edge_widthaxiom new_edgesinference_rulesrAagendar5rrs r" chart_parsez"IncrementalChartParser.chart_parseZs~ =KKE//f $$V,!!&)-- 22u7G7G7IA7MN  %,,-=> ?\\EU[[89I E5)US0"MM(3%., 1 r-r)rcrdre__doc__BU_LC_INCREMENTAL_STRATEGYrrrrr-r"rrs  $,$ )V!r-rceZdZdZy)EarleyChartParserc <tj||tfi|yr)rrEARLEY_STRATEGYr+r~ parser_argss r"rzEarleyChartParser.__init__s''gV+Vr-Nrcrdrerrr-r"rr~sWr-rceZdZdZy)IncrementalTopDownChartParserc <tj||tfi|yr)rrTD_INCREMENTAL_STRATEGYrs r"rz&IncrementalTopDownChartParser.__init__ '' '2 6A r-Nrrr-r"rr r-rceZdZdZy)IncrementalBottomUpChartParserc <tj||tfi|yr)rrBU_INCREMENTAL_STRATEGYrs r"rz'IncrementalBottomUpChartParser.__init__rr-Nrrr-r"rrrr-rceZdZdZy)(IncrementalBottomUpLeftCornerChartParserc <tj||tfi|yr)rrrrs r"rz1IncrementalBottomUpLeftCornerChartParser.__init__s '' '5 9D r-Nrrr-r"rrrr-rceZdZdZy) IncrementalLeftCornerChartParserc r|js tdtj||tfi|y)NzNIncrementalLeftCornerParser only works for grammars without empty productions.) is_nonemptyrPrrLC_INCREMENTAL_STRATEGYrs r"rz)IncrementalLeftCornerChartParser.__init__s@""$-  '' '2 6A r-Nrrr-r"rrs r-rceZdZedefdZy)FeatureIncrementalChartParserc :tj||f|||d|y)N)rrr)rr)r+r~rrrrs r"rz&FeatureIncrementalChartParser.__init__s2 ''   /#    r-N)rcrdre"BU_LC_INCREMENTAL_FEATURE_STRATEGYrgrrr-r"rrs4+  r-rceZdZdZy)FeatureEarleyChartParserc <tj||tfi|yr)rrEARLEY_FEATURE_STRATEGYrs r"rz!FeatureEarleyChartParser.__init__s %.. '2 6A r-Nrrr-r"rrrr-rceZdZdZy)$FeatureIncrementalTopDownChartParserc <tj||tfi|yr)rrTD_INCREMENTAL_FEATURE_STRATEGYrs r"rz-FeatureIncrementalTopDownChartParser.__init__ %.. ': >I r-Nrrr-r"rrrr-rceZdZdZy)%FeatureIncrementalBottomUpChartParserc <tj||tfi|yr)rrBU_INCREMENTAL_FEATURE_STRATEGYrs r"rz.FeatureIncrementalBottomUpChartParser.__init__rr-Nrrr-r"rrrr-rceZdZdZy)/FeatureIncrementalBottomUpLeftCornerChartParserc <tj||tfi|yr)rrrrs r"rz8FeatureIncrementalBottomUpLeftCornerChartParser.__init__s %.. '= AL r-Nrrr-r"rrrr-rc.ddl}ddl}ddlm}|} |rt dt | t dt ||j } t | t t | |} t} | j| } t| j| j}t| z } |rt||k(sJd|r|D] }t |nt dt||r t d | yy) z0 A demonstration of the Earley parsers. rN) demo_grammarz * Grammarz * Sentence:)rzNot all parses foundz Nr trees:zTime:) systimenltk.parse.chartrrsplitrrrr/parsesrwlen) print_times print_grammar print_treesrsent numparsesrrrr~rearleytr}rtrees r"demors-nG k g - $K ZZ\F &M Gwe 4FA   v &E %,,w}}/ 0FA6{i'?)??'D $K k3v;' gqr-__main__N)TFTz$I saw John with a dog with my cookie);rrrrrrrrrr r r r r rrrnltk.parse.featurechartrrrrrrrrrrgrtrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrcrr-r"r s     >,u>,B.9. .9l 7 J+JJ)J , B&GB G%E G9=999 4 NOMO N N N  N&(#%\[\~W.W  $:  %;  /E   '=  "N N"$ #N "$ #N%'"$ &" $: