
    gp                     L    d dl Z d dlZd dlmZ d dlmZ d dlmZ  G d d      Zy)    N)Path)AsyncLogger)AsyncLLMTextManagerc                   R    e Zd ZddZd ZdefdZdee   fdZddZ	dded	e
fd
Zy)DocsManagerNc                 L   t        j                         dz  dz  | _        t        t              j                  j                  dz  dz  | _        | j                  j                  dd       |xs t        d      | _        t        | j                  | j                        | _
        y )Nz	.crawl4aidocszllm.txtT)parentsexist_ok)verbose)r   homedocs_dir__file__parent
local_docsmkdirr   loggerr   llm_text)selfr   s     J/var/www/openai/venv/lib/python3.12/site-packages/crawl4ai/docs_manager.py__init__zDocsManager.__init__   sx    		k1F:x.//66?)KD489D 9+DMM4;;G    c                    K   t        | j                  j                               s| j                          d{    yy7 w)zFetch docs if not presentN)anyr   iterdir
fetch_docs)r   s    r   ensure_docs_existzDocsManager.ensure_docs_exist   s3     4==((*+//### ,#s   7AA Areturnc                   K   	 | j                   j                         rt        | j                   j                  d            s$t        | j                   j                  d            r~| j                  j                  d      D ]  }|j                           | j                   j                  d      D ]/  }t        j                  || j                  |j                  z         1 yt        j                  dddi      }|j                          |j                         D ]z  }|d   d	k(  s|d
   j                  d      s!t        j                  |d         j                  }t        | j                  |d
   z  dd      5 }|j!                  |       ddd       | y# 1 sw Y   xY w# t"        $ r-}| j$                  j'                  dt)        |               d}~ww xY ww)z,Copy from local docs or download from GitHub*.mdz*.tokensTzEhttps://api.github.com/repos/unclecode/crawl4ai/contents/docs/llm.txtAcceptzapplication/vnd.github.v3+json)headerstypefilenamez.mddownload_urlwzutf-8)encodingNzFailed to fetch docs: )r   existsr   globr   unlinkshutilcopy2r%   requestsgetraise_for_statusjsonendswithtextopenwrite	Exceptionr   errorstr)r   	file_pathresponseitemcontentfes          r   r   zDocsManager.fetch_docs   s    	%%'S1E1Ef1M-NRUVZVeVeVjVjkuVvRw!%!3!3F!;I$$& "< "&!5!5f!=ILLDMMINN,JK ">   ||W!#CDH %%' <6)d6l.C.CE.J&ll4+?@EEGdmmd6l:C'RVW( SR (
  SR  	KK 6s1vh?@	s[   G CF' #G $AF' )F' >A F' >F
F' G F$	 F' '	G0(GGG c                 \   | j                   j                  d      D cg c]  }|j                   }}|D cg c],  }|d   j                         r|j	                  dd      d   n|. }}|D cg c](  }|j                  d      r|j                  d      r'|* }}|S c c}w c c}w c c}w )zList available topicsr    r   _   z.xsz.q)r   r*   stemisdigitsplitr2   )r   r9   namesr%   s       r   listzDocsManager.list6   s    151C1CF1KL1KI1KLRWXRW$$q'//*;C#A&ERWX"'`%$t}}U/CDMMZ^L_%` MX`s   B1B$.B)B)B)c                 :    | j                   j                  ||      S N)r   generate)r   sectionsmodes      r   rI   zDocsManager.generate?   s    }}%%h55r   querytop_kc                 :    | j                   j                  ||      S rH   )r   search)r   rL   rM   s      r   rO   zDocsManager.searchB   s    }}##E511r   rH   )extended)   )__name__
__module____qualname__r   r   boolr   rF   r8   rI   intrO    r   r   r   r      sB    H$
 $  Dd3i 62C 2 2r   r   )	r.   r,   pathlibr   crawl4ai.async_loggerr   crawl4ai.llmtxtr   r   rW   r   r   <module>r[      s       - /<2 <2r   