g8ddlZddlZddlZddlZddlmZddlmZmZm Z m Z ddl m Z ddl mZmZmZddlZddlZddlmZddlmZmZdd lmZdd ldd ldd ldd lmZmZm Z dd l!m"Z"m#Z#m$Z$dd l%m&Z&m'Z'ddl(m)Z)ddl*m+Z+ddl,m-Z-m.Z.ddl/m0Z0m1Z1m2Z2ddl3m4Z4m5Z5m6Z6m7Z7m8Z8ddl9m:Z:ddl;Z;ddly)N)Enum)initForeBackStyle)Path)OptionalListUnion)asynccontextmanager) CrawlResultMarkdownGenerationResult)async_db_manager)*)AsyncCrawlerStrategyAsyncPlaywrightCrawlerStrategyAsyncCrawlResponse) CacheMode CacheContext_legacy_to_cache_mode)DefaultMarkdownGeneratorMarkdownGenerationStrategy)WebScrapingStrategy) AsyncLogger) BrowserConfigCrawlerRunConfig)MIN_WORD_THRESHOLD$IMAGE_DESCRIPTION_MIN_WORD_THRESHOLDURL_LOG_SHORTEN_LENGTH)sanitize_input_encodeInvalidCSSSelectorError format_htmlfast_format_htmlcreate_box_message)urlparse) __version__ceZdZdZiZddddeejdejdfde e de e de de e d ed e f d Zd Zd ZdZdZdZedZdededddddddddddfdede edededede ede de de de dede de ded efd!Zded"ed#ededed$ed%e d efd&Zdededdddddddf d'e ede edededede ede dede de ded e efd(Z!d)Z"d*Z#d+Z$y),AsyncWebCrawleraB Asynchronous web crawler with flexible caching capabilities. There are two ways to use the crawler: 1. Using context manager (recommended for simple cases): ```python async with AsyncWebCrawler() as crawler: result = await crawler.arun(url="https://example.com") ``` 2. Using explicit lifecycle management (recommended for long-running applications): ```python crawler = AsyncWebCrawler() await crawler.start() # Use the crawler multiple times result1 = await crawler.arun(url="https://example.com") result2 = await crawler.arun(url="https://another.com") await crawler.close() ``` Migration Guide: Old way (deprecated): crawler = AsyncWebCrawler(always_by_pass_cache=True, browser_type="chromium", headless=True) New way (recommended): browser_config = BrowserConfig(browser_type="chromium", headless=True) crawler = AsyncWebCrawler(config=browser_config) Attributes: browser_config (BrowserConfig): Configuration object for browser settings. crawler_strategy (AsyncCrawlerStrategy): Strategy for crawling web pages. logger (AsyncLogger): Logger instance for recording events and errors. always_bypass_cache (bool): Whether to always bypass cache. crawl4ai_folder (str): Directory for storing cache. base_directory (str): Base directory for storing cache. ready (bool): Whether the crawler is ready for use. Methods: start(): Start the crawler explicitly without using context manager. close(): Close the crawler explicitly without using context manager. arun(): Run the crawler for a single source: URL (web, local file, or raw HTML). awarmup(): Perform warmup sequence. arun_many(): Run the crawler for multiple sources. aprocess_html(): Process HTML content. Typical Usage: async with AsyncWebCrawler() as crawler: result = await crawler.arun(url="https://example.com") print(result.markdown) Using configuration: browser_config = BrowserConfig(browser_type="chromium", headless=True) async with AsyncWebCrawler(config=browser_config) as crawler: crawler_config = CrawlerRunConfig( cache_mode=CacheMode.BYPASS ) result = await crawler.arun(url="https://example.com", config=crawler_config) print(result.markdown) NFCRAWL4_AI_BASE_DIRECTORYcrawler_strategyconfigalways_bypass_cachealways_by_pass_cachebase_directory thread_safec |}|2tfddDr3|jjddntj}||_t tjj|dd|j jd |_jD cic] \} } | d vs | | } } } |xstd||jd | |_ |jjs|j|j_|6jd drtj dt"d||_n||_|rt'j(nd|_tjj|d|_tj.|j,dtj.|j,ddd|_ycc} } w)ax Initialize the AsyncWebCrawler. Args: crawler_strategy: Strategy for crawling web pages. If None, will create AsyncPlaywrightCrawlerStrategy config: Configuration object for browser settings. If None, will be created from kwargs always_bypass_cache: Whether to always bypass cache (new parameter) always_by_pass_cache: Deprecated, use always_bypass_cache instead base_directory: Base directory for storing cache thread_safe: Whether to use thread-safe operations **kwargs: Additional arguments for backwards compatibility Nc3&K|]}|v ywN).0kkwargss N/var/www/openai/venv/lib/python3.12/site-packages/crawl4ai/async_webcrawler.py z+AsyncWebCrawler.__init__..sj(i11;(is) browser_typeheadlessviewport_widthviewport_heightz`Both browser_config and legacy browser parameters provided. browser_config will take precedence.WARNINGmessagetagz .crawl4aiz crawler.log )log_fileverbose tag_width)browser_congiglogger)browser_configrGwarningTz'always_by_pass_cache' is deprecated and will be removed in version 0.5.0. Use 'always_bypass_cache' instead. Pass warning=False to suppress this warning. stacklevel)exist_okz/cacheFr4)anyrGrIr from_kwargsrHrospathjoinrDitemsrr+getwarningswarnDeprecationWarningr-asyncioLock_lockcrawl4ai_foldermakedirsready) selfr+r,r-r.r/r0r7rHr6vparamss ` r8__init__zAsyncWebCrawler.__init__ks.   %j(ijj ##~!$ +66v>N,"WW\\.+}M''//  #LLN *DAqa3O.OAaCN  !1! 4R5 );;5 5 $$+++/;;D ! ! ( +zz)T* C' (K|jd{S7wr3)rfres r8rczAsyncWebCrawler.__aenter__sZZ\!!!s c@K|jd{y7wr3)ri)r^exc_typeexc_valexc_tbs r8rhzAsyncWebCrawler.__aexit__sjjls cbK|jjdtdd|_yw)z Initialize the crawler with warm-up sequence. This method: 1. Logs initialization info 2. Sets up browser configuration 3. Marks the crawler as ready z Crawl4AI INIT)rATN)rGinfocrawl4ai_versionr]res r8rdzAsyncWebCrawler.awarmups/ 9%5$67VD s-/cKdyw)u异步空上下文管理器Nr4res r8 nullcontextzAsyncWebCrawler.nullcontexts  s Turlextraction_strategychunking_strategycontent_filter cache_mode bypass_cache disable_cache no_cache_readno_cache_write css_selector screenshotpdf user_agentreturnc K|}t|tr|s td|jxs|j 4d{ ||}n'||||||| | | | | ||d |}t j |}t|| | | grN|jddrtjdtd|jt| || | |_ |jtj|_ t!||j|j"}d}d}d}d}d}t%j&}|j)rt+j,|d{}|rt/|j0}t/|j2xsd }|r|d k(rdn|}|j4}|j6}|j4r| r|j6r|sd}|j8j;|j<t?|t%j&|z d |rsHt%j&}|r|j@jC||j@jE||d{}t/|j0}|j4}|jF}t%j&}|j8j;|j<t?|||z d |jHd(|||||||jJ|jMdrdndd|d{}|jN|_'|jP|_(|jR|_)|jT|_*t?||_+tY|dd|_-|j8jWdd|j<|jVt%j&|z ddd|jVrt\j^nt\j`t\jbd|jer(t?|st+jf|d{|cdddd{S|j8jWdd|j<dt%j&|z dddt\j^t\jbdt?||_+tY|dd|_-|cdddd{S7777777#th$r}tktmjn} d| dd| dd| dd t|d!| d" }!|j8jq|ts|!d#$d%&tu|d d|!'cYd}~cdddd{7Sd}~wwxYw#1d{7swYyxYww))a Runs the crawler for a single source: URL (web, local file, or raw HTML). Migration Guide: Old way (deprecated): result = await crawler.arun( url="https://example.com", word_count_threshold=200, screenshot=True, ... ) New way (recommended): config = CrawlerRunConfig( word_count_threshold=200, screenshot=True, ... ) result = await crawler.arun(url="https://example.com", crawler_config=config) Args: url: The URL to crawl (http://, https://, file://, or raw:) crawler_config: Configuration object controlling crawl behavior [other parameters maintained for backwards compatibility] Returns: CrawlResult: The result of crawling and processing z4Invalid URL, make sure the URL is a non-empty stringN) word_count_thresholdrvrwrxryrzr{r|r}r~rrrDrITztCache control boolean flags are deprecated and will be removed in version 0.5.0. Use 'cache_mode' parameter instead.rJrK)r{rzr|r}z[]FETCH)rusuccesstimingrA)r,zraw:F)ruhtmlextracted_contentr,rpdf_datarD is_raw_html session_idz1{url:.50}... | Status: {status} | Total: {timing}COMPLETE.2fs)rustatusr)rrr@rAr`colorsz'Unexpected error in _crawl_web at line line_noz in functionz (filenamez ): Error: z Code context: code_contexterror)typeERROR)rurrA)rurr error_messager4); isinstancestr ValueErrorrZrtrrOrNrTrUrVrWryrrENABLEDrr-time perf_counter should_readraget_cached_urlr!rrrrrG url_status display_urlboolr+update_user_agentcrawlr aprocess_htmlrD startswith status_coderesponse_headersdownloaded_filesssl_certificatergetattrrrGREENREDYELLOW should_write acache_url Exceptionget_error_contextsysexc_info error_statusr%r)"r^rur,rrvrwrxryrzr{r|r}r~rrrrDr7crawler_config config_kwargs cache_contextasync_response cached_resultscreenshot_datarr start_timert1t2 crawl_resulte error_contextrs" r8arunzAsyncWebCrawler.arunsf$Nc3's !WXXzz7T%5%5%777W%1"05I3F1B.<*4,8-:-:.<,8*4#&'.)%) "2!=!=m!LL-WX!::i6$MM!F 2+, ",,40E.;-9.;/= 1F-((0,5,=,=)%1f6G6GIaIa$bM:>N15M&*O#H(,%!%!2!2!4J%002.>.N.Ns.S(S $4]5G5GH,A-BaBaBgeg,h)8IM^bfMfDl})*7*B*B#0#4#4!,,Z6::VY,0M .. - 9 9$(J#'#4#4#6#C ' /)!..0% 11CCJO04/D/D/J/J#)0K0* 5^5H5HI*8*C*C#1#:#:!..0 .. - 9 9$(J#%7 ' /.@T-?-? . #!%.?#)'6%-$*NN25..2H$e .% . ( 4B3M3M 08F8W8W 58F8W8W 57E7U7U 4,04Dz ,29&,PT2U / ++$W *'4'@'@*6*>*>-1->->-@:-Mc,RRS*T$ 9E8L8L$**RVRZRZ*.++$, )557]@S"2"="=l"KKK+Y877^ ++$W *'4'@'@*.-1->->-@:-Mc,RRS*T$ +/***.++$, 15T -3:6$'abeaf%ghh"1NB1OPL0NB1OPL,VZZ B-GHHJJw+EJJw+Ezz*b1HHNG`G`G~d|d~  9K8\8\)9]9O *K,_-I-IJH KK  C!4#4#4#6#;t"CD  ")**((v99;OP&&("(!;!;!H!H!^3O>'2$*$>$>$B$B3$Q!$(JJ/@TWfk$l!   J!#"&"3"3"5":!+5d*O#+tH/ = )'!)!!*"3  YXX"+ ) Q(( v #]^a]bbklopqlrks!tuu vs`OA!N % M=2M=7!N N 3N 8,N $JO= N ON&& O2O  OOurlsc  K}|:td|||||| | | fDrjjdd|n$||||||| | | | d |}tj||rO|j ddrt jd td jtj_ jxsd }tj| fd }jjdddt!|it#j$}|Dcgc] }|| }}tj&|ddid{}t#j$}jj)ddt!|||z ddddt*j,i|Dcgc]}t/|t0s|n t3|!c}Scc}w7cc}ww)a Runs the crawler for multiple URLs concurrently. Migration Guide: Old way (deprecated): results = await crawler.arun_many( urls, word_count_threshold=200, screenshot=True, ... ) New way (recommended): config = CrawlerRunConfig( word_count_threshold=200, screenshot=True, ... ) results = await crawler.arun_many(urls, crawler_config=config) Args: urls: List of URLs to crawl crawler_config: Configuration object controlling crawl behavior for all URLs [other parameters maintained for backwards compatibility] Returns: List[CrawlResult]: Results for each URL Nc3$K|]}|du ywr3r4)r5params r8r9z,AsyncWebCrawler.arun_many..s7UuD(7szXBoth crawler_config and legacy parameters provided. crawler_config will take precedence.r>r?) rrvrwrxryrzr~rrrDrITz'bypass_cache' is deprecated and will be removed in version 0.5.0. Use 'cache_mode=CacheMode.BYPASS' instead. Pass warning=False to suppress this warning.rJrKcLKt|j}tj}jj ddd|ij }j }|jvrM|j|z }||kr6|tjd|z}tj|d{|j|< 4d{j| d{cdddd{S7K7377 #1d{7swYyxYww)NzStarted task for {url:.50}...PARALLELrurr)rr) r&netlocrrGdebug mean_delay max_range_domain_last_hitrandomuniformrXsleepr) rudomain current_timerrtime_since_lastdelayr,r^ semaphorers r8crawl_with_semaphorez7AsyncWebCrawler.arun_many..crawl_with_semaphores !#--#yy{  !!;"!3<"$.. ",, T222&2T5J5J65R&RO&3 *V^^Ay-I I%mmE2220<%%f-$9!%'-#-"+"%99 3%%999slB8D$;D<D$D D$D1D 2D5 D$D D$ D$ D D$D!D D!D$z0Starting concurrent crawling for {count} URLs...rpcountrreturn_exceptionszEConcurrent crawling completed for {count} URLs | Total time: {timing}rrr)rrrr)rNrGrIrrOrTrUrVrWryrBYPASSsemaphore_countrX Semaphorerqlenrrgatherrrrrrr)r^rr,rrvrwrxryrzr~rrrrDr7rrrrrrutasksresultsend_timerrs` ` ` @r8 arun_manyzAsyncWebCrawler.arun_manys\$N)(*=?P"J j#7KK'' z%((-A+>):&4",$0$0",& ! ! *55mD::i.MMG+#$ $$,(1(8(8F%$449O))/:I @ KK  JT+  **,J:>?$3)#.$E?#NNEJTJJG((*H KK  _ Y!)J!6s ;1= dkk `gg_fU[*VY"?FS[P_fg g#@J hs1D"G#(G7G#GAG#0$G G#G#cHKtjd{y7w)zClear the cache database.N)rcleanupres r8 aclear_cachezAsyncWebCrawler.aclear_cache7s&&((( " "cHKtjd{y7w)zFlush the cache database.N)r aflush_dbres r8 aflush_cachezAsyncWebCrawler.aflush_cache;s((***rcFKtjd{S7w)z%Get the total number of cached items.N)raget_total_countres r8aget_cache_sizezAsyncWebCrawler.aget_cache_size?s%668888s !!)%__name__ __module__ __qualname____doc__rrrPgetenvrhomer rrrrarfrircrhrdr rtr RegexChunkingrExtractionStrategyChunkingStrategyRelevantContentFilterrrrrr rrrrr4r8r)r))s>~<@*.$)/3!)")),F  "TU!P"#78P'P" P 'tn P  PPd" @" 26!36:2?/48.2!&"'"'#( $$")OO-.O "4 O 0O2O!+OO O O!O !O"#O$%O&'O,-Ob___ # _ % _  ____H26!36:2?/48.2!& $$"Shs)Sh-.Sh "4 Sh 0Sh2Sh!+ShShShShShSh"+ #Shj)+9r$r))?rPrrrUenumrcoloramarrrrpathlibrtypingr r r rrX contextlibr modelsrrasync_databaserrwcontent_filter_strategyrvasync_crawler_strategyrrrrrrrmarkdown_generation_strategyrrcontent_scraping_strategyr async_loggerr async_configsrrr,rrr utilsr!r"r#r$r% urllib.parser&rr'rrr)r4r$r8r4s{ ,,(( *9, &"llII^:%: " 8X 9X 9r$