g1) ddlZddlZdejd<ddlmZddlmZmZddlm Z m Z m Z m Z m Z ddlddlddlddldd lmZdd lmZdd lmZddlddlZddlZej6d d GddZy)NfalseTOKENIZERS_PARALLELISM)Path)UrlModel CrawlResult)init_dbget_cached_url cache_urlDB_PATHflush_db)*)List)ThreadPoolExecutor)WebScrapingStrategyignorezBField "model_name" has conflict with protected namespace "model_".)messagecHeZdZddededefdZdZeddedddde f d e d e d e d ed e dedede de defdZeddedddde f dee d e d e d eded e dede de deefdZede dddddfde de de ded e dede defdZde de de dede de d e dedededefdZy) WebCrawlerNFcrawler_strategyalways_by_pass_cacheverbosec~|xs t||_||_tjj tj dtjd|_ tj|jdtj|jddtd|_ y)N)rCRAWL4_AI_BASE_DIRECTORYz .crawl4aiT)exist_okz/cacheF) LocalSeleniumCrawlerStrategyrrospathjoingetenvrhomecrawl4ai_foldermakedirsr ready)selfrrrs I/var/www/openai/venv/lib/python3.12/site-packages/crawl4ai/web_crawler.py__init__zWebCrawler.__init__s 0 a4PY`4a$8!!ww||BII6PRVR[R[R],^`kl D((48 t++,F3dC  cztd|jddtddd|_tdy)Nu([LOG] 🌤️ Warming up the WebCrawlerzhttps://google.com/F)urlword_count_thresholdextraction_strategy bypass_cacherTu'[LOG] 🌞 WebCrawler is ready to crawl)printrunNoExtractionStrategyr$)r%s r&warmupzWebCrawler.warmups@ 89 %!" 4 6    78r(T url_modelprovider api_tokenextract_blocks_flag css_selector screenshotuse_cached_htmlr-chunking_strategyreturnc  v|j|j|| xs t| f|j||d| S)N)r.r7r8)r0r+r1forced) r%r3r4r5r6r,r7r8r9r-r:kwargss r& fetch_pagezWebCrawler.fetch_page*sPtxx MM  9#7#9   #))%!    r( url_modelsc  | xs t} fd} t5} t| j| ||gt |z|gt |z|gt |z|gt |z|gt |z|gt |z|gt |z| gt |z| gt |zg | gt |z}ddd|S#1swYSxYw)Nc0j|g|i|S)N)r?)r3argsr>r%s r&fetch_page_wrapperz2WebCrawler.fetch_pages..fetch_page_wrapperSs"4??9>t>v> >r()r1rlistmaplen)r%r@r4r5r6r,r9r7r8r-r:r>rDexecutorresultss` r& fetch_pageszWebCrawler.fetch_pagesDs2K5I5K ? !X &JZ0K#j/1()C O;)*S_<!NS_4L3z?2$%J7()C O;&'#j/9 XJ/ G"$%"$s B)CCr+r. user_agentc   |xs t}| |_t|ts t dt|t s t dt |t}d} d} d} |s|js t|} | jddr |jsy| r't| d}t| d} |r | d} | sd} | rs|r|jj|tj}t|jj |fi| }tj}| r"t#d|d t%|d ||z d d |r|jj'} |j(||| ||||| | t%| f i| }t%||_|S#t,$rZ}t/|d st1||_t#d|d|j2t5|dd|j2cYd}~Sd}~wwxYw)NzUnsupported extraction strategyzUnsupported chunking strategyr2Tr u[LOG] 🚀 Crawling done for z , success: , time taken: .2f secondsmsgu[ERROR] 🚫 Failed to crawl z , error: F)r+htmlsuccess error_message)r1r isinstanceExtractionStrategy ValueErrorChunkingStrategymaxMIN_WORD_THRESHOLDrr getr$sanitize_input_encoderupdate_user_agenttimecrawlr/booltake_screenshot process_htmlrU ExceptionhasattrstrrRr)r%r+r,r-r:r.r7r8rKrr>cachedscreenshot_dataextracted_contentrTt1t2 crawl_resultes r&r0zWebCrawler.runjs:. Y&9&S=Q=S#.5#+!"57IJ$%FGG!"35EF$%DEE'*+?AS'T$"&$(!#D,E,E+C0F::h-djj0;D(=fQi(H%!*0).%)FT!--?? KB01L1F1F1L1LS1[TZ1[\DB =cU+dSWj\Yghjmohopsgtt|}~!*.*?*?*O*O*Q 1t00dtrkscrapping_strategykv extra_paramsresultrnrsrtrurvrwsectionss r&rdzWebCrawler.process_htmls A )YY[%8%:"17 CA1MBDB1 C1+11 *>!-$jje<9?>@d: # =cUB_`d`i`i`knp`pqt_uu}~>$'STWSX%YZZ" 1NB1OPL,VZZ B-GHHJJw+EJJw+Ezz*b1H (FseH>A:I>I I& I!!I&)NFF)__name__ __module__ __qualname__CrawlerStrategyrbr'r2DEFAULT_PROVIDERr\ RegexChunkingrrgrXrZrr?rrJr0intrdr(r&rrs/W[nr 9)$(/  %26.;o     "     0 ,   :)$(/ %  26.;o$N$$ $ " $$$$0$,$ k $R"46:2?/!& $$";Y;Y"4 ;Y 0 ;Y  ;Y;Y;Y;Y;YzVVV # V #& V "4 V 0VVVVVVr(r)rr`environpathlibrmodelsrrdatabaser r r r r utilsr:r-rtypingrconcurrent.futuresrcontent_scraping_strategyrconfigwarningsrfilterwarningsrrr(r&rs`'. #$)KK "1: *noiir(