gWddlZddlZddlZddlmZmZddlmZmZm Z m Z m Z m Z m Z ddlZddlZddlZddlZddlZddlmZmZmZmZmZddlmZddlmZddlmZmZm Z ddl!m"Z"dd lm#Z#dd l$m%Z%ddl&Z&ddl'Z'ddl(Z(d d l)m*Z*d d l+m,Z,d dl-m.Z.d dl/m0Z0d dl1m2Z2m3Z3d dl4m5Z5m6Z6d dl7m8Z8ddl9m:Z:m;Z;d dlgdZ?GddZ@GddZAGddeZBGddeBZCy) N)ABCabstractmethod)CallableDictAnyListOptional AwaitableUnion)async_playwrightPageBrowserErrorBrowserContext) TimeoutError)BytesIO)Image ImageDraw ImageFont)Path ProxySettings) BaseModel)load_js_script)AsyncCrawlResponse)get_error_context)UserAgentGenerator)SCREENSHOT_HEIGHT_TRESHOLDDOWNLOAD_PAGE_TIMEOUT) BrowserConfigCrawlerRunConfig) AsyncLogger) StealthConfig stealth_async)SSLCertificateT) webdriver chrome_app chrome_csichrome_load_timeschrome_runtimenavigator_languagesnavigator_pluginsnavigator_permissions webgl_vendorouterdimensionsnavigator_hardware_concurrency media_codecs)z--disable-background-networking%--disable-background-timer-throttlingz(--disable-backgrounding-occluded-windowsz--disable-breakpadz(--disable-client-side-phishing-detectionz4--disable-component-extensions-with-background-pagesz--disable-default-appsz--disable-extensionsz--disable-features=TranslateUIz--disable-hang-monitor!--disable-ipc-flooding-protectionz--disable-popup-blockingz--disable-prompt-on-repostz--disable-sync--force-color-profile=srgbz--metrics-recording-only--no-first-runz--password-store=basicz--use-mock-keychainc eZdZUdZeed<eed<eed<ejed<eed<e ed<eed< ddede ededede f d Z d efd Z d Z d efdZd eefdZdZy )ManagedBrowsera6 Manages the browser process and context. This class allows to connect to the browser using CDP protocol. Attributes: browser_type (str): The type of browser to launch. Supported values: "chromium", "firefox", "webkit". Default: "chromium". user_data_dir (str or None): Path to a user data directory for persistent sessions. If None, a temporary directory may be used. Default: None. headless (bool): Whether to run the browser in headless mode (no visible GUI). Default: True. browser_process (subprocess.Popen): The process object for the browser. temp_dir (str): Temporary directory for user data if not provided. debugging_port (int): Port for debugging the browser. host (str): Host for debugging the browser. Methods: start(): Starts the browser process and returns the CDP endpoint URL. _get_browser_path(): Returns the browser executable path based on OS and browser type. _get_browser_args(): Returns browser-specific command line arguments. _get_user_data_dir(): Returns the user data directory path. _cleanup(): Terminates the browser process and removes the temporary directory. browser_type user_data_dirheadlessbrowser_processtemp_dirdebugging_porthostNc||_||_||_d|_d|_||_||_||_d|_y)a; Initialize the ManagedBrowser instance. Args: browser_type (str): The type of browser to launch. Supported values: "chromium", "firefox", "webkit". Default: "chromium". user_data_dir (str or None): Path to a user data directory for persistent sessions. If None, a temporary directory may be used. Default: None. headless (bool): Whether to run the browser in headless mode (no visible GUI). Default: True. logger (logging.Logger): Logger instance for logging messages. Default: None. host (str): Host for debugging the browser. Default: "localhost". debugging_port (int): Port for debugging the browser. Default: 9222. NF) r9r:r;r<r=r>r?logger shutting_down)selfr9r:r;rAr?r>s T/var/www/openai/venv/lib/python3.12/site-packages/crawl4ai/async_crawler_strategy.py__init__zManagedBrowser.__init__asH.)*  # ,  "returncZK|js,tjd|_|j|_|j }|j } t j|t jt j|_ tj|jtjdd{d|jd|jS7 #t $r,}|j#d{7t!d|d}~wwxYww) z Starts the browser process and returns the CDP endpoint URL. If user_data_dir is not provided, creates a temporary directory. zbrowser-profile-)prefix)stdoutstderrNhttp://:zFailed to start browser: )r:tempfilemkdtempr=_get_browser_path_get_browser_args subprocessPopenPIPEr<asyncio create_task_monitor_browser_processsleepr?r> Exceptioncleanup)rC browser_pathargses rDstartzManagedBrowser.starts!!$,,4FGDM!%D --/ %%' =#-#3#3Z__Z__$D     = = ? @--" " "TYYKq)<)<(=> > # =,,. 7s;< < =sIAD+A4C3C1C30D+1C33 D(<D#DD##D((D+c dK|jrK tjtj|jjj tj|jj j d{\}}|jj|jsk|jjdd|jj|j|jd|jd{y|jjddd|jjiyyy77>#t$rD}|js.|jjd dd t!|iYd}~yYd}~yd}~wwxYww) a Monitor the browser process for unexpected termination. How it works: 1. Read stdout and stderr from the browser process. 2. If the process has terminated, log the error message and terminate the browser. 3. If the shutting_down flag is set, log the normal termination message. 4. If any other error occurs, log the error message. Note: This method should be called in a separate task to avoid blocking the main event loop. Nz\Browser process terminated unexpectedly | Code: {code} | STDOUT: {stdout} | STDERR: {stderr}ERROR)coderJrKmessagetagparamsz2Browser process terminated normally | Code: {code}INFOrbz)Error monitoring browser process: {error}error)r<rVgather to_threadrJreadrKpollrBrArh returncodedecoder[inforZstr)rCrJrKr^s rDrXz'ManagedBrowser._monitor_browser_processsw    '.~~%%d&:&:&A&A&F&FG%%d&:&:&A&A&F&FG(" '',,.:-- ))%C '(,(<(<(G(G*0--/*0--/$*#lln,, (($X &$*D,@,@,K,K#L); ""- ))KK%% K# 'Q0&* s`F0A;E E BE E E $F0%4E F0E E F-)5F( F0(F--F0ctjdk(rdddd}n tjdk(rddd d}nd d d d}|j|jS) z@Returns the browser executable path based on OS and browser typedarwinzr:r;appendrpNotImplementedError)rC base_argsr]s rDrRz ManagedBrowser._get_browser_argss++-.    **4+>+>*?@"4#5#5"67D}} ,-4  ) +)D''("" D }} L)4& d6G6G5H&WX XrFc NKd|_|jr |jjtdD];}|jj nt j dd{=|jj 7|jjt j dd{|jrKtjj|jr! t!j"|jyyy77^#t$r3}|jjdddt|iYd}~d}~wwxYw#t$r3}|jjd ddt|iYd}~yd}~wwxYww) z/Cleanup browser process and temporary directoryT N皙?z"Error terminating browser: {error}rarhrcz+Error removing temporary directory: {error})rBr< terminaterangerlrVrYkillrZrArhrpr=ospathexistsshutilrmtree)rC_r^s rDr[zManagedBrowser.cleanupsb"    $$..0rA++002>!--,,,# '',,.6((--/!--,,, ==RWW^^DMM:  dmm,;=- -  !!@#SV,"   !!I#SV," s~F%AD'3D#4AD'D%D' 5F%E& F%#D'%D'' E#0)EF%E##F%& F"/)FF%F""F%)rsNFN localhosti$)__name__ __module__ __qualname____doc__rp__annotations__boolrSrTintr rEr_rXrQrrRr[rFrDr8r8Bs.N%%%M I''+"## }# #  ##B=S=8,\,3,. 49 2"rFr8cneZdZdZddefdZdZdefdZ dde d e fd Z d Z d e fd Z d efdZdZdZy)BrowserManageraO Manages the browser instance and context. Attributes: config (BrowserConfig): Configuration object containing all browser settings logger: Logger instance for recording events and errors browser (Browser): The browser instance default_context (BrowserContext): The default browser context managed_browser (ManagedBrowser): The managed browser instance playwright (Playwright): The Playwright instance sessions (dict): Dictionary to store session information session_ttl (int): Session timeout in seconds Nbrowser_configc||_||_d|_d|_d|_d|_i|_d|_|jjrpt|jj|jj|jj|j|jj|_yy)a Initialize the BrowserManager with a browser configuration. Args: browser_config (BrowserConfig): Configuration object containing all browser settings logger: Logger instance for recording events and errors Ni)r9r:r;rAr>)configrAbrowserdefault_contextmanaged_browser playwrightsessions session_ttluse_managed_browserr8r9r:r;r>)rCrrAs rDrEzBrowserManager.__init__/s&4   ##  ;; * *#1![[55"kk77--{{#{{99 $D  +rFcK|j(ddlm}|jd{|_|jj r|j jd{}|jjj|d{|_ |jj}|r |d|_ n|jd{|_ |j|jd{y|j}|jjdk(r4|jj j"di|d{|_ n|jjdk(r4|jj$j"di|d{|_ n3|jjj"di|d{|_ |j|_ y777c7&777`7.w)a Start the browser instance and set up the default context. How it works: 1. Check if Playwright is already initialized. 2. If not, initialize Playwright. 3. If managed browser is used, start it and connect to the CDP endpoint. 4. If managed browser is not used, launch the browser and set up the default context. Note: This method should be called in a separate task to avoid blocking the main event loop. Nr)r rtrur)rplaywright.async_apir r_rrrrsconnect_over_cdprcontextsrcreate_browser_context setup_context_build_browser_argsr9rtlaunchru)rCr cdp_urlr browser_argss rDr_zBrowserManager.startNs ?? " =$4$6$<$<$>>DO ;; * * 006688G!%!9!9!J!J7!SSDL||,,H'/{$-1-H-H-J'J$$$T%9%9: : :335L{{''94%CT__%<%<%C%C%Sl%SS ))X5%BT__%;%;%B%B%R\%RR %DT__%=%=%D%D%T|%TT #'<  2 23H I{{))AE!!**+=>KK..!!**+;< ;; ! !"kk44![[55    # #DKK$7$7 8001AB B B!!&@P@T@T U V    / /--%%)).9N*OP P P&G F ; 3$ C  Qsn?IIAII /I4I 5DII'I9I:AI?II I IIIIc K|jjjd|jj}|jj|jj d}|jj rd|jj ind}gd}||||jj|jj|jjd|jjd}|jjrddd }|j||jjd i|d{}|jjr%|D] }|jd |d d{"|S7A7 w) a Creates and returns a new browser context with configured settings. Applies text-only mode settings if text_mode is enabled in config. Returns: Context: Browser context object with the specified configurations rwidthheightrN),jpgjpegpnggifwebpsvgicobmptiffpsdwoffwoff2ttfotfeotmp4webmoggavimovwmvflvm4vmp3wavaacm4aopusflacpdfdocdocxxlsxlsxpptpptxziprar7ztargzxmlswfwasmg?)rviewportrrrignore_https_errorsdevice_scale_factorjava_script_enabledF) has_touch is_mobilez**/*.c"|jSN)abort)routes rDz7BrowserManager.create_browser_context..Os rFr)rrryrrrrrrrrrrr new_contextr) rCrviewport_settingsrblocked_extensionscontext_settingstext_mode_settingsrexts rDrz%BrowserManager.create_browser_contextsZ[[((,,\4;;;Q;QR [[//kk11 ;?++:K:K(DKK$5$56QU (%)# $ < <![[66#';;#B#B#&#';;#B#B   ;; """   # #$6 71 00D3CDD ;; )mmeC5M3NOOO*E Ps$EF F8F =F> F F cK|j|jrg|j|jvrO|j|j\}}}||tjf|j|j<||fS|jj r%|j }|jd{}nJ|jd{}|j||d{|jd{}|jr.||tjf|j|j<||fS77v7^7Hw)ai Get a page for the given session ID, creating a new one if needed. Args: crawlerRunConfig (CrawlerRunConfig): Configuration object containing all browser settings Returns: Page: The page object for the given session ID. BrowserContext: The browser context for the given session ID. N) _cleanup_expired_sessions session_idrtimerrrnew_pagerr)rCrrpagers rDget_pagezBrowserManager.get_pageSs &&(  & &+;+F+F$--+W#}}-=-H-HI GT1:A49UDMM*55 6= ;; * ***G ))++D 7799G$$W.>? ? ? ))++D  & &:A49UDMM*55 6W},9 ?+sIB9E;E <EE E.E /EEAE E EEr%cK||jvrg|j|\}}}|jd{|jjs|jd{|j|=yy7A7w)z Kill a browser session and clean up resources. Args: session_id (str): The session ID to kill. N)rcloserr)rCr%rr(rs rD kill_sessionzBrowserManager.kill_sessionrsl  &#}}Z8 GT1**,  ;;22mmo%% j) ' %s!5A=A9-A=%A;&A=;A=ctj}|jjDcgc]\}\}}}||z |jkDr|}}}}|D]&}t j |j |(ycc}}}w)z'Clean up expired sessions based on TTL.N)r&ritemsrrVrWr,)rC current_timesidr last_usedexpired_sessionss rDr$z(BrowserManager._cleanup_expired_sessionssyy{ +/--*=*=*? *?&&aIi'$*:*:: *?  $C    1 1# 6 7$  s"BcK|jjrtjdd{t |j j }|D]}|j|d{|jr)|jjd{d|_|jrFtjdd{|jjd{d|_ |jr*|jjd{d|_ yy7777k7K7w)z)Close all browser resources and clean up.g?N)rsleep_on_closerVrYlistrkeysr,rr+rr[rstop)rC session_idsr%s rDr+zBrowserManager.closes ;; % %--$ $ $4==--/0 %J##J/ / /& <<,,$$& & &DL   --$ $ $&&..0 0 0#'D ??//&&( ( ("DO  % 0 ' % 0 )sj/ED7AE2D:3/E"D<#/ED>!E4E54E)E*E:E<E>EEEr)F)rrrrr!rEr_dictrrr"rrr)rpr,r$r+rrFrDrr!st }>20hDTDT HQHQ+HQT>B/?> *S * 8#rFrc*eZdZdZededefdZy)AsyncCrawlerStrategyze Abstract base class for crawler strategies. Subclasses must implement the crawl method. rrGc Kywrr)rCrkwargss rDcrawlzAsyncCrawlerStrategy.crawls  sN)rrrrrrprr>rrFrDr;r;s* s 1C  rFr;c ,eZdZdZ d8dedefdZdZdZdZ d Z d e fd Z d e d e fdZd e fdZde fdZdee e ffdZd9dede defdZd9dede defdZdZde fdZde dedefdZde dedefd Zd:ded!efd"Zd#Zdeddfd$Zdede fd%Z!de fd&Z"d'e de fd(Z#dede fd)Z$dede fd*Z%d;d+e de&fd,Z'ded-e(e e)e fdee e*ffd.Z+ded-e(e e)e fdee e*ffd/Z,d0Z-d:ded1e.d2e.d3efd4Z/ded1e.d2e.dee e*ffd5Z0defd6Z1dede2fd7Z3y)<AsyncPlaywrightCrawlerStrategya Crawler strategy using Playwright. Attributes: browser_config (BrowserConfig): Configuration object containing browser settings. logger (AsyncLogger): Logger instance for recording events and errors. _downloaded_files (List[str]): List of downloaded file paths. hooks (Dict[str, Callable]): Dictionary of hooks for custom behavior. browser_manager (BrowserManager): Manager for browser creation and management. Methods: __init__(self, browser_config=None, logger=None, **kwargs): Initialize the AsyncPlaywrightCrawlerStrategy with a browser configuration. __aenter__(self): Start the browser and initialize the browser manager. __aexit__(self, exc_type, exc_val, exc_tb): Close the browser and clean up resources. start(self): Start the browser and initialize the browser manager. close(self): Close the browser and clean up resources. kill_session(self, session_id): Kill a browser session and clean up resources. crawl(self, url, **kwargs): Run the crawler for a single URL. NrrAc |xstj||_||_g|_ddddddddd|_t |j|j|_y)a Initialize the AsyncPlaywrightCrawlerStrategy with a browser configuration. Args: browser_config (BrowserConfig): Configuration object containing browser settings. If None, will be created from kwargs for backwards compatibility. logger: Logger instance for recording events and errors. **kwargs: Additional arguments for backwards compatibility and extending functionality. N)on_browser_createdon_page_context_createdon_user_agent_updatedon_execution_started before_goto after_gotobefore_return_htmlbefore_retrieve_html)rrA)r! from_kwargsrrA_downloaded_fileshooksrbrowser_manager)rCrrAr=s rDrEz'AsyncPlaywrightCrawlerStrategy.__init__sn-Q 0I0I&0Q "$#''+%)$("&$(    ...t{{ rFcBK|jd{|S7wr)r_rCs rD __aenter__z)AsyncPlaywrightCrawlerStrategy.__aenter__sjjl  s c@K|jd{y7wr)r+)rCexc_typeexc_valexc_tbs rD __aexit__z(AsyncPlaywrightCrawlerStrategy.__aexit__sjjls cK|jjd{|jd|jj|jjd{y7I7w)zG Start the browser and initialize the browser manager. NrBr)rMr_ execute_hookrrrOs rDr_z$AsyncPlaywrightCrawlerStrategy.startsi""((***  ( (((88     + s"A.A*AA.$A,%A.,A.cTK|jjd{y7w)z; Close the browser and clean up resources. N)rMr+rOs rDr+z$AsyncPlaywrightCrawlerStrategy.closes ""((***s (&(r%cK|jjdd|jj|d{y7w)z Kill a browser session and clean up resources. Args: session_id (str): The ID of the session to kill. Returns: None zSSession auto-kill is enabled in the new version. No need to manually kill sessions.WARNINGrdreN)rAwarningrMr,)rCr%s rDr,z+AsyncPlaywrightCrawlerStrategy.kill_sessionsA i  ""// ;;;s? ?rFcK|jj|}|r-tj|r||i|d{S||i|S|r|dSdS7w)aH Execute a hook function for a specific hook type. Args: hook_type (str): The type of the hook. *args: Variable length positional arguments. **kwargs: Keyword arguments. Returns: The return value of the hook function, if any. Nr)rLryrViscoroutinefunction)rCr^r]r=r_s rDrXz+AsyncPlaywrightCrawlerStrategy.execute_hook0sdzz~~i( **40!426222T,V,,tAw(D(3s>AAArc||_y)z Update the user agent for the browser. Args: user_agent (str): The new user agent string. Returns: None N)r)rCrs rDupdate_user_agentz0AsyncPlaywrightCrawlerStrategy.update_user_agentDs %rFrc||_y)z Set custom headers for the browser. Args: headers (Dict[str, str]): A dictionary of headers to set. Returns: None N)r)rCrs rDset_custom_headersz1AsyncPlaywrightCrawlerStrategy.set_custom_headersPs  rFr(wait_fortimeoutcPK|j}|jdr.|ddj}|j|||d{S|jdr0|ddj} |j||d{y|jd s|jd r|j|||d{S |j||d{y77a#t$r3}dt |vrt d|d |d td |d d}~wwxYw7d7J#t$rf}dt |vrt d|d |d  |j|d|d|d{7cYd}~S#t$rtd|dwxYwd}~wwxYww)a Wait for a condition in a smart way. This functions works as below: 1. If wait_for starts with 'js:', it assumes it's a JavaScript function and waits for it to return true. 2. If wait_for starts with 'css:', it assumes it's a CSS selector and waits for it to be present. 3. Otherwise, it tries to evaluate wait_for as a JavaScript function and waits for it to return true. 4. If it's not a JavaScript function, it assumes it's a CSS selector and waits for it to be present. This is a more advanced version of the wait_for parameter in CrawlerStrategy.crawl(). Args: page: Playwright page object wait_for (str): The condition to wait for. Can be a CSS selector, a JavaScript function, or explicitly prefixed with 'js:' or 'css:'. timeout (float): Maximum time to wait in milliseconds Returns: None zjs:Nzcss:rjTimeoutzTimeout after zms waiting for selector ''zInvalid CSS selector: 'z()functionz() => {}zInvalid wait_for parameter: 'zp'. It should be either a valid CSS selector, a JavaScript function, or explicitly prefixed with 'js:' or 'css:'.)strip startswithcsp_compliant_waitwait_for_selectorrrprra)rCr(rirjjs_code css_selectorr^s rD smart_waitz)AsyncPlaywrightCrawlerStrategy.smart_wait\s$>>#   u %qrl((*G00wHH H   (#AB<--/L P,,\7,KKK""4(H,?,? ,K!44T8WMMM0070KKK+I L PA&&( 1J<.XYZ%'>|nA%NOO  PNL CF**,WI5NxjXYZ )-)@)@ $ "&=w*$$ %","?zJO!O#sA F& C-(F&7C1 C/C19F& D0 F&D4'D2(D4,F&/C11 D-:.D((D--F&2D44 F#=FF7E:8F<F#=F&FFF##F&user_wait_functioncKd|d|d} |j|d{}|S7#t$r.}dt|vrtdt|Yd}~yd}~wwxYww)a Wait for a condition in a CSP-compliant way. Args: page: Playwright page object user_wait_function: JavaScript function as string that returns boolean timeout: Maximum time to wait in milliseconds Returns: bool: True if condition was met, False if timed out Raises: RuntimeError: If there's an error evaluating the condition z8 async () => { const userFunction = z; const startTime = Date.now(); try { while (true) { if (await userFunction()) { return true; } if (Date.now() - startTime > aR) { return false; // Return false instead of throwing } await new Promise(resolve => setTimeout(resolve, 100)); } } catch (error) { throw new Error(`Error evaluating condition: ${error.message}`); } } NzError evaluating conditionz#Failed to evaluate wait condition: F)evaluaterZrp RuntimeError)rCr(rzrj wrapper_jsresultr^s rDruz1AsyncPlaywrightCrawlerStrategy.csp_compliant_waits~""4!5623: ;  ( ==44FM5 +s1v5"%HQ#QRR  s6 A$*(*A$* A!$AA$A!!A$c K|jdd{}t|D]\}} |jd|dd{|jd{}|ro|j ddd{|jdd{}d |}|j d d }|jd |d |d|dd{n |j jddd|i|S77777k73#t$r5} |j jdd|t| dYd} ~ d} ~ wwxYww)a Process iframes on a page. This function will extract the content of each iframe and replace it with a div containing the extracted content. Args: page: Playwright page object Returns: Playwright page object iframeNz"(element) => element.id = "iframe-"load0urnz() => document.body.innerHTMLzextracted-iframe-content-`z\`zl () => { const iframe = document.getElementById('iframe-zx'); const div = document.createElement('div'); div.innerHTML = `z0`; div.className = 'zf'; iframe.replaceWith(div); } z1Could not access content frame for iframe {index}SCRAPEindexrcz(Error processing iframe {index}: {error}ra)rrh) query_selector_all enumerater| content_framewait_for_load_statereplacerAr]rZrhrp) rCr(iframesirframeiframe_content class_name_iframer^s rDprocess_iframesz.AsyncPlaywrightCrawlerStrategy.process_iframess//99"7+IAv- oo(J1#Q&OPPP%2244334 ,1>>7,&N $=QC!@J-44S%@G--LLM3O..5Y7..8\:    KK'' S$ '|(I,b g: Q5 & "  !!F%&Q8" sEC8EDC:DC<D;C><DD9DD%D4E:D<D>DDD E *D=7E=EErGc,K|jd{|jdxsttj}|jd|j }|j j||d{\}}|S7y7 w)a Creates a new browser session and returns its ID. A browse session is a unique openned page can be reused for multiple crawls. This function is asynchronous and returns a string representing the session ID. Args: **kwargs: Optional keyword arguments to configure the session. Returns: str: The session ID. Nr%r)r_ryrpuuiduuid4rrMr))rCr=r%rr(rs rDcreate_sessionz-AsyncPlaywrightCrawlerStrategy.create_sessionszjjlZZ -BTZZ\1B ZZ doo> "22;;J SS g  Ts"BBA/BB BBrrcK|xstj|}i}d}d}|jdr|j||d{S|jdr|dd}tj j |std|t|dd 5}|j} ddd|jr|j d{}t |||d S|jd s|jd rI|dd d k(r|d dn|dd} | } |jr|j| d{}t| |||d Std7#1swYxYw770w)a Crawls a given URL or processes raw HTML/local file content based on the URL prefix. Args: url (str): The URL to crawl. Supported prefixes: - 'http://' or 'https://': Web URL to crawl. - 'file://': Local file path to process. - 'raw://': Raw HTML content to process. **kwargs: Additional parameters: - 'screenshot' (bool): Whether to take a screenshot. - ... [other existing parameters] Returns: AsyncCrawlResponse: The response containing HTML, headers, status code, and optional screenshot. N)rMzhttps://zfile://zLocal file not found: rutf-8)encoding)htmlresponse_headers status_code screenshotget_delayed_contentzraw:zraw://rmz?URL must start with 'http://', 'https://', 'file://', or 'raw:')r"rJrt _crawl_webrrrFileNotFoundErroropenrkr_generate_screenshot_from_htmlrra) rCrrr=rrscreenshot_datalocal_file_pathfrraw_htmls rDr>z$AsyncPlaywrightCrawlerStrategy.crawls ?+77?  >>1 2f55 5 ^^I &!!"gO77>>/2'*@@Q(RSSosW=vvx>  (,(K(KD(Q"Q%!1'*$(  ^^F #s~~h'?"%bq'V"3s12wQRHD  (,(K(KD(Q"Q%!1'*$(  Q C6>=#R#RsJAE5E" AE5E%/(E5E1A*E5E3 E5%E.*E53E5c nK|_i}d}g_jj}|jrZjj dk7rAt jdNijjxsij_jj|d{\}|jdddgd{|js|js |jr"|jtdd{j!d| d{|j"r. dOfd j%d j%d fd  d}|j&rt)j*}jj,rj%dfd|j.sj!d|d{ t1j2t5j6dj9}j;dd|did{j=|j>|j@d{} j!d|| d{| d}i}n| jH}| jJ}nd}i} jMdddd{jOdd d{} | s3|jPs'jSd{} tCd!| jj\s|j^s |j`r}jcd&d{tejfd'd{jOd(d) d{} | s)jXrjXjid*d+,jj\s|j`r jkd{}|d-}|d.}jjl}to||z|z d/z}jq||d0d{ts||z ||z }jtjwd{}|jyd1||d2d3|d4d{|j|r$j|jd{|jrtj|jd{}|d8s/jXjid9d:d7|jd7i%j!d;| d{|js |jrjjdtG| d} ~ wwxYw7#t$rYwxYw7#tz$r4} jXjdAdBd7tG| i%Yd} ~ d} ~ wwxYw777Y777"7777p7#tz$r} | d} ~ wwxYw#|jsjd{7wwxYww)Qac Internal method to crawl web URLs with the specified configuration. Args: url (str): The web URL to crawl config (CrawlerRunConfig): Configuration object controlling the crawl behavior Returns: AsyncCrawlResponse: The response containing HTML, headers, status code, and optional data Nrandom)rrrrrrCrWc|dk(r.jjd|dd|jiy|dk(r.jjd|dd|jiyy)NrhzConsole error: CONSOLEmsgrcdebugz Console: )rArhtextr)rconsole_log_typerCs rD log_consolz=AsyncPlaywrightCrawlerStrategy._crawl_web..log_consols|$w.KK%%"1# 7% %sxx0& &0KK%%"+C5 1% %sxx0&1rFconsole pageerrorc|dS)Nrhr)r^rs rDrz;AsyncPlaywrightCrawlerStrategy._crawl_web..s :a+ArFdownloadcLtjj|Sr)rVrW_handle_download)rrCs rDrz;AsyncPlaywrightCrawlerStrategy._crawl_web..sW%8%8--h7&rFrF)rr zContent-Security-Policyz-default-src 'self'; script-src 'self' 'nonce-z' 'strict-dynamic') wait_untilrjzFailed on navigating ACS-GOTO: rG)rrresponserbodyattachedr)staterja() => { const element = document.body; if (!element) return false; const style = window.getComputedStyle(element); const isVisible = style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'; return isVisible; }rnzBody element is hidden: zBody visibility info: {info}DEBUGrorcdomcontentloadedrzQ() => Array.from(document.getElementsByTagName('img')).every(img => img.complete)z)Some images failed to load within timeoutrr\rrgffffff?rz"Emulation.setDeviceMetricsOverriderF)rrdeviceScaleFactormobilescalez-Failed to adjust viewport to content: {error}VIEWPORTrhsuccessz)User script execution had issues: {error}JS_EXECrEd ArrowDownzWait condition failed: update_image_dimensionsz(Error updating image dimensions: {error}rarIrH)r(rr)screenshot_height_thresholdz8Exporting PDF and taking screenshot took {duration:.2f}sEXPORTdurationdelayrGcKjjdd|dtj|d{j d{S77w)Nz?Waiting for {delay} seconds before retrieving content for {url}rg)rrrc)rArorVrYcontent)rr(rCrs rDrzFAsyncPlaywrightCrawlerStrategy._crawl_web..get_delayed_contentsX   ]%*37! mmE***!\\^++++s!:AAAAAA)rrrrpdf_datarssl_certificatedownloaded_filesr)r)g@)`rrKrrruser_agent_modergenerateuser_agent_generator_configrMr)rrrrrrX log_consoleonfetch_ssl_certificater&from_urlrjs_onlyhashlibsha256rurandom hexdigestrgotor page_timeoutrr}rpstatusrrvruignore_body_visibilitycheck_visibilityrverboserArrwait_for_imagesadjust_viewport_to_contentrrVrYr]get_page_dimensionsrrset_viewport_sizeminrnew_cdp_sessionsendrZscan_full_page_handle_full_page_scan scroll_delayrwrobust_execute_user_scriptrymousemovedownupkeyboardpressriryPlaywrightTimeoutErrorr|rhrdelay_before_return_htmlremove_overlay_elementsrr& perf_counterr export_pdfrscreenshot_wait_fortake_screenshotrrofloatrr%r+)rCrrrrrrssl_certnoncerr^ is_visiblevisibility_info images_loaded dimensions page_height page_width target_width target_heightrcdpexecution_resultupdate_image_dimensions_jsrstart_export_timerrrrr(s`` @@rDrz)AsyncPlaywrightCrawlerStrategy._crawl_webWs   "$((33 <>'' tWRU'VVV T#NN2::b>:DDFE5515bchbii{3|7&*YY(9(96CVCV&/& H '' dGQT_g'hhh#"%K')$"*//K'/'7'7$" #% ! N,,V:u,UUU$(#:#:"$; $  "&*G*G,0,A,A$,G&GO":?:K LMMv&&00&&&*K*K../ABBBmmC((('+&=&=g '>'! %KK'' K$( &&00V5V5V"'+'?'?'E!EJ",X"6K!+G!4J$(#6#6#E#EL$' z(AK(ORV(V$WM00".-H z 9=;;VWE $ < g'VVV##v||jjooc3///jjoo'''jjmmo%%mm))+666K//foov7J7J*&&00-;O(OP! , , ,&!1'*!$7 (.2.D.DD**$ $$$jjl""%q U Q RRW T&)I#a&'RSSTiV 'H N(,(=(=d(C"C"C;;&&KK%% ># &8& 44":?:K LMM5 NxC(!"F C !KK'' O& 'Q0(M$_W0'%6 !K&)@Q'IJJKV1C KK%% J# 'Q0&8 SD9( d7D#R#  G  $$jjl""%sB.j54b$5 j5b'Aj5b*j57b-8>j57A2i7)b0*i7/Ab9?b3/b9/b60b94i7 c!(i77c-c$c-*c'+&c-c*c-$Ai7&e.'i7e1i7e4 Ai72fe7Afe:4f e= f+f,f0+i7g/i7 g Ai7g ;i7g !i79g:!i7g"i7>g?i7+g;g<g!i7#h9h:h>hhh i77i8i7i2i7i$i7+i",i7i%i7 i(!i7i.$i7i1Aj5i4j5'j5*j5-j50i73b96b99 ccci7$c-'c-*c-- e+6e& d  Ae& i7&e++i71i74i77f:f=ff g )f;5i7;gi7i7 i7 i7i7i7i7g h$g;;hi7h hhhh i")i i7ii7i7i7"i7%i7(i7+i7.i71i74j57 jjjj j2*j-+j22j5rc K |jjd|jj}|}|j |d||d{|j |d{}|d}||krWt ||z|}|j |d||d{|j |d{}|d}||kDr|}||krW|j |ddd{|j |d|d{y777j7S7(7#t$r3}|jjdddt|iYd}~yd}~wwxYww) a Helper method to handle full page scanning. How it works: 1. Get the viewport height. 2. Scroll to the bottom of the page. 3. Get the total height of the page. 4. Scroll back to the top of the page. 5. Scroll to the bottom of the page again. 6. Continue scrolling until the bottom of the page is reached. Args: page (Page): The Playwright page object scroll_delay (float): The delay between page scrolls rr)rNz)Failed to perform full page scan: {error} PAGE_SCANrhrc) viewport_sizeryrr safe_scrollrrrZrAr]rp) rCr(rrcurrent_positionr total_height new_heightr^s rDrz5AsyncPlaywrightCrawlerStrategy._handle_full_page_scans"' :"0044$--==O / ""4, V B / :  KK  CQ(    sEA DDD'D(5DDD6D 7D D#D $D(E>D?EDDD D DE E )EEE  Ec HK |j}tjj|j|}|j j dd||dtj}|j|d{tj}|jj||j jdd||||z dd d y7]#t$r3}|j jd d d t|iYd}~yd}~wwxYww)a Handle file downloads. How it works: 1. Get the suggested filename. 2. Get the download path. 3. Log the download. 4. Start the download. 5. Save the downloaded file. 6. Log the completion. Args: download (Download): The Playwright download object Returns: None z Downloading {filename} to {path}FETCH)filenamerrcNz"Downloaded {filename} successfullyCOMPLETEz.2fs)rrrz"Failed to handle download: {error}rarh)suggested_filenamerrrrrAror&rsave_asrKr|rrZrhrp)rCrr download_path start_timeend_timer^s rDrz/AsyncPlaywrightCrawlerStrategy._handle_downloads$ !)!PQM KK  :$6 N   **,J""=1 1 1((*H  " " ) )- 8 KK  < 2)#+j#8"=Q ?   2  KK  <Q(    sBD"A?C#C!AC# D"!C## D,)DD"DD"c Ktd} |jd|dd{|jdd{y77#t$r3}|jj dddt |i Yd}~yd}~wwxYww) z Removes popup overlays, modals, cookie notices, and other intrusive elements from the page. Args: page (Page): The Playwright page instance rzL (() => { try { a\ return { success: true }; } catch (error) { return { success: false, error: error.toString(), stack: error.stack }; } })() Niz*Failed to remove overlay elements: {error}rrhrc)rr|wait_for_timeoutrZrAr]rp)rCr(remove_overlays_jsr^s rDrz6AsyncPlaywrightCrawlerStrategy.remove_overlay_elements7s,,EF --%,, -  !   '', , ,  -  KK  DQ(    sP B A AA AA B A A B)B<B BB cFK|jdd{}|S7w)z Exports the current page as a PDF. Args: page (Page): The Playwright page object Returns: bytes: The PDF data T)print_backgroundN)r)rCr(rs rDrz)AsyncPlaywrightCrawlerStrategy.export_pdfWs%4889s !!cK|j|d{}|s|j|d{S|j|fi|d{S7:7!7w)a Take a screenshot of the current page. Args: page (Page): The Playwright page object kwargs: Additional keyword arguments Returns: str: The base64-encoded screenshot data N)page_need_scrolltake_screenshot_naivetake_screenshot_scroller)rCr(r= need_scrolls rDrz.AsyncPlaywrightCrawlerStrategy.take_screenshotdsa!11$77 33D99 9766tFvFF F8:Gs1AAAAA AAAArcK ddlm}||}|djd}t}|j |dt j |jjdS#t$r}dt|}|jjdd d |i tjdd d }tj |} t#j$} | j'd|d| t}|j |dt j |jjdcYd}~Sd}~wwxYww)a  Convert the first page of the PDF to a screenshot. Requires pdf2image and poppler. Args: pdf_data (bytes): The PDF data Returns: str: The base64-encoded screenshot data r)convert_from_bytesRGBJPEGformatrz%Failed to take PDF-based screenshot: zPDF Screenshot failed: {error}rarhrci iXblackcolorrrr6r6fillfontN) pdf2imager+convertrsavebase64 b64encodegetvaluernrZrprArhrnewrDrawr load_defaultr) rCrr+images final_imgbufferedr^ error_messageimgdrawr9s rDtake_screenshot_from_pdfz7AsyncPlaywrightCrawlerStrategy.take_screenshot_from_pdfys/ I 4'1Fq ))%0IyH NN8FN 3##H$5$5$78??H H ICCF8LM KK  8/   ))E:W=C>>#&D))+D IIh O$I OyH HHXfH -##H$5$5$78??H H Is6EA0A54E5 E>CEEEEEcK |j|d{}|d}|d}t||jdt}|j ||dd{g}|j }|d} || zdz} t | D]} | | z} |jd| dd{tjd d{|jd d{} tjt| jd }|j|t!d |D}tj"d |dj$|f}d}|D]4}|j'|jd d|f||j(z }6t}|jd }|j+|ddt-j.|j1j3d}||j5d{S777o7U7>7#t6$r}dt9|}|j:j=ddd|itj"d dd}t?j@|}tCjD}|jGd|d|t}|j+|dt-j.|j1j3dcYd}~|j5d{7Sd}~wwxYw#|j5d{7wxYww) a Attempt to set a large viewport and take a full-page screenshot. If still too large, segment the page as before. Requires pdf2image and poppler. Args: page (Page): The Playwright page object kwargs: Additional keyword arguments Returns: str: The base64-encoded screenshot data Nrrrrrzwindow.scrollTo(0, )g{Gz?F full_pager,c34K|]}|jywr)r).0rGs rD zJAsyncPlaywrightCrawlerStrategy.take_screenshot_scroller..s>XcszzXsrBMPU)r/qualityrz*Failed to take large viewport screenshot: z)Large viewport screenshot failed: {error}rarhrcr0r1r2r4r5r7r-r.)$rrryrrrrr|rVrYrrrrr;r|sumr@rpasterr<r=r>r?rnr+rZrprArhrrArrBr)rCr(r=rrrlarge_viewport_heightsegmentsrr num_segmentsry_offsetseg_shotrGrstitchedoffsetrEencodedr^rFrHr9s rDr(z7AsyncPlaywrightCrawlerStrategy.take_screenshot_scrollers= #77==J#G,J$X.K %( 8:TU% !(($0EF   H ..M+H5O'?:a?L<(.mm&9(1$EFFFmmD)))!%5!AAjj!23;;EB$ )>X>>Lyy!):):L(IJHFs{{51Av;?#**$ yH''.H MM(5"M =&&x'8'8':;BB7KG"**,  w> G)AD ! IHQQM KK  C/   ))E:W=C>>#&D))+D IIh O$I OyH HHXfH -##H$5$5$78??H H**,  ! I **,  sMH#HAH#HA H#)H*H#HH# H!DH#9M H! MH#H#H#H#H#!M# L",CL>L"?L%MLML""L%%M9L<:MMcK |jdd{}tj|jd|j d{S7@7#t $r}dt |}|jjddd|i tjd d d }tj|}tj}|jd|d|t!}|j#|dtj|j%jdcYd}~|j d{7Sd}~wwxYw#|j d{7wxYww)z Takes a screenshot of the current page. Args: page (Page): The Playwright page instance Returns: str: Base64-encoded screenshot image FrLNrzFailed to take screenshot: zScreenshot failed: {error}rarhrcr,r0r1r2r4r5r7r-r.)rr=r>rnr+rZrprArhrr@rrArrBrrr<r?) rCr(rr^rFrGrHr9rEs rDr'z4AsyncPlaywrightCrawlerStrategy.take_screenshot_naives= #??J##J/66w?&**,  )@( % I9#a&BM KK  4/  ))E:W=C>>#&D))+D IIh O$I OyH HHXfH -##H$5$5$78??H H**,  % I$**,  sE>AA'AE>AE>AE> E'CE9E:E >E>EE>EE E;4E75E;;E>rcK|jrF|jj|d{}|jjddd|i|S|jj dd y7Dw) a, Exports the current storage state (cookies, localStorage, sessionStorage) to a JSON file at the specified path. Args: path (str): The path to save the storage state JSON file Returns: dict: The exported storage state rNz Exported storage state to {path}rgrrcz5No default_context available to export storage state.r[r\)rrrAror])rCrrs rDexport_storage_statez3AsyncPlaywrightCrawlerStrategy.export_storage_state s{   ..<<$<GGE KK  :~   L KK  O  Hs,A5A3AA5rwc K |jdd{t|tr|g}n|}g}|D]} d} |jd|dd{}tj} |jdd d{tdtj| z |j|r|nddid|dS77u#t$r,}dt|vr|j j dd |jd d d{7n?#t$r3}|j jd dd t|iYd}~nd}~wwxYw |jdd d{7n?#t$r3}|j jddd t|iYd}~nd}~wwxYwddd}n7|j jddd t|idt|d}Yd}~d}~wwxYw7#t$r4}|j jddd t|iYd}~d}~wwxYw#t$rQ}|j jddd t|i|jdt|dYd}~hd}~wwxYw#t$rA}|j jddd t|idt|dcYd}~Sd}~wwxYww)a Executes user-provided JavaScript code with proper error handling and context, supporting both synchronous and async user code, plus navigations. How it works: 1. Wait for load state 'domcontentloaded' 2. If js_code is a string, execute it directly 3. If js_code is a list, execute each element in sequence 4. Wait for load state 'networkidle' 5. Return results Args: page (Page): The Playwright page instance js_code (Union[str, List[str]]): The JavaScript code to execute Returns: Dict[str, Any]: The results of the execution rNzj (async () => { try { a return { success: true }; } catch (err) { return { success: false, error: err.toString(), stack: err.stack }; } })(); zExecution context was destroyedz6Navigation triggered by script, waiting for load stater)rerrrnzNavigation wait failed: {error}rhrc networkidlez!Network idle wait failed: {error}Tz6Navigation triggered, ignoring context destroyed error)rro#Playwright execution error: {error}Frrh,DOM content loaded after script execution inz!DOM content load timeout: {error}rzScript chunk failed: {error}rresults Script execution failed: {error}) r isinstancerpr|rrAror]rhr&printr|rZ) rCr(rwscriptsrhscriptrr^nav_errt1s rDrz9AsyncPlaywrightCrawlerStrategy.robust_execute_user_script%s&f 7**+=> > >'3'")!G!RH"F+I'+}}:!"() 6 ( "ZB"667ISW6XXXLdiik\^N^_&NNV6)T9JKU"j $8 8{ ? "! I<AF KK,,-ekt,u"&*&>&>vu&>&U U U#(" $ 3 3,M(1,3S\+B!4!"!"" "&*&>&>}V[&>&\ \ \#(" $ 3 3,O(1,3S\+B!4!"!"",0(`&F !KK--(M$-(/Q'8. 27Q%HFA IJY  ++$G )$+SV#4,(!HKK%% >% 'Q0& NNus1v#FGGH 7 KK  :Q(   %s1v6 6  7sRK4J'C !J'I CC CI 0H H%H ,I J' K4 J' C H*G?D DD G? E))EG?EG? E=6E97E=<G?= F9)F4/G?4F99AG?9I ?HI H  I)I<I II  J$AJJ'J$$J'' K106K,&K1'K4,K11K4c $K |jdd{t|tr|g}n|}g}|D]} |jd|dd{}t j}|jddd{t dt j|z t j}|jddd{t d t j|z |j |r|nd d id |dS7777I#t$rQ}|jjd d dt|i|j dt|dYd}~ { return new Promise((resolve) => { try { const result = (function() { a' })(); // If result is a promise, wait for it if (result instanceof Promise) { result.then(() => { // Wait a bit for any triggered effects setTimeout(() => resolve({ success: true }), 100); }).catch(error => { resolve({ success: false, error: error.toString(), stack: error.stack }); }); } else { // For non-promise results, still wait a bit for effects setTimeout(() => resolve({ success: true }), 100); } } catch (error) { resolve({ success: false, error: error.toString(), stack: error.stack }); } }); })() rernrfrbz&Network idle after script execution inrTrcrrhrcFrdrgri) rrjrpr|r&rkr|rrArhrZ) rCr(rwrlrhrmrror^s rDexecute_user_scriptz2AsyncPlaywrightCrawlerStrategy.execute_user_scriptsU 7**+=> > >'3'")!G!7H#'==6) *01 !2!$!FHB223Et2TTTH$))+XZJZ[B22=$2OOOBDIIKRTDTUNNV6)T9JK_"t $8 8G ?!JUP HKK%% E% 'Q0& NNus1v#FGGH 7 KK  :Q(   %s1v6 6 7 KK  :Q(   %s1v6 6  7sHE;D!E;DD.DDADD { const element = document.body; if (!element) return false; const style = window.getComputedStyle(element); const isVisible = style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'; return isVisible; } Nr|rCr(s rDrz/AsyncPlaywrightCrawlerStrategy.check_visibilitys(]] $    xyrcK|j|||d{}|dr|j|dzd{|S7'7w)z Safely scroll the page with rendering time. Args: page: Playwright page object x: Horizontal scroll position y: Vertical scroll position Nrr) csp_scroll_tor!)rCr(rvrwrrs rDrz*AsyncPlaywrightCrawlerStrategy.safe_scrollsM))$155 ) '' 5 5 5 6 5sAA AAAAcnK |jd|d|d|d|d|d|d d{}|d s/|jjd d d |jd i |S7:#t$rA}|jj dd d t |i dt |dcYd}~Sd}~wwxYww)aM Performs a CSP-compliant scroll operation and returns the result status. Args: page: Playwright page object x: Horizontal scroll position y: Vertical scroll position Returns: Dict containing scroll status and position information z() => { try { const startX = window.scrollX; const startY = window.scrollY; window.scrollTo(z, a); // Get final position after scroll const endX = window.scrollX; const endY = window.scrollY; return { success: true, startPosition: { x: startX, y: startY }, endPosition: { x: endX, y: endY }, targetPosition: { x: z, y: z\ }, delta: { x: Math.abs(endX - z6), y: Math.abs(endY - a) } }; } catch (e) { return { success: false, error: e.toString() }; } }Nrz Scroll operation failed: {error}SCROLLrhrcz!Failed to execute scroll: {error}Frd)r|rAr]ryrZrhrp)rCr(rvrwrr^s rDryz,AsyncPlaywrightCrawlerStrategy.csp_scroll_to&s1 ==)*+2aS 1345#U1#>445374453 7#F<)$ ##> #VZZ%89$ MKN  KK  ;Q(   !Q  sDB5'A(A&9A(%B5&A(( B216B-'B2(B5-B22B5c@K|jdd{S7w)z Get the dimensions of the page. Args: page: Playwright page object Returns: Dict containing width and height of the page z () => { const {scrollWidth, scrollHeight} = document.documentElement; return {width: scrollWidth, height: scrollHeight}; } Nrsrts rDrz2AsyncPlaywrightCrawlerStrategy.get_page_dimensionses(]]$     ruc K |jdd{}|S7#t$r3}|jjdddt |iYd}~yd}~wwxYww)z Determine whether the page need to scroll Args: page: Playwright page object Returns: bool: True if page needs scrolling z () => { const scrollHeight = document.documentElement.scrollHeight; const viewportHeight = window.innerHeight; return scrollHeight > viewportHeight; } NzDFailed to check scroll need: {error}. Defaulting to True for safety.r{rhrcT)r|rZrAr]rp)rCr(r)r^s rDr&z/AsyncPlaywrightCrawlerStrategy.page_need_scrollvso  $ /!K   KK  ^Q(     s6A !!A ! A)AA AA )NN)r)rr)4rrrrr!r#rErPrUr_r+rpr,rrbrXrfrrhr rryrurrr"rr>rrrrbytesrrrIr(r'r9r`r rrrrqrrrryrrr&rrFrDr@r@s8KO" +" T>S>5>@*d**V[*X>@(99-=9M_9vs#Ss#2Bs#HZs#j 8:8:U8:t/b$4@ T e GsG*"Iu"I"IHK4KcKZ   Dsd2y7Ty7E#tTWy.DYy7^bcfhkck^ly7v`7d`7U3S >=R`7W[\_ad\dWe`7D , d s s 5 ====c3h=~ d "4DrFr@)DrVr=r&abcrrtypingrrrrr r r rrwrrOrSrr r rrrrriorPILrrrpathlibrrpydanticrrjsonr js_snippetrmodelsrutilsruser_agent_generatorrrrr async_configsr!r" async_loggerr#playwright_stealthr$r%rr&stealth_configrr8rr;r@rrFrDrs #HHHWWG++. &&$4E:%;+#' .\\~~#~#B  3 b%9brF