g#VddlmZddlmZddlZddlZddlZddlZddlddl Z ddl Z ddl m Z ejjejj!ej"ejj%eZedZedZed Zd Zed Zed Zedd efdZedZedZedZedZ ddZ!dZ"e#dk(re"yy)) lru_cache)PathN)*)MODEL_REPO_BRANCHcddl}|jdk(r%|jj|jS|jdk(ryy)Nrcudampsl0)torchtyper get_device_properties total_memory)devicer s J/var/www/openai/venv/lib/python3.12/site-packages/crawl4ai/model_loader.pyget_available_memoryr s> {{fzz//7DDD  c|t|}|jdk(ry|jdvr|dk\ry|dk\ry|dk\ry y y) Ncpu)r r lll@ )rr )ravailable_memorys rcalculate_batch_sizersO+F3 {{e  ' ~ -  /  .rcddl}|jjr|jd}|S|jj jr|jd}|S|jd}|S)Nrr r r)r r is_availablerbackendsr )r rs r get_devicer)sl zz f% M    ( ( *e$ Me$ Mrc@t}|j|||fS)N)rto)modelrs rset_model_devicer#4s \F HHV &=rc(tjjtjdt j d}tj |dtj |ddtj |dd|S)NCRAWL4_AI_BASE_DIRECTORYz .crawl4aiT)exist_okz/cachez/models)ospathjoingetenvrhomemakedirs) home_folders rget_home_folderr.9sg'',,ryy)CTYY[QS^_KKK d+KK;-v&6KK;-w'$7 rcddlm}m}m}m}|j dd}|j dd}|j t|\}}||fS)Nr BertTokenizer BertModel AutoTokenizer AutoModelzbert-base-uncasedresume_download transformersr1r2r3r4from_pretrainedevalr#)r1r2r3r4 tokenizerr"rs rload_bert_base_uncasedr<AsWOO--.ASW-XI  % %&94 % PE JJL$U+ME6 e rreturncddlm}m}m}m}|j |d}|j |d}|j t|\}}||fS)zLoad the Hugging Face model for embedding. Args: model_name (str, optional): The model name to load. Defaults to "BAAI/bge-small-en-v1.5". Returns: tuple: The tokenizer and model. rr0Nr5r7) model_namer1r2r3r4r;r"rs rload_HF_embedding_modelr@JsWPO--j$-OI  % %j$ % GE JJL$U+ME6 e rcddlm}m}ddlm}ddl}|j d}|j d}|j t|\}}|d||}|S)Nr)r3"AutoModelForSequenceClassification)pipelinez1dstefa/roberta-base_topic_classification_nyt_newsztext-classification)r"r;)r8r3rBrCr r9r:r#)r3rBrCr r;r"rpipes rload_text_classifierrE[sXN%--.abI . > >?r sE JJL$U+ME6 )) LD Krc ddlm}m}ddl}ddlmddl d}|j|d |j|djt\jjd fd }|fS)Nr)rBr3)expitzcardiffnlp/tweet-topic-21-multir5c|ddd|}|jDcic]\}}||j}}}j5di|}dddjj j j }|}||k\dz}g} |D]9} t| D cgc]\} } | dk(s | } } } | j| ;| Scc}}w#1swYxYwcc} } w)NptT)return_tensorspadding truncation max_lengthr) itemsr!no_gradlogitsdetachrnumpy enumerateappend)texts thresholdrMtokenskeyvaloutputscores predictions batch_labels predictionivaluelabels class_mappingrrGr"r;r s r _classifierz4load_text_multilabel_classifier.._classifiers5tPTakl6 >uVZ > [E JJL$U+ME6LL))M$  rcddl} |jjd|jjdS#t$r|j dY7wxYw)Nrztokenizers/punktpunkt)nltkdatafind LookupErrordownload)rms rload_nltk_punktrrsN )* 99>>, --  gs<AAc 6ddl}d}t}t||z }|jrt |j sd}t }t|dz }td|jr; tj||jrtj| tjdd d ||t|gtjtjd t|d z }|jd d |d z dz }tj ||tj|td |j't|S#t$r*tdtd|td|YywxYw#tj"$r} td| Yd} ~ yd} ~ wt$$r} td| Yd} ~ yd} ~ wwxYw#t$$r} td| Yd} ~ yd} ~ wwxYw)Nrmodels/reutersz)https://github.com/unclecode/crawl4ai.gitcrawl4aiu7[LOG] ⏬ Downloading Spacy model for the first time...zh[WARNING] Unable to remove existing folders. Please manually delete the following folders and try again:z- gitclonez-bT)stdoutstderrcheckmodels)parentsr&reutersu-[LOG] ✅ Spacy Model downloaded successfullyz0An error occurred while cloning the repository: zAn error occurred: zError loading spacy model: )spacyr.rexistsanyiterdirrprintshutilrmtreePermissionError subprocessrunstrDEVNULLmkdircopytreeCalledProcessError Exceptionload) r~namer- model_folderrepo_urlbranch repo_folder models_folder source_folderes rload_spacy_modelrs D!#K $t+L    !c,*>*>*@&A>";'*4  GH      k*&&(MM,/  NNvx[9IJ!))!))  !-8M   t  <((2Y>M OOM< 8 MM+ & A Bzz#l+,,G# AB;-()<.)*  6,,  DQCH I  's+ ,   +A3/0sO;:E46B#F*G740F'&F'*G4=G G4G//G47 HHHc|rtdt}tjj |dtjj |dg}|D]1}t |j stj|3tdtdt\}}td|tdttdy ) z*Download all models required for Crawl4AI.z![LOG] Removing existing models...rtr{z[LOG] Existing models removed.z$[LOG] Downloading text classifier...z [LOG] Text classifier loaded on z,[LOG] Downloading custom NLTK Punkt model...u-[LOG] ✅ All models downloaded successfully.N) rr.r'r(r)rrrrrjrr)remove_existingr- model_foldersfolder_rs rdownload_all_modelsrs 12%' GGLL&6 7 GGLLh / $FF|""$ f%$ ./ 01/1IAv ,VH 56 89 9:rctdtdtjd}|jddd|j }t |j y) Nz/[LOG] Welcome to the Crawl4AI Model Downloader!zE[LOG] This script will download all the models required for Crawl4AI.zCrawl4AI Model Downloader) descriptionz--remove-existing store_truez)Remove existing models before downloading)actionhelp)r)rargparseArgumentParser add_argument parse_argsrr)parserargss rmainrsX ;< QR  $ $1L MF +LGrs    D(<(<=r__main__)zBAAI/bge-small-en-v1.5)F)$ functoolsrpathlibrrr'rtarfile model_loaderrurllib.requesturllibcrawl4ai.configrr(realpathr)getcwddirname__file__ __location__rrrr#r.r<tupler@rErjrrrrr__name__rNrrrs^ -ww YRYY["''//(:S TU     $        E       ) )V . . : :x;8> zFr