g[TddlZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddlm Z ddl mZmZmZmZmZddlmZddlmZmZmZdd lmZmZmZmZdd lmZmZdd l m!Z!m"Z"m#Z#dd l$m%Z%dd l&m'Z'erddl(m)Z)ejTe+Z,dZ-dZ.dZ/ddddddddddde0dee0e fde0dee0dee1deeee0e0fdeeee0e0fdee2de1d e2fd!Z3Gd"d#ejhZ5eeefZ6Gd$d%Z7d&e7ddde0de0de0f d'Z8d&e7d(eee5ee6ffd)Z9d*e6d(dfd+Z:d,ee6ddde0de0de0d(df d-Z;d*e6ddde0de0de0d(df d.Zd*e6d(e>fd2Z?d3d4d(ee6fd5Z@d3d4d6e2d(ee6fd7ZAd3d4d(ee6fd8ZBd9e0d(dfd:ZCy);N)datetime)Path)Lock) TYPE_CHECKINGListOptionalTupleUnion) constants)CommitOperationAdd UploadInfo_fetch_upload_modes)LocalUploadFileMetadataLocalUploadFilePathsget_local_upload_pathsread_upload_metadata)DEFAULT_REVISION REPO_TYPES)DEFAULT_IGNORE_PATTERNSfilter_repo_objectstqdm) _format_size) sha_fileobj)HfApi KT<)revisionprivateallow_patternsignore_patterns num_workers print_reportprint_report_everyapirrepo_id folder_path repo_typer r!r"r#r$r%r&c | td|tvrtdt|t}tj j j stdd|g}nt|tr|g}|tz }|'tjxsd} t| dz d}|j|||d } tjd | | j }t#fd j%d D|| } | Dcgc]}t'|}}tjdt)|dt+|dDcgc]}|t-|j.f}}t1|}t3|Dcgc]#}t5j6t8|||||d%}}|D]}|j;| rt=d|j?ztAj@} tAjBdtAj@|z | k\r/| rtE|j?tAj@}|jGrtIjdn|D]}|jKtj|j?tIjdycc}wcc}wcc}w)zUpload a large folder to the Hub in the most resilient way possible. See [`HfApi.upload_large_folder`] for the full documentation. NzFor large uploads, `repo_type` is explicitly required. Please set it to `model`, `dataset` or `space`. If you are using the CLI, pass it as `--repo-type=model`.z"Invalid repo type, must be one of zProvided path: 'z' is not a directoryr T)r(r*r!exist_okzRepo created: c3~K|]4}|js|jj6ywN)is_file relative_toas_posix).0pathr)s Y/var/www/openai/venv/lib/python3.12/site-packages/huggingface_hub/_upload_large_folder.py z/upload_large_folder_internal..bs4i>VdZ^ZfZfZh  + & / / 1>Vs=$=z**/*)r"r#zFound z candidate files to uploadzRecovering from metadata files)desc)statusr'r(r*r )targetkwargsz zIs done: exiting main loopzUpload is complete!)& ValueErrorrrr expanduserresolveis_dir isinstancestrros cpu_countmax create_repologgerinfor(rglobrlenrr path_in_repoLargeUploadStatusrange threadingThread _worker_jobstartprintcurrent_reporttimesleep_print_overwriteis_doneloggingjoin)r'r(r)r*r r!r"r#r$r%r&nb_coresrepo_urlfiltered_paths_listrelpath paths_listpathsitemsr8_threadsthreadlast_report_tss ` r5upload_large_folder_internalrc0s& I   "=j\JKK#{#..088:K    +K=8LMNN OS )*+..O<<>&Q(Q,* w)W_cdH KK. +,G.ik>N>Nv>Vi%' ObbNa7(g>NaJb KK&Z))CDE *+KL LE $[%2D2DEFL   u %F{# $A  "&$  $   fv,,../YY[N  1 99; '+= = !6!6!89!YY[N >>  LL5 6    KK%%'( LL&']c  s$K,*K1#(K6ceZdZejZejZejZejZejZ y) WorkerJobN) __name__ __module__ __qualname__enumautoSHA256GET_UPLOAD_MODE PREUPLOAD_LFSCOMMITWAITr5reresC TYY[FdiikODIIKM TYY[F 499;Drqrec:eZdZdZdeefdZdefdZde fdZ y)rJzBContains information, queues and tasks for a large upload process.r^cN||_tj|_tj|_tj|_tj|_t|_d|_ d|_ d|_ d|_ d|_ d|_tj |_|jD]}|\}}|j$|jj'|0|j(|jj'|X|j(dk(r(|j*s|j j'||j,s|j j'|t.j1d|j2dy)NrlfszSkipping file z! (already uploaded and committed))r^queueQueue queue_sha256queue_get_upload_modequeue_preupload_lfs queue_commitrlocknb_workers_sha256nb_workers_get_upload_modenb_workers_preupload_lfsnb_workers_commitnb_workers_waitinglast_commit_attemptrnow _started_atsha256put upload_mode is_uploaded is_committedrEdebugrI)selfr^itemr]metadatas r5__init__zLargeUploadStatus.__init__sC 7<{{}@E ">Ckkm 7<{{}F &'/0'-.%&''(48 #<<>JJD"OE8&!!%%d+%%-**..t4%%.x7K7K((,,T2**!!%%d+ ~e.@.@-AAbcdrqreturnc d}d}d}d}d}d}d}d}d} d} d} |j5|jD]\} } | jr| dz } | | jz } | dz } | j|dz }|| jz }| j dk(r|dz }| j |dz }| j r|dz }|| jz }| js|dz }|| jz }t| }tj}|jd}||jz }t|jdd}d}|d|d |d z }|d z }|d z }|d |d| d t|d|d z }|d|d|d t|d|d z }|dkDr |d|dz }|d|d| d t|d|d z }|d| dz }|dz }|d|jdz }|d|j dz }|d|j"dz }|d|j$dz }|d|j&dz }|dz }|cdddS#1swYyxYw)z >G *4+J+J*K3O OG )F)F(GsK KG d&<&<%=SA AG 4#:#:";2> >G x G_YYsB*H4EH44H=c~|j5td|jDcdddS#1swYyxYw)Nc3VK|]!\}}|jxs |j#ywr/rrr3r_rs r5r6z,LargeUploadStatus.is_done.. s+eZd;1hx,,F0F0FFZd'))r{allr^rs r5rUzLargeUploadStatus.is_dones( YYeZ^ZdZdeeYYs3<N) rfrgrh__doc__r JOB_ITEM_Trr@rQboolrUrprqr5rJrJs3Led:.e>==~ffrqrJr8c d}t|}|y|\}}|tjk(rX|d} t||jj ||j5|xjdzc_dddn|tjk(r t!||||||D]}|\} } | j"r| j$dk(r|j&j |@| j$dk(r|j(j |k|jj ||j5|xj*dzc_dddn-|tj,k(r\|d} t/||||||j(j ||j5|xj0dzc_dddn|tj2k(rU t5||||||j5t7j6|_|xj:dzc_dddnV|tj<k(rCt7j>t@|j5|xjBdzc_!ddd#t $rt$rR} tjd| tj|jj |Yd} ~ d} ~ wwxYw#1swYrxYw#t $rt$r7} tjd| tjYd} ~ d} ~ wwxYw#1swYxYw#t $rt$rR} tjd | tj|j&j |Yd} ~ d} ~ wwxYw#1swY>xYw#t $rt$rY} tjd | tj|D]}|j(j |Yd} ~ Ad} ~ wwxYw#1swYxYw#1swYxYw) a Main process for a worker. The worker will perform tasks based on the priority list until all files are uploaded and committed. If no tasks are available, the worker will wait for 10 seconds before checking again. If a task fails for any reason, the item(s) are put back in the queue for another worker to pick up. Read `upload_large_folder` docstring for more information on how tasks are prioritized. NrzFailed to compute sha256: r )r'r(r*r zFailed to get upload mode: rtregularzFailed to preupload LFS: zFailed to commit: )"_determine_next_jobrerk_compute_sha256rxrKeyboardInterrupt ExceptionrEerror traceback format_excrwr{r|rl_get_upload_moderrryrzr}rm_preupload_lfsr~rn_commitrRrrrorSWAITING_TIME_IF_NO_TASKSr) r8r'r(r*r next_jobjobr^rer_rs r5rNrN s AE'v.    U )"" "8D .%,,006((A-(I-- - ' CI`hi" 8))''50..2248))Y6''++D10044T:11Q61I++ +8D 5tg]ef##''-//14/I$$ $ 239W_`-1YY[*((A-(INN " JJ/ 0))Q.)i %  . 9!=>$$&##''-- .  %  ' :1#>?$$&& '(%  5 8<=$$&**..t44 5  %  2 1!56$$&!D''++D1" 2  s&J "K2K><M 5+M,N=O ;/P8+Q K/AK**K/2K;>M,MM MN:(AN55N:=O P5AP00P58QQrc0|j5|jdk(r|jjdkDr|j|t j |jz dkDrX|xjdz c_t jdtjt|jfcdddS|jdk(ru|jjdk\rX|xjdz c_t jdtjt|jfcdddS|jjdk\rY|xjdz c_ t jdtjt|jd fcdddS|jjdkDrg|j dk(rX|xj dz c_t jd tj"t%|jfcdddS|j&jdkDrg|j(dk(rX|xj(dz c_t jd tj*t%|j&fcdddS|jjdkDrh|jdk(rY|xjdz c_ t jd tjt|jd fcdddS|jjdkDrw|j dk(st,j.sX|xj dz c_t jd tj"t%|jfcdddS|j&jdkDrX|xj(dz c_t jdtj*t%|j&fcdddS|jjdkDrY|xjdz c_ t jdtjt|jd fcdddS|jdk(r|jjdkDr|j|t j |jz dkDrX|xjdz c_t jdtjt|jfcdddS|jdk(r|jjdkDr|j&jdk(r|jjdk(r|jjdk(r|j(dk(rv|jdk(rg|j dk(rX|xjdz c_t jdtjt|jfcdddSt1d|j2Drt j5d dddy|xj6dz c_t jdt8dtj:gfcdddS#1swYyxYw)Nri,r z;Job: commit (more than 5 minutes since last commit attempt)rzJob: commit (>100 files ready)rz&Job: get upload mode (>10 files ready)2z5Job: preupload LFS (no other worker preuploading LFS)z.Job: sha256 (no other worker computing sha256)z:Job: get upload mode (no other worker getting upload mode)zJob: preupload LFSz Job: sha256zJob: get upload moderz-Job: commit (1 min since last commit attempt)z Job: commitc3VK|]!\}}|jxs |j#ywr/rrs r5r6z&_determine_next_job..s+aT`[Q&&@(*@*@@T`rz.All files have been processed! Exiting worker.zNo task available, waiting... (zs))r{rrzqsizerrRrErrern_get_items_to_commitrxr}rl_get_nryr~rm_get_onerwr|rkr HF_HUB_ENABLE_HF_TRANSFERrr^rFrrro)r8s r5rrts   $ $ )##))+a/**6 f8886A  $ $ ) $ LLV W$$&:6;N;N&OP  % % *v/B/B/H/H/Jc/Q  $ $ ) $ LL9 :$$&:6;N;N&OP! & ) ) / / 1R 7  - - 2 - LLA B--vf6R6RTV/WX- 2 ' ' - - /! 38W8W[\8\  + +q 0 + LLP Q++Xf6P6P-QR9 > & & (1 ,1I1IQ1N  $ $ ) $ LLI J$$hv/B/B&CDE J ) ) / / 1A 5&:[:[_`:`  - - 2 - LLU V--vf6R6RTV/WXQ X ' ' - - /! 3  + +q 0 8[8[  + +q 0 + LL- .++Xf6P6P-QRc h & & (1 ,  $ $ ) $ LL '$$hv/B/B&CDo t ) ) / / 1A 5  - - 2 - LL/ 0--vf6R6RTV/WX{ B  $ $ )##))+a/**6 f8886A  $ $ ) $ LLH I$$&:6;N;N&OPQ Z  $ $ )##))+a/##))+q0,,2249**002a7((A-11Q6//14  $ $ ) $ LL '$$&:6;N;N&OPq vaTZT`T`a a KKH I{ B  % % * % LL:;S:TTVW XNNB'G s_B+\ A:\ A,\ \ 2\ ?A\  \rc|\}}|jB|jjd5}t|j |_ddd|j |y#1swYxYw)z1Compute sha256 of a file and save it in metadata.Nrb)r file_pathopenrhexsave)rr]rfs r5rrsWOE8 __ ! !$ '1)!n002HO( MM%( 's A&&A/r^c|Dcgc] }t|}}t||||j|t||D]=\}}|\}} |j| _|j | _| j|?ycc}w)zmGet upload mode for each file and update metadata. Also receive info if the file should be ignored. ) additionsr*r(headersr N) _build_hacky_operationr_build_hf_headerszip _upload_moder_should_ignorerr) r^r'r(r*r rradditionr]rs r5rrs ;@@%$'-%I@%%' eY/hx'44!)!8!8 e 0AsBc|\}}t|}|j||||gd|_|j|y)z'Preupload LFS file and update metadata.)r(r*r rTN)rpreupload_lfs_filesrr)rr'r(r*r r]rrs r5rrsLOE8%d+H*  H MM%rqc|Dcgc] }t|}}|j||||d|D]\}}d|_|j|ycc}w)zCommit files to the repo.z(Add files using upload-large-folder tool)r(r*r operationscommit_messageTN)r create_commitrr) r^r'r(r*r rrr]rs r5rr sh:?@%$'-%I@A !x $ e!AsAceZdZddZy)HackyCommitOperationAddNcnt|jtrt|j|_yyr/)r?path_or_fileobjrr@rs r5 __post_init__z%HackyCommitOperationAdd.__post_init__!s* d**D 1#&t';';#!=AIEIIK!= >> >sAcg}d\}} |jdk(r|S|tk\s |tk\r|S|j}|j ||\}}|j dk(r|dz }n|dz }l)zXSpecial case for commit job: the number of items to commit depends on the type of files.)rrrrtr )rMAX_NB_LFS_FILES_PER_COMMITMAX_NB_REGULAR_FILES_PER_COMMITrappendr)rur^r nb_regularrr_rs r5rr>s!EFJ  5;;=A L 0 0JBa4aLuyy{ T 8   5 ( aKF !OJ! rqreportc$|dz }tjjtfd|j D}t |D]@}t jjdt jjdBt jj|t jjdt|j dz zt jjy)zPrint a report, overwriting the previous lines. Since tqdm in using `sys.stderr` to (re-)write progress bars, we need to use `sys.stdout` to print the report. Note: works well only if no other process is writing to `sys.stdout`! rc3@K|]}t|zdzyw)r N)rH)r3lineterminal_widths r5r6z#_print_overwrite..cs"S?Rt3t9.2?Rsz zrN) shutilget_terminal_sizecolumnssum splitlinesrKsysstdoutwriterHflush)rnb_linesr_rs @r5rTrTVs dNF--/77NSv?P?P?RSSH8_ $ " JJVJJSNS1B1B1DR1H-IIJKJJrq)DrirVrArurrrLrRrrpathlibrrtypingrrrr r r _commit_apir rr _local_folderrrrrrrutilsrrrutils._cache_managerr utils.sharhf_apir getLoggerrfrErrrr@rintrcEnumrerrJrNrrrrrrrrrrrTrprqr5rs   >>LLvv3EE."   8 $"$!#"6:7;!% d( d( d(sDy!d(  d( sm d(d^d(U49c>23d(eDIsN34d(#d(d(d(X ')@@ A cfcfLd/ d/ d/d/ d/  d/Nd( 1d(huYPTU_P`E`?a6bd(X*D,7SUXdglp(  ' C C [^ cg  4 # ' C C [^ cg (=0= 0G -$z2B?+??Z8H? 9d:>N0STrq