g NUddlmZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z ddl m Z ddlmZddlmZmZmZmZmZmZmZmZmZerddlZddlmZedZnedZed Zej>d Z ee!e!ge"fZ#Gd d Z$Gd de$Z%Gdde$Z&Gdde$Z'Gdde$Z(Gdde$Z)Gdde$Z*Gdde$Z+GddeeefZ,Gdde$Z-de$iZ.de/d <d"d#d!Z0e$e%e)e&e(e'e*e+e-f D] Z1e0e1 y)$) annotationsN)FutureThreadPoolExecutor)groupby) itemgetter) TYPE_CHECKINGAnyCallableClassVarGeneric NamedTupleOptional OrderedDictTypeVar) ParamSpecPTfsspeccHeZdZUdZdZded<d dZd dZd dZddZ dd Z y ) BaseCacheagPass-though cache: doesn't keep anything, calls every time Acts as base class for other cachers Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is none ClassVar[str]namecf||_d|_||_||_d|_d|_d|_yNr) blocksizenblocksfetchersize hit_count miss_counttotal_requested_bytes)selfrrrs C/var/www/openai/venv/lib/python3.12/site-packages/fsspec/caching.py__init__zBaseCache.__init__<s4"   %&"ct|d}| |j}||jk\s||k\ry|j||S)Nrr&)rrr#startstops r$_fetchzBaseCache._fetchFs@ =E <99D DII $||E4((r&c.d|_d|_d|_y)zAReset hit and miss counts for a more ganular report e.g. by file.rN)r r!r"r#s r$ _reset_statszBaseCache._reset_statsOs%&"r&c |jdk(r|jdk(ryd|jd|jd|jd|jd S)z2Return a formatted string of the cache statistics.rz , z: z hits, z misses, z total requested bytes)r r!rr"r-s r$ _log_statszBaseCache._log_statsUs^ >>Q 4??a#7TYYKr$..!18ISWSmSmRnoEF Fr&cd|jjd|jd|jd|jd|j d|j d|jdS) Nz ) __class____name__rrrr r!r"r-s r$__repr__zBaseCache.__repr__\s .. ! ! "#!^^,-!\\N+!YYK(!^^,-!__-.$$($>$>#?@   r&NrintrFetcherrr7returnNoner) int | Noner*r<r9bytesr9r:)r9str) r4 __module__ __qualname____doc__r__annotations__r%r+r.r1r5r&r$rr*s- !D- ')' F  r&rcdeZdZdZdZ d d fd Zd dZd dZd dZd dZ xZ S) MMapCachezmemory-mapped sparse file cache Opens temporary file, which is filled blocks-wise when data is requested. Ensure there is enough disc space in the temporary location. This cache method might only work on posix mmapct||||| tn||_||_|j |_yN)superr%setblockslocation _makefilecache)r#rrrrMrLr3s r$r%zMMapCache.__init__ts: GT2%~ce6   ^^% r&c4ddl}ddl}|jdk(r tS|j)t j j|js|j |j}t|_ nt|jd}|j|jdz |jd|jnt|jd}|j|j|jS)Nrzwb+1zr+b)rGtempfiler bytearrayrMospathexists TemporaryFilerKrLopenseekwriteflushfileno)r#rGrSfds r$rNzMMapCache._makefiles 99>;  == t}}(E}}$++-!e $--/ GGDIIM " HHTN HHJdmmU+Btyydii00r&c tjd|d||d}| j}|jk\s||k\ry|jz}|jz}t ||dz}fd|D}xj t fd|Dz c_tt|d D]\}}tttd|}|djz} t|d jzjzj} xj| | z z c_ tjd |dd|d d | d| d j| | j| | j j#|xj$t'|z c_j||S)NzMMap cache fetching -rr&rQc3@K|]}|jvs|ywrIrL.0ir#s r$ z#MMapCache._fetch..s?;a!4;;*>;c3@K|]}|jvsdyw)rQNrbrcs r$rfz#MMapCache._fetch..sIAT[[8Hargc|d|dz S)NrrQrD)xs r$z"MMapCache._fetch..s1!r&)keyzMMap get blocks z ())loggerdebugrrranger sumr enumeratetuplemaprminr"rrOrLupdater!len) r#r)end start_block end_block block_rangeneed__blockssstartsends ` r$r+zMMapCache._fetchs +E7!C59: =E ;))C DII #t~~- 4>>) KQ7 ?;? #IIII")D/7LMMJAwC 1 w78GQZ$..0Fwr{T^^3dnnDdiiPD  & &$- 7 & LL"71:,a }BvhavQO '+ll64&@DJJvd # KK  w ' OOs7| +O%N(zz%$$r&c@|jj}|d=|S)NrO)__dict__copyr#states r$ __getstate__zMMapCache.__getstate__s  ""$ 'N r&cd|jj||j|_yrI)rrwrNrOrs r$ __setstate__zMMapCache.__setstate__s" U#^^% r&)NN) rr7rr8rr7rMz str | NonerLzset[int] | Noner9r:)r9zmmap.mmap | bytearrayr)r<ryr<r9r=r9dict[str, Any]rrr9r:) r4r@rArBrr%rNr+rr __classcell__r3s@r$rFrFisk D $"& & & & &  &  &  &1,,%\ &r&rFc0eZdZdZdZdfd ZddZxZS)ReadAheadCachea!Cache which reads only when we get beyond a block of data This is a much simpler version of BytesCache, and does not attempt to fill holes in the cache or keep fragments alive. It is best suited to many small reads in a sequential order (e.g., reading lines from a file). readaheadcRt||||d|_d|_d|_y)Nr&r)rJr%rOr)ryr#rrrr3s r$r%zReadAheadCache.__init__s) GT2  r&c^|d}|||jkDr |j}||jk\s||k\ry||z }||jk\rM||jkr>|xjdz c_|j||jz ||jz S|j|cxkr|jkrOnnL|xj dz c_|j||jz d}|t |z}|j}n|xj dz c_d}t|j||jz}|xj||z z c_ |j|||_||_|jt |jz|_||jd|zSNrr&rQ) rr)ryr rOr!rxrvrr"r)r#r)rylparts r$r+zReadAheadCache._fetchse =E ;# /))C DII # %K DJJ 3$((? NNa N::edjj033CD D ZZ5 +488 + OOq O::edjj023D TNAHHE OOq OD$))S4>>12 ""cEk1"\\%-  ::DJJ/djj!n$$r&r6rr4r@rArBrr%r+rrs@r$rrs D %r&rc0eZdZdZdZdfd ZddZxZS)FirstChunkCachezCaches the first block of a file only This may be useful for file types where the metadata is stored in the header, but is randomly accessed. firstcD||kDr|}t||||d|_yrI)rJr%rOrs r$r%zFirstChunkCache.__init__s( t I GT2#' r&cn|xsd}||jkDrtjdyt||j}||jkr&|j |xj dz c_||jkDr@|xj|z c_|jd|}|d|j|_||dS|jd|j|_|xj|jz c_|j ||}||jkDrA|xj||jz z c_||j|j|z }|xjdz c_ |S|xj dz c_|xj||z z c_|j||S)Nrz,FirstChunkCache: requested start > file sizer&rQ) rrorprvrrOr!r"rr )r#r)rydatars r$r+zFirstChunkCache._fetchsh  499  LLG H#tyy! 4>> !zz!1$'..#5.<<3/D!%&6!7DJ<'!\\!T^^< **dnn<*::eC(DT^^#**cDNN.BB* T^^S99 NNa NK OOq O  & &#+ 5 &<<s+ +r&r6rrrs@r$rrs D(,r&rceZdZdZdZ d d fd ZdZd dZd dZddZ dfd Z dd Z xZ S) BlockCachea Cache holding memory as a set of blocks. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. blockcachect||||tj||z |_||_t j||j|_ yrI) rJr%mathceilr maxblocks functools lru_cache _fetch_block_fetch_block_cachedr#rrrrr3s r$r%zBlockCache.__init__FsR GT2yy !12 "#A9#6#6y#A$BSBS#T r&c6|jjSz The statistics on the block cache. Returns ------- NamedTuple Returned directly from the LRU Cache used internally. r cache_infor-s r$rzBlockCache.cache_infoN''2244r&c$|j}|d=|S)Nrrrs r$rzBlockCache.__getstate__Ys  ' ( r&c|jj|tj|d|j|_y)Nr)rrwrrrrrs r$rzBlockCache.__setstate__^s< U##J9#6#6u[7I#J   $  r&c|d}| |j}||jk\s||k\ry||jz}||jz}t||dzD]}|j||j ||||S)Nrr&rQstart_block_numberend_block_number)rrrqr _read_cache)r#r)ryrr block_numbers r$r+zBlockCache._fetchds =E ;))C DII ##dnn4$..0""46F6JKL  $ $\ 2L  1-    r&c@||jkDrtd|d|jd||jz}||jz}|xj||z z c_|xjdz c_t j d|t|!||}|S)= Fetch the block of data for `block_number`. 'block_number=(' is greater than the number of blocks (rnrQzBlockCache fetching block %d) r ValueErrorrr"r!roinforJr+)r#rr)ryblock_contentsr3s r$rzBlockCache._fetch_block{s $,, & /))-a9  t~~-dnn$ ""cEk1" 1 2LAs3r&c ||jz}||jz}|xjdz c_||k(r|j|}|||S|j||dg}|jt |jt |dz||j |j|d|dj|Sz Read from our block cache. Parameters ---------- start, end : int The start and end byte positions. start_block_number, end_block_number : int The start and end block numbers. rQNr&rr rextendrurqappendjoin r#r)ryrr start_posend_posblockouts r$rzBlockCache._read_cachesDNN* & ! !1 1334FGE7+ +++,>? KLC JJ,,,q02BC  JJt//0@A(7K L88C= r& rr7rr8rr7rr7r9r:rrr)rr7r9r= r)r7ryr7rr7rr7r9r=) r4r@rArBrr%rrrr+rrrrs@r$rr-s, DMOUU'.U69UFIU U 5   .$&!&!"&!8;&!OR&! &!r&rcZeZdZUdZdZded< d d fd Zd dZd dZxZ S) BytesCacheaKCache which holds data in a in-memory bytes object Implements read-ahead by the block size, for semi-random reads progressing through the file. Parameters ---------- trim: bool As we read more data, whether to discard the start of the buffer when we are more than a blocksize ahead of it. r=rrc`t||||d|_d|_d|_||_y)Nr&)rJr%rOr)rytrim)r#rrrrr3s r$r%zBytesCache.__init__s2 GT2 !% # r&c|d}| |j}||jk\s||k\ry|jc||jk\rT|jH||jkr9||jz }|xjdz c_|j|||z|z S|j r$t |j||j z}n|}||k(s||jkDry|j||jkrh|j||jkDrM|xj||z z c_|xjdz c_|j|||_||_n|jJ|jJ|xjdz c_||jkr|j|j|z |j kDr8|xj||z z c_|j|||_||_n4|xj|j|z z c_|j||j}||_||jz|_n|j||jkDr|j|jkDrn||jz |j kDr7|xj||z z c_|j|||_||_nR|xj||jz z c_|j|j|}|j|z|_|jt|jz|_||jz }|j|||z|z }|jrq|j|jz |j dzz}|dkDrC|xj|j |zz c_|j|j |zd|_|Sr) rr)ryr rOrrvr"r!rrxr)r#r)ryoffsetbendnewrnums r$r+zBytesCache._fetchsJ =E ;))C DII # JJ "#$dhhTZZ'F NNa N::fv|e';< < >>tyy#"67DD 5=EDII- JJ %$**"4 HH dhh  & &$, 6 & OOq OeT2DJDJ::) ))88' '' OOq Otzz!88#txx#~'F..$,>.!%eT!:DJ!&DJ..$**u2DD.,,udjj9C!&DJ!$tzz!1DJ%$/88dii'488^dnn4..$,>.!%eT!:DJ!&DJ..$/A.,,txx6C!%c!1DJ::DJJ/#jj&3,"67 9988djj(dnnq.@ACQw dnns22 !ZZ(<(>?  r&c,t|jSrI)rxrOr-s r$__len__zBytesCache.__len__s4::r&)T) rr7rr8rr7rboolr9r:r)r9r7) r4r@rArBrrCr%r+rrrs@r$rrsT "D-!IM'.69AE GRr&rcXeZdZUdZdZded< d dfd Zd dZxZS) AllBytesz!Cache entire contents of the fileallrrct|||||P|xjdz c_|xj|jz c_|j d|j}||_y)NrQr)rJr%r!r"rrr)r#rrrrr3s r$r%zAllBytes.__init__ sY GT2 < OOq O  & &$)) 3 &<<499-D r&cJ|xjdz c_|j||S)NrQ)r rr(s r$r+zAllBytes._fetch.s! !yyt$$r&)NNNN) rr<rzFetcher | Nonerr<rz bytes | Noner9r:r; r4r@rArBrrCr%r+rrs@r$rrsX+D-!%"&!         %r&rc\eZdZUdZdZded< d dfd Zd fd ZxZS) KnownPartsOfAFilea Cache holding known file parts. Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is data: dict A dictionary mapping explicit `(start, stop)` file-offset tuples with known bytes. strict: bool, default True Whether to fetch reads that go beyond a known byte-range boundary. If `False`, any read that ends outside a known part will be zero padded. Note that zero padding will not be used for reads that begin outside a known byte-range. partsrrc t||||||_|rt|j }|dg}|j |dg} |ddD]m\} } |d\} } | | k(r&| | f|d<| dxx|j | | fz cc<9|j | | f| j |j | | fott|| |_ yi|_ y)NrrQrm) rJr%strictsortedkeyspoprdictzipr)r#rrrrrr~ old_offsetsoffsetsrLr)r*start0stop0r3s r$r%zKnownPartsOfAFile.__init__Ls GT2   -K"1~&Ghh{1~./F*12 t ' E>#)4.GBK2J$((E4="99JNNE4=1MM$((E4="9: /S&12DIDIr&ct|d}| |j}d}|jjD]t\\}}}||cxkr|ksn||z }||||z|z }|jr||cxkr|kr3nn0|d||z t |z zz }|xj dz c_|cS|}n|j td||fdtjd||fdtjd|d ||xj||z z c_ |xjdz c_ |t|=||zS) Nrr&rQz&Read is outside the known file parts: z. z%. IO/caching performance may be poor!z!KnownPartsOfAFile cache fetching r`)rritemsrrxr rrwarningswarnrorpr"r!rJr+) r#r)r*rloc0loc1roffr3s r$r+zKnownPartsOfAFile._fetchjsX =E <99D"&))//"3 LT4$u#t#dl3te!34{{dd&:d&: 7dUlSX&=>>CNNa'NJ !E'#40 << Eudm_TVWX X  4eT]OD2 3   8qGH ""dUl2" 1UW^E4000r&)NT) rr7rr8rr7rz&Optional[dict[tuple[int, int], bytes]]rrr~r r;rrs@r$rr3se,"D-!8<   5   <+1+1r&rcPeZdZdZGddeZd d dZd dZd dZddZ ddZ y ) UpdatableLRUzg Custom implementation of LRU cache that allows updating keys Used by BackgroudBlockCache c6eZdZUded<ded<ded<ded<y)UpdatableLRU.CacheInfor7hitsmissesmaxsizecurrsizeN)r4r@rArCrDr&r$ CacheInfors    r&rctj|_||_||_d|_d|_tj|_ yr) collectionsr_cache_func _max_size_hits_misses threadingLock_lock)r#funcmax_sizes r$r%zUpdatableLRU.__init__s<+6+B+B+D  !  ^^% r&c^|rtd|j|j5||jvrH|jj ||xj dz c_|j|cdddS ddd|j |i|}|j5||j|<|xjdz c_t|j|jkDr|jjdddd|S#1swYxYw#1swY|SxYw)Nz Got unexpected keyword argument rQFlast) TypeErrorrr r move_to_endrrrrxrpopitem)r#argskwargsresults r$__call__zUpdatableLRU.__call__s >v{{}oNO O ZZt{{" ''- a {{4( Z" T,V, ZZ &DKK  LLA L4;;$..0 ###/   Z  sA D)A#D"D"D,cb|j5||jvcdddS#1swYyxYwrI)r r)r#rs r$ is_key_cachedzUpdatableLRU.is_key_cacheds ZZ4;;&ZZs%.c|j5||j|<t|j|jkDr|jj ddddy#1swYyxYw)NFr)r rrxrr)r#rrs r$add_keyzUpdatableLRU.add_keysK ZZ &DKK 4;;$..0 ###/ZZs AA$$A-c|j5|j|jt|j|j |j cdddS#1swYyxYw)N)rrrr)r rrrxrrrr-s r$rzUpdatableLRU.cache_infosE ZZ>>T[[)ZZ|| "ZZs AAA&N))r zCallable[P, T]r r7r9r:)rzP.argsrzP.kwargsr9r)rr r9r)rrrr r9r:r9r) r4r@rArBr rr%rrrrrDr&r$rrs, J &&'0 r&rceZdZUdZdZded< d d fd ZddZddZddZ dd Z ddfd Z dd Z xZ S)BackgroundBlockCachea Cache holding memory as a set of blocks with pre-loading of the next block in the background. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. If the next block is not in cache, it is loaded in a separate thread in non-blocking way. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. backgroundrrct||||tj||z |_||_t |j||_td|_ d|_ d|_ tj|_y)NrQ max_workers)rJr%rrrrrrrr_thread_executor_fetch_future_block_number _fetch_futurerr _fetch_future_lockrs r$r%zBackgroundBlockCache.__init__ss GT2yy !12 "#/0A0A9#M 2q A6:'37"+.."2r&c6|jjSrrr-s r$rzBackgroundBlockCache.cache_inforr&c<|j}|d=|d=|d=|d=|d=|S)Nrr$r%r&r'rrs r$rz!BackgroundBlockCache.__getstate__s<  ' ( $ % . / / " & ' r&c|jj|t|j|d|_t d|_d|_d|_tj|_ y)NrrQr") rrwrrrrr$r%r&rr r'rs r$rz!BackgroundBlockCache.__setstate__sZ U##/0A0A5CU#V 2q A*.'!"+.."2r&c|d}| |j}||jk\s||k\ry||jz}||jz}d}d}|j5|j|jJ|jj rbt jd|jj|jj|jd|_d|_nKt||jcxkxr|knc}|r&|j}|j}d|_d|_ddd|?t jd|jj|j|t||dzD]}|j||dz} |j5|j]| |jkrN|jj| s3| |_|jj!|j"| d|_ddd|j%||||S#1swYxYw#1swY+xYw)Nrr&z3BlockCache joined background fetch without waiting.z(BlockCache waiting for background fetch.rQasyncr)rrr'r&r%donerorrrrrrqrrr$submitrr) r#r)ryrrfetch_future_block_number fetch_future must_joinrend_block_plus_1s r$r+zBackgroundBlockCache._fetchsL =E ;))C DII ##dnn4$..0$(!  $ $!!-66BBB%%**,KK UV,,44**113T5T5T7;D3)-D&!%*::,+,!I !594S4S1'+'9'9 ;?7-1*7%<  # KKB C  $ $ , ,##%'@  ""46F6JKL  $ $\ 2L ,a/  $ $""*$ 400>>?OP2B/%)%:%:%A%A%%'7&"%  1-    o% $X% $sC"H>7A*I >I IcB||jkDrtd|d|jd||jz}||jz}tj d|||xj ||z z c_|xj dz c_t|!||}|S)rrrrnz!BlockCache fetching block (%s) %drQ) rrrrorr"r!rJr+)r#rlog_infor)ryrr3s r$rz!BackgroundBlockCache._fetch_blockes $,, & /))-a9  t~~-dnn$ 7<P ""cEk1" 1s3r&c ||jz}||jz}|xjdz c_||k(r|j|}|||S|j||dg}|jt |jt |dz||j |j|d|dj|Srrrs r$rz BackgroundBlockCache._read_cachewsDNN* & ! !1 1,,-?@E7+ +++,>? KLC JJ,,,q02BC  JJt//0@A(7K L88C= r&rrrrr>r)sync)rr7r4r?r9r=r)r4r@rArBrrCr%rrrr+rrrrs@r$rrs2'D-&MO 3 3'. 369 3FI 3  3 53J X$(!(!"(!8;(!OR(! (!r&rz!dict[str | None, type[BaseCache]]cachescr|j}|s |tvrtd|dt||t|<y)z'Register' cache implementation. Parameters ---------- clobber: bool, optional If set to True (default is False) - allow to overwrite existing entry. Raises ------ ValueError zCache with name z is already known: N)rr7r)clsclobberrs r$register_cacher;s= 88D tv~+D83Fvd|nUVVF4Lr&)F)r9ztype[BaseCache]r:rr9r:)2 __future__rrrloggingrrUrrconcurrent.futuresrr itertoolsroperatorrtypingrr r r r r rrrrGtyping_extensionsrrr getLoggerror7r=r8rrFrrrrrrrrr7rCr;crDr&r$rEsa" 9   +#A A CL   8 $ C:u$ %< < ~e& e&P+%Y+%\+,i+,\F!F!RbbJ%y%0b1 b1J971a4=9xK!9K!` )-) (   A1 r&