
    g?!                        U d dl mZ d dlZd dlZd dlZd dlmZ d dlmZ 	 d dl	Z
er$d dlmZmZmZmZ d dlmZ ddlmZ eeef   Zd	ed
<    G d d      Zy# e$ r	 esd dl
Z
Y >w xY w)    )annotationsN)TYPE_CHECKING)atomic_write)AnyDictIteratorLiteral)	TypeAlias   )CachingFileSystemr
   Detailc                  ~    e Zd ZdZddZddZddZ	 d	 	 	 ddZ	 	 	 	 	 	 ddZddZ	ddZ
dd	Zdd
ZddZddZy)CacheMetadataa\  Cache metadata.

    All reading and writing of cache metadata is performed by this class,
    accessing the cached files and blocks is not.

    Metadata is stored in a single file per storage directory in JSON format.
    For backward compatibility, also reads metadata stored in pickle format
    which is converted to JSON when next saved.
    c                J    |st        d      || _        i g| _        d| _        y)z

        Parameters
        ----------
        storage: list[str]
            Directories containing cached files, must be at least one. Metadata
            is stored in the last of these directories by convention.
        z3CacheMetadata expects at least one storage locationFN)
ValueError_storagecached_files_force_save_pickle)selfstorages     Z/var/www/openai/venv/lib/python3.12/site-packages/fsspec/implementations/cache_metadata.py__init__zCacheMetadata.__init__%   s.     RSS+-$ #(    c                   	 t        |d      5 }t        j                  |      }ddd       j                         D ]3  }t        |j                  d      t              s#t        |d         |d<   5 |S # 1 sw Y   QxY w# t        $ r: t        |d      5 }t	        j                  |      }ddd       n# 1 sw Y   nxY wY w xY w)z6Low-level function to load metadata from specific filerNrbblocks)
openjsonloadr   picklevalues
isinstancegetlistset)r   fnfloadedcs        r   _loadzCacheMetadata._load8   s    	(b#!1 
 A!%%/40!!H+.( !   	(b$1Q  	(s?   B  A4B  4A=9B   CB4+	C4B=	9CCc                    | j                   r+t        |      5 }t        j                  ||       ddd       yt        |d      5 }t	        j                  ||       ddd       y# 1 sw Y   yxY w# 1 sw Y   yxY w)z4Low-level function to save metadata to specific fileNw)mode)r   r   r!   dumpr   )r   metadata_to_saver'   r(   s       r   _savezCacheMetadata._saveE   sc    ""b!Q,a0 "! bs+q		*A. ,+ "! ,+s   A%A1%A.1A:c              #     K   t        | j                        }t        | j                        D ]7  \  }}||dz
  k(  }|r|st        j                  j                  |d      ||f 9 yw)a  Yield locations (filenames) where metadata is stored, and whether
        writable or not.

        Parameters
        ----------
        writable: bool
            Set to True to only yield writable locations.

        Returns
        -------
        Yields (str, str, bool)
        r   cacheN)lenr   	enumerateospathjoin)r   writable_onlynir   writables         r   _scan_locationszCacheMetadata._scan_locationsN   s^      #DMM2JAwAEzHX'',,w0'8CC	 3s   A&A(c                   t        | j                         | j                        D ]  \  \  }}}}||vr||   j                         }|b|j                  r"|d   |j
                  j                  |      k7  rR|j                  r(t        j                         |d   z
  |j                  kD  rt        j                  j                  ||d         }t        j                  j                  |      s||fc S  y)zIf path is in cache return its details, otherwise return ``False``.

        If the optional CachingFileSystem is specified then it is used to
        perform extra checks to reject possible matches, such as if they are
        too old.
        uidtimer'   F)zipr=   r   copycheck_filesfsukeyexpiryr@   r6   r7   r8   exists)r   r7   cfsr'   base_r3   details           r   
check_filezCacheMetadata.check_filed   s     %((<(<(>@Q@Q$R MRq55 4[%%'F??ve}D8I'I::$))+v">"KdF4L1Bww~~b!rz! %S  r   c                   g }| j                   d   j                         j                         D ]  \  }}t        j                         |d   z
  |kD  s$|j	                  dd      }|st        d|       t        j                  j                  | j                  d   |      }|j                  |       | j                   d   j                  |        | j                   d   rLt        j                  j                  | j                  d   d      }| j                  | j                   d   |       | j                   d    }||fS )zRemove expired metadata from the cache.

        Returns names of files corresponding to expired metadata and a boolean
        flag indicating whether the writable cache is empty. Caller is
        responsible for deleting the expired files.
        r@   r'    z)Cache metadata does not contain 'fn' for r3   )r   rB   itemsr@   r$   RuntimeErrorr6   r7   r8   r   appendpopr1   )r   expiry_timeexpired_filesr7   rK   r'   
cache_pathwritable_cache_emptys           r   clear_expiredzCacheMetadata.clear_expired   s"     --b1668>>@LD&yy{VF^+k9ZZb)&CD6J  WW\\$--"3R8$$R(!!"%))$/ A R dmmB&7AJJJt((,j9#'#4#4R#88222r   c                    g }| j                         D ]W  \  }}}t        j                  j                  |      r!|j	                  | j                  |             G|j	                  i        Y |xs i g| _        y)z>Load all metadata from disk and store in ``self.cached_files``N)r=   r6   r7   rG   rR   r+   r   )r   r   r'   rJ   s       r   r    zCacheMetadata.load   se    ,,.HB1ww~~b!##DJJrN3##B' / )0RDr   c                    | j                   d   |   }|d   dur/t        |d         |j                  z  |j                  k\  rd|d<   yyy)zPerform side-effect actions on closing a cached file.

        The actual closing of the file is the responsibility of the caller.
        rN   r   TN)r   r4   	blocksizesize)r   r(   r7   r*   s       r   on_close_cached_filez"CacheMetadata.on_close_cached_file   sT     b!$'X;d"s1X;'7!++'E'OAhK (P"r   c                    | j                  |d      }|sy|\  }}|j                  | j                  d         r0| j                  d   j	                  |       | j                          |S t        d      )zRemove metadata of cached file.

        If path is in the cache, return the filename of the cached file,
        otherwise return ``None``.  Caller is responsible for deleting the
        cached file.
        NrN   z<Can only delete cached file in last, writable cache location)rL   
startswithr   r   rS   savePermissionError)r   r7   detailsrJ   r'   s        r   pop_filezCacheMetadata.pop_file   ss     //$-2==r*+b!%%d+IIK
 	 "N r   c                ,   t        | j                         | j                        D ]f  \  \  }}}}|st        j                  j                  |      r| j                  |      }|j                         D ]e  \  }}||v s|d   du s
||   d   du rd|d<   n!||   d   }|j                  |d          ||d<   t        |d   ||   d         |d<   ||   d   |d<   g |j                         D ]  \  }}||vs|||<    n|}|j                         D 	ci c]  \  }}	||	j                          }}}	|j                         D ]'  }t        |d   t              st        |d         |d<   ) | j                  ||       || j                  d<   i yc c}	}w )zSave metadata to diskr   Tr@   r?   rN   N)rA   r=   r   r6   r7   rG   r+   rP   updatemaxrB   r"   r#   r&   r%   r1   )
r   r'   rJ   r<   r3   r   kr*   r   vs
             r   r`   zCacheMetadata.save   s   (+D,@,@,BDDUDU(V$RHuww~~b!#zz"~(..0DAqEzX;$.%(82D2L*.AhK &+1Xh%7F"MM!H+6*0AhK$'&	58F3C$D&	#(8E?% 1  "KKMDAq,*+Q *  %-9-?-?-AB-ATQQ[-AEB\\^ak3/"&q{"3AhK $ JJub!$0Db!A )W6 Cs   Fc                (    || j                   d   |<   y)z8Update metadata for specific file in memory, do not saverN   N)r   )r   r7   rK   s      r   update_filezCacheMetadata.update_file   s    &,"d#r   N)r   z	list[str])r'   strreturnr   )r0   r   r'   rk   rl   None)F)r9   boolrl   zIterator[tuple[str, str, bool]])r7   rk   rH   zCachingFileSystem | Nonerl   z#Literal[False] | tuple[Detail, str])rT   intrl   ztuple[list[str], bool])rl   rm   )r(   r   r7   rk   rl   rm   )r7   rk   rl   z
str | None)r7   rk   rK   r   rl   rm   )__name__
__module____qualname____doc__r   r+   r1   r=   rL   rX   r    r]   rc   r`   rj    r   r   r   r      so    (&/ %*D!D	(D,6	,634	1("1H-r   r   )
__future__r   r6   r!   r@   typingr   fsspec.utilsr   ujsonr   ImportErrorr   r   r   r	   typing_extensionsr
   cachedr   rk   r   __annotations__r   rt   r   r   <module>r}      sf    " 	     %
 33+)S#XFI&N- N-  s   A A$#A$