
    ugS)                        d dl mZ d dlmZmZmZmZmZmZ d dl	Z	d dl
mZmZmZ d dlZd dlmZmZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZ d d
lmZmZmZm Z m!Z! d dl"m#Z#m$Z$m%Z%m&Z&m'Z' d dl(m)Z)m*Z* d dl+Z+d dl,m-Z- d dl.m/Z/  e+j`                         dk7  rd dl1Z1n e+j`                         dk(  rd dl2Z2ejf                  dejh                  dejj                  diZ6 G d de      Z7dede dedefdZ8y)    )Lock)SegmentImplementationSegmentManagerMetadataReaderSegmentTypeVectorReaderSN)SegmentLRUCache
BasicCacheSegmentCache)System	get_class)SysDB)override)PersistentLocalHnswSegment)OpenTelemetryClientOpenTelemetryGranularitytrace_method)
Collection	OperationSegmentSegmentScopeMetadata)DictTypeSequenceOptionalcast)UUIDuuid4)LRUCache)get_directory_sizeWindowsz;chromadb.segment.impl.metadata.sqlite.SqliteMetadataSegmentz8chromadb.segment.impl.vector.local_hnsw.LocalHnswSegmentzMchromadb.segment.impl.vector.local_persistent_hnsw.PersistentLocalHnswSegmentc                       e Zd ZU eed<   eed<   eed<   eee	f   ed<   e
eef   ed<   ej                  Zeed<   eed<   eed<   d	ef fd
Z edej(                        deddfd       Zed% fd       Zed% fd       Zed% fd       Z edej(                        ededee   fd              Z edej(                        ededee   fd              ZdedefdZ edej(                        dede defd       Z! edej(                        dede"e#   de#fd       Z$ ed ej(                        eded!e%ddfd"              Z&dede"e	   fd#Z'dede	fd$Z( xZ)S )&LocalSegmentManager_sysdb_system_opentelemetry_client
_instances#_vector_instances_file_handle_cache_vector_segment_type_lock_max_file_handlessystemc                 6    t            |        j                  t               _        | _        |j                  t               _        t        j                  t               _        i  _        t        j                  t               i _        |j"                  j$                  dk(  r]|j"                  j&                  dkD  rDt)        |j"                  j&                   fd fd       j                   t        j*                  <   n%t                j                   t        j*                  <   t-                _         j
                  j"                  j                  d      rt0        j2                   _        t7        j8                         dk7  r,t;        j<                  t:        j>                        d    _         n-tB        jD                  jF                  jI                          _          j@                  tK        jL                         z  }tO        |d 	       _(        y y )
NLRUr   c                 &    j                  |      S N)callback_cache_evict)kvselfs     X/var/www/openai/venv/lib/python3.12/site-packages/chromadb/segment/impl/manager/local.py<lambda>z.LocalSegmentManager.__init__.<locals>.<lambda>N   s    d&?&?&B    c                 &    j                  |       S r2   )_get_segment_disk_size)r4   r6   s    r7   r8   z.LocalSegmentManager.__init__.<locals>.<lambda>O   s    D$?$?$Br9   )capacitycallback	size_funcis_persistentr#   c                 "    |j                         S r2   )close_persistent_index)_r5   s     r7   r8   z.LocalSegmentManager.__init__.<locals>.<lambda>d   s    Q5M5M5Or9   )r=   ))super__init__requirer   r&   r'   r   r(   logging	getLogger__name__loggerr)   r   METADATAr   segment_cachesettingschroma_segment_cache_policychroma_memory_limit_bytesr
   VECTORr   r,   r   HNSW_LOCAL_PERSISTEDr+   platformr.   resource	getrlimitRLIMIT_NOFILEr-   ctypeswindllmsvcrt_getmaxstdior   get_file_handle_countr!   r*   )r6   r.   segment_limit	__class__s   `  r7   rD   zLocalSegmentManager.__init__>   s    ll5)%+^^4G%H"''1!!:<@
 OO775@99A=6EBBBB7D|223 7AlD|223V
 <<  ((9(3(H(HD% I-)1););H<R<R)STU)V&)/)=)=)J)J)L&&&-CCEF 
 8@(O8D4 :r9   z(LocalSegmentManager.callback_cache_evictsegmentreturnNc                     |d   }| j                   j                  d|        | j                  |      }|j                          | j                  |d   = y )N
collectionzLRU cache evict collection id)rI   info	_instancestopr)   )r6   r\   collection_idinstances       r7   r3   z(LocalSegmentManager.callback_cache_evictg   sO    
  -6}oFG>>'*OOGDM*r9   c                     | j                   j                         D ]  }|j                           t        | 	          y r2   )r)   valuesstartrC   r6   re   r[   s     r7   rh   zLocalSegmentManager.startr   s-    ..0HNN 1r9   c                     | j                   j                         D ]  }|j                           t        | 	          y r2   )r)   rg   rc   rC   ri   s     r7   rc   zLocalSegmentManager.stopx   s,    ..0HMMO 1r9   c                    | j                   j                         D ]"  }|j                          |j                          $ i | _         | j                  t
        j                     j                          t        |           y r2   )	r)   rg   rc   reset_staterK   r   rO   resetrC   ri   s     r7   rl   zLocalSegmentManager.reset_state~   s`    ..0HMMO  " 1 <../557r9   z7LocalSegmentManager.prepare_segments_for_new_collectionr_   c                     t        | j                  t        j                  |      }t        t        j
                  t        j                  |      }||gS r2   )_segmentr+   r   rO   r   SQLITErJ   )r6   r_   vector_segmentmetadata_segments       r7   #prepare_segments_for_new_collectionz7LocalSegmentManager.prepare_segments_for_new_collection   sO     "%%|':':J
 $ 5 5z
  011r9   z#LocalSegmentManager.delete_segmentsrd   c                    | j                   j                  |      }|D ]5  }|d   | j                  v r|d   t        j                  j
                  k(  r'| j                  |t              }|j                          nF|d   t        j                  j
                  k(  r&| j                  |t              }|j                          | j                  |d   = |d   t        j                  u r,| j                  t        j                     j                  |       |d   t        j                  u s
| j                  t        j                     j                  |       8 |D cg c]  }|d   	 c}S c c}w )N)r_   r`   typescope)r&   get_segmentsr)   r   rP   valueget_segmentr   deleterp   r   r   rO   rK   poprJ   )r6   rd   segmentsr\   re   ss         r7   delete_segmentsz#LocalSegmentManager.delete_segments   s/    ;;++}+EGt}/6?k&F&F&L&LL#//|LHOO%V_(:(:(@(@@#//~NHOO%OOGDM2w<#6#66""<#6#67;;MJw<#8#88""<#8#89==mL   "**A$***s   E+c           	      0   | j                   j                  |t        j                        }t	        |      dk(  ryt        t        j                  j                  | j                  j                  j                  d      t        |d   d                     }|S )Nr_   rv   r   persist_directoryr`   )r&   rw   r   rO   lenr"   ospathjoinr'   rL   rE   str)r6   rd   r|   sizes       r7   r;   z*LocalSegmentManager._get_segment_disk_size   s    ;;++$L,?,? , 
 x=A!GGLL%%--.ABHQK%&
 r9   z&LocalSegmentManager._get_segment_sysdbrv   c                     | j                   j                  ||      }t        t        j	                         D cg c]  }|j
                   c}      t        t        fd|            }|S c c}w )Nr   c                     | d   v S Nru    )r}   known_typess    r7   r8   z8LocalSegmentManager._get_segment_sysdb.<locals>.<lambda>   s    &	[(@r9   )r&   rw   setSEGMENT_TYPE_IMPLSkeysrx   nextfilter)r6   rd   rv   r|   r4   r\   r   s         @r7   _get_segment_sysdbz&LocalSegmentManager._get_segment_sysdb   sd    
 ;;++}E+R,>,C,C,EF,Eq177,EFGv@(KL Gs   A.zLocalSegmentManager.get_segmentru   c                    |t         k(  rt        j                  }n(|t        k(  rt        j                  }nt        d|       | j                  |   j                  |      }|1| j                  ||      }| j                  |   j                  ||       | j                  5  | j                  |      }d d d        t        t              S # 1 sw Y   xY w)NzInvalid segment type: )r   r   rJ   r   rO   
ValueErrorrK   getr   r   r,   rb   r   r	   )r6   rd   ru   rv   r\   re   s         r7   ry   zLocalSegmentManager.get_segment   s    
 >! ))E\! ''E5dV<==$$U+//>?--mUCGu%))-A ZZ~~g.H Ax   Zs    C

Cz'LocalSegmentManager.hint_use_collection	hint_typec                 "   t         t        fD ]  }| j                  ||      }|t        k(  s| j                  j                  j                  d      sEt        t        |      }|j                          | j                  j                  ||        y )Nr?   )r   r   ry   r'   rL   rE   r   r   open_persistent_indexr*   r   )r6   rd   r   ru   re   s        r7   hint_use_collectionz'LocalSegmentManager.hint_use_collection   su     $\2D''t<H |#(=(=(E(Eo(V :HE..088<<]HU 3r9   c                 P    t         t        |d            }t        |t              }|S r   )r   r   r   r   )r6   r\   	classnameclss       r7   _clszLocalSegmentManager._cls   s'    &{76?'CD		#89
r9   c                     |d   | j                   vrF| j                  |      } || j                  |      }|j                          || j                   |d   <   | j                   |d      S )Nr`   )r)   r   r'   rh   )r6   r\   r   re   s       r7   rb   zLocalSegmentManager._instance   s_    4=/))G$C4<<1HNN-5DOOGDM*wt}--r9   )r]   N)*rH   
__module____qualname__r   __annotations__r   r   r   r   r   r!   r   r   HNSW_LOCAL_MEMORYr+   r   intrD   r   r   OPERATION_AND_SEGMENTr   r3   r   rh   rc   rl   r   r   rs   r~   r;   r   r   r   r	   ry   r   r   r   rb   __classcell__)r[   s   @r7   r%   r%   2   sZ   MO..T0011)1((*  )4(E(E+EK'v 'R 2 66+G + +	+  
  
   A 66 2j 2XV]M^ 2 	
2 - 66 +T +htn + 	
+"D S  0 66 \ g 	 ) 66! !T!W ! !	!& 1 66 V V) VPT V 	
VG -B(C 
. .-B .r9   r%   ru   rv   r_   r]   c                     t        t        |    t              }|j                  }d}|r|j	                  |      }t        t               | j                  ||j                  |i       S )zRCreate a metadata dict, propagating metadata correctly for the given segment type.N)r`   ru   rv   r_   metadata
file_paths)	r   r   r   r   propagate_collection_metadatar   r    rx   r`   )ru   rv   r_   r   collection_metadatar   s         r7   ro   ro      sc    
&t,.C
DC$--#'H445HI7ZZ== r9   )9	threadingr   chromadb.segmentr   r   r   r   r   r	   rF   )chromadb.segment.impl.manager.cache.cacher
   r   r   r   chromadb.configr   r   chromadb.db.systemr   	overridesr   2chromadb.segment.impl.vector.local_persistent_hnswr    chromadb.telemetry.opentelemetryr   r   r   chromadb.typesr   r   r   r   r   typingr   r   r   r   r   uuidr   r    rQ   chromadb.utils.lru_cacher!   chromadb.utils.directoryr"   r.   rR   rU   rp   r   rP   r   r%   ro   r   r9   r7   <module>r      s       
 
 - $  
 R Q 7 7   - 7 8??	!X__)# U!!#]$$&u G.. G.T; |  PW r9   