
    ug/                     6   d dl mZ d dlmZmZmZmZmZmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZmZ d dlmZ d dlmZ d d	lmZmZmZ d d
lmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ d dl%m&Z& d dl'Z'd dl(m)Z)m*Z*m+Z+ d dl,Z,d dl-Z. e,j^                  e0      Z1dZ2 G d de      Z3y)    )override)OptionalSequenceDictSetListcast)UUID)VectorReader)Consumer)SystemSettings)Batch)
HnswParams)OpenTelemetryClientOpenTelemetryGranularitytrace_method)
	LogRecordRequestVersionContextVectorEmbeddingRecordVectorQueryVectorQueryResultSeqIdSegmentMetadata	OperationVector)InvalidDimensionExceptionN)ReadWriteLock
ReadRWLockWriteRWLocki  c            
           e Zd ZU eed<   eed<   ee   ed<   ee   ed<   eed<   eed<   ee	j                     ed<   ee   ed<   eed	<   eed
<   eed<   eeef   ed<   eeef   ed<   eeef   ed<   eed<   dedefdZeededee   fd              Z edej4                        ed2 fd              Z edej4                        ed2 fd              Z edej4                        e	 d3dedeee      dee   fd              Z  edej4                        ed e!deee"      fd!              Z#edefd"       Z$ededefd#       Z% ed$ej4                        d%eddfd&       Z& ed'ej4                        d(ed)eddfd*       Z' ed+ej4                        d,e(ddfd-       Z) ed.ej4                        d/ee*   ddfd0       Z+ed2d1       Z, xZ-S )4LocalHnswSegment_id	_consumer_collection_subscription	_settings_params_index_dimensionality_total_elements_added_max_seq_id_lock_id_to_label_label_to_id_id_to_seq_id_opentelemtry_clientsystemsegmentc                    |j                  t              | _        |d   | _        |d   | _        d | _        |j                  | _        t        |d   xs i       | _	        d | _
        d | _        d| _        | j                  j                         | _        i | _        i | _        i | _        t%               | _        |j)                  t*              | _        y )Nid
collectionmetadatar   )instancer   r%   r$   r&   r'   settingsr(   r   r)   r*   r+   r,   	min_seqidr-   r1   r/   r0   r   r.   requirer   r2   )selfr3   r4   s      \/var/www/openai/venv/lib/python3.12/site-packages/chromadb/segment/impl/vector/local_hnsw.py__init__zLocalHnswSegment.__init__=   s    24="<0!!'*"5";<#%&">>335"_
$*NN3F$G!    r8   returnc                 0    t        j                  |       }|S N)r   extract)r8   segment_metadatas     r>   propagate_collection_metadataz.LocalHnswSegment.propagate_collection_metadataQ   s     &--h7r@   zLocalHnswSegment.startNc                     t         |           | j                  rH| j                         }| j                  j                  | j                  | j                  |      | _        y y )N)start)superrH   r&   	max_seqidr%   	subscribe_write_recordsr'   )r=   seq_id	__class__s     r>   rH   zLocalHnswSegment.startX   sW     	^^%F!%!9!9  $"5"5V ": "D r@   zLocalHnswSegment.stopc                     t         |           | j                  r&| j                  j	                  | j                         y y rC   )rI   stopr'   r%   unsubscribe)r=   rN   s    r>   rP   zLocalHnswSegment.stopb   s4     	NN&&t'9'9: r@   zLocalHnswSegment.get_vectorsrequest_version_contextidsc                    |$t        | j                  j                               }n6g }|D ]/  }|| j                  v s|j	                  | j                  |          1 g }| j
                  t        t        t           t        j                  | j
                  j                  |                  }t        ||      D ]0  \  }}| j                  |   }|j	                  t        ||             2 |S )N)r6   	embedding)listr0   keysr/   appendr*   r	   r   r   nparray	get_itemszipr   )	r=   rR   rS   labelsr6   resultsvectorslabelvectors	            r>   get_vectorszLocalHnswSegment.get_vectorsi   s     ;$++0023FF***MM$"3"3B"78  ;;" "((4;;+@+@+H"IG "%VW!5v&&u-4fMN "6 r@   zLocalHnswSegment.query_vectorsqueryc           
      4   | j                   %t        t        |d               D cg c]  }g  c}S |d   }t        | j                        }||kD  r t        j                  d| d| d|        |}t               |d   }|E|D ch c]   }|| j                  v s| j                  |   " c}t              |k  rt              }dt        dt        ffd	}|d   }t        | j                        5  | j                   j                  t        j                  |t        j                  
      ||r|nd       \  }	}
g }t        t        |	            D ]  }g }t        |	|   |
|         D ]y  \  }}| j                   |   }|d   r3t        j                  | j                   j#                  |g      d         }nd }|j%                  t'        ||j)                         |             { |j%                  |        |cd d d        S c c}w c c}w # 1 sw Y   y xY w)Nr_   kzNumber of requested results z- is greater than number of elements in index z, updating n_results = allowed_idsr`   rA   c                     | v S rC    )r`   r]   s    r>   filter_functionz7LocalHnswSegment.query_vectors.<locals>.filter_function   s    F?"r@   )dtype)re   filterinclude_embeddingsr   )r6   distancerU   )r*   rangelenr/   loggerwarningsetintboolr    r.   	knn_queryrY   rZ   float32r\   r0   r[   rX   r   item)r=   rc   _re   sizerS   r6   ri   query_vectorsresult_labels	distancesall_resultsresult_ir^   r`   rm   rU   r]   s                    @r>   rz   zLocalHnswSegment.query_vectors   s*   
 ;; %c%	*:&; <= <1B <==#J4$$%t8NN.qc1^_c^dd{  }A  |B  C A5M"?69UcR4CTCT=Td''+cUF6{QK	#3 	#4 	# i(

#'+{{'<'<bjj9*-4 (= ($M9 :<K!#m"4535'*!(+Yx-@(OE8 **51B12$&HH KK115':1=%	 %)	NN)!%-]]_&/(" ""7+' 6* C $#/ > V $#s   	HH	H	4DHHc                     | j                   S rC   )r-   r=   s    r>   rJ   zLocalHnswSegment.max_seqid   s    r@   c                 ,    t        | j                        S rC   )ro   r/   )r=   rR   s     r>   countzLocalHnswSegment.count   s    4$$%%r@   zLocalHnswSegment._init_indexdimensionalityc                    t        j                  | j                  j                  |      }|j	                  t
        | j                  j                  | j                  j                         |j                  | j                  j                         |j                  | j                  j                         || _        || _        y )N)spacedim)max_elementsef_constructionM)hnswlibIndexr)   r   
init_indexDEFAULT_CAPACITYconstruction_efr   set_ef	search_efset_num_threadsnum_threadsr*   r+   )r=   r   indexs      r>   _init_indexzLocalHnswSegment._init_index   s     ,,$$.
 	) LL88llnn 	 	

 	T\\++,dll667-r@   zLocalHnswSegment._ensure_indexnr   c                    | j                   s|| _        | j                  |       n/|| j                  k7  r t        d| dd| j                   dz         t	        t
        j                  | j                         }| j                  |z   |j                         kD  rOt        | j                  |z   | j                  j                  z        }|j                  t        |t                     yy)zCCreate or resize the index as necessary to accomodate N new recordszDimensionality of (z) does not match indexzdimensionality ()N)r*   r+   r   r   r	   r   r   r,   get_max_elementsrs   r)   resize_factorresize_indexmaxr   )r=   r   r   r   new_sizes        r>   _ensure_indexzLocalHnswSegment._ensure_index   s     {{#&D S!d***/)#.DE()=)=(>a@A 
 W]]DKK0&&*e.D.D.FF++a/4<<3M3MMH s8-=>?	 Gr@   zLocalHnswSegment._apply_batchbatchc                 L   |j                         }|j                         }|j                  |      }dgt        |      z  }t        |      dkD  rt	        t
        j                  | j                        }t        t        |            D ]]  }||   }|| j                  vr| j                  |   }	|j                  |	       | j                  |= | j                  |	= | j                  |= _ t        |      dkD  r/| j                  |j                  t        |d                | j                  dz   }
t        t        |            D ]3  }||   | j                  vr|
||<   |
dz  }
| j                  ||      ||<   5 t	        t
        j                  | j                        }|j!                  ||       t#        |      D ]J  \  }}|j%                  |      d   | j                  |<   ||   | j                  |<   || j                  ||   <   L | xj                  |j                  z  c_        yy)z4Apply a batch of changes, as atomically as possible.r      
log_offsetN)get_deleted_idsget_written_idsget_written_vectorsro   r	   r   r   r*   rn   r/   mark_deletedr0   r1   r   	add_countr,   	add_items	enumerate
get_record)r=   r   deleted_idswritten_idsvectors_to_writelabels_to_writer   ir6   r`   
next_labels              r>   _apply_batchzLocalHnswSegment._apply_batch   s    ++-++- 44[A#$4 55{a4E3{+, ^T...))"-""5)%%b)%%e,&&r* - {au4DQ4G0HI33a7J3{+,q>):)::)3OA&!OJ)-):):;q>)JOA& - 4E OO,o> #;/2).)9)9")=l)K""2&(7(:!!"%8:!!/!"45 0 &&%//9&/  r@   zLocalHnswSegment._write_recordsrecordsc                 X   | j                   st        d      t        | j                        5  t	               }|D ]H  }t        | j                  |d         | _        |d   d   }|d   d   }| j                  j                  |d      }|t        j                  k(  r-|r|j                  |       ut        j                  d|        |t        j                  k(  r<|d   d   ||j                  |       t        j                  d	|d   d           |t        j                  k(  r0|s|j                  |d
       t        j                  d|         |t        j                   k(  s5|j                  ||du       K | j#                  |       ddd       y# 1 sw Y   yxY w)z&Add a batch of embeddings to the indexz*Cannot add embeddings to stopped componentr   recordr6   	operationNz$Delete of nonexisting embedding ID: rU   z$Update of nonexisting embedding ID: FzAdd of existing embedding ID: )_runningRuntimeErrorr!   r.   r   r   r-   r/   getr   DELETEapplyrp   rq   UPDATEADDUPSERTr   )r=   r   r   r   r6   opr`   s          r>   rL   zLocalHnswSegment._write_records"  sy    }}KLL $GE!#&t'7'79M#N H%d+H%k2))--b$7)))F+)MbT'RS9+++h'4@ ,!KK/"NN"FvhGWX\G]F^ _ 9==( FE2)Gt'LM9+++KKT(9:5 "8 e$? %$$s   B6F $B	F /(F  F)c                     t               rC   )NotImplementedErrorr   s    r>   deletezLocalHnswSegment.deleteJ  s    !##r@   )rA   NrC   ).__name__
__module____qualname__r
   __annotations__r   r   r   r   r   r   rs   r   r   r   strr   r   r   r?   staticmethodr   r   rF   r   r   ALLrH   rP   r   r   r   rb   r   r   rz   rJ   r   r   r   r   r   r   rL   r   __classcell__)rN   s   @r>   r#   r#   %   s   	I$D>!W]]##c]"sCx. sCx.  U
##--Hv H H(    Xh=O     
 *,D,H,HI  J )+C+G+GH;  I;
 02J2N2NO (,!6 hsm$ 
'	(	  P2 24L4P4PQ< <	(,-	.<  R<|  5     &-B &s & & 02J2N2NO.# .$ . P." 24L4P4PQ@s @ @ @ R@( 13K3O3OP,:% ,:D ,: Q,:\ 35M5Q5QR%%hy&9 %%d %% S%%N $ $r@   r#   )4	overridesr   typingr   r   r   r   r   r	   uuidr
   chromadb.segmentr   chromadb.ingestr   chromadb.configr   r   "chromadb.segment.impl.vector.batchr   (chromadb.segment.impl.vector.hnsw_paramsr    chromadb.telemetry.opentelemetryr   r   r   chromadb.typesr   r   r   r   r   r   r   r   r   r   chromadb.errorsr   r   chromadb.utils.read_write_lockr   r    r!   loggingnumpyrY   	getLoggerr   rp   r   r#   rh   r@   r>   <module>r      ss     < <  ) $ , 4 ? 
   6  Q Q  			8	$ g$| g$r@   