
    uguE                        d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlZd dlmZmZ d d	lmZmZmZmZmZmZmZmZmZ d d
lmZ d dlZd dl mZ! d dl"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z) d dl*m+Z+m,Z, d dl-m.Z.  ed      Z/dee/ee/   f   dee/   fdZ0dedefdZ1de%deddfdZ2de%dedeed   ed   ed   f   ddfdZ3de%deddfdZ4de%deddfd Z5de%deddfd!Z6de%deddfd"Z7de%ddfd#Z8e.jr                  fd$e#jt                  d%e#jt                  d&ee!jv                  e!jv                  ge<f   deeee=      eee<      f   fd'Z>d(e=ddfd)Z?	 	 	 	 	 d:de%ded*e=d+e<d,ee#j                     d-eee=      d.ee#jt                     ddfd/ZAd0e#j                  d*e=ddfd1ZCd2ede=fd3ZDd4ed5ee%   d6eEddfd7ZFd4ed5ee%   deee=f   fd8ZGd4ed5ee%   d6eEddfd9ZHy);    N)UUID)create_topic_name)System)get_sql)SqliteDB)sleep)NormalizedRecordSet	RecordSet)	CallableOptionalTupleUnionListTypeVarcastAnyDict)Literal)types)
Collection)note)InvalidArgument)Table	functions)distance_functionsTvaluereturnc                 F    | t        d      t        | t              r| S | gS )z*Wrap a value in a list if it is not a listzvalue cannot be None)r   
isinstancer   )r   s    V/var/www/openai/venv/lib/python3.12/site-packages/chromadb/test/property/invariants.pywrapr"      s)    }455	E4	 w    
record_setc                 |   | d   d}nt        | d   t              r| d   J t        | d         dkD  rt        d | d   D              rt	        t
        j                  | d         }nt        d | d   D              rt	        t
        j                  | d         }nht        d | d   D              rt	        t
        j                  | d   g      }n4t        d      t	        t
        j                  | d         }nt        d      t        | d	         | d
   t        | d
         nd| d   t        | d         |dS d|dS )z9Ensure that an embedding set has lists for all its values
embeddingsNr   c              3   <   K   | ]  }t        |t                y wN)r    list.0	embeddings     r!   	<genexpr>zwrap_all.<locals>.<genexpr>/   s      =U	
9d+=Us   c              3   P   K   | ]  }t        |t        j                           y wr(   )r    npndarrayr*   s     r!   r-   zwrap_all.<locals>.<genexpr>3   s#      !9I 9bjj1!9s   $&c              3      K   | ]8  }t        |t        t        t        j                  t        j
                  f       : y wr(   )r    intfloatr/   integerfloatingr+   es     r!   r-   zwrap_all.<locals>.<genexpr>9   s0      5 q3rzz2;;"GH5s   >A z-an embedding must be a list of floats or intszVembeddings must be a list of lists, a list of numpy arrays, a list of numbers, or Noneids	documents	metadatas)r8   r9   r:   r&   )	r    r)   lenallr   r   
Embeddingsr   r"   )r$   embedding_lists     r!   wrap_allr?   &   s    ,'	J|,d	3,'333z,'(1, =G=U  "&e&6&6
<8P!Q !+L!9  "&e&6&6
<8P!Q '5  &*%*:*:Z=U<V%WN)G  "%"2"2J|4LMNd
 	

 Ju%&k". *[12k". *[12 %	 	 $	 	r#   
collectionc                 `    | j                         }t        |      }|t        |d         k(  sJ y)z?The given collection count is equal to the number of embeddingsr8   N)countr?   r;   )r@   r$   rB   normalized_record_sets       r!   rB   rB   U   s4    E$Z0C-e45555r#   rC   
field_namer9   r:   r&   c                 f   | j                  |d   |g      t        |d         D ci c]  \  }}||
 c}}|   }t        |d         dk(  r>|dk(  r1t        t        j
                  t           |      j                  dk(  sJ y|g k(  sJ y|J t        t        |      fd      }|D cg c]  \  }}|	 }	}}||   }
|
dgt        |d         z  }
|dk(  r?t        j                  t        j                  |	      t        j                  |
            sJ y|	|
k(  sJ yc c}}w c c}}w )zq
    The actual embedding field is equal to the expected field
    field_name: one of [documents, metadatas]
    r8   r8   includer   r&   Nc                      d   | d         S )Nr8   r    )index_and_field_valueembedding_id_to_indexresults    r!   <lambda>z _field_matches.<locals>.<lambda>y   s    *?5M/23+
r#   key)get	enumerater;   r   nptNDArrayr   sizesortedr/   allclosearray)r@   rC   rD   iidactual_fieldsorted_field_field_valuefield_valuesexpected_fieldrK   rL   s              @@r!   _field_matchesr`   \   s^    ^^ 5e <zl^SF 1::OPU:V0WX0Wuq"RU0WX*%L
 '(A-%C(,7<<AAA 	  2%%% ###,
L 7CClNA{KlLC*:6N #&;E&B"CC\!{{288L1288N3KLLL~---; Y& Ds   D'3D-c                     t        |      }| j                  |d   g       d   }t        |d         D ci c]  \  }}||
 c}}t        |fd      }||d   k(  sJ yc c}}w )z5The actual embedding ids is equal to the expected idsr8   rF   c                     |    S r(   rI   )rY   rK   s    r!   rM   zids_match.<locals>.<lambda>   s    3H3Lr#   rN   N)r?   rP   rQ   rU   )r@   r$   rC   
actual_idsrX   rY   rK   s         @r!   	ids_matchrd      s    $Z0$9%$@"MeTJ 1::OPU:V0WX0Wuq"RU0WX
(LMJ.u5555 Ys   A#c                 4    t        |      }t        | |d       y)z?The actual embedding metadata is equal to the expected metadatar:   Nr?   r`   r@   r$   rC   s      r!   metadatas_matchrh          $Z0:4kBr#   c                 4    t        |      }t        | |d       y)AThe actual embedding documents is equal to the expected documentsr9   Nrf   rg   s      r!   documents_matchrl      ri   r#   c                 4    t        |      }t        | |d       y)rk   r&   Nrf   rg   s      r!   embeddings_matchrn      s    $Z0:4lCr#   c                 n    | j                         d   }t        |      t        t        |            k(  sJ y )Nr8   )rP   r;   set)r@   r8   s     r!   no_duplicatesrq      s-    
..
5
!Cs8s3s8}$$$r#   querytargetsdistance_fnc                 6   t        j                  | t         j                        }t        j                  |t         j                        t        j                  fdd|      }t        j                  |      j                         |j                         fS )zGReturn the ordered indices and distances from each query to each target)dtypec                 4    t        j                  d|       S )N   )r/   apply_along_axis)rr   rt   
np_targetss    r!   rM   z"_exact_distances.<locals>.<lambda>   s    b))+q*eLr#   rx   )r/   rW   float32ry   argsorttolist)rr   rs   rt   np_query	distancesrz   s     `  @r!   _exact_distancesr      sp     xxRZZ0H'4J ##L	I ::i '')9+;+;+===r#   threadpool_sizec                 D   t        j                         }|j                         }d}d}t        |D cg c]  }d|j                  v s|j                   c}      dz
  | kD  rw||k  rrt        j                          |j                         }|dz  }t        d       t        |D cg c]  }d|j                  v s|j                   c}      dz
  | kD  r||k  rrt        |D cg c]  }d|j                  v s|j                   c}      dz
  | k  sJ yc c}w c c}w c c}w )zs
    Checks that the open file descriptors are not exceeding the threadpool size
    works only for SegmentAPI
       r   sqlite3rx   N)psutilProcess
open_filesr;   pathgccollectr   )r   current_processr   max_retriesretry_countps         r!    fd_not_exceeding_threadpool_sizer      s   
 nn&O ++-JKK 	Z?Z9+>QVVZ?@1DV+%


$//1
qa 	Z?Z9+>QVVZ?@1DV+% 	Z?Z9+>QVVZ?@1DWW @? @s#   DD#D7D!D5D	n_results
min_recallembedding_functionquery_indicesquery_embeddingsc           
         t        |      }t        |d         dk(  ry|d   }|duxr t        |      dkD  }	|	s+|J |d   J t        |d   t              sJ  ||d         }t        j
                  }
d}| j                  J |J d| j                  v r| j                  d   }t        d |D              sJ t        |d         }|t        j                  d	t        t        j                  |                  z  }|d
k(  rt        j                  }
|dk(  rt        j                  }
|||n|D cg c]  }||   	 c}}|d   }|||D cg c]  }||   	 }}t        |||
      \  }}| j                  |	r|nd|	s|nd|g d      }t!        ||       |d   J |d   J |d   J |d   J |d   J t#        |d         D ci c]  \  }}||
 }}}d}t#        t%        ||            D ]   \  }\  }}t'        j(                  |d         |d|    }|t        t+        |      t+        |d   |         z
        z  }t#        |d   |         D ]  \  }}||v}||   }t'        j,                  ||   |d   |   |   |      }|r	|r|dz  }n>|sJ t'        j,                  ||   |d   |   |         sJ |d   |d   |   |d   |   |   k(  sJ |d   |d   |   |d   |   |   k(  rJ   t        |d         }||z
  |z  }	 t/        d| d| d| d|        ||k\  sJ |d   D ]-  }t'        j,                  t'        j2                  |      |      r-J  yc c}w c c}w c c}}w # t0        $ r Y Xw xY w)zBValidate that the API performs nearest_neighbor searches correctlyr8   r   Nr&   r9   gư>z
hnsw:spacec              3   \   K   | ]$  }t        |t        t        j                  f       & y wr(   )r    r)   r/   r0   r6   s     r!   r-   zann_accuracy.<locals>.<genexpr>   s!     Ij:a$

!34js   *,
   cosineip)rt   )r&   r9   r:   r   )r   query_textsr   rG   r   r:   )atolrx   z
# recall: z
, missing z out of z, accuracy threshold )r?   r;   r    r)   r   l2metadatar<   mathpowr2   log10r   r   r   rr    _query_results_are_correct_shaperQ   zipr/   rW   rp   rV   r   r   sort) r@   r$   r   r   r   r   r   rC   r&   have_embeddingsdistance_functionaccuracy_thresholdspacedimrX   query_documentsindicesr   query_resultsrY   id_to_indexmissing	indices_idistances_iexpected_idsjunexpected_idindexcorrect_distancerT   recalldistance_results                                    r!   ann_accuracyr      s    %Z0
 '(A--B<-PJ ,DZ11DO!---$[1===/<dCCC'(=k(JK
 +--***!!!z***##L1
 IjIIII*Q- /$((2s4::c??S2TTH 2 9 9D= 2 5 5  $ )67A*Q-7 	
 ,K8O _%@7DE}!?1-}E)*2CGY $$-<)$+:OE	 % M %]I> +++%111&222%111%111 '00Ee0L&MN&MUQ2q5&MKNG'0Wi1H'I##I{xx 5e <=i
>ST3s<(3}U/CA/F+GGHH }U3A67EArl2MOE!{{E"k*1-a0' 
  $qLG''';;z%0-2Ma2PQR2STTT$[1=)+6u=$[1!4Q788 %[1=)+6u=$[1!4Q7889 8 (JL $U+,DWn$F
7)8D6AVWiVjk	
 Z )5{{277?3_EEE 6a 8 F. O^  s$   0M M%
M*M0 0	M<;M<r   c                 T    dD ]"  }| |   J t        fd| |   D              r"J  y )N)r   r&   r9   r:   c              3   :   K   | ]  }t        |      k(    y wr(   )r;   )r+   rL   r   s     r!   r-   z3_query_results_are_correct_shape.<locals>.<genexpr>b  s      
3MCK9$3Ms   )r<   )r   r   result_types    ` r!   r   r   ]  sF     M[)555 
3@3M
 
 	
 
 Mr#   sqlitec                    t        d      }| j                         j                  |      }| j                         5 }t	        |j                  t        j                  |j                              | j                               \  }}|j                  ||      }t        t        |j                         d         cd d d        S # 1 sw Y   y xY w)Nembeddings_queuer   )r   querybuilderfrom_txr   selectr   Countseq_idparameter_formatexecuter   r2   fetchone)r   tqcursqlparamsrL   s          r!   _total_embedding_queue_log_sizer   g  s     !A##A&A	HHY__QXX./1H1H1J
V S&)C*1-. 
s   A<CC
systemcollectionshas_collection_mutatedc                     | j                  t              }|rKt        |      dk\  sJ t        d |D              }t        d |D              }t        |      dz
  ||z   k  sJ y t        |      dk(  sJ y )Nrx   c              3   p   K   | ].  }|j                   |j                   j                  dd      nd 0 y w)Nzhnsw:sync_thresholdi  r   rP   r+   r@   s     r!   r-   z%log_size_below_max.<locals>.<genexpr>}  sF      !
 *
 "". ##$94@ *   46c              3   p   K   | ].  }|j                   |j                   j                  dd      nd 0 y w)Nhnsw:batch_sized   r   r   s     r!   r-   z%log_size_below_max.<locals>.<genexpr>  sF      
 *
 "". ##$5s; *r   r   )instancer   r   sum)r   r   r   r   sync_threshold_sumbatch_size_sums         r!   log_size_below_maxr   s  s     __X&F.v6!;;; ! !
 *	!
 
  
 *	
 
 ,F3a7!N23	
3 /v6!;;;r#   c           	         | j                  t              }t        d      }|j                         j	                  |      }| j
                  j                  d      }| j
                  j                  d      }|D ci c]  }t        |||j                        | }}|j                         5 }	t        |j                  |j                  t        j                  |j                              j!                  d      |j#                               \  }
}|	j%                  |
|      }i }|j'                         D ]  }|d   |||d      j                  <    |cd d d        S c c}w # 1 sw Y   y xY w)Nr   	tenant_idtopic_namespacetopicrx   r   )r   r   r   r   r   settingsrequirer   rY   r   r   r   r   r   r   r   groupbyr   r   fetchall)r   r   r   r   r   _tenant_topic_namespacer@   topic_mappingsr   r   r   rL   outress                  r!   ._total_embedding_queue_log_size_per_collectionr     sB    __X&F !A##A&Aoo%%k2G../@A &%J 	'#3Z]]CZO%   
HHQWWiooahh78@@I##%
V S&)??$C-0VCs1v&))* % 
	 
s   :E+B&E  E)c                 8   | j                  t              }|rt        |      dk\  sJ |D ci c]7  }|j                  |j                  |j                  j                  dd      nd9 }}|D ci c]0  }|j                  |j                         ||j                     z  dz   2 c}t        | |      t        j                               t        j                               k(  sJ t        fd|D              sJ y t        |      dk(  sJ y c c}w c c}w )Nrx   r   r   c              3   \   K   | ]#  }|j                      |j                      k(   % y wr(   )rY   )r+   r@   actual_sizesexpected_sizess     r!   r-   z:log_size_for_collections_match_expected.<locals>.<genexpr>  s0      
)
 '>*--+HH)s   ),r   )r   r   r   rY   r   rP   rB   r   rp   keysr<   )r   r   r   r   r@   r   r   r   s         @@r!   'log_size_for_collections_match_expectedr     sA    __X&F.v6!;;; *	
 *
 MM"". &..223DcJ *	 	 
 *
)
 MM:++-z}}0MMPQQQ)

 FK
 <$$&'3~/B/B/D+EEEE 
)
 
 	
 
 /v6!;;;+

s   <D05D)rx   gGz?NNN)Ir   r   uuidr   chromadb.ingest.impl.utilsr   chromadb.configr   chromadb.db.baser   chromadb.db.impl.sqliter   timer   r   !chromadb.test.property.strategiesr	   r
   typingr   r   r   r   r   r   r   r   r   typing_extensionsr   numpyr/   numpy.typingrR   chromadb.apir   chromadb.api.models.Collectionr   
hypothesisr   hypothesis.errorsr   pypikar   r   chromadb.utilsr   r   r"   r?   rB   r`   rd   rh   rl   rn   rq   r   r=   	ArrayLiker3   r2   r   r   EmbeddingFunctionr   QueryResultr   r   boolr   r   r   rI   r#   r!   <module>r     sE   	   8 " $ ,   L S S S %    5  - # -CLaaj! d1g , ,': ,^6j 6i 6D 6+.+..+. gk2GL4II+. 
+.\6* 6) 6 6C
 C	 Cd CC
 C	 Cd CD D Dt D%j %T % 	>>> 	&-> 4S	?De--.>*c d 4 <@)-37BFBFBF BF 	BF
 !!8!89BF DI&BF u//0BF 
BFJ
$$
14
	
	/H 	/ 	/<<!%j!1<KO<	<Bj! 
$)_2<<!%j!1<KO<	<r#   