
    ugQ!              
       2   d dl Z d dlmZmZmZ d dlZd dlZd dlmZm	Z	m
Z
mZmZmZmZ d dlZd dlmZ d dlmc mc mZ d dlmZ d dlmZ d dlmZ d dlmZ d	eeef   fd
Zdeded	efdZ dedededed	df
dZ!dedededed	df
dZ"ded	dfdZ#ded	dfdZ$y)    N)FutureThreadPoolExecutorwait)AnyDictListOptionalSetTuplecast)	ClientAPI)
SegmentAPI)	RecordSet)test_hnsw_config)Metadatareturnc                  b    t        j                  dd      } t        j                  dd      }| |fS )N
   i'     )randomrandint)NDs     U/var/www/openai/venv/lib/python3.12/site-packages/chromadb/test/test_multithreaded.pygenerate_data_shaper      s+    r5!Ar3Aq6M    r   r   c                 <   t        |       D cg c]  }t        |       }}t        |       D cg c]  }| |i	 }}t        |       D cg c]  }d| 	 }}t        j                  j	                  | |      j                         }||||d}|S c c}w c c}w c c}w )Nzdoc ids
embeddings	metadatas	documents)rangestrnpr   randtolist)r   r   ir   r!   r"   r    normalized_record_sets           r   generate_record_setr*      s     8
$8a3q68C
$<A!H&EHq1#{HI&E%*1X.X4sXI.1%,,.J  	( !  %&E.s   BB
Bclientnum_workersc           
         t        ||      }|d   }|d   }|d   }|d   }t        d| d| d| d       | j                          | j                  d	t        
      }	t        |      5 }
g }d}|t        |      k  rt        j                  d|dz        }t        |t        |      |z
        }|dz   }||z   dz   }|t        |||       dk(  rn]|
j                  |	j                  ||| |||| nd |||| nd |||| nd       }|j                  |       ||z  }|t        |      k  rd d d        t               |D ]  }|j                         }|| t        j                   |	|       t        j"                  |	|       t        j$                  |	|       t        j&                  |	       t        j(                  t+        |      D cg c]  }| c}d      }d}t        j,                  |	|||       y # 1 sw Y   xY wc c}w )Nr   r    r!   r"   Adding  records with  dimensions on  workerstestnamemetadatamax_workers   r   r   r      	n_resultsquery_indices)r*   printresetcreate_collectionr   r   lenr   r   minsubmitaddappendr   	exception
invariantscount	ids_matchmetadatas_matchno_duplicatessampler#   ann_accuracy)r+   r   r   r,   records_setr   r    r!   r"   collexecutorfutures
total_sent
batch_sizeto_sendstartendfuturerF   r(   r=   r<   s                         r   _test_multithreaded_addrX   )   s4    &a+K
e
C\*JK(IK(I	GA3nQC{m8
LM LLN##:J#KD		4%'
3s8#173J*c#h&;<GNEw&*C%#js.C*D*I__cN4>4J:eC0PT2;2G)E#.T2;2G)E#.T % F NN6"'!J! 3s8# 
5* 	M$$&	 O  T;'{+t[1T" MMeAh"7h1h"7<MI#	M 
5	4H #8s   )CG9	H9Hc           	         | j                          | j                  dt              t        ||      }t	        t
        t           |d         t	        t
        t           |d         t	        t
        t           |d         |d   t               t        j                         t        d| d| d	| d
       	 ddt        dt        t
        t              ddffd}t        |      5 }g }d}|t!              dz
  k  rt#        j$                  dd      }	|	dk(  rkt#        j$                  d|dz        }
t'        |
t!              |z
        }|dz   }||z   dz   }|j)                  ||	||       }|j+                  |       ||z  }n(|	dk(  r#|j)                  ||	      }|j+                  |       |t!              dz
  k  rddd       t-               |D ]  }|j/                         }|| t1        | t2              r1| j5                         j6                  du rt9        j:                  |       t9        j<                  |       t9        j>                  |       t9        j@                  |       t9        jB                         t#        jD                  tG        |      D cg c]  }| c}d      }d}t9        jH                  |||       y# 1 sw Y   xY wc c}w )zeTest that will use multiple threads to interleave operations on the db and verify they work correctlyr2   r3   r   r    r!   r"   r.   r/   r0   r1   N	operationids_to_modifyr   c           
      .   | dk(  r|J |D cg c]  }j                  |       }}t        |      dk(  ryj                  ||D cg c]  }|   	 c}nd|D cg c]  }|   	 c}nd|D cg c]  }|   	 c}nd       5  
j                  |       ddd       y| dk(  rg }d}5  t	        
j                               }ddd       |D cg c]  }j                  |       }}t        |      dk(  st        |      |k  ryt        j                  |t        dt        |                  }|D cg c]  }|   	 }	}j                  |	|       yyc c}w c c}w c c}w c c}w # 1 sw Y   yxY w# 1 sw Y   xY wc c}w c c}w )z,Perform a random operation on the collectionr   Nr   r9   r:   r   )r<   )
indexrA   rD   updatelistcopyr   rL   rB   query)rZ   r[   idindices_to_modifyr(   currently_added_idsr<   currently_added_indicesr=   query_vectors	added_idsrO   r"   r    r   lockr!   s             r   perform_operationz6_test_interleaved_add_query.<locals>.perform_operationx   s    > ,,,9F G22 G$%*HH!) 4EE3DaJqM3DE( 2CC1BA9Q<1BC( 2CC1BA9Q<1BC     / !^"$I&*9>>+;&<# ?R&S?Rsyy}?R#&S'(A-./);
 #MM'B345M 5BBMqZ]MMBJJ#  + % !H F D D 
 &S Cs;   E!
E&
 E+6E0E54FF>F5E>F
r6   r8   r9   r      r   Tr:   r;   )N)%r?   r@   r   r*   r   r   r$   floatr   set	threadingLockr>   intr	   r   rA   r   r   rB   rC   rE   r   rF   
isinstancer   get_settingsis_persistentrG    fd_not_exceeding_threadpool_sizerH   rI   rJ   rK   rL   r#   rM   )r+   r   r   r,   rN   ri   rP   rQ   rR   rZ   rS   rT   rU   rV   rW   rF   r(   r=   r<   rg   rO   r"   r    r   rh   r!   s                      @@@@@@@r   _test_interleaved_add_queryrt   e   s   
 LLN##:J#KD%a+K
tCy+e,
-Cd5k;|#<=JT(^[%=>IK(I%I>>D	GA3nQC{m8
LM >B00'/S	':0	0 0d 
	4%'
3s8a<'q!,IA~#^^AqBw7
j#c(Z*?@"Q 7*Q.!):Is5QT~Vv&g%
a!% v&! 3s8a<' 
5* 	M$$&	 O 
 	6:&6+>+>+@+N+NRV+V33K@T;'{+t[1T" MMeAh"7h1h"7<MI#	Q 
5	4L #8s   7CK/	K)K&c                     t        d      D ]H  }t        j                  dt        j                         dz        }t               \  }}t        | |||       J y N   rj   )r#   r   r   multiprocessing	cpu_countr   rX   r+   r(   r,   r   r   s        r   test_multithreaded_addr{      sH    1XnnQ(A(A(Ca(GH"$11k: r   c                     t        d      D ]H  }t        j                  dt        j                         dz        }t               \  }}t        | |||       J y rv   )r#   r   r   rx   ry   r   rt   rz   s        r   test_interleaved_add_queryr}      sH    1XnnQ(A(A(Ca(GH"$1#FAq+> r   )%rx   concurrent.futuresr   r   r   r   rm   typingr   r   r   r	   r
   r   r   numpyr%   chromadb.apir   !chromadb.test.property.invariantsr2   propertyrG   chromadb.api.segmentr   !chromadb.test.property.strategiesr   r   chromadb.typesr   ro   r   r*   rX   rt   r{   r}    r   r   <module>r      s     ? ?   > > >  " 6 6 + 7 > #U38_ !3 !3 !9 !&999"%9479	9xrrr"%r47r	rj;9 ; ;?y ?T ?r   