
    ug)1              	          d dl Z d dlZd dlZd dlZd dlmZ d dlZd dlmZ d dl	m
Z
mZmZmZmZmZmZmZmZ d dlmZ d dlmZmZ d dlmZ d dlmZ d d	lmZmZmZm Z  d d
l!m"Z"m#Z# d dlm$Z$m%Z% d dl m&Z&m'Z'm(Z( d dl)Z*de
eeef   ddf   fdZ+de
eeef   ddf   fdZ,deeg e
eeef   ddf   f      fdZ- ej\                  d e-             de$de
eeef   ddf   fd       Z/ ej\                  d      dee   fd       Z0 G d d      Z1dee2   dee2   ddfdZ3dee   dee   ddfdZ4ejj                  j                   deeef   d ee   d!eddfd"       Z6ejj                  j                   deeef   d ee   ddfd#       Z7ejj                  j                   deeef   d ee   ddfd$       Z8ejj                  j                   deeef   d ee   d!eddfd%       Z9ejj                  j                   deeef   d ee   d!eddfd&       Z:ejj                  j                   deeef   d ee   ddfd'       Z;ejj                  j                   deeef   d ee   d!eddfd(       Z<ejj                  j                   deeef   d ee   ddfd)       Z=y)*    N)UUID)count)		GeneratorListCallableOptionalDictUnionIteratorSequenceTuple)BatchSizeExceededError)ProducerConsumer)SqliteDB)
ProducerFn)OperationRecord	Operation	LogRecordScalarEncoding)SystemSettings)FixtureRequestapprox)Eventwait_forTimeoutErrorreturnc               #      K   t        t        d            } | j                  t              }| j	                          ||f | j                          yw)z0Fixture generator for sqlite Producer + ConsumerT)allow_resetN)r   r   requirer   startstop)systemdbs     `/var/www/openai/venv/lib/python3.12/site-packages/chromadb/test/ingest/test_producer_consumer.pysqliter'   #   s>     H./F		!B
LLN
b&L
KKMs   AAc               #   B  K   t        j                         } t        t        dd|             }|j	                  t
              }|j                          ||f |j                          t        j                  j                  |       rt        j                  |        yyw)z;Fixture generator for sqlite_persistent Producer + ConsumerT)r    is_persistentpersist_directoryN)tempfilemkdtempr   r   r!   r   r"   r#   ospathexistsshutilrmtree)	save_pathr$   r%   s      r&   sqlite_persistentr3   ,   sv       "ITSF 
	!B
LLN
b&L
KKM	ww~~i i  !s   BBc                  F    t         t        g} dt        j                  v rg } | S )NCHROMA_CLUSTER_TEST_ONLY)r'   r3   r-   environ)fixturess    r&   r7   r7   :   s#    )*H!RZZ/O    module)scopeparamsrequestc              #   B   K   t        | j                                y wN)nextparam)r<   s    r&   producer_consumerrA   C   s      w}}
s   )r:   c                  J     dt         dt        fd  fdt               D        S )Nir   c                     t        j                  | | dz  z   | dz   | dz  z   g      }| dz  dk(  rd }nd|  | | | dz  z   d}t        d|  |t        j                  |t
        j                        }|S )	Ng?      r   value_)str_keyint_key	float_key
embedding_)id	embeddingencodingmetadata	operation)nparrayr   r   FLOAT32r   ADD)rC   vectorrO   records       r&   create_recordz(sample_embeddings.<locals>.create_recordL   s    1q3w;AC89q5A:H%+A3<AAPQTWPWKXH A3#++mm
 r8   c              3   .   K   | ]  } |        y wr>    ).0rC   rW   s     r&   	<genexpr>z$sample_embeddings.<locals>.<genexpr>]   s     .gM!gs   )intr   r   )rW   s   @r&   sample_embeddingsr]   J   s%      " /eg..r8   c                   r    e Zd ZU ee   ed<   eeeef      ed<   d
dZ	de
e   ddfdZddedede
e   fd	Zy)CapturingConsumeFn
embeddingswaitersr   Nc                 R    g | _         g | _        t        j                         | _        y)zA function that captures embeddings and allows you to wait for a certain
        number of embeddings to be available. It must be constructed in the thread with
        the main event loop
        N)r`   ra   asyncioget_event_loop_loop)selfs    r&   __init__zCapturingConsumeFn.__init__d   s"    
 ++-
r8   c                     | j                   j                  |       | j                  D ]C  \  }}t        | j                         |k\  s| j                  j                  |j                         E y r>   )r`   extendra   lenre   call_soon_threadsafeset)rf   r`   nevents       r&   __call__zCapturingConsumeFn.__call__m   sM    z*HAu4??#q(

//		: %r8   rm   timeout_secsc                   K   t        | j                        |k\  r| j                  d| S t               }| j                  j	                  ||f       t        |j                         |       d{    | j                  d| S 7 w)zJWait until at least N embeddings are available, then return all embeddingsN)rj   r`   r   ra   appendr   wait)rf   rm   rp   rn   s       r&   getzCapturingConsumeFn.gett   sq     t1$??2A&&GELLE
+5::<666??2A&& 7s   A,B.B/B)r   N)
   )__name__
__module____qualname__r   r   __annotations__r   r\   r   rg   r   ro   rt   rY   r8   r&   r_   r_   `   s\    Y%U
#$$.;8I#6 ;4 ;	'3 	'c 	'8I;N 	'r8   r_   abc                 ^    t        | |      D ]  \  }}t        |      t        |      k(  rJ  y r>   )zipr   )rz   r{   rC   js       r&   assert_approx_equalr      s,    Aq	1ayF1I%%% r8   inserted_recordsconsumed_recordsc                 (   t        |      t        |       k(  sJ t        | |      D ]j  \  }}|d   |d   d   k(  sJ |d   |d   d   k(  sJ |d   |d   d   k(  sJ |d   |d   d   k(  sJ |d   L|d   d   J t        |d   |d   d          l y)zCGiven a list of inserted and consumed records, make sure they matchrL   rV   rP   rN   rO   rM   N)rj   r}   r   )r   r   insertedconsumeds       r&   assert_records_matchr      s    
  C(8$9999!"24DE(~(!3D!9999$(:;(GGGG
#x'9*'EEEE
#x'9*'EEEEK ,H%k2>>> 5x7I+7VW Fr8   rA   r]   produce_fnsc                 .  K   | \  }}|j                          |j                          t        d      } ||||d      d   }t               }|j                  |||j	                                |j                  d       d {   }t        ||       y 7 w)N$00000000-0000-0000-0000-000000000000   r   r"   )reset_stater   r_   	subscribe	min_seqidrt   r   )	rA   r]   r   producerconsumercollection_idr`   
consume_fnrecieveds	            r&   test_backfillr      s      +Hh?@MX}6GKANJ#%J}j8J8J8LM^^A&&HX. 's   A?BBBc                   K   | \  }}|j                          |j                          t        d      }g }t               }|j                  |||j	                                t        d      D ]X  }t        |      }|j                  |       |j                  ||       |j                  |dz          d {   }	t        ||	       Z y 7 w)Nr   r   ru   rE   r   r   r_   r   r   ranger?   rr   submit_embeddingrt   r   )
rA   r]   r   r   r   r`   r   rC   ereceiveds
             r&   test_notificationsr      s     
 +Hh?@M(*J#%J}j8J8J8LM2Y"#!!!-3#A..Z2  /s   B2C
4C5C
c                   K   | \  }}|j                          |j                          t        d      }t        d      }g }g }t               }t               }	|j                  |||j	                                |j                  ||	|j	                                t        d      D ]  }
t        |      }|j                  |       |j                  ||       |j                  |
dz          d {   }t        ||       t        |      }|j                  |       |j                  ||       |	j                  |
dz          d {   }t        ||        y 7 i7 w)Nz$00000000-0000-0000-0000-000000000001z$00000000-0000-0000-0000-000000000002r   ru   rE   r   )rA   r]   r   r   collection_1collection_2embeddings_1embeddings_2consume_fn_1consume_fn_2rC   e_1	results_2e_2s                 r&   test_multiple_collectionsr      sC    
 +Hh>?L>?L*,L*,L%'L%'L|\9K9K9MN|\9K9K9MN2Y$%C !!,4&**1q511	\95$%C !!,4&**1q511	\95  2 2s%   C+E-E.AEEEEc                 D  K   | \  }}|j                          |j                          t        d      }t               }t               }|j                  |||j	                                 ||||d      d   }|j                  d       d {   }	t        ||	       |j                  d   d   }
|j                  |||
        ||||d      d   }t        |t              sJ |j                  |       |j                  d       d {   }t        |dd  |       y 7 7 w)Nr   r      r   
log_offset)r   r   r_   r   r   rt   r   r`   
isinstancelistri   )rA   r]   r   r   r   
collectionr   r   r`   	results_1r"   second_embeddingsr   s                r&   test_start_seq_idr      s%     +Hh<=J%'L%'Lz<x7I7I7KLXz3DaHKJ"&&q))IY/##B'5Ez<u=#Hj:KQOPQRj$''''("&&q))IBC)4 * *s%   B	D DA;D DD D c                   K   | \  }}|j                          |j                          t        d      }t               }t               }|j                  |||j	                                 ||||d      d   }|j                  d       d {   }	t        ||	       |j                  d   d   }
|j                  |||j	                         |
       |j                  d       d {   }t        |d d |       t        j                  t              5  t        |j                  d	      d
       d {   }d d d        y 7 7 \7 # 1 sw Y   y xY ww)Nr   r   ru   r   r   r   )r"   end      rE   )timeout)r   r   r_   r   r   rt   r   r`   pytestraisesr   r   )rA   r]   r   r   r   r   r   r   r`   r   r   r   _s                r&   test_end_seq_idr     s9     +Hh<=J%'L%'Lz<x7I7I7KLXz3DbI!LJ"&&r**IY/

!
!"
%l
3Cz<x7I7I7KQTU"&&q))IBQ3 
|	$<++A.:: 
%	$ + *
 ; 
%	$sO   B	EE AE%E&,E E2E3E7
EEEEEc                   K   | \  }}|j                          |j                          t        d      }t        d      D cg c]  }t        |       }}|j	                  ||       t               }|j                  |||j                                |j                  d       d {   }t        ||       y c c}w 7 w)Nr   d   r`   r   )
r   r   r   r?   submit_embeddingsr_   r   r   rt   r   )	rA   r]   r   r   r   r   r`   r   r   s	            r&   test_submit_batchr   !  s     
 +Hh<=J38:>:a$():J>zjA#%Jz:X5G5G5IJ^^C((HX. ? )s   >C  B9AC 'B>(C c                 t  K   | \  }}|j                          |j                          d}t        |      D cg c]  }t                }}t        |      D ]3  }|j                  t	        d|       ||   |j                                5 t        |      D cg c]  }g  }	}d}
d}d}t        ||
z        D ]q  }t        |      D ]\  }|	|   j                   ||t	        d|       ||
      d          ||   j                  ||
z          d {   }t        |	|   |       ^ ||
z  }s y c c}w c c}w 7 'w)NrF   z#00000000-0000-0000-0000-00000000000r   ru   r   r   )	r   r   r_   r   r   r   ri   rt   r   )rA   r]   r   r   r   N_TOPICSr   consume_fnsrC   embeddings_nPRODUCE_BATCH_SIZEN_TO_PRODUCEtotal_producedrm   r   s                  r&   test_multiple_collections_batchr   6  sf     +HhH16xAA%'KA8_6qc:;N$$& 	 	
  >C8_0M__L0MLN<#556xAO"">qcBC%&	
  )^//AS0STTH a(; ! 	,, 7 B 1N Us*   5D8D,AD8	D1!A-D8D6(D8c                   K   | \  }}|j                          |j                          t        d      }|j                  }|dkD  sJ t        |      D cg c]  }t	        |       }}t               }|j                  |||j                                |j                  ||       |j                  |d       d {   }	t        ||	       t        |dz         D cg c]  }t	        |       }}t        j                  t              5 }
|j                  ||       d d d        dt        
j                        v sJ y c c}w 7 c c}w # 1 sw Y   /xY ww)	Nr   r   r   r   x   )rp   rE   zCannot submit more than)r   r   max_batch_sizer   r?   r_   r   r   r   rt   r   r   r   r   strvalue)rA   r]   r   r   r   r   r   r`   r   r   r   s              r&   test_max_batch_sizer   ]  sD    
 +Hh<=J,,NA 493HI3Ha$()3HJI#%Jz:X5G5G5IJzjA^^N^EEHX.38!9K3LM3La$()3LJM	-	.!"":*"E 
/$AGG444 J F N	.	.sI   AEE %AE<E= EE/E
E(EEEE)>rc   r-   r0   r+   uuidr   r   	itertoolsr   typingr   r   r   r   r	   r
   r   r   r   chromadb.errorsr   chromadb.ingestr   r   chromadb.db.impl.sqliter   chromadb.test.conftestr   chromadb.typesr   r   r   r   chromadb.configr   r   r   r   r   r   r   numpyrQ   r'   r3   r7   fixturerA   r]   r_   floatr   r   markr   r   r   r   r   r   r   r   rY   r8   r&   <module>r      s    	     
 
 
 3 . , -  - ) 1 1 	%( 23T4?@ !9U8X+=%>d%JK !$xIeHh4F.Gt.S$T TUV  hxz2  uXx'($45  3  h/8O4 /  /*' '@&8E? &x &4 &
X/Xy)X 
X" /Xx/0/0/ / 
	/ /$ 3Xx/0303 
3 3. 6Xx/0606 
6 6B 5Xx/0505 5 
	5 5: ;Xx/0;0; ; 
	; ;> /Xx/0/0/ 
/ /( #-Xx/0#-0#- #- 
	#- #-L 5Xx/0505 
5 5r8   