
    ugy                        d dl Z d dlZd dlZd dlZd dlmZmZmZmZm	Z	m
Z
mZmZ d dlmZ d dlmZmZ d dlmZmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZm Z m!Z! d d	l"m#Z# d d
l$m%Z% d dl&m'Z' d dl(Z(d dl)Z)d dl*m+Z+ d dlm,Z, d dl-m.Z. deeddf   fdZ/deeddf   fdZ0deeg eeddf   f      fdZ1 ejd                  d e1             de,deeddf   fd       Z3 ejd                  d      dee   fd       Z4ddddddd d!d"d#d$
Z5d%e6de7fd&Z8 e e(jr                         d'e jt                   e(jv                  d (      di )      Z< e e(jr                         d'e jt                   e(jv                  d*(      di )      Z=d+e'd,e!ddfd-Z>d.ed/ee   d0eddfd1Z?d2ee   d3ee   ddfd4Z@d.ed/ee   d0eddfd5ZAd.ed/ee   d0eddfd6ZBd.ed/ee   d0eddfd7ZCd.ed/ee   ddfd8ZDd.ed/ee   d0eddfd9ZEd/ee   d:e%d+e'd;e(jv                  d<eddfd=ZFd.ed/ee   d0eddfd>ZGd.ed/ee   d0eddfd?ZHd.ed/ee   d0eddfd@ZId.ed/ee   d0eddfdAZJdCdBZKy)D    N)	GeneratorListCallableIteratorDictOptionalUnionSequence)validate_metadata)SystemSettings)ParameterValueget_sql)SqliteDB)
ProducerFn)OperationRecordMetadataEmbeddingRecord	OperationRequestVersionContextScalarEncodingSegmentSegmentScopeSeqId)Table)Producer)MetadataReader)SqliteMetadataSegment)FixtureRequest)countreturnc               #      K   t        dd      } t        |       }|j                          | |j                          yw)Fixture generator for sqlite DBTF)allow_resetis_persistentN)r   r   startstop)settingssystems     X/var/www/openai/venv/lib/python3.12/site-packages/chromadb/test/segment/test_metadata.pysqliter*   +   s3     D>HHF
LLN
L
KKMs   >A c               #     K   t        j                         } t        dd|       }t        |      }|j	                          | |j                          t        j                  j                  |       rt        j                  |        yyw)r"   T)r#   r$   persist_directoryN)tempfilemkdtempr   r   r%   r&   ospathexistsshutilrmtree)	save_pathr'   r(   s      r)   sqlite_persistentr5   4   si       "I	H HF
LLN
L
KKM	ww~~i i  !s   BB
c                      t         t        gS N)r*   r5        r)   system_fixturesr:   B   s    %&&r9   module)scopeparamsrequestc              #   B   K   t        | j                                y wr7   )nextparam)r>   s    r)   r(   r(   F   s     
w}}
s   function)r<   c                  J     dt         dt        fd  fdt               D        S )Nir    c                 
   | | dz  z   | dz   | dz  z   g}| dk(  rd }n8d|  | | | dz  z   dd}| dz  dk(  rd|d	<   | d
z  dk(  rd|d<   t        |       |d<   t        d|  |t        j                  |t        j
                        }|S )Ng?   r   value_T)str_keyint_key	float_keybool_key   truediv_by_three   FrK   chroma:document
embedding_id	embeddingencodingmetadata	operation)_build_documentr   r   FLOAT32r   ADD)rD   vectorrV   records       r)   create_recordz(sample_embeddings.<locals>.create_recordM   s    a#g+q1uq3w/6H $A3<S[ 	H 1uz+1(1uz',$*9!*<H&' A3#++mm
 r9   c              3   .   K   | ]  } |        y wr7   r8   ).0rD   r]   s     r)   	<genexpr>z$sample_embeddings.<locals>.<genexpr>h   s     .gM!gs   )intr   r   )r]   s   @r)   sample_embeddingsrb   K   s%      6 /eg..r9   zeroonetwothreefourfivesixseveneightnine)
0123456789rD   c                 Z    t        t        |             }dj                  d |D              S )N c              3   .   K   | ]  }t         |     y wr7   )
_digit_map)r_   ds     r)   r`   z"_build_document.<locals>.<genexpr>{   s     26aJqM6s   )liststrjoin)rD   digitss     r)   rX   rX   y   s$    #a&\F8826222r9   	test_type)ra   )rS   typer<   
collectionrV   
file_pathsrF   segmentseq_idc                    t        j                          }t        j                          |z
  dk  rD| j                         |k\  ry t        j                  d       t        j                          |z
  dk  rDt        d|       )N   g      ?zTimed out waiting for seq_id )time	max_seqidsleepTimeoutError)r   r   r%   s      r)   syncr      sj    IIKE
))+

!&(

4 ))+

! 6vh?
@@r9   r(   rb   produce_fnsc                    | j                  t              }| j                          t        d   }|} ||||d      d   d   }t	        | t              }|j                          t        ||       |j                  t        dd            dk(  sJ t        d      D ]  }|j                  |t        |            } t        ||       |j                  t        dd            dk(  sJ y )	Nr   rL   rF   r   collection_versionlog_positionrequest_version_context   )instancer   reset_statesegment_definitionr   r%   r   r   r   rangesubmit_embeddingr@   )r(   rb   r   producercollection_idmax_idr   rD   s           r)   test_insert_and_countr      s   
 x(H
&|4M!M=2CQGJ2NF#F,>?GMMO& 	$9#$1% 	 	

 		 1X**=$?P:QR  	& 	$9#$1% 	 	

 		r9   expectedactualc                     t        |       t        |      k(  sJ t        | d       }t        |d       }t        ||      D ]  \  }}|d   |d   k(  sJ |d   |d   k(  rJ  y )Nc                     | d   S NrS   r8   rs    r)   <lambda>z&assert_equiv_records.<locals>.<lambda>   s    QtWr9   )keyc                     | d   S r   r8   r   s    r)   r   z&assert_equiv_records.<locals>.<lambda>   s    4r9   rS   rV   )lensortedzip)r   r   sorted_expectedsorted_actualeas         r)   assert_equiv_recordsr      su     x=CK'''X+<=O6'89MO]31w!D'!!!}*--- 4r9   c                    | j                  t              }| j                          t        d   }|} ||||d      \  }}t	        | t              }|j                          t        ||d          t        dd      }|j                  ddi|      }	t        |	      d	k(  sJ |j                  dd
i|      }	t        |	      dk(  sJ |j                  |      }
t        ||
       |j                  |dd	 D cg c]  }|d   	 c}|      }	t        |dd	 |	       g }|j                  |j                  d|             t        |      dk(  sJ |j                  |j                  dd|             t        |      dk(  sJ |j                  |j                  dd|             t        |      dk(  sJ |j                  |j                  dd|             t        |      dk(  sJ t        ||       |j                  ddi|      }	t        |	      dk(  sJ |j                  ddd	ii|      }	t        |	      dk(  sJ |j                  ddd	ii|      }	t        |	      d	k(  sJ |j                  ddd	ii|      }	t        |	      dk(  sJ |j                  ddd	ii|      }	t        |	      d	k(  sJ |j                  dddii|      }	t        |	      d	k(  sJ |j                  dddii|      }	t        |	      dk(  sJ |j                  ddd	ii|      }	t        |	      d	k(  sJ |j                  ddd	ii|      }	t        |	      dk(  sJ |j                  dddii|      }	t        |	      dk(  sJ |j                  dddii|      }	t        |	      dk(  sJ |j                  ddd	ii|      }	t        |	      dk(  sJ |j                  ddd	id|      }	t        |	      d k(  sJ |j                  d!dd"idd igi|      }	t        |	      d k(  sJ |j                  d#ddiddd	iigi|      }	t        |	      dk(  sJ |j                  d#ddiddd	iigi|      }	t        |	      d"k(  sJ y c c}w )$Nr   
   r   r   r   rK   Twherer   r   F   r   rS   idsr   rL   limitr   r   offsetr   r   	   rN   rM   rI   z$gtz$gtez$ltz$lterJ   g
ףp=
@g(\@z$ne)rN   rI   rO   $orrF   $and)r   r   r   r   r   r%   r   r   get_metadatar   r   extend)r(   rb   r   r   r   
embeddingsseq_idsr   r   resultresultsr   rets                r)   test_getr      s   
 x(H
&|4M!M%h?PRTUJ#F,>?GMMO'"+31 !!4 :Q " F v;!!!5!;R " F v;! "";R"SGW- !!(1o.oQtWo. 7 " F Aa&1
 *,CJJ1>UV s8q==JJA7N 	 	

 s8q==JJA7N 	 	

 s8q==JJA7N 	 	

 s8r>>S) !!v&@W " F v;! !!5!*% 7 " F v;!!!61+& 7 " F v;!!!5!*% 7 " F v;!!!61+& 7 " F v;! !!UDM* 7 " F v;!!!UDM* 7 " F v;! !!UAJ' 7 " F v;!!!UAJ' 7 " F v;! !!5$-( 7 " F v;!!!5$-( 7 " F v;! !!5!*% 7 " F v;! !!%5!*= 7 " F v;! !!	1~	1~67 7 " F v;! !!Auaj)
 !8 " F v;!!!Auaj)
 !8 " F v;!W /s   5Q2c           
      8   | j                  t              }| j                          t        d   }|}t	        | t              }|j                           ||||d      d   d   }t        ||       t        dd      }|j                  ddi|	      }|j                  d
g|      }	||	k(  sJ |j                  ddi|      }t        |      dk(  sJ |j                  ddi|      }t        |      t        t        dd      D 
cg c]  }
dt        |
      vs|
 c}
      dz   k(  sJ |j                  ddi|      }t        |      dk(  sJ |j                  ddi|      }t        |      t        t        dd      D 
cg c]  }
dt        |
      vs|
 c}
      dz   k(  sJ |j                  dddiddigi|      }t        |      dk(  sJ t        |D cg c]  }|d   	 c}      d
dhk(  sJ |j                  dddiddigi|      }t        |      t        t        dd      D 
cg c]  }
dt        |
      vrdt        |
      vr|
  c}
      dz   k(  sJ |j                  dddiddigi|      }t        dd      D 
cg c]  }
dt        |
      v s|
 }}
t        dd      D 
cg c]  }
dt        |
      v s|
 }}
t        t        ||z         D 
cg c]  }
d|
 	 c}
      }t        |D cg c]  }|d   	 c}      |k(  sJ |j                  dddiddigi|      }t        |      t        t        dd      D 
cg c]  }
dt        |
      vsdt        |
      vr|
  c}
      dz   k(  sJ |j                  dddiiddi|      }t        |      dk(  sJ |j                  dddiiddi|      }t        |      dk(  sJ |j                  ddi|      }t        |      dk(  sJ y c c}
w c c}
w c c}w c c}
w c c}
w c c}
w c c}
w c c}w c c}
w )Nr   d   rF   r   r   r   rP   zfour twor   embedding_42r   	$containswhere_documentr   z$not_containsrc   r   r   rg   re   rO   rS   embedding_24r   rd   rQ   rI   z$eq*   )r   r   r   zer)r   r   r   r   r   r%   r   r   r   r   r   rX   set)r(   rb   r   r   r   r   r   r   r   result2rD   r   oneszerosr   s                  r)   test_fulltextr     s]   
 x(H
&|4M!M#F,>?GMMO=2CSI!LRPF&31 !! *- 7 " F ""6M # G W !!#Z0 7 " F v;! !!'4 7 " F
 	F5C=Q=aJoa>P,P=QRUVV	W	W
 !!#V, 7 " F v;! !!'0 7 " F
 	F5C=M=aF/!:L,L=MNQRR	S	S
 !!+v!6e8L MN 7 " F v;!(A$()nn-MMMM!!/6!:_e<T UV 7 " F
 	F q#&A!33_UVEW8W &
 	
	 !!f 5U7KLM 7 " F Q}D}!1C(CA}DDaF1?13E)EQEFc$,.?@.?j$.?@AH(A$()X555!!& 9OU;STU 7 " F
 	F q#&A!33uOTUDV7V &
 	
	 !!5"+&#V, 7 " F
 v;! !!5"+&#V, 7 " F
 v;! !!#U+ 7 " F v;!w R" N ) EF@(sN   O/
O/
>O4
O4
O9)#O>
PP*P=PP4P#P
c                    | j                  t              }| j                          t        d   }|}t	        | t              }|j                           ||||d      \  }}|d   }t        ||       t        dd      }	|j                  |	      dk(  sJ |j                  dg|	      }
t        |d d	 |
       t        dd d d t        j                  
       |||fdt        d	      D        d	      d	   d   }t        ||       t        dd      }	|j                  |	      dk(  sJ |j                  dg|	      g k(  sJ  |||fdt        d	      D        d	      d	   d   }t        ||       |j                  |	      dk(  sJ |j                  dg|	      g k(  sJ |j                  ||d         }t        ||       |j                  |	      dk(  sJ |j                  dg|	      }
y )Nr   r   r   r   r   r   embedding_0r   rF   rR   c              3   "   K   | ]  }  y wr7   r8   r_   _delete_embeddings     r)   r`   ztest_delete.<locals>.<genexpr>1       !EHq"2H   r   c              3   "   K   | ]  }  y wr7   r8   r   s     r)   r`   ztest_delete.<locals>.<genexpr>A  r   r   )r   r   r   r   r   r%   r   r   r   r   r   r   r   DELETEr   r   )r(   rb   r   r   r   r   r   r   r   version_contextr   r   s              @r)   test_deleter     sJ   
 x(H
&|4M!M#F,>?GMMO%h?PRTUJR[F&+qqQO===ARGGG""O_ # G BQ1 '"" -!EE!H!Eq		F 	&+qqQO===AQFFF 	 	
 		 -!EE!H!Eq		F 	&===AQFFF 	 	
 		 &&}jmDF&===ARGGG""O_ # Gr9   c                    | j                  t              }| j                          t        d   }|}t	        | t              }|j                          t        ||||t        j                         t        dddid d t        j                        }|j                  ||      }t        ||       t        dd      }|j                  dg|      }t        |      dk(  sJ |j                  |	      d
k(  sJ y )Nr   
no_such_idfoobarrS   rV   rT   rU   rW   r   r   r   r   rL   )r   r   r   r   r   r%   _test_updater   UPDATEr   r   r   r   r   r   r   )	r(   rb   r   r   r   update_recordr   r   r   s	            r)   test_updater   V  s    x(H
&|4M!M#F,>?GMMO"Hg}iFVFVW $""M &&}mDF&31 ""N4K # G w<1==1H=IQNNNr9   c                   	 | j                  t              }| j                          t        d   }|}t	        | t              }|j                          t        ||||t        j                         t        dddid d t        j                        	 |||	fdt        d      D        d      d   d	   }t        ||       t        d
d
      }|j                  dg|      }|d
   d   ddik(  sJ y )Nr   r   r   r   r   c              3   "   K   | ]  }  y wr7   r8   )r_   r   r   s     r)   r`   ztest_upsert.<locals>.<genexpr>  s     48aM8r   rF   )r   r   r   nr   r   r   r   rV   )r   r   r   r   r   r%   r   r   UPSERTr   r   r   r   r   )
r(   rb   r   r   r   r   r   r   r   r   s
            @r)   test_upsertr   v  s   
 x(H
&|4M!M#F,>?GMMO"Hg}iFVFVW $""M #4584
	
 	
 
F 	&31 ""N4K # G 1:j!eU^333r9   r   r   opc                    t        d      D cg c]  }t        |        }}d}|D ]  }|j                  ||      } t        ||       t	        dd      }	|j                  dg|	      }
t        |dd |
       t        ddd	idd|
      }|j                  ||      }t        ||       |j                  dg|	      }
|
d   d   dd	ik(  sJ |j                  ddi|	      }
|
d   d   dd	ik(  sJ t        dddidd|
      }|j                  ||      }t        ||       |j                  dg|	      }
|
d   d   ddik(  sJ |j                  ddi|	      }
|
d   d   ddik(  sJ |j                  ddi|	      }
t        |
      dk(  sJ t        dddidd|
      }|j                  ||      }t        ||       |j                  dg|	      }
|
d   d   dddk(  sJ t        dddidd|
      }|j                  ||      }t        ||       |j                  dg|	      }
|
d   d   ddik(  sJ |j                  ddi|	      }
t        |
      dk(  sJ yc c}w )z0test code common between update and upsert pathsrL   r   r   r   r   NrF   rP   zfoo barr   rV   r   r   r   zbiz buzbizbazr   )rP   r   )	r   r@   r   r   r   r   r   r   r   )rb   r   r   r   r   rD   r   r   r   r   r   r   s               r)   r   r     s`    498<8a$()8J<F**=!<  	&31 ""O5L # G BQ1 $#Y/M &&}mDF&""O5L # G 1:j!&7%CCCC""#U+ 7 # G 1:j!&7%CCCC $#Y/M &&}mDF&""O5L # G 1:j!&7%CCCC""#U+ 7 # G 1:j!&7%CCCC""#U+ 7 # G w<1 $M &&}mDF&""O5L # G 1:j!2%NNNN $#T*M &&}mDF&""O5L # G 1:j!eR[000""#U+ 7 # G w<1{ =s   Ic                 Z   | j                  t              }| j                          t        d   } ||||d      d   d   }t        d   } ||||d      d   d   }t        | t              }|j                          t        | t              }	|	j                          t        ||       t        |	|       t        dd      }
|j                  |
      dk(  sJ t        d      D ]  }|j                  |t        |            } t        ||       |j                  |
      dk(  sJ |j                  d|
	      }t        |      dk(  sJ t        j                   t"              5  |j                  d|
	       d d d        |j                  dd
|
      }t        |      dk(  sJ y # 1 sw Y   .xY w)Nr   rL   rF   r   r   r   r   r   r   r   r   )r   r   r   r   segment_definition2r   r%   r   r   r   r   r   r@   r   r   pytestraises
ValueError)r(   rb   r   r   r   r   collection_id_2max_id2r   segment2r   rD   ress                r)   
test_limitr     s   
 x(H
&|4M=2CQGJ2NF),7O(O5FJ1MbQG#F,>?GMMO$V-@AHNN&731 ==1H=IQNNN1X**=$?P:QR  	&==1H=IQNNN


Q@W

XCs8q== 
z	"2?VW 
# 

4K  C s8q== 
#	"s    F!!F*c                    | j                  t              }| j                          t        d   }|}t	        | t              }|j                           ||||d      \  }}|d   }t        ||       t        dd      }	|j                  |	      dk(  sJ |j                  dg|	      }
t        |d d	 |
       |j                  }|j                          | j                  t              }t        d
      }|j                         j!                  |      j#                  |j$                        j'                  |j(                  t+        |j-                  |            k(        }t/        |      \  }}|j1                         5 }|j3                  ||      }t5        |j7                               dk(  sJ 	 d d d        t        d      }|j                         j!                  |      j#                         j'                  |j8                  j;                  |j                         j!                  |      j#                  |j$                        j'                  |j(                  t+        |j-                  |            k(                    }t/        |      \  }}|j1                         5 }|j3                  ||      }t5        |j7                               dk(  sJ 	 d d d        y # 1 sw Y   -xY w# 1 sw Y   y xY w)Nr   r   r   r   r   r   r   r   rF   r   embedding_fulltext_search)r   r   r   r   r   r%   r   r   r   r   r   _iddeleter   r   querybuilderfrom_selectrS   r   
segment_idr   
uuid_to_dbr   txexecuter   fetchallrowidisin)r(   rb   r   r   r   r   r   r   r   r   r   r   _dbtqsqlr=   curr   fts_tq_ftss                        r)   test_delete_segmentr  8  sj   
 x(H
&|4M!M#F,>?GMMO%h?PRTUJR[F&31 ==1H=IROOO""O5L # G BQ1
++CNN
//(
#ClA	q		q||~cnnS.ABB	C	  !*KC	Skk#v&3<<>"a''' 

 -.E	u		KK  "qq||~cnnS6I'JJK	

	 
 %.KC	Skk#v&3<<>"a''' 
) 
( 
s   =1K1K'K$'K0c                    | j                  t              }| j                          t        d   }|}t	        | t              }|j                           ||||d      \  }}|d   }t        ||       t        dd      }	|j                  |	      dk(  sJ |j                  dg|	      }
t        |d d	 |
       |j                  }| j                  t              }t        dd d d t        j                  
       |||fdt!        d	      D        d	      d	   d   }t#        d      }t        ||       t#        d      }|j%                         j'                  |      j)                         j+                  |j,                  j/                  |j%                         j'                  |      j)                  |j0                        j+                  |j2                  t5        |j7                  |            k(        j+                  |j8                  t5        d         k(                    }t;        |      \  }}|j=                         5 }|j?                  ||      }tA        |jC                               dk(  sJ 	 d d d        y # 1 sw Y   y xY w)Nr   r   r   r   r   r   r   r   rF   rR   c              3   "   K   | ]  }  y wr7   r8   r   s     r)   r`   z0test_delete_single_fts_record.<locals>.<genexpr>  r   r   r   r   rS   )"r   r   r   r   r   r%   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r   r	  r
  rS   r  r   r  embedding_idr   r  r  r   r  )r(   rb   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r=   r  r   r   s                       @r)   test_delete_single_fts_recordr  x  s;   
 x(H
&|4M!M#F,>?GMMO%h?PRTUJR[F&31 ==1H=IROOO""O5L # G BQ1
++C
//(
#C&"" -!EE!H!Eq		F 	lA&-.E	u		KK  "qq||~cnnS6I'JJKq~~8H8N)OOP

	 
 %.KC	Skk#v&3<<>"a''' 
s   1I<<Jc                 >   | j                  t              }| j                          t        d   }|}t	        | t              }|j                           ||||d      \  }}|d   }t        ||       t        dd      }	|j                  |	      dk(  sJ |j                  dg|	      }
t        |d d	 |
       |j                  dgd
|	      }
t        |
      d	k(  sJ |
d   d   J |j                  dgd|	      }
t        |
      d	k(  sJ |
d   d   |d   d   k(  sJ y )Nr   r   r   r   r   r   r   r   rF   F)r   include_metadatar   rV   T)r   r   r   r   r   r%   r   r   r   r   r   r   )r(   rb   r   r   r   r   r   r   r   r   r   s              r)   test_include_metadatar    si   
 x(H
&|4M!M#F,>?GMMO%h?PRTUJR[F&31 ==1H=IROOO""O5L # G BQ1 ""O 7 # G
 w<11:j!))) ""O 7 # G
 w<11:j!Z]:%>>>>r9   c                  ~    t        j                  t        d      5  t        ddi       d d d        y # 1 sw Y   y xY w)NrP   )matchz,this is not the document you are looking for)r   r   r   r   r8   r9   r)   &test_metadata_validation_forbidden_keyr    s.    	z):	; NO	
 
<	;	;s   3<)r    N)Lr/   r2   r-   r   typingr   r   r   r   r   r   r	   r
   chromadb.api.typesr   chromadb.configr   r   chromadb.db.baser   r   chromadb.db.impl.sqliter   chromadb.test.conftestr   chromadb.typesr   r   r   r   r   r   r   r   pypikar   chromadb.ingestr   chromadb.segmentr   uuidr   %chromadb.segment.impl.metadata.sqliter   r   	itertoolsr   r*   r5   r:   fixturer(   rb   rz   ra   r}   rX   uuid4METADATAUUIDr   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r8   r9   r)   <module>r.     s9   	   	 	 	 1 , 4 , -	 	 	  $ +   G ! 	&$,- !9VT4%78 !'hr9VT45G+H'HIJ ' h'89 N  yt1C'D   :  j!/8O4 / "/@ 
									
3s 3s 3
 tzz|	


tyyQ  tzz|	


tyyQ A. A% AD A((0( ( 
	(V.'.19:Q1R.	.pp0p p 
	pfGG0G G 
	GTEE0E E 
	EPO O8O3L OQU O@%4%40%4 %4 
	%4Pf0ff f 99	f
 	f 
fR..0. . 
	.b=(=(0=( =( 
	=(@>(>(0>( >( 
	>(B,?,?0,? ,? 
	,?^
r9   