
    ugZ                        d dl Z d dlZd dlmZ d dlZd dlZd dlmZmZm	Z	m
Z
mZ d dlmZ d dlmZ d dlmZ d dlZd dlZd dlmZmZ d dlmZmZ d dlmZ d d	lmZ d dlm c m!c mZ d dl"m c m!c m#Z# d d
lm$Z$ d dl%m&Z&m'Z'm(Z( d dl)m*Z*m+Z+m,Z,m-Z-m.Z. d dl/Z/d dl0Z0d dl1Z1d dl2m3Z4 d dl5m6Z6 d dl7Z8eg ef   Z9 eddddddd e1jt                               gZ; ejx                  de;      dejz                  deeddf   fd       Z> ej~                   ej                  d ej                  d      dd      d      ZBej                  dej                  de
ej                  ej                  f   fd       ZG e ej                   eG       dd  d! f"      #      d$ed%e	e
ej                  ej                  f      ddfd&       ZId$eddfd'ZJd$ed(eKd)ej                  d*eddf
d+ZLd, ZM G d- d.e&      ZNd/ZO G d0 d1e(      ZPd2ej                  d$eddfd3ZRd2ej                  d$eddfd4ZSd$eddfd5ZTd2ej                  d$eddfd6ZUy)7    N)
Connection)	GeneratorCallableListTuplecast)UUID)given)	ClientAPI	ServerAPI)SettingsSystem)VectorReader)LocalSegmentManager)hashing_embedding_function)EmbeddingStateMachineStatestraceEmbeddingStateMachineBase)run_state_machine_as_testruleprecondition
initializeMultipleResults)Client)DefaultEmbeddingFunctionzchromadb.api.segment.SegmentAPIz chromadb.db.impl.sqlite.SqliteDBz7chromadb.segment.impl.manager.local.LocalSegmentManagerT)chroma_api_implchroma_sysdb_implchroma_producer_implchroma_consumer_implchroma_segment_manager_implallow_resetis_persistentpersist_directorymodule)scopeparamsrequestreturnc              #     K   | j                   }|j                  }t        j                  j	                  |      st        j
                  |d       | t        j                  j	                  |      rt        j                  |d       y y w)NT)exist_ok)ignore_errors)paramr#   ospathexistsmakedirsshutilrmtree)r'   configuration	save_paths      X/var/www/openai/venv/lib/python3.12/site-packages/chromadb/test/property/test_persist.pysettingsr6   6   sa     MMM//I77>>)$
I-
	ww~~i it4 !s   BB
   with_hnsw_paramswith_persistent_hnsw_paramsmax_hnsw_batch_sizemax_hnsw_sync_thresholdcoll)keydrawc                      | t        j                  dt        j                  d      dd            } | t        j                  t        j                  |                  }||fS )NTr7   r8   )
strategiescollectionsstjust
recordsets)r?   
collection	recordsets      r5   !collection_and_recordset_strategyrH   P   s[     !(* "$&	
J Z**277:+>?@Iy         c                      | d   j                   S Nr   namexs    r5   <lambda>rQ   e   s    QqTYYrI   c                      | d   j                   S rL   rM   rO   s    r5   rQ   rQ   e   s    !A$))rI   )min_size	unique_by)#collection_and_recordset_strategiesr6   rU   c                    t        |       }|j                          t        j                  |      }|j	                          |D ]  \  }}|j                  |j                  |j                  |j                        } |j                  di | t        j                  ||       t        j                  ||       t        j                  ||       t        j                  ||       t        j                  |||j                          |j!                          ~~t        |       }|j                          t        j                  |      }|D ]  \  }}|j#                  |j                  |j                        }t        j                  ||       t        j                  ||       t        j                  ||       t        j                  ||       t        j                  |||j                          |j!                          ~~y )NrN   metadataembedding_function)rY   rN   rY    )r   startClientCreatorfrom_systemresetcreate_collectionrN   rX   rY   add
invariantscountmetadatas_matchdocuments_match	ids_matchann_accuracystopget_collection)	r6   rU   system_1client_1collection_strategyrecordset_strategyr=   system_2client_2s	            r5   test_persistrp   a   s    hHNN((2HNN 
-	))$))(112EE * 
 	&%&12""4);<""4);<T#562EE	
 
-& MMOhHNN((2H
 
-	&&$))2EE ' 
 	12""4);<""4);<T#562EE	
 
- MMOrI   c                 0   t        |       }|j                          t        j                  |      }|j	                  dddd      }|j                  t              }|j                  |j                  t              dt        ffd} |       }|j                  ddgd	gd
gg        |       |k(  sJ  |       }|j                  dgdgg        |       |kD  sJ  |       }|j                  g dd	gd
gdgg        |       |kD  sJ  |       }|j                  dgdgg       |j                  ddgd	gd
gg        |       |kD  sJ  |       }|j                  dgdgg       |j                  ddgd	gd
gg        |       |kD  sJ  |       }y )Ntest   )hnsw:batch_sizehnsw:sync_threshold)rN   rX   r(   c                      t        j                  d       	 t        j                  j	                   j                               S # t        $ r Y yw xY w)Ng?)timesleepr-   r.   getmtime_get_metadata_fileFileNotFoundError)segments   r5   get_index_last_modified_atz7test_sync_threshold.<locals>.get_index_last_modified_at   sC    

3	77##G$>$>$@AA  		s   ,A 	AA12      ?       @)ids
embeddings3      @)r   r   r   4      @5      @)r   r\   r]   r^   r`   instancer   get_segmentidr   floatra   upsert)r6   systemclientrF   managerr~   last_modified_atr}   s          @r5   test_sync_thresholdr      s   HF
LLN&&v.F))!AN * J oo12G!!*-->G  23NNSzusenN= &'+;;;;13 NNu3%N1 &'*::::13 /usecU6KL &'*::::13 NNu3%N13*3%#@ &'*::::13 NNu3%N1NNSzusenN= &'*::::13rI   collection_name
record_setconnc                    	 t        |       }|j                          t        j                  |      }|j	                  |t        j                               }t        j                  ||       t        j                  ||       t        j                  ||       t        j                  ||       t        j                  ||       |j                          y # t        $ r}|j                  |       |d }~ww xY w)NrZ   )r   r\   r]   r^   ri   rA   "not_implemented_embedding_functionrb   rc   rd   re   rf   rg   rh   	Exceptionsend)r6   r   r   r   r   r   r=   es           r5   load_and_checkr      s    !**62$$ )LLN % 
 	z*""44""44T:.j1 		!s   CC 	C7C22C7c                      	 t        j                  d      } | j                  dg       | S # t        $ r t        j                  d      cY S w xY w)N
forkserverchromadbspawn)multiprocessingget_contextset_forkserver_preloadr   )ctxs    r5   get_multiprocessing_contextr      sP    
4 )),7 	""J<0
 4**7334s   (+ AAc                       e Zd ZdZy)#PersistEmbeddingsStateMachineStatespersistN)__name__
__module____qualname__r   r[   rI   r5   r   r     s    GrI   r      c                        e Zd Zdedef fdZ ee      dej                  fd       Z e
d        e       dd
              Zdedd	f fdZddZ xZS )PersistEmbeddingsStateMachiner   r6   c                     || _         || _        t        | _        | j                   j	                          t
        |   | j                          y N)r   r6    MIN_STATE_CHANGES_BEFORE_PERSIST(min_state_changes_left_before_persistingr_   super__init__)selfr   r6   	__class__s      r5   r   z&PersistEmbeddingsStateMachine.__init__  s:     8X5%rI   rF   rF   c                 l   | j                   j                          | j                   j                  |j                  |j                  |j
                        | _        |j
                  | _        t        d       | j                  t        j                         t        j                  g g g g       | _        y )NrW   init)r   	metadatas	documentsr   )r   r_   r`   rN   rX   rY   rF   r   on_state_changer   r   rA   StateMachineRecordSetrecord_set_state)r   rF   s     r5   r   z(PersistEmbeddingsStateMachine.initialize  s    ++77(()<< 8 

 #-"?"?f8CCD * @ @bB2!
rI   c                 Z    t        | j                  d         dk\  xr | j                  dk  S )Nr   rJ   r   )lenr   r   r   s    r5   rQ   z&PersistEmbeddingsStateMachine.<lambda>,  s1    S..u56!; ?99Q>?rI   r(   Nc                    | j                  t        j                         | j                  j                  }t        j                         \  }}t               }|j                  t        | j                  || j                  |f      }|j                          |j                          |j                         r|j                         }||j!                          y )N)targetargs)r   r   r   rF   rN   r   Piper   Processr   r6   r   r\   joinpollrecvclose)r   r   conn1conn2r   pr   s          r5   r   z%PersistEmbeddingsStateMachine.persist+  s     	@HHI//..&++-u)+KK!--$2G2GO  
 	
		::<

AG		rI   	new_statec                     t         |   |       |t        j                  k(  rt        | _        y | xj
                  dz  c_        y )NrJ   )r   r   r   r   r   r   )r   r   r   s     r5   r   z-PersistEmbeddingsStateMachine.on_state_changeB  s=    	*;CCC0 9 99Q>9rI   c                 8    | j                   j                          y r   )r   r_   r   s    r5   teardownz&PersistEmbeddingsStateMachine.teardownK  s    rI   )r(   N)r   r   r   r   r   r   r   collection_strA   
Collectionr   r   r   strr   r   __classcell__)r   s   @r5   r   r     s~    &y &H & =)
Z%:%: 
 *
 	? 
V 	
$? ? ?rI   r   caplogc                     | j                  t        j                         t        j                        t        fd       y )Nc                      t               S )Nr6   r   )r   )r   r6   s   r5   rQ   z/test_persist_embeddings_state.<locals>.<lambda>U  s    -xOrI   )	set_levelloggingERRORr   r   r   )r   r6   r   s    `@r5   test_persist_embeddings_stater   O  s0     W]]#__X&FOrI   c                    t        j                  |      }t        ||      }|j                  t	        j
                  dddddddd t        d      dt        j                  i g dd	

             |j                          |j                          |j                          |j                          |j                          |j                  dgddggd gd d      \  }|j                          |j                          |j                          |j                          |j                          |j                  ddgddgddggd d gd d      \  }}|j                          |j                          |j                          |j                          |j                          |j                  |g       |j                          |j!                          y )Nr   A00   rs   hnsw:construction_efhnsw:search_efhnsw:Mru   rt   z$2d3eddc7-2314-45f4-a951-47a9a8e099d2   FT
rN   rX   rY   r   	dimensiondtypeknown_metadata_keysknown_document_keywordshas_documentshas_embeddingsr   0g      ?g     ?r   r   r   r   r   r   r   g     P?g     ?g     ÿg     ?r   )r   r   r   r   rA   r   r	   npfloat16rg   rc   fields_matchlog_size_below_maxno_duplicatesadd_embeddingsdelete_by_idsr   )r   r6   r   stateembedding_ids_0embedding_ids_1embedding_ids_2s          r5   test_delete_less_than_kr   Y  s    __X&F)8FKE	(((+"%'(#$  $:;** "$&!
  ( 
	KKM			--#XbdrWsVt  EI  DJ  Y]  :^-  __		KKM			',';';PSUXzkx  {K  kL  O_  ao  Np  jq  AE  GK  @L  [_  H`';  (a$O_		KKM				_-.		NNrI   c                    t        j                  |       }t        | |      }|j                  t	        j
                  dddddddt               t        d      dd i g dd	

             |j                  g dddgddgddggg dd d       |j                  dg       |j                  dgddggd gd d       |j                          y )Nr   r   r   rs   )r   r   r   rt   ru   z$0851f751-2f11-4424-ab23-4ae97074887ar   FTr   r   )r   r   r   r   )NNNr   r   r   r   rJ   )r   r   r   r   rA   r   r   r	   r   r   r   )r6   r   r   s      r5   test_delete_add_after_persistr     s   __X&F)8FKE	(((+"%#$'(  89:; "$&#
  , 
 #q6Aq6Aq62+
   
SE"	5q6(	
   
rI   c                 
   t        j                  |      }t        ||      }|j                  t	        j
                  dddddddt        dt        j                        t        d	      dt        j                  i g d
d
             |j                          |j                          |j                          |j                          |j                          t        t         t"           |j%                  g dg dg dg dg dg dg dg dgd d d            \  }}}}}}	}
|j                          |j                          |j                          |j                          |j                          t'        d       |j%                  dgg dgd d d      }|j                          |j                          |j                          |j                          |j                          t        t         t"           |j%                  g dg dg dg dg dg d g d!g d"g d#g d$g d%g d&g d'g d(g d)g d*g d+g d,gd d d            \  }}}}}}}}}}}}}}}}}|j                          |j                          |j                          |j                          |j                          |j%                  d-d.gg d/g d0gd d gd d       |j                          |j                          |j                          |j                          |j                          |j%                  g d1g d2g d3g d4g d5g d6g d7gd d d       |j                          |j                          |j                          |j                          |j                          |j)                          |j                          |j                          |j                          |j                          |j                          |j%                  d8g d9gd d d       |j                          |j                          |j                          |j                          |j                          |j%                  d:d;gg d<g d=gd d gd d       |j                          |j                          |j                          |j                          |j                          |j%                  d>d?d@dA|dBdCdDdE||dFdGdH|dIdJgg dg dg dg dg d g d!g d"g d#g d$g d%g d&g d'g d(g d)g d*g d+g d,gd d d       |j                          |j                          |j                          |j                          |j+                          y )KNr   zJqzMs4pPm14c
r   	      r   \   )dimr   z$45c5c816-0a90-4293-8d01-4325ff860040FTr   r   )Ne8r6r   YaoqFjA2cjHCvr   )        r
  r
  )r   r   r   )r   r   r   )r   r   r   )r   r   r   )r   r   r   )      @r  r  r   r   z

	MVu393QTc)      @r  r  )CyF0Mk-q_Fwu2D2sQSFogDgPLkcfTSrwuQHQ6w4f51qWr2enLPQw8uKYs1Gwdzt5W8tpsnfJbV7zr   V1iFkoJXZw4uFc7vEEwrPYf)       @r  r  )      "@r  r  )      $@r   r   )      &@r!  r!  )      (@r"  r"  )      *@r#  r#  )      ,@r$  r$  )      .@r%  r%  )      0@r&  r&  )      1@r'  r'  )      2@r(  r(  )      3@r)  r)  )      4@r*  r*  )      5@r+  r+  )      6@r,  r,  )      7@r-  r-  )      8@r.  r.  r   df_RWhR0HelOcv)      9@r0  r0  )      :@r1  r1  )3R9_44u3BMZCXZDSUelx)      ;@r8  r8  )      <@r9  r9  )      =@r:  r:  )      >@r;  r;  )      ?@r<  r<  )      @@r=  r=  YlVm)     @@r?  r?  Rk1TPL)      A@rB  rB  )     A@rC  rC  r  r  r  r  r  r  r  r  r  r  r  r  r  )r   r   r   r   rA   r   r   r   float64r	   rg   rc   r   r   r   r   r   r   r   printr   r   )r   r6   r   r   r   r   r   embedding_ids_3embedding_ids_4embedding_ids_5embedding_ids_6_embedding_ids_12embedding_ids_17embedding_ids_18embedding_ids_22s                   r5   Mtest_batch_size_less_than_sync_with_duplicate_adds_results_in_skipped_seq_idsrO    s{    __X&F)8FKE	((!(+"%'(#$  :b

S:;** "$&!
  ( 
	KKM			 	G####### "! 	 	
	( 
	KKM				&M


=*+	
  Q 
	KKM			( 	( $#&&&&&&&&&&&&&&&#& "!Q) 	 +	
.	%													` 
	KKM				)*-/AB	
   
	KKM				?"""""" 
   
	KKM				MMO		KKM				-.	
   
	KKM				5>-/AB	
   
	KKM				 #/    #(  """""""""""""""#& Q)
  +X 
	KKM			NNrI   )Vr   r   multiprocessing.connectionr   multiprocessing.contextrx   typingr   r   r   r   r   uuidr	   
hypothesisr
   hypothesis.strategiesrA   rC   pytestr   chromadb.apir   r   chromadb.configr   r   chromadb.segmentr   #chromadb.segment.impl.manager.localr   !chromadb.test.property.strategiesrr   property!chromadb.test.property.invariantsrb   r   &chromadb.test.property.test_embeddingsr   r   r   hypothesis.statefulr   r   r   r   r   r-   r1   tempfilechromadb.api.clientr   r]   "chromadb.utils.embedding_functionsr   numpyr   CreatePersistAPImkdtempconfigurationsfixtureFixtureRequestr6   sharedrB   rD   r   	compositeDrawFnr   	RecordSetrH   listsrp   r   r   r   r   r   r   r   LogCaptureFixturer   r   r   rO  r[   rI   r5   <module>ro     s     1   9 9   "   - , ) C 6 6 6 6 1 
  
   7 G B	M*  9<??$]*(**,	 h~6	5f++ 	5	(D$:N0O 	5 7	5 		J$+BGGDM  " 	
 !
))!
:  *"6"667! !  (0)+&(;<)==)-j##Z%9%99:*=
 
==@84( 84t 84v $$ 	
 
64*E  $%  9$= 9x$$08	.$$.08.	.d0H 0 0fQ$$Q08Q	QrI   