gׅ BdZddlZddlZddlZddlZddlmZmZmZm Z m Z m Z m Z m Z mZmZddlmZddlZddlmZmZddlmZddlmZddlmZdd lmZmZmZm Z m!Z!m"Z"er dd l#m$Z%dd l&m'Z'neZ%eZ'Gd d eZ(GddZ) dde de eedfde e*effdZ+y)a This contains LLMCachingHandler This exposes two methods: - async_get_cache - async_set_cache This file is a wrapper around caching.py This class is used to handle caching logic specific for LLM API requests (completion / embedding / text_completion / transcription etc) It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache, DiskCache) based on what the user has setup In each method it will call the appropriate method from caching.py N) TYPE_CHECKINGAnyAsyncGeneratorCallableDict GeneratorListOptionalTupleUnion) BaseModel) print_verboseverbose_logger)S3Cache)1_assemble_complete_response_from_streaming_chunks)RerankResponse) CallTypes EmbeddingEmbeddingResponse ModelResponseTextCompletionResponseTranscriptionResponse)LoggingCustomStreamWrappercHeZdZUdZdZeeed<dZee ed<dZ e ed<y)CachingHandlerResponsea This is the response object for the caching handler. We need to separate embedding cached responses and (completion / text_completion / transcription) cached responses For embeddings there can be a cache hit for some of the inputs in the list and a cache miss for others N cached_resultfinal_embedding_cached_responseF embedding_all_elements_cache_hit) __name__ __module__ __qualname____doc__rr r__annotations__rrr boolT/var/www/openai/venv/lib/python3.12/site-packages/litellm/caching/caching_handler.pyrr<s7 $(M8C='CG#X.?%@G %dr(rcpeZdZdedeeefdejfdZ d'dedede dejded eeefd e e ed fd e fd Z d'dedede dejded eeefd e e ed fd e fdZde edee eeefd eeefde dejded e e eeffdZde dedejdejd ef dZde dedejdejdef dZded eeefd e ed fd e efdZ d'deded eeefde ded e ed fde ed e eeeeeeeffdZdedede ded ef dZ d'deded eeefd e e ed ffdZ d'ded eeefd e e ed ffdZded eeefd efdZ ded efd Z!d!efd"Z"d!efd#Z# d(de ded eeefded$ed%ef d&Z$y))LLMCachingHandleroriginal_functionrequest_kwargs start_timecJg|_g|_||_||_||_yN)async_streaming_chunkssync_streaming_chunksr-r,r.)selfr,r-r.s r)__init__zLLMCachingHandler.__init__Ks. <>#:<",!2$ r(Nmodel logging_obj call_typekwargsargs.returnc Kddlm}|xsd}d} d} d} |jddtj|jddduru|jdijd ddurPtj?|j | r,t jd |j||| d{} | jt| tsYt jd d} tjj} tj||jdd|jdd|jdd\}}}}|j|||| d|j}|j!| |||||jdd|} |jdddur|j#|| || | tjj$di|}t| t&s t| |rt)| dr|| j*d<t-| S|t.j0j2k(rq| ot| tr_tjOttjjt4s'|j7| | ||||\} } t-| | St jd| t-| | S7%w)aC Internal method to get from the cache. Handles different call types (embeddings, chat/completions, text_completion, transcription) and accordingly returns the cached response Args: model: str: original_function: Callable: logging_obj: LiteLLMLoggingObj: start_time: datetime.datetime: call_type: str: kwargs: Dict[str, Any]: args: Optional[Tuple[Any, ...]] = None: Returns: CachingHandlerResponse: Raises: None rrr'NFcachingTcachezno-cacher,Checking Cache)r7r8r9z Cache Hit!custom_llm_providerapi_baseapi_keyr5r@rArBr6r5r8ris_asyncrr7r8r6r5r@r9streamr6rr.end_time cache_hit_hidden_params cache_keyr)rrr8r6r.r5)rr zCACHE RESULT: )rr) litellm.utilsrgetlitellmr= _is_call_type_supported_by_cacherdebug_retrieve_from_cache isinstancelistdatetimenowget_llm_provider'_update_litellm_logging_obj_environmentr!(_convert_cached_result_to_model_response!_async_log_cache_hit_on_callbacks!_get_preset_cache_key_from_kwargsr hasattrrKrr aembeddingvaluer(_process_async_embedding_cached_response)r3r5r,r6r.r7r8r9rrr rrJrI_rLs r)_async_get_cachez"LLMCachingHandler._async_get_cacheXs< 6zrGK'16('+ ZZ 4 ( 0W]]5Nzz)U+t3 JJw # ' ' E :$ F}}(T-R-R"3.S.$$%56&*&?&?'!'@'! !,Z t5T"((6 $I'00446H%,%=%=#,2JJ7Ld,S!'J!= & 9d ; &NE1a @@$/#%&3!% A!2 : :I$($Q$Q&3"+%$/#,2JJ7Ld,S!%R%Mzz(E2e;>>(3*7'1%-&/ ?!( O O! !I#=)<%m5HI!-1ABDM 44[A1 NN!5!5!;!;;%1"=$7 1& ++WEE8W&3%$/#-# F7828W9Y ~m_=>%',K  ]!sB?K)K&H%K)c ddlm}|xsd}|j} | jt |j |d} t j|j|rtdt jjdi| } | Wd| vrnQ|j}|j| |||||jdd|} d } tjj} t j |xsd |jdd|jd d|jd d \}} }}|j#|||| dt%j&|j(| || | fj+t jj,di|}t/| t0s t/| |rt3| dr|| j4d<t7| St7| S)Nrrr'r>r?detailr@rFTrArBrCFrDtargetr9rKrLrM)rNrcopyupdateconvert_args_to_kwargsr,rPr=rQr get_cacher!rZrOrVrWrXrY threadingThreadsuccess_handlerstartr\rTr r]rKr)r3r5r,r6r.r7r8r9r new_kwargsrrJrIr@dynamic_api_keyrArLs r)_sync_get_cachez!LLMCachingHandler._sync_get_caches 6zr[[]  "&&  (, == $)N)N/*O*  * +#MM33AjAM(}, 1 : :I$($Q$Q&3"+%$/#,2JJ7Ld,S!%R%M!%I'00446H  00#kr,2JJ7Ld,S!'J!= & 9d ;  +' @@$/#%&3!& A$$*::+Z9Meg ' O O! !I#=)<%m5HI!-1ABDM 44[A1 NN%MBBr(rrc d}g}g} t|D]2\} } | |j|d|  | j| | f4|d} ||d<t| dkDr|tdt| t |j ddgt| z}d|j d <| D](} | \} } |  t| d | d |j| <*t|dk(rd}d}tjj}tj||j d d|j d d|j dd\}}}}|j||||dd|j|||||||fS||fS)a Returns the final embedding cached response and a boolean indicating if all elements in the list have a cache hit For embedding responses, there can be a cache hit for some of the inputs in the list and a cache miss for others This function processes the cached embedding responses and returns the final embedding cached response and a boolean indicating if all elements in the list have a cache hit Args: final_embedding_cached_response: Optional[EmbeddingResponse]: cached_result: List[Optional[Dict[str, Any]]]: kwargs: Dict[str, Any]: logging_obj: LiteLLMLoggingObj: start_time: datetime.datetime: model: str: Returns: Tuple[Optional[EmbeddingResponse], bool]: Returns the final embedding cached response and a boolean indicating if all elements in the list have a cache hit FNinputrzEMBEDDING CACHE HIT! - r5)r5dataTrJ embedding)rvindexobjectr@rArBrC)r6r5r8rrE is_embeddingrH) enumerateappendlenrrrOrKrrurVrWrPrXrYr[)r3rrr8r6r.r5r remaining_list non_null_listidxcroriginal_kwargs_inputvalrJrIr@rqrAs r)r`z:LLMCachingHandler._process_async_embedding_cached_response*s:27(  /GCz%%fWoc&:;$$c2Y/ 0 !'w(w }  ! 3C 4F3GH I.?jj)Vc"788/ +KO + : :; G$R>@I"$[/!*A388=% ~ ! #I/3 ,((,,.H (($*JJ/Dd$KJ5 9d3   #  8 8'=! 9   2 2'=%!# 3 34TT T.0PPPr(_caching_handler_responseembedding_responserIc|j|Sd}g}|jjD]E}|0|j$|j|j||dz }5|j|G||j_d|jjd<||z j dz|j_|jS)a Combines the cached embedding response with the API EmbeddingResponse For caching there can be a cache hit for some of the inputs in the list and a cache miss for others This function combines the cached embedding response with the API EmbeddingResponse Args: caching_handler_response: CachingHandlerResponse: embedding_response: EmbeddingResponse: Returns: EmbeddingResponse: rTrJi)rrur{rK total_seconds _response_ms)r3rrr.rIrfinal_data_listitems r)2_combine_cached_embedding_response_with_api_resultzDLLMCachingHandler._combine_cached_embedding_response_with_api_results( % D D L% %-MMRRD| 2 7 7 C&&'9'>'>s'CDq&&t, SJY!AAF  "AAPP   z ! -/DR!!AAN)HHHr(rJctj|j||||tj|j ||||fj y)a Helper function to log the success of a cached result on callbacks Args: logging_obj (LiteLLMLoggingObj): The logging object. cached_result: The cached result. start_time (datetime): The start time of the operation. end_time (datetime): The end time of the operation. cache_hit (bool): Whether it was a cache hit. rfN)asyncio create_taskasync_success_handlerrlrmrnro)r3r6rr.rIrJs r)r[z3LLMCachingHandler._async_log_cache_hit_on_callbackssW$   - -z8Y  ..XyA  %'r(c2Ktjy|j}|jt |j |d}|t jjk(rt|dtrg}t|dD]Y\}}tjjdii|d|i} |jtjj| [tj |d{}|$t|trt#d|Drd}|Stjj%dur)tjjdi|d{}|Stjj&di|}|S77)w)a Internal method to - get cache key - check what type of cache is used - Redis, RedisSemantic, Qdrant, S3 - async get cache value - return the cached value Args: call_type: str: kwargs: Dict[str, Any]: args: Optional[Tuple[Any, ...]] = None: Returns: Optional[Any]: Raises: None Nrt)rLc3$K|]}|du ywr0r').0results r) z9LLMCachingHandler._retrieve_from_cache..sBM&v~MsTr')rPr=rhrirjr,rr^r_rTrUrz get_cache_keyr{async_get_cachergatherall_supports_asyncrk) r3r7r8r9rprtasksripreset_cache_keys r)rSz&LLMCachingHandler._retrieve_from_cachesx( == [[]  "&&  (,  ,,22 2z w 8 E#Jw$78Q#*==#>#>$00Wa0$  W]]::EU:VW 9 #*..%"88M(Z t-LBMBB$(M  }},,.$6&-mm&C&C&Qj&Q Q !( 7 7 E* E 9!Rs%C:F<F=A.F+F,(FFr@cddlm}|tjjk(s|tj jk(rLt |tr<|jdddur|j||||}n||t}|tjjk(s|tjjk(rHt |tr8|jdddur|j||||}n%tdi|}n|tjjk(s|tjjk(r$t |tr||t!d }n|tj"jk(s|tj$jk(rt |tr ||d d }nd|tj&jk(s|tj(jk(r*t |trd |dd } ||t+d| }t-|dr5|j.)t |j.trd|j.d<|S)aH Internal method to process the cached result Checks the call type and converts the cached result to the appropriate model response object example if call type is text_completion -> returns TextCompletionResponse object Args: cached_result: Any: call_type: str: kwargs: Dict[str, Any]: logging_obj: LiteLLMLoggingObj: model: str: custom_llm_provider: Optional[str] = None: args: Optional[Tuple[Any, ...]] = None: Returns: Optional[Any]: r) convert_to_model_response_objectrGFT)rr7r6r5)response_objectmodel_response_objectrv)rr response_typeNrerankz whisper-1)r5r@rJaudio_transcription)rrr hidden_paramsrKrJr')rNrr acompletionr_ completionrTdictrO_convert_cached_stream_responseratext_completiontext_completionrr^rvrarerankratranscription transcriptionrr]rK) r3rr7r8r6r5r9r@rrs r)rZz:LLMCachingHandler._convert_cached_result_to_model_responses+J C ..44 4I00666-zz(E*d2 $ D D"/' + !E! !A$1*7/! 3399 9I55;;;-zz(E*d2 $ D D"/' + !E! !7 G G --33 3I//555-< -&7&9)M **00 0IAQAQAWAW4W-< -&*&M 1177 7I33999-$':!M = -&;&=3+ M M#3 4,,8=77>8>vTT  3 3W=$$&}}66$Y)eg'' 55"2248B ##GMM$A$A&$WJ$WXC sHHc |j}|jt|j|tj y|j |j|r t j j|fi|y)zE Sync internal method to add the result to the cache Nr)rhrirjr,rPr=rr)r3rr8r9rps r)sync_set_cachez LLMCachingHandler.sync_set_cachesz[[]  "&&  ==   - -"44Z .  MM # #F 9j 9r(ctjduxrstjjduxrUt|jtjjvxr$|j dij ddduS)z Helper function to determine if the result should be stored in the cache. Returns: bool: True if the result should be stored in the cache, False otherwise. Nr=zno-storeFT)rPr=supported_call_typesstrr!rO)r3r,r8s r)rz/LLMCachingHandler._should_store_result_in_cachesy]]$ & M 22$> M&//0GMM4V4VV MGR(,,Z?tK  r(ctjJtjj0t|jtjjvryy)aO Helper function to determine if the call type is supported by the cache. call types are acompletion, aembedding, atext_completion, atranscription, arerank Defined on `litellm.types.utils.CallTypes` Returns: bool: True if the call type is supported by the cache, False otherwise. TF)rPr=rrr!)r3r,s r)rQz2LLMCachingHandler._is_call_type_supported_by_cachesB MM % 22>%../7==3U3UUr(processed_chunkc Kt||jtjj|j|j d}|1|j ||j|jd{yy7w)z Internal method to add the streaming response to the cache - If 'streaming_chunk' has a 'finish_reason' then assemble a litellm.ModelResponse object - Else append the chunk to self.async_streaming_chunks Trr.rIr-streaming_chunksrEN)rr,r8)rr.rVrWr-r1rr,r3rcomplete_streaming_responses r) _add_streaming_response_to_cachez2LLMCachingHandler._add_streaming_response_to_caches >"&&**,..!88   $ ' 2&&2"&"8"8**'   3 sA9B;B<Bct||jtjj|j|j d}||j ||jyy)zQ Sync internal method to add the streaming response to the cache FrN)rr8)rr.rVrWr-r2rrs r)%_sync_add_streaming_response_to_cachez7LLMCachingHandler._sync_add_streaming_response_to_cache+sn >"&&**,..!77   $ ' 2   2**   3r(rEryc6|jdd||jdd|jdi|jdi|jdd|jdid }tj#tjjdi||d <nd|d <|j ||jd di||s|jd dn|jd d|jddt |d|jdd y)a- Helper function to update the LiteLLMLoggingObj environment variables. Args: logging_obj (LiteLLMLoggingObj): The logging object to update. model (str): The model being used. kwargs (Dict[str, Any]): The keyword arguments from the original function call. cached_result (Any): The cached result to log. is_async (bool): Whether the call is asynchronous or not. is_embedding (bool): Whether the call is for embeddings or not. Returns: None logger_fnNrAremetadata model_infoproxy_server_requeststream_response)rrrArrrrrusermessagesrtrBrGF) r5roptional_paramslitellm_paramsrtrBoriginal_responseadditional_argsrGr')rOrPr=r\update_environment_variablesr)r3r6r5r8rrEryrs r)rYz9LLMCachingHandler._update_litellm_logging_obj_environmentAs0 K6# :r2 :r2 **\26$*JJ/Et$L%zz*;R@  == $ ??I&I - .26N- .00FD))$ :r*ZZ,JJy$/!-0 ::h. 1 r(r0)F)%r!r"r#rrrrrVr4LiteLLMLoggingObjr r rrbrrrr r&r`rr[rSr rrrrrrZrrrrrQrrrYr'r(r)r+r+Js  #  S#X  %%  *+/@ @ $@ ' @ %% @  @ S#X@ uS#X'@  @ T+/NCNC$NC' NC %% NC  NCS#XNCuS#X'NC NC`XQ)12C)DXQHT#s(^45XQS#X XQ ' XQ %% XQXQ x)*D0 1XQt'I#9'I.'I%% 'I ## 'I  'IR&%%  ##   833&*38n3  # ,m8]:#6 &6 6 S#X 6  6  6 6 r(r+r,r9.r:ctj|}t|jj }i}|r,t |D]\}}|t |ks||}|||< |Sr0)inspect signaturerU parameterskeysrzr|)r,r9r param_namesargs_to_kwargsrwarg param_names r)rjrjzsu !!"34Iy++0023KN #D/JE3s;''(/ -0z** r(r0),r$rrVrrltypingrrrrrrr r r r pydanticr rPlitellm._loggingrrlitellm.caching.cachingr(litellm.litellm_core_utils.logging_utilsrlitellm.types.rerankrlitellm.types.utilsrrrrrr*litellm.litellm_core_utils.litellm_loggingrrrNrrr+rrjr'r(r)rs    :+0W1 Y m  m  d'+ 5c? # #s(^r(