
    g                       U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlmZmZ d dlmZmZ d dlmZ d dlmZ d dl m!Z!m"Z"m#Z# d dl$Z$d dl%Z%d dl&Z&d dl'Z'd dl(Z(d dl&m)Z) d dl*m+Z+ d d	l,m-Z-m.Z. d d
l/m0Z0 d dl1m2Z2 d dl(m3Z3 d dl4m5Z5 d dl6Z6d dl7Z6d dl8Z6d dl9Z6d dl:Z6d dl;m<Z< d dl=m>Z>m?Z? d dl@mAZA d dlBmCZCmDZD d dlEmFZF d dlGmHZHmIZImJZJ d dlKmLZLmMZM d dlNmOZO d dlPmQZQ d dlRmSZSmTZTmUZUmVZVmWZW d dlXmYZY d dlZm[Z[ d dl\m]Z] d dl^m_Z_m`Z` d dlambZb d dlcmdZd d dlemfZfmgZg d dlhmiZimjZj d d lkmlZlmmZm d d!lnmoZo d d"lpmqZqmrZrmsZsmtZtmuZumvZvmwZw d d#lxmyZy d d$lzm{Z{ d d%lzm|Z|m}Z}m~Z~mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ  ej,                  d&d'      5 Z e
j0                  e      Zddd        e
j4                  e      Zd dlZd d(lmZ d d)lmZmZmZmZmZmZmZmZmZmZmZmZmZ d d*l'mZ d d+lmZ d d,lmZ d d-lmZ d d.lmZ d d/lmZ d d0lmZ d1d2lmZ d1d3lmZmZmZmZmZ d1d4lmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ d1d5lmZmZ d1d6lmZmZmZ d1d7lmZ d8Z ee۬9      ZdZdZdadZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZg aeee      ed:<   dai Zeeeef      ed;<   i Zeeeef      ed<<   dZdZ	 	 dd=ed>ed?   fd@ZdA Z dBefdCZdDe_dEefdFZdG Z	 ddHee   dIedJefdKZ	 ddLedMee   fdNZ edOP      dLefdQ       ZddMee   fdSZdRg dfdTee	   dMee   fdUZ
	 	 	 	 	 	 	 	 ddVee   dWee   dXee   dYeeeu      dZeet   d[ee   d\ee   fd]Z	 dd^ed_ee   fd`ZdaefdbZdc Zdd Zde Z	 	 	 	 	 	 	 	 ddMee   dWeeeee   f      dVee   d[ee   dYeeeu      dZeet   d\ee   dJe	fdfZdgedJefdhZdLedgee   dJefdiZ	 ddLedgee   dJefdjZ	 ddLedgee   dJefdkZdLedgee   dledJefdmZddLedgee   dJefdnZddLedgee   dJefdoZ	 ddLedgee   dJefdpZ	 ddLedgee   dJefdqZddLedgee   dJefdrZ	 ddLedgee   dJefdsZdLefdtZduedvedJefdwZdxeeef   fdyZ 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d dzee   dgee   d{ee   d|ee   d}ee   d~ee   dee   dee   dee   dee   dee   dee   dee   fdZ!dJefdZ"dededee   dJefdZ#	 	 	 	 	 	 	 d!dLedee   dee   dee   dee	   deeed         dgee   dee   fdZ$	 	 	 	 	 	 	 	 	 d"dLee   dee	   dee   dee   dee   dee   dee   dgee   dee   fdZ%	 	 	 	 	 	 d#dLedee   dee   dee	   dee   dee   fdZ&d Z'd Z(dedeee      dJefdZ)	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d$dLedVeeeq      fdZ*dedJefdZ+dee	   dee	   dee	   dee	   dJee	   f
dZ,dee   dJefdZ-dgededJee   fdZ.dedJee   fdZ/dedJefdZ0dedJefdZ1dededJefdZ2dedee   dJee   fdZ3dHedJefdZ4dWedJe	fdZ5dedJefdZ6dedee   fdZ7d Z8dLedJee	   fdZ9dJefdZ:dJefdZ;dedJefdZ<dLedgee   dJefdZ=dledJefdZ>dedgee   dJefdÄZ?d dlm@Z@  G dń dƐe@      ZAdLedgee   dJeAfdǄZBdedJee	   fdȄZC	 ddLedgee   dJefdɄZDddLedgee   dJefdʄZEdefd̄ZFd̈́ ZGd΄ ZH	 	 	 	 	 d%dLedged{ededϐe	dАe	fdфZIdgedJee   fd҄ZJdӄ ZK	 	 	 d&dLee   dzee   d{ee   dJefdԄZLdՄ ZMdք ZNdׄ ZOdLedzefd؄ZPdِe	fdڄZQdeeee2   ef      dJee   fdۄZR	 ddee&j                     fd݄ZT	 	 d'dސe	dߐe	dee&j                     de	dJeeUe	f   f
dZV	 	 d(dLedededefdZW G d d      ZX	 ddee	   fdZY	 ddee	   fdZZdJefdZ[d Z\d Z]d Z^d Z_d Z`d Zad ZbdLee   fdZc	 	 	 	 d)dLee   deUdefdZdddedJee   fdZed Zfd Zgd|ee   dJee   fdZhddZi G d d      Zj G d d      Zk G d  de      ZldeqdJefdZmdedJefdZndedJefdZodVeeq   dJefdZpdgedJeeu   fdZqd d	lpmrZrmsZsmtZtmuZumvZvmwZw dee2ef   dJefd
ZxdVeeq   fdZydVeeq   fdZzdZeeeef      dJeeeef      fdZ{ G d d      Z|	 d*deded   dJee   fdZ}	 	 d+dLedVeeeq      dYeeeu      dgee   dJef
dZ~dedJeeU   fdZd dl&Z&d{ededJefdZdHedJefdZy# 1 sw Y   {xY w(,      N)	dataclassfield)	lru_cachewraps)	resources)iscoroutine)abspathdirnamejoin)Proxy)get_environment_proxies)_parsing	_pydantic)ResponseFormat)	BaseModel)Encoding)	Tokenizer)	DualCache)CachingHandlerResponseLLMCachingHandler)CustomLogger)map_finish_reasonprocess_response_headers)encoding)_get_response_headersexception_typeget_error_message)_is_non_openai_azure_modelget_llm_provider)get_supported_openai_params)_ensure_extra_body_is_safe)LiteLLMResponseObjectHandler#_handle_invalid_parallel_tool_calls convert_to_model_response_objectconvert_to_streaming_response#convert_to_streaming_response_async)get_api_base)get_formatted_prompt)get_response_headers)LiteLLMLoggingObject(redact_message_input_output_from_logging)Rules)CustomStreamWrapper)calculate_img_tokensget_modified_max_tokens)AsyncHTTPHandlerHTTPHandler)!get_num_retries_from_retry_policyreset_retry_policy)
get_secret)AllMessageValuesAllPromptValuesChatCompletionAssistantToolCall"ChatCompletionNamedToolChoiceParamChatCompletionToolParam$ChatCompletionToolParamFunctionChunkOpenAITextCompletionUserMessage)RerankResponse)	FileTypes)OPENAI_RESPONSE_HEADERS	CallTypesChatCompletionDeltaToolCallChatCompletionMessageToolCallChoicesCostPerTokenCustomHuggingfaceTokenizerDelta	EmbeddingEmbeddingResponseFunctionImageResponseLlmProvidersLlmProvidersSetMessage	ModelInfoModelInfoBaseModelResponseModelResponseStreamProviderFieldStreamingChoicesTextChoicesTextCompletionResponseTranscriptionResponseUsageall_litellm_paramsz%litellm.litellm_core_utils.tokenizerszanthropic_tokenizer.json)ThreadPoolExecutor)TYPE_CHECKINGAnyCallableDictIterableListLiteralOptionalTupleTypeUnioncastget_args)OpenAIError)BaseAudioTranscriptionConfig)BaseLLMModelInfo)
BaseConfig)BaseTextCompletionConfig)BaseEmbeddingConfig)BaseRerankConfig   )verbose_logger)CacheQdrantSemanticCache
RedisCacheRedisSemanticCacheS3Cache)APIConnectionErrorAPIErrorAuthenticationErrorBadRequestErrorBudgetExceededErrorContentPolicyViolationErrorContextWindowExceededErrorNotFoundErrorrf   PermissionDeniedErrorRateLimitErrorServiceUnavailableErrorTimeoutUnprocessableEntityErrorUnsupportedParamsError)AllowedModelRegionKeyManagementSystem) ChatCompletionDeltaToolCallChunkChatCompletionToolCallChunk#ChatCompletionToolCallFunctionChunk)LiteLLM_Paramsd   )max_workerscallback_listadditional_detailslocal_cachelogger_only	log_level)DEBUGINFOERRORc                 
   	 |dk(  rt        j                  |        n5|dk(  rt        j                  |        n|dk(  rt        j                  |        t        j
                  du r|du rt        |        y y y # t        $ r Y y w xY w)Nr   r   r   TF)rn   debuginfoerrorlitellmset_verboseprint	Exception)print_statementr   r   s      B/var/www/openai/venv/lib/python3.12/site-packages/litellm/utils.pyprint_verboser     s    

  1& 0'!  1$&;%+?/" ,@& s   A1A6 6	BBc                     t         j                  D ]q  } | d   t         j                  vr"t         j                  j                  | d          | d   t         j                  vsPt         j                  j                  | d          s y)z2
    Add custom_llm provider to provider list
    providerN)r   custom_provider_mapprovider_listappend_custom_providers)
custom_llms    r   custom_llm_setupr   *  sl     11
j!)>)>>!!((J)?@j!)B)BB%%,,Z
-CD 2    original_functionc                    ddl m} ddlm} t         j                  du rt        j                  d       	 t                d|v r|d   nd }t        t         j                        dkD  rt         j                  D ][  t        t              rQt         j                  j                  j                  d d       "t        fdt         j                   D              ret         j"                  vrt         j"                  j%                         t         j&                  vrt         j&                  j%                         t         j(                  vrt         j(                  j%                         t         j                   vrt         j                   j%                         t         j*                  vs=t         j*                  j%                         ^ t-        d	t         j                           t        t         j"                        dkD  s8t        t         j&                        dkD  st        t         j(                        dkD  rdt        t.              dk(  rRt1        t3        t         j"                  t         j&                  z   t         j(                  z               a |t.        |
       t        t         j"                        dkD  rg }t5        t         j"                        D ]K  \  }	t7        j8                        st         j:                  j%                         |j%                  |	       M t=        |      D ]!  }	t         j"                  j?                  |	       # t        t         j&                        dkD  rg }t5        t         j&                        D ]  \  }	t7        j8                        r1t         j                   j%                         |j%                  |	       Mdk(  sdk(  r1t         j                   j%                         |j%                  |	       t         j@                  v sddlm}
  |
d d       t        fdt         j                  D              rt         j                  j%                         t         j"                  j%                         t         j&                  j%                         t         j(                  j%                         t         j                   j%                         t         j*                  j%                          t=        |      D ]!  }	t         j&                  j?                  |	       # t        t         j(                        dkD  rg }t5        t         j(                        D ]K  \  }	t7        j8                        st         j*                  j%                         |j%                  |	       M t=        |      D ]!  }	t         j(                  j?                  |	       # d }d }d }d }|jC                  dd       t        |d   t0              rg }t5        |d         D ]]  \  }	t7        j8                        sdk(  sdk(  s&|"t        |t0              r|j%                         ng}|j%                  |	       _ t=        |      D ]  }	|d   j?                  |	        |j?                  d      }|jC                  dd       $t        |d   t0              r|j?                  d      }tD        rp	 tG        jH                  |      }t         jL                  r6|j?                  dd        |j?                  dd        |j?                  dd        tE        dd| d| d       d|v r|d   a't        |      dkD  r|d   n|jC                  dd       }| }|tP        jR                  jT                  k(  s|tP        jV                  jT                  k(  rd }t        |      dkD  r|d   }n|jC                  dd       r|d   }t        |t0              rt        |      dkD  rt        |d   tX              rd|d   v r|j[                  dj]                  d |D              |        n|tP        j^                  jT                  k(  s|tP        j`                  jT                  k(  r't        |      dkD  r|d   n|jC                  dd       }n4|tP        jb                  jT                  k(  s|tP        jd                  jT                  k(  rt        |      dkD  r|d   n|d   }n|tP        jf                  jT                  k(  s|tP        jh                  jT                  k(  rt        |      dkD  r|d   n|d   }n|tP        jj                  jT                  k(  s|tP        jl                  jT                  k(  rt        |      dkD  r|d   n|d   }n8|tP        jn                  jT                  k(  s|tP        jp                  jT                  k(  r|jC                  d!      }n|tP        jr                  jT                  k(  s|tP        jt                  jT                  k(  rct        |      dkD  r|d   n|d"   }t         j                  jv                  jx                  j{                  |#      }d$|v r	||d$   d%<   nd%|i|d$<   |}nO|tP        j|                  jT                  k(  s|tP        j~                  jT                  k(  r|jC                  dd&      }nd'}d(|v r	|d(   du rdnd)} |||||d*   |jC                  d+      |xs d|||||||,      }d-di}d$|v r|d$   |d$<   |j                  |di ||jC                  d.d       /       ||fS # tJ        $ r |}Y ow xY w# tJ        $ r7}t        j                  d0t        j                          d1| d2|        |d }~ww xY w)3Nr   )Logging)set_callbacksTze`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs.id)internal_usage_cache
llm_routerc              3   H   K   | ]  }t        |t                      y wN
isinstancetype).0cbcallbacks     r   	<genexpr>z!function_setup.<locals>.<genexpr>Q  s$      /"AB #2tH~6"A   "z8Initialized litellm callbacks, Async Success Callbacks: )r   function_iddynamodb	openmeter)$_init_custom_logger_compatible_classc              3   H   K   | ]  }t        |t                      y wr   r   )r   r   callback_classs     r   r   z!function_setup.<locals>.<genexpr>  s"      >GX
2tN';<GXr   success_callbacks3failure_callbackmessagesinputpromptzlitellm.llm_callzPositional Args: z, Keyword Args: r   )categorymessagelevel	logger_fnmodelrm   content c              3   n   K   | ]-  }d |v r't        |d    t              r|j                  d d       / yw)r   r   Nr   strget)r   ms     r   r   z!function_setup.<locals>.<genexpr>  s7      "!)A$>j9s.K i,!)s   35r   r   queryfile)file_objmetadatafile_checksumspeechzdefault-message-valuestreamFlitellm_call_idlitellm_trace_id)r   r   r   r   r   r   	call_type
start_timedynamic_success_callbacksdynamic_failure_callbacksdynamic_async_success_callbacksdynamic_async_failure_callbackskwargsapi_basestream_options)r   useroptional_paramslitellm_paramsr   z4litellm.utils.py::function_setup() - [Non-Blocking] z	; args - z; kwargs - )Dr   r   *litellm.litellm_core_utils.litellm_loggingr   r   rn   warningr   len	callbacksr   r   litellm_core_utilslitellm_loggingr   any_async_success_callbackinput_callbackr   r   r   _async_failure_callbackr   r   listset	enumerateinspectiscoroutinefunction_async_input_callbackreversedpop)_known_custom_logger_compatible_callbacksr   add_breadcrumbcopydeepcopyr   turn_off_message_logginguser_logger_fnr?   
completionvalueacompletiondictpre_call_rulesr   	embedding
aembeddingimage_generationaimage_generation
moderationamoderationatext_completiontext_completionrerankarerankatranscriptiontranscriptionaudio_utilsutilsget_audio_file_nameaspeechr   update_environment_variablesr   	traceback
format_exc)r   	rules_objr   argsr   LiteLLMLoggingr   r   removed_async_itemsindexr   r   r   r   r   details_to_logr   r   r   	_file_objr   r   logging_objr   er   r   s                            @@r   function_setupr  6  s
    2Hd"s	
N 	 6:V^VD\w  !A%#--h,&99IInn t  o  H  '3 /")"A"A/ , !7#9#99**11(;7#;#;;,,33H=7#;#;;,,33H=7#B#BB33::8D7#B#BB33::8D) .* J7KjKjJkl
 &&'!+7++,q07++,q0
 !**../../M ;Ow%%&*"$#,W-C-C#Dx..x81188B'..u5 $E ""56&&**51 7w''(1,"$#,W-E-E#Fx..x833::8D'..u5+x;/F 33::8D'..u5!R!RR &J -1#'&N &1# >GNGXGX> ;  ))00@..55nE0077G0077G77>>~N77>>~N; $G@ ""56((,,U3 7 w''(1,"$#,W-E-E#Fx..x833::8D'..u5 $G ""56((,,U3 7
  	"
  	(
  	"
  	( ::($/;
%&A
 #%#,V4F-G#Hx//9:-4'6Bz7H 8>>xH;C*7'..u5 $I ""56)*..u5 7(.

3E(F%::($/;
%&A
 )/

3E(F%(!%v!6 //"":t4""7D1""8T2++D61A.AQR
 & #K0Nt9q=Qfjj$.G%	--333I11777H4y1}7J-!*- 8T*MA%x{D1!,(('' "!)" 
   )  ,,222I00666"%d)a-tAwVZZ5NH33999I77==="%d)a-tAwVH5EH--333I11777"%d)a-tAwVG_H33999I55;;;"%d)a-tAwVH5EH))///9	@Q@Q@W@W3Wzz'*H11777I33999.1$i!m47I**66<<PP& Q  
 V#6Cz"?3&5}%Ez"$H**000IAQAQAWAW4Wzz'84H.H!V+x0@D0He$"#45#ZZ(:;#)r!&?&?,K,K
" +5b)9)/
);N:&00)!::&6= 	1 	
 F""g  (!'(h  B9CWCWCYBZZcdhciitu{t|}	
 	sl   E=n! 6En! ?D3n! 3n! "n! *D>n! )Cn! -B+n! n .Q n! nn! nn! !	o!*2oo!r  is_completion_with_fallbacksc                    K   |du rSt        d| j                          t        j                  | j                  |||             | j	                  |||       y y w)NFz>Async Wrapper: Completed Call, calling async_success_handler: resultr   end_time)r   async_success_handlerasynciocreate_task-handle_sync_success_callbacks_for_async_callsr  r"  r   r#  r  s        r   _client_async_logging_helperr)  R  so      	%-L[MnMnLop	
 	--fj(K	
 	AA! 	B 	
 	.   AAc           
      r    t               dt        fddt        t           f fddt        t
        t        f   dt        dt        t        t           t        t
        t        f   f   fdt                fd       }t                fd	       }t        j                         }|r|S |S )
Nreturnc                 \    t        j                  |       ryt        j                  |       ryyNTF)r   r   r   )r   s    r   check_coroutinezclient.<locals>.check_coroutinem  s'    u%((/r   r   c                 T   	 | y 	j                   }|t        j                  j                  k(  s|t        j                  j                  k(  r |       }|du ry t        | t              rt        | j                        dkD  r| j                  d   j                  j                  }|Z
j                  ||       t        j                  du r	 |d|v r|d   	 d }t        |d   t              r|d   j                  d      	 |d   }n1t         j"                  j%                  |d         rt'        |d         }|1t        j(                  j*                  j-                  |d   d   |       |d|v rt        |d   t              rxd	|d   v rp|d   d	   d
k(  rdd|d   v r\t        |d   d   t              rEd|d   v r=|d   d   du r2t        j(                  j*                  j-                  |d   d   |       y y y y y y y y y y y y y y # t.        $ r Y w xY w# t0        $ r}|d }~ww xY w)NTr   r   response_formatjson_schemar1  schema)r4  responser   json_objectresponse_schemaenforce_validation)__name__r?   r   r   r   r   rO   r   choicesr   r   post_call_rulesr   enable_json_schema_validationr   r   r   _completionsis_basemodel_typetype_to_response_format_paramr   json_validation_rulevalidate_schema	TypeErrorr   )original_responser   r   r   is_coroutinemodel_responsejson_response_formatr  r/  r   r  s           r   post_call_processingz$client.<locals>.post_call_processingu  s   d	 (-66	!5!5!;!;; I$9$9$?$??#23D#EL#t+ ''8-H #$5$=$= > B<M<U<U !=%ggg +  .9 ) 9 9*8 !: !" $+#H#HD#P'%-,;,G0A_0T0?@Q0R3718 TX,@0:4CDU4V4812 5D4E5225#m2D7;5<
 HW4EH20D 2:1F1F1X1X0?@Q0R2. 5RHW<MI:56 1E 0D/O070J0J0_0_0o0o;O8E<66><@ >L	 1p 12 %4$?(9_(L(2(78I(JD)& )//BS2T(T(78I(J6(R'4)5(9'67H'I)J(2(78I(J,=)* )-	)& )='67H'I)J(78I(J(<)& (,), %,$>$>$S$S$c$c/>?P/Q,=0* 2@	 %d %&),)J)&)J)5 )U)& )M %@a  : !C I @t ,5 %-(,%-D  	G	sJ   H AH A5H B H 0B
H 	HH HH 	H' H""H'r   	exceptionc                     | j                  dd       xs t        j                  xs d }| j                  dd       r)t        || j                  d            }t	               | d<   || fS )Nnum_retriesretry_policyrH  rK  )r   r   rJ  r2   r3   )r   rH  rJ  s      r   _get_num_retriesz client.<locals>._get_num_retries  sf     jj5T9L9LTPT::nd+;##ZZ7K &8%9F>"F""r   c                  :   j                   }t        |      rt        j                  K|j	                  di       j	                  dd       }|'t        j                  t        |      k  rt        d       | i |}d|v ra|d   du rZd|v rT|d   du rMg }t        |      D ]  \  }}|j                  |        t        j                  ||j	                  dd             S |S |S t        | |       t        j                  j                         }d }|j	                  d	d       }	d
|vr t        t        j                               |d
<   t        |       dkD  r| d   n|j	                  dd       }
	 |	t!        j                   |g| i |\  }	}|	|d	<   t#        ||      }||	_        |j'                         D ]C  \  }}|	t)        |t              s|j+                  d      s,t        j,                  |      ||<   E t        j.                  rJt        j0                  t        j.                  kD  r)t3        t        j0                  t        j.                        t        j                  K|j	                  di       j	                  dd       }|'t        j                  t        |      k  rt        d      t5        d|j	                  dd       dt        j6                   d|j	                  di       j	                  dd              |j	                  dd       t        j6                  |j	                  dd      du r|j	                  di       j	                  dd      dur|j	                  dd      dur|j	                  dd      dur|j	                  dd      dur|j	                  dd      dur|j	                  dd      durq|j	                  dd      dur]|j	                  dd      durIt9        j:                  d       |j=                  |
xs d|	||||        }|j>                  |j>                  S |j	                  d!d       |
t        j@                  du r|tB        jD                  jF                  k(  s|tB        jH                  jF                  k(  r}	 |
}|j	                  d"d       d#|j	                  d"       }d }t        |       d$kD  r| d$   }n|j	                  dd       r|d   }|j	                  d!      }tK        |
|||d d %      }||d!<    | i |}t        j                  j                         }d|v ra|d   du rZd|v rT|d   du rMg }t        |      D ]  \  }}|j                  |        t        j                  ||j	                  dd             S |S d|v r	|d   du r|S d|v r	|d   du r|S d|v r	|d   du r|S d|v r	|d   du r|S d'|v r	|d'   du r|S  ||
xs d |(       |jM                  || |)       t9        jN                  d*       tP        jS                  |	jT                  |||       tW        |d+      r|j	                  d,i       j	                  d-d       |jX                  d.<   t[        |
xs dt]        |	d/i       0      |jX                  d1<   |	j_                  |2      |jX                  d3<   ta        |jX                  j	                  d4      xs i       |jX                  d4<   |||z
  jc                         d5z  |_2        |S # t        $ r"}t5        d&t        |              Y d }~#d }~ww xY w# t        $ r}j                   }|tB        jH                  jF                  k(  r]|j	                  d6d       xs t        jf                  xs d }|j	                  d7d       r)ti        ||j	                  d7      8      }tk               |d7<   d t        _3        |j	                  d9i       }d:|j	                  di       v }|ro|smt)        |tl        jn                        s4t)        |tl        jp                        st)        |tl        jr                        rw||d6<   t        jt                  | i |cY d }~S t)        |t        jv                  jx                        r4|r2|
|v r.|s,t        |       dkD  r	||
   | d<   n||
   |d<    | i |cY d }~S t{        j|                         }t        j                  j                         }|	r|	j                  ||||       |d }~ww xY w);Nr   previous_modelszMax retries per request hit!r   Tcomplete_responser   r   litellm_logging_objr   r   r   r   request_kwargsr   zos.environ/current_cost
max_budgetzSYNC kwargs[caching]: cachingF; litellm.cache: z#; kwargs.get('cache')['no-cache']: cachezno-cacher  r  r   aimg_generationr  r  
_arealtimezINSIDE CHECKING SYNC CACHEr   r   r   r  r   r   r   r  
max_tokenshf_model_namezhuggingface/rm   )r   
base_modelr   user_max_tokens
buffer_numbuffer_percz&Error while checking max token limit: r  rC  r   r   )r"  r  r   z0Wrapper: Completed Call, calling success_handler_hidden_params
model_infor   model_idr   r   r   r   r"  response_costadditional_headers  rJ  rK  rL  context_window_fallback_dictmodel_group)@r9  _is_async_requestr   num_retries_per_requestr   r   r   r   r   stream_chunk_builderprint_args_passed_to_litellmdatetimenowr   uuiduuid4r  r   _llm_caching_handleritemsr   
startswithr4   rW  _current_costrx   r   rZ  rn   r   _sync_get_cachecached_resultmodify_paramsr?   r   r   r   r/   sync_set_cacher   executorsubmitsuccess_handlerhasattrre  r'   getattr_response_cost_calculatorr   total_seconds_response_msrJ  r2   r3   openairu   r   rt   completion_with_retries
exceptionsrz   r  r  failure_handler)r  r   r   rO  r"  chunksidxchunkr   r  r   rw  kvcaching_handler_responser`  r   ra  modified_max_tokensr  r#  rJ  rm  _is_litellm_router_calltraceback_exceptionr   rG  r  s                            r   wrapperzclient.<locals>.wrapper  s	    &..	V$..:"(**Z"<"@"@%t# #.66#o:NN'(FGG '77F6!fX&6$&>'6123t;F&/&7
Ue, '8"77J)E  "MM 	%%6fE&&**,
6<jj!47

 F*(+DJJL(9F$%*-d)a-tAwVZZQU=Vg	"&4%..	:'HL'PV'#V -8F()6G"3%%7 
 0DK, 1=Z3%7ALL<W ' 2 21 5F1I ' !!((7+=+==-%,%:%:#*#5#5  ..:"(**Z"<"@"@%t# #.66#o:NN'(FGG (Iu)E(FFWX_XeXeWf  gJ  KQ  KU  KU  V]  _a  Kb  Kf  Kf  gq  sx  Ky  Jz  { #JJy$7? ' 9!::i74?

7B/33JFdRJJ|U34?JJ159EJJ}e4D@JJ0%8DJJ/7tCJJy%0<JJ|U34? $$%AB(88#kr*;$/#-"+%! 9  ) ,99E3AAA 

<.:%)) !6!6!<!<< I$8$8$>$>>U!&Jzz/48D'3FJJ4O3P%Q
#H4y1}#'7J5#)*#5&,jj&>O*A##-!)(7#'$(+' ,?F<( '77F((,,.H6!fX&6$&>'6123t;F&/&7
Ue, '8"77J)E  "M&(VM-Bd-J'F<,@D,H"f,8I1Jd1R!V+7G0HD0Pf$	):d)B !"(mt & !// 0   RSOO++	 v/04:JJ|R4P4T4T$5%%j1 5A+2$+K9JB$O5%%j1
  999H %%o6 ?W))--.BCIr?%%&:; !z)-/D')# MO ! U!$J3q6("STTUP  5	)22II00666JJ}d3Rw7J7JRd  ::nd3"C"#%+ZZ%?#K
 +, >*  # 06zz2B0, +86::< +'  (? #1foo6%a8%a)B)BC0;}-&>>OOOq'"4"4"O"OP4!==34y1}">u"EQ*Fu*Mw,d=f=="+"6"6"8((,,.H ++*J Gk5	s   ,A\: ?\: \: "I\: 2A \: A<\ B\: \: \: '\: 4\: \: \: D0\: 	\7\2,\: 2\77\: :
dD)d-d3AddA
ddc                    K   t        | |       t        j                  j                         }d }|j                  dd       }t	        ||      }j
                  }d|vr t        t        j                               |d<   t        |       dkD  r| d   n|j                  dd       }|j                  d      d u}	 |t        j
                  |g| i |\  }}||d<   ||_        t        j                  rJt        j                  t        j                  kD  r)t        t        j                  t        j                        t!        d|j                  d	d
       dt        j"                   d|j                  dd               |j%                  |xs d|||||        d {   }	|	j&                  |	j(                  |	j&                  S |	j*                  du r|	j(                  S  | i | d {   }t        j                  j                         }
d|v ra|d   du rZd|v rT|d   du rMg }t-        |      D ]  \  }}|j/                  |        t        j0                  ||j                  dd             S |S |t2        j4                  j6                  k(  r|S t9        |d      rt;        |dd       |j<                  d<   |j                  di       j                  dd       |j<                  d<   t?        |xs d|      |j<                  d<   |jA                  |      |j<                  d<   tC        |j<                  j                  d      xs i       |j<                  d<   tE        |tF              s tE        |tH              stE        |tJ              r!tM        |d|
|z
  jO                         dz          |||        |jQ                  ||| !       d {    tS        jT                  tW        ||||
|"             |jY                  |||
#       tE        |tH              r"|	j(                  	 |j[                  |	|||
$      S |S 7 7 U7 u# t\        $ r}t_        j`                         }t        j                  j                         }
|r[	 |jc                  ||||
       n# t\        $ r}|d }~ww xY w	 |je                  ||||
       d {  7   n# t\        $ r}|d }~ww xY wj
                  } ||%      \  }}|t2        jf                  j6                  k(  r|j                  d&i       }d'|j                  d(i       v }|r|s	 d t        _4        ||d)<   |d*<   tE        |tj        jl                        rd+|d,<   ntE        |tj        jn                        rd-|d,<   t        jp                  | i | d {  7  cY d }~S # t\        $ r Y ncw xY wtE        |t        jr                  jt                        r;|r9||v r5t        |       dkD  r	||   | d<   n||   |d<    | i | d {  7  cY d }~S tM        |d)|       |d }~ww xY ww).NrR  rS  r   r   r   	fallbacksrU  zASYNC kwargs[caching]: rX  FrY  z; kwargs.get('cache'): rZ  r   r]  Tr   rP  r   rQ  re  rf  r   rg  rh  r   ri  rj  rk  r  rl  rd  )r"  r   r   r  r(  r!  )_caching_handler_responseembedding_responser   r#  )r   rH  rm  rn  r   rJ  r   exponential_backoff_retryretry_strategyconstant_retry);rr  rs  rt  r   r   r9  r   ru  rv  r   r  rw  r   rW  rz  rx   r   rZ  _async_get_cacher|  final_embedding_cached_response embedding_all_elements_cache_hitr   r   rq  r?   	arealtimer   r  r  re  r'   r  r   r   rO   rG   rU   setattrr  async_set_cacher%  r&  r)  r'  2_combine_cached_embedding_response_with_api_resultr   r  r  r  async_failure_handlerr   rJ  r  r}   ru   acompletion_with_retriesr  rz   )r  r   r   r"  r  rw  r   r   r  r  r#  r  r  r  r  r  rJ  rm  r  rM  r   rG  r  s                      r   wrapper_asynczclient.<locals>.wrapper_async  s    $%6fE&&**,
6<jj!47
 3D/!!3
 &..	F*(+DJJL(9F$%*-d)a-tAwVZZQU=V'-zz+'>d'J$|	"&4%..	:'HL'PV'#V -8F()/CK,!!((7+=+==-%,%:%:#*#5#5  )&**Y*F)GGXY`YfYfXgg~  @F  @J  @J  KR  TX  @Y  Z  [ +;;+2&7 +)'! <   & *77C-MMU0>>>*KKtS0PPP -d=f==F((,,.H6!fX&6$&>'6123t;F&/&7
Ue, '8"77J)E  "Mi11777 v/0;B!2D<%%&78 5;JJ|R4P4T4T$5%%j1 5A+2$*5%%j1
  999H %%o6 ?W))--.BCIr?%%&:; 6=1f&78f&;<"
*99;dB !"(v
 '66"3	 7    , +!)%1M EE%! F  6#45-MM ,^^.G'-)%	 _   MY( >jH  ;	"+"6"6"8((,,.H//.
H ! G%;;.
H   ! G *22I"2&A"NKI11777/5zz2B0, +86::< +'  (?   + 1<}-6G23%v44 8SF#34'6??;7GF#34%,%E%Et%Vv%VVVV$  q'"4"4"O"OP4!==4y1}">u"EQ*Fu*Mw!2D!CF!CCCC=+ Gw;	sW  B6Y :C'Q	 !Q"'Q	 	Y 
Q	 #Y $Q	 /Q0BQ	 4Y 5Q	 6Y 7Q	 Y D;Q	 QA,Q	 >Y ?Q	  Y Q	 Q	 Q	 	
X=4X8RX8	R-&R((R--X81SS	SX8	SSSA$X8A,V;0V31V;5X=6Y ;	WX8WAX8X!X8#X=$Y )X88X==Y )r,   boolr`   r   r\   r   rZ   r   ra   intr   r   r   )r   r  r  rD  rM  r/  rG  r  s   `   @@@@r   clientr  j  s    I$ eQU eN#S#X#+4#	x}d38n,	-# V Vp P Pd ../@AL r   r   is_pass_throughr,  c                    | y| j                  dd      du s| j                  dd      du s| j                  dd      du s| j                  dd      du s|| j                  dd      du sh| j                  dd      du sT| j                  d	d      du s@| j                  d
d      du s,| j                  dd      du s| j                  dd      du s|du ryy)aj  
    Returns True if the call type is an internal async request.

    eg. litellm.acompletion, litellm.aimage_generation, litellm.acreate_batch, litellm._arealtime

    Args:
        kwargs (dict): The kwargs passed to the litellm function
        is_pass_through (bool): Whether the call is a pass-through call. By default all pass through calls are async.
    Fr   Tr  r[  r  r  r  r  r\  acreate_batchacreate_fine_tuning_jobr   )r   r  s     r   ro  ro    s     ~

=%(D0::lE*d2::'/47::mU+t3::(%0D8::&.$6::i'4/::lE*d2::ou-5::/74?d"r   r   custom_tokenizerc                 P    |t        |d   |d   |d         }|S t        |       S )N
identifierrevision
auth_token)r  r  r  r   )create_pretrained_tokenizer_select_tokenizer_helper)r   r  
_tokenizers      r   _select_tokenizerr    s?     #0'5%j1'5


 #%00r      )maxsizec                    | t         j                  v rd| v rt        j                  d      }d|dS | t         j                  v r"d| vrt        j
                  t              }d|dS d| j                         v sd| j                         v rt        j                  d      }d|dS d	| j                         v rt        j                  d
      }d|dS dt        dS )Nz	command-rz#Xenova/c4ai-command-r-v01-tokenizerhuggingface_tokenizerr   	tokenizerzclaude-3zllama-2	replicatez#hf-internal-testing/llama-tokenizerzllama-3zXenova/llama-3-tokenizeropenai_tokenizer)	r   cohere_modelsr   from_pretrainedanthropic_modelsfrom_strclaude_json_strlowerr   )r   cohere_tokenizerclaude_tokenizerr  s       r   r  r    s    %%%+*>$441
 0>NOO	'**	*z/F$--o>/>NOO	ekkm	#{ekkm'C--.ST	/iHH	ekkm	#--.HI	/iHH '!
 	
r   r   c                     |xs t        |       }t        |d   t              r|d   j                  |d      }|S |d   j                  |      }|S )a  
    Encodes the given text using the specified model.

    Args:
        model (str): The name of the model to use for tokenization.
        custom_tokenizer (Optional[dict]): A custom tokenizer created with the `create_pretrained_tokenizer` or `create_tokenizer` method. Must be a dictionary with a string value for `type` and Tokenizer for `tokenizer`. Default is None.
        text (str): The text to be encoded.

    Returns:
        enc: The encoded text.
    r  r   disallowed_special)r  r   r   encode)r   textr  tokenizer_jsonencs        r   r  r  '  s`     &G):)GN.-x8[)00"0M J [)006Jr   tokensc                 N    |xs t        |       }|d   j                  |      }|S )Nr  r  )r  decode)r   r  r  r  decs        r   r  r  ;  s,    %G):)GN

%
,
,V
4CJr   r   r  is_tool_calltoolstool_choicecount_response_tokensuse_default_image_token_countc           
         t        d|        	 d|v rt        j                  d      }nt        j                  |      }|dk(  rd}	d}
n=|t
        j                  v rd	}	d
}
n&|t
        j                  v rd	}	d
}
nt        d| d      d}d}|r!|t         |j                  |d            }n| P| D ]I  }||	z  }|j                  dd      dk(  rd}|j                         D ]  \  }}t        |t              r,|t         |j                  |d            z  }|dk(  s=||
z  }Ct        |t              sT|D ]  }|d   dk(  r,||d   z  }|t         |j                  |d   d            z  }7|d   dk(  s@t        |d   t               r>|d   }|j                  dd      }|j                  d      }|t#        |||xs d      z  }t        |d   t              s|d   }|t#        |d|xs d      z  }  L nD|#|du rt         |j                  |d            }|S |t         |j                  |d            }|d	z  }|r,|t         |j                  t%        |                  z  }|dz  }|r|r|dz  }|dk(  r|d
z  }|S t        |t               r)|dz  }|t         |j                  |d    d               z  }|S # t        $ r$ t        d       t        j                  d      }Y w xY w)!z
    Return the number of tokens used by a list of messages.

    Borrowed from https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb.
    z2LiteLLM: Utils - Counting tokens for OpenAI model=zgpt-4o
o200k_basez5Warning: model not found. Using cl100k_base encoding.cl100k_basezgpt-3.5-turbo-0301      rm   z8num_tokens_from_messages() is not implemented for model zz. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.r   FNr  r  rolesystemTnamer   r  	image_urldetailautourldatamoder  	   none   function)r   tiktokenget_encodingencoding_for_modelKeyErrorr   open_ai_chat_completion_models
azure_llmsNotImplementedErrorr   r  r   rx  r   r   r^   r   r.   _format_function_definitions)r   r   r  r  r  r  r  r  r   tokens_per_messagetokens_per_name
num_tokensincludes_system_messager   keyr   cimage_url_dictr  r  image_url_strs                        r   openai_token_counterr  A  si   " FugNO8u,,\:H2259H $$ 	 	'88	8	'$$	$!H  PJ  M
 	
 J#("EF
		G,,J{{64(H4*.'%mmo
UeS)#ohooePR&S"TTJf}"o5
t,"V9. AfI-D&# /&	b Q+ J vY+5)!K.$?12;)7););Hf)M&4&8&8&? *.B),)/B_ C-',	/" !"
 ",AkNC!@01+ *.B)6)/B_ C-',	/" !"
' # .	  F 
	3t;"EF
		"EF
!OJc/(//*Fu*MNOO
a
(a
 fa

 	 
K	&a
c/(//+j*A&*IJKK
g  8MN((78s   /K )K43K4r  r  c                     	 t        j                  | ||      }d|dS # t        $ r:}t        j                  d| d       t        j                  | |      }Y d}~Cd}~ww xY w)a  
    Creates a tokenizer from an existing file on a HuggingFace repository to be used with `token_counter`.

    Args:
    identifier (str): The identifier of a Model on the Hugging Face Hub, that contains a tokenizer.json file
    revision (str, defaults to main): A branch or commit id
    auth_token (str, optional, defaults to None): An optional auth token used to access private repositories on the Hugging Face Hub

    Returns:
    dict: A dictionary with the tokenizer and its type.
    )r  r  z%Error creating pretrained tokenizer: z-. Defaulting to version without 'auth_token'.)r  Nr  r  )r   r  r   rn   r   )r  r  r  r  r  s        r   r  r    so    M--j
	 ,)DD  M3A36cd	
 --j8L		Ms    	A"0AA"jsonc                 6    t        j                  |       }d|dS )z
    Creates a tokenizer from a valid JSON string for use with `token_counter`.

    Args:
    json (str): A valid JSON string representing a previously serialized tokenizer

    Returns:
    dict: A dictionary with the tokenizer and its type.
    r  r  )r   r  )r  r  s     r   create_tokenizerr    s     ""4(I+)DDr   c                 h   g }|j                  d       |j                  d       | D ]  }|j                  d      }|j                  d      x}r|j                  d|        |j                  d      }|j                  di       }|j                  d      }|rR|j                         rB|j                  d	| d
       |j                  t        |d             |j                  d       n|j                  d	| d       |j                  d        |j                  d       dj	                  |      S )zFormats tool definitions in the format that OpenAI appears to use.
    Based on https://github.com/forestwanglin/openai-java/blob/main/jtokkit/src/main/java/xyz/felh/openai/jtokkit/utils/TikTokenUtils.java
    znamespace functions {r   r  description// r  
parameters
propertiesztype z = (_: {r   z
}) => any;z = () => any;z} // namespace functions
)r   r   keys_format_object_parametersr   )r  linestoolr  function_descriptionfunction_namer  r  s           r   r  r    s    E	LL()	LL88J'#+<<#>>>LL33456 V,\\,3
^^L1
*//+LL5y9:LL2:qABLL&LL5}=>R  
LL+,99Ur   c           
         | j                  d      }|sy| j                  dg       }g }|j                         D ]Y  \  }}|j                  d      }|r|j                  d|        d}|r||v rd}|j                  | | dt        ||       d       [ d	j	                  |D 	cg c]  }	d
t        d|      z  |	z    c}	      S c c}	w )Nr  r   requiredr  r  ?z: ,r   r   )r   rx  r   _format_typer   max)
r  indentr  required_paramsr
  r  propsr  questionlines
             r   r	  r	    s    -J nnZ4OE &&(
Uii.LL3{m,-so5HuXJbeV)D(EQGH ) 99eDedcC6N*T1eDEEDs   $Cc                 n   | j                  d      }|dk(  r+d| v r&dj                  | d   D cg c]  }d| d
 c}      S y|dk(  rt        | d   |       dS |d	k(  rd
t        | |dz          dS |dv r+d| v r&dj                  | d   D cg c]  }d| d
 c}      S y|dk(  ry|dk(  ryyc c}w c c}w )Nr   stringenumz | "arrayrx  z[]objectz{
   z
})integernumberr"  booleannullr   )r   r   r  r	  )r  r  r   items       r   r  r    s    99VDxU?::uV}E}t4&{}EFF	uW~v67r::		/vzBC4HH	&	&U?::uV}E}t4&{}EFF			 # F Fs   B-B2c                    d}d}	|E|7t        d|        d}|D ]   }
|
j                  dd      |
j                  d      }t        |t              r	||
d   z  }nt        |t              r|D ]  }|d   dk(  r	||d   z  }|d   d	k(  st        |d	   t
              r>|d	   }|j                  d
d      }|j                  d      }|	t        |||xs d      z  }	nt        |d	   t              s|d	   }|	t        |d|xs d      z  }	 |
j                  d      sd}|
d   D ]  }d|v s|d   d   }||z  } # nFt        d      t        |t              rdj                  d |D              }nt        |t              rd}| ||xs t        |       }|d   dk(  r+|d   j                  |      }t        |j                        }	|	S |d   dk(  r| t        j                  v s| t        j                  v rK| t        j                  v r| j!                  dd      } t        d|         t#        || ||||||xs d      }	|	S t        d|         t#        |d||||||xs d      }	|	S t        t%        j                  |d            }	|	S ) a  
    Count the number of tokens in a given text using a specified model.

    Args:
    model (str): The name of the model to use for tokenization. Default is an empty string.
    custom_tokenizer (Optional[dict]): A custom tokenizer created with the `create_pretrained_tokenizer` or `create_tokenizer` method. Must be a dictionary with a string value for `type` and Tokenizer for `tokenizer`. Default is None.
    text (str): The raw text string to be passed to the model. Default is None.
    messages (Optional[List[Dict[str, str]]]): Alternative to passing in text. A list of dictionaries representing messages with "role" and "content" keys. Default is None.

    Returns:
    int: The number of tokens in the text.
    Fr   Nz!token_counter messages received: r   r   r   r  r  r  r  r  r  
tool_callsTr  	argumentsz%text and messages cannot both be Nonec              3   B   K   | ]  }t        |t              s|  y wr   )r   r   )r   ts     r   r   z token_counter.<locals>.<genexpr>\  s     =$Q*Q*<q$   r  r  r  r  z-35z-3.5z6Token Counter - using OpenAI token counter, for model=)r  r   r   r  r  r  r  r  z7Token Counter - using generic token counter, for model=zgpt-3.5-turbor  r  )r   r   r   r   r^   r   r.   
ValueErrorr   r  r  r   idsr   r  r  replacer  r   )r   r  r  r   r  r  r  r  r  r  r   r   r  r  r  r  r  	tool_callfunction_argumentsr  r  s                        r   token_counterr1    s   . LJ|=hZHID#;;y$/;%kk)4G!'3/	 22#GT2!(A yF2 $&	 1!"6k!9#-and#C56{^N-;-?-?&-QF*8*<*<U*CC$.2F-0-3Fc G1+0	3& %&J &0+%D45kNM$.2F-:-3Fc G1+0	3& %&J! "), ;;|,#'L%,\%:	%21::1F{1S. $66D &;= $F DEE	D$	ww=$==	D#	 $,8)K->U-K&!%<< -44T:CSWWJR Q F#'99???G...G...!MM%8ELUGT 2%!-*? +2O 3

8 ! MeWU 2)%!-*? +2O 3

  "EF
r   custom_llm_providerc                     g d}| |v ryy)zU
    Helper function to know if a provider implementation supports httpx timeout
    )r  azurebedrockTFr  )r2  supported_providerss     r   supports_httpx_timeoutr7    s     911r   c                     t        | |d      S )a  
    Check if the given model supports system messages and return a boolean value.

    Parameters:
    model (str): The model name to be checked.
    custom_llm_provider (str): The provider to be checked.

    Returns:
    bool: True if the model supports system messages, False otherwise.

    Raises:
    Exception: If the given model is not found in model_prices_and_context_window.json.
    supports_system_messagesr   r2  r  _supports_factoryr   r2  s     r   r9  r9    s     /& r   c                     t        | |      \  } }}}t        j                  j                  t        j                  j                  g}||v ryt        | |d      S )ak  
    Check if the given model + provider supports 'response_schema' as a param.

    Parameters:
    model (str): The model name to be checked.
    custom_llm_provider (str): The provider to be checked.

    Returns:
    bool: True if the model supports response_schema, False otherwise.

    Does not raise error. Defaults to 'False'. Outputs logging.error.
    r=  Tsupports_response_schemar:  )r   r   rJ   	PREDIBASEFIREWORKS_AIr<  )r   r2  _*PROVIDERS_GLOBALLY_SUPPORT_RESPONSE_SCHEMAs       r   r?  r?    sj      (8)<($E1 	&&))2.
 HH/& r   c                     t        | |d      S )  
    Check if the given model supports function calling and return a boolean value.

    Parameters:
    model (str): The model name to be checked.
    custom_llm_provider (Optional[str]): The provider to be checked.

    Returns:
    bool: True if the model supports function calling, False otherwise.

    Raises:
    Exception: If the given model is not found or there's an error in retrieval.
    supports_function_callingr:  r;  r=  s     r   rF  rF    s      /' r   r  c                    	 t        j                  | |      \  } }}}t        | |      }|j                  |d      du ryy# t        $ r4}t        j                  d| d|  d| dt        |              Y d}~yd}~ww xY w)	rE  r=  FTz%Model not found or error in checking z support. You passed model=, custom_llm_provider=	. Error: N)r   r   _get_model_info_helperr   r   rn   r   r   )r   r2  r  rB  rf  r  s         r   r<  r<    s    +2+C+C-@,
("Aq ,-@

 >>#u%- 3C58STYSZZp  rE  qF  FO  PS  TU  PV  OW  X	
 	s   =A 	A>
*A99A>c                     t        | |d      S )zECheck if a given model supports audio input in a chat completion callsupports_audio_inputr:  r;  r=  s     r   rL  rL    s    )<BX r   c                     t        | |d      S )zCCheck if a given model supports pdf input in a chat completion callsupports_pdf_inputr:  r;  r=  s     r   rN  rN    s    )<BV r   c                     t        | |d      S )zFCheck if a given model supports audio output in a chat completion callrL  r:  r;  r=  s     r   supports_audio_outputrP    s     )<BX r   c                     t        | |d      S )a  
    Check if the given model supports prompt caching and return a boolean value.

    Parameters:
    model (str): The model name to be checked.
    custom_llm_provider (Optional[str]): The provider to be checked.

    Returns:
    bool: True if the model supports prompt caching, False otherwise.

    Raises:
    Exception: If the given model is not found or there's an error in retrieval.
    supports_prompt_cachingr:  r;  r=  s     r   rR  rR  #  s      /% r   c                    	 t        j                  | |      \  } }}}t        j                  | |      }|j                  dd      du ryy# t        $ r1}t        j                  d|  d| dt        |              Y d}~yd}~ww xY w)	a  
    Check if the given model supports vision and return a boolean value.

    Parameters:
    model (str): The model name to be checked.
    custom_llm_provider (Optional[str]): The provider to be checked.

    Returns:
    bool: True if the model supports vision, False otherwise.
    r=  supports_visionFTzFModel not found or error in checking vision support. You passed model=rH  rI  N)r   r   get_model_infor   r   rn   r   r   )r   r2  rB  rf  r  s        r   rT  rT  :  s    +2+C+C-@,
("Aq ++-@

 >>+U3t; TUZT[[q  sF  rG  GP  QT  UV  QW  PX  Y	
 	s   AA 	B'B  Bc                     t        | |d      S )z]
    Check if the given model supports embedding image input and return a boolean value.
    supports_embedding_image_inputr:  r;  r=  s     r   rW  rW  X  s     /, r   c                     | t         j                  v r)t         j                  |    }|j                  dd      du ryyt        d|  d      )a  
    Check if the given model supports parallel function calling and return True if it does, False otherwise.

    Parameters:
        model (str): The model to check for support of parallel function calling.

    Returns:
        bool: True if the model supports parallel function calling, False otherwise.

    Raises:
        Exception: If the model is not found in the model_cost dictionary.
    "supports_parallel_function_callingFTz?Model not supports parallel function calling. You passed model=.)r   
model_costr   r   )r   rf  s     r   rY  rY  e  sV     """''.
>>>F$NMeWTUV
 	
r   existing_dictnew_dictc                 @    |j                         D ]
  \  }}|| |<    | S r   rx  )r\  r]  r  r  s       r   _update_dictionaryr`  ~  s)     1a ! r   r[  c                 L	   i }t        | t              r| }n&t        | t              rt        j                  |       }|j                         D ]@  \  }}	 t        t        t        |            }|d   }t        ||      }t        j                  j                  |i       j                  |       t        j                  | d       |j                  d      dk(  r3|t        j                   vst        j                   j#                  |       |j                  d      dk(  r4|t        j$                  vst        j$                  j#                  |       	|j                  d      dk(  r5|t        j&                  vs1t        j&                  j#                  |       R|j                  d      d	k(  r5|t        j(                  vszt        j(                  j#                  |       |j                  d      d
k(  rJ|j+                  dd      }|t        j,                  vst        j,                  j#                  |d          |j                  d      dk(  r5|t        j.                  vs!t        j.                  j#                  |       B|j                  d      dk(  r5|t        j0                  vsjt        j0                  j#                  |       |j                  d      dk(  r5|t        j2                  vst        j2                  j#                  |       |j                  d      dk(  r5|t        j4                  vst        j4                  j#                  |       |j                  d      dk(  r5|t        j6                  vsEt        j6                  j#                  |       f|j                  d      dk(  r5|t        j8                  vst        j8                  j#                  |       |j                  d      dk(  r5|t        j:                  vst        j:                  j#                  |       |j                  d      dk(  s|t        j<                  vs"t        j<                  j#                  |       C | S # t        $ r i }|}Y /w xY w)a  
    Register new / Override existing models (and their pricing) to specific providers.
    Provide EITHER a model cost dictionary or a url to a hosted json blob
    Example usage:
    model_cost_dict = {
        "gpt-4": {
            "max_tokens": 8192,
            "input_cost_per_token": 0.00003,
            "output_cost_per_token": 0.00006,
            "litellm_provider": "openai",
            "mode": "chat"
        },
    }
    r  r  r  z added to model cost maplitellm_providerr  text-completion-openaicohere	anthropic
openrouter/rm   zvertex_ai-text-modelszvertex_ai-code-text-modelszvertex_ai-chat-modelszvertex_ai-code-chat-modelsai21	nlp_cloudaleph_alphar5  )r   r   r   r   get_model_cost_maprx  rd   rU  r   r`  r[  
setdefaultupdatern   r   r   r  r   open_ai_text_completion_modelsr  r  splitopenrouter_modelsvertex_text_modelsvertex_code_text_modelsvertex_chat_modelsvertex_code_chat_modelsai21_modelsnlp_cloud_modelsaleph_alpha_modelsbedrock_models)r[  loaded_model_costr  r   existing_modelmodel_cost_keyupdated_dictionarysplit_strings           r   register_modelr    s    *d#&	J	$#66:F'--/
U	!#'n3.G#HN+E2N
 0F%%nb9@@AST//GHI99'(H4'@@@66==cBYY)*.FF'@@@66==cBYY)*h6'///%%,,S1YY)*k9'222((//4YY)*l:99S!,L'333))00aAYY)*.EE'444**11#6YY)*.JJ'999//66s;YY)*.EE'444**11#6YY)*.JJ'999//66s;YY)*f4'---##**3/YY)*k9'222((//4YY)*m;'444**11#6YY)*i7'000&&--c2i 0j a  	!N N	!s   RR#"R#api_keyr   r   r`  r   r_  custom_prompt_dictlitellm_metadata(disable_add_transform_inline_image_blockdrop_params	prompt_idprompt_variablesc$                     i d|d| d|d|d|d|d|	d|
d	|d
|d|d|d|d|d|di d|i d|d|d|d|d|d|d|d|xs t        |      d|d|d|d|d| d |!d!|"d"|#}$|$S )#Nr   r  force_timeoutr   verboser2  r   r   model_alias_mapcompletion_call_idr   rf  proxy_server_requestpreset_cache_keyzno-logstream_responseinput_cost_per_tokeninput_cost_per_secondoutput_cost_per_tokenoutput_cost_per_secondcooldown_timer	  azure_ad_token_provideruser_continue_messager`  r   r   r_  r  r  r  r  r  r  )*_get_base_model_from_litellm_call_metadata)%r  r  r4  r   r  hugging_facer  together_air2  r   r   r  r  r   rf  r  r   r  no_logr  r  r  r  r  r	  r  r  r`  r   r_  r  r  r  r  r  r  r   s%                                        r   get_litellm_paramsr    s   L#{#7# 	# 	Y	#
 	7# 	2# 	H# 	?# 	?# 	0# 	H# 	j# 	 4# 	,# 	&#  	2!#" 	 4##$ 	 !6%#& 	 !6'#( 	!"8)#* 	+#, 	?-#. 	"#:/#0 	 !61#2 	j I5xH5#6 	,7#8 	9#: 	0;#< 	,=#> 	34\?#@ 	{A#B 	YC#D 	,E#NH r   c                 2    |t        |t              r| |v ryyr.  )r   r   r  additional_drop_paramss     r   _should_drop_paramr    s"    *-t4''r   passed_paramsdefault_paramsr  c                     i }| j                         D ](  \  }}||v s|||   k7  st        ||      du s$|||<   * |S )Nr  F)rx  r  )r  r  r  non_default_paramsr  r  s         r   _get_non_default_paramsr  *  s^     ##%1^A&&"Q?UV %&q! & r   languager   r1  temperaturetimestamp_granularities)wordsegmentc                 2   t               }	|	j                  d      |	j                  d      |	j                  d      }
|
j                         D ]
  \  }}||	|<    d d d d d}|	j                         D ci c]  \  }}||v r|||   k7  r|| c}}i }fd}d } t        j	                  | t                    }dk(  sdk(  r}nd	k(  rWt        j                         j                         } ||
       t        j                         j                  || nd      }n6|4|j                  |       } ||
       |j                  || nd      }|	j                         D ]  }||j                         vs|	|   ||<    |S c c}}w )Nr2  r  r   )r  r   r1  r  c                    t        j                               dkD  rdt        j                               }|D ]D  }du st        j                  du r|| vrj                  |d        0|| vs5t        dd d       S y )Nr   T  z1Setting user/encoding format is not supported by =. To drop it from the call, set `litellm.drop_params = True`.status_coder   r   r  r   r   r  r   r   )supported_paramsr  r  r2  r  r  s      r   _check_valid_argz;get_optional_params_transcription.<locals>._check_valid_arg\  s    !&&()A-*//12D4'7+>+>$+F//&**1d3..0$'"STgSh  if  !g   &% .r   r   r   r  r4  groqr  Fr  r   r   r  r  )localsr   rx  ProviderConfigManager'get_provider_audio_transcription_configrJ   r   GroqSTTConfigget_supported_openai_params_sttmap_openai_params_sttr    map_openai_paramsr  )r   r  r   r1  r  r  r2  r  r   r  special_paramsr  r  r  r   r  provider_configr  r  s         ``          @r   !get_optional_params_transcriptionr  :  s    HM'++,AB##M2K"&&x0N$$&1a ' 	N "''))DAqA):$: 	
1)
 O& ?CO&/WW!"56 X 

 h&*=*H,		&"002RRT*:;!//1GG1+'2'>E	 H 
 
	$*FFUFS*:;);;1+'2'>E	 < 
 !N''))!.q!1OA " ks   8Fnqualitysizestyler   c	                 r   t               }
|
j                  dd       } |
j                  d      |
j                  dd       }|
j                  d      }|j                         D ]M  \  }}|j                  d      rdk7  rdk7  r"|dk(  rdk7  r-|j                  d	      rd
k7  rdk7  rI||
|<   O d d d d d d d}t	        |
||      i }fd}dk(  sdk(  st
        j                  v r}ndk(  rot
        j                  j                  |       rt
        j                  nt
        j                  }|j                  |       } ||       |j                  i       }n"d
k(  rdg}	  ||       |t        |      |d<   |
j                         D ]  }||j                         vs|
|   ||<    |S )Nr   r2  r  r   aws_r5  	sagemakerr_  vertex_	vertex_aivertex_ai_beta)r  r  r1  r  r  r   r  r  r  c           	         t        j                               dkD  rct        j                               }|D ]C  }t        j                  du r|| vrj                  |d        ,|| vs1t        dd| d d       S y )Nr   Tr  z	Setting `z` is not supported by r  r  r  )r  r  r  r2  r  s      r   r  z7get_optional_params_image_gen.<locals>._check_valid_arg  s    !&&()A-*//12D''4/A=M4M&**1d3..0$'"+A3.DEXDY  ZW  !X   &% .r   r  r4  r  r  r  r   r  sampleCount)r  r   rx  ry  r  r   openai_compatible_providersAmazonStability3Config_is_stability_3_modelAmazonStabilityConfigr    r  r  r  )r   r  r  r1  r  r  r   r2  r  r   r  r  r  r  r  r   r  config_classr  r  s          `           @r   get_optional_params_image_genr    s    HMgt,E'++,AB*../GN"&&x0N$$&1<<9,1D1S/!&9[&HLL##{2#'77a '  N 1#%5
 O&  	x'')'"E"EE,			) --CC%CP **.. 	
 (CC%CP*:;&8812 9 
 
	+5	 	*:;=-0VOM*!N''))!.q!1OA " r   encoding_format
dimensionsc                 	    t               }|j                  dd       |j                  d      }	|	j                         D ]
  \  }
}|||
<    |j                  dd       |j                  dd       }d d d d}dt        t           f fd}t        |||      d	k(  r( d
 vrdj                         v rt        dd      dk(  rJt         d      } ||       t        j                         j                  i  nd      }i ||}|S dk(  rHt         xs ddd      } ||       t        j                         j                  i       }i ||}|S dk(  r?t         xs ddd      } ||       t        j                  j                  i |      }|S dk(  rHt         dd      } ||       t        j                         j                  i |      \  }}i ||}|S dk(  rXt        j                         j                         } ||       t        j                         j                  i       }i ||}|S dk(  rd v rt        j                          }n\d v rt        j"                         }nCd v rt        j$                         }n*d  v rt        j&                         }ng } ||       i |}|S |j                         } ||       |j                  i       }i ||}|S d!k(  rDt         d!d      } ||       t        j(                         j                  i       }i ||}|S d"k(  rDt         d"d      } ||       t        j*                         j                  i       }i ||}|S d#k(  rJt         d#d      } ||       t        j,                         j                  i  nd      }i ||}|S d$k(  rEt         d$d      } ||       t        j.                         j                  i  %      }i ||}|S d	k7  rd&k7  rt        j0                  vryt3        j                               d'kD  r]t        j4                  d(u sd(u r3t	        j                               }|D ]  }
j                  |
d         nt        dd) d* d+      i |}|S ),Nr2  r   r  r  )r   r  r  r  c           	          | y i }j                         D ]  }|| vs|   ||<    |r/t        j                  du sdu ry t        d d| d d      y )NTr   does not support parameters: , for model=f. To drop these, set `litellm.drop_params=True` or for proxy:

`litellm_settings:
 drop_params: true`
r  )r  r   r  r   )r  unsupported_paramsr  r2  r  r   r  s      r   r  z8get_optional_params_embeddings.<locals>._check_valid_arg	  s    ##((*A(((:1(="1% + ""d*'K4,?, #233QRdQeeqrwqx  yc  d  r   r  r  ztext-embedding-3r  r  zSetting dimensions is not supported for OpenAI `text-embedding-3` and later models. To drop it from the call, set `litellm.drop_params = True`.r  triton
embeddings)r   r2  request_typer  Fr  
databricksr   r  
nvidia_nim)r  r   r   r  	lm_studior5  zamazon.titan-embed-text-v1zamazon.titan-embed-image-v1zamazon.titan-embed-text-v2:0zcohere.embed-multilingual-v3mistraljina_aivoyagefireworks_ai)r  r   r   r4  r   TzSetting z is not supported by r  )r  r   rx  r`   r   r  r  r   r    r   TritonEmbeddingConfigr  DatabricksEmbeddingConfignvidiaNimEmbeddingConfigVertexAITextEmbeddingConfigLmStudioEmbeddingConfigAmazonTitanG1Config&AmazonTitanMultimodalEmbeddingG1ConfigAmazonTitanV2ConfigBedrockCohereEmbeddingConfigMistralEmbeddingConfigJinaAIEmbeddingConfigVoyageEmbeddingConfigFireworksAIEmbeddingConfigr  r   r  )r   r   r  r  r2  r  r  r   r  r  r  r  r  r  r  r   final_paramsr  r  r  s   `   ``             @r   get_optional_params_embeddingsr    s    HM'++,A4H"&&x0N$$&1a '  ##M48K*../GN"t4PN8D>  $ 1#%5 h& "%/ 2 7 7 99( j  
	(6 3%

 	*:;!779KK1'2'>E	 L 
 5/4V4		,6+2 ,%

 	*:;!;;=OO12 P 
 5/4V4		,6+2 ,%

 	*:;!::LL12f M 
 		+6 +%

 	*:; //1CC12f D 
	
 5/4V4		+++-IIK 	 	*:;!99;MM12 N 
 5/4V4			)'50!557F*e3CCEF+u4002F+u499;F!.>?%f:L!==?*:; 2212 3 
 5/4V4			)6 )%

 	*:;!88:LL12 M 
 5/4V4			)6 )%

 	*:;!779KK12 L 
 5/4V4		(6 (%

 	*:;!779KK1'2'>E	 L 
 5/4V4		.6 .%

 	*:;!<<>PP12U Q 
 5/4V4 	x'7*w'J'JJ!&&()A-##t+{d/B.3356A&**1d3  - #&'9&::OPcOd  eb  c  4(3F3Lr   c                     t        | t              r3d| v r
| d   du r| d= | j                         D ]  \  }}t        |        | S t        | t              r| D ]  }t        |        | S )a  
    clean out 'additionalProperties = False'. Causes vertexai/gemini OpenAI API Schema errors - https://github.com/langchain-ai/langchainjs/issues/5240

    Relevant Issues: https://github.com/BerriAI/litellm/issues/6136, https://github.com/BerriAI/litellm/issues/6088
    additionalPropertiesF)r   r   rx  _remove_additional_propertiesr   r4  r  r   r%  s       r   r  r  	  sw     &$!V+7M0NRW0W-. !,,.JC)%0 ) M 
FD	!D)$/  Mr   c                     t        | t              r,d| v r| d= | j                         D ]  \  }}t        |        | S t        | t              r| D ]  }t        |        | S )zy
    Relevant Issues: https://github.com/BerriAI/litellm/issues/6136, https://github.com/BerriAI/litellm/issues/6088
    strict)r   r   rx  _remove_strict_from_schemar   r  s       r   r  r  	  sh     &$vx  !,,.JC&u- ) M 
FD	!D&t,  Mr   r  supported_openai_paramsc                     g }|i S | j                         D ]  }||vs|j                  |        |D ]  }| j                  |d        | S )z;
    Remove unsupported params from non_default_params
    N)r  r   r   )r  r  remove_keysparamr  s        r   _remove_unsupported_paramsr  	  s`     K&	#((*//u% + sD) r   c                  n'   23 t               j                         33j                  d      }!|!j                         D ]M  \  }"}#|"j	                  d      rdk7  rdk7  r"|"dk(  rdk7  r-|"j	                  d      rdk7  rdk7  rI|#3|"<   O i }$t
        j                  }%|%d	   v r	 d
k(  r&t        j                         j                  3|$      }$ndk(  r&t        j                         j                  3|$      }$nZdk(  sdk(  r&t        j                         j                  3|$      }$n*dk(  r%t        j                         j                  3|$      }$i dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd d dd d d d d d d d d d d d d d}&3j                         D "#ci c]A  \  }"}#|"d k7  r7|"d!k7  r2|"d"k7  r-|"d#k7  r(|"d$k7  r#|"d%k7  r|"|&v r|#|&|"   k7  rt        |"|&      d'u r|"|#C c}#}"2d2v s	d2v sd(2v r d)k(  rd*k7  rd
k7  rdk7  rd+k7  rd,k7  rd-k7  rd.k7  rd/k7  rd0k7  rd1k7  rd2k7  r߉d3k7  rډd4k7  rՉd5k7  rЉd6k7  rˉd7k7  rƉd8k7  rdk7  rd9k7  rd:k7  rt
        j                  vrd)k(  rTd;|$d<<   d=t
        _        d(2v r'2j                  d(      |$d><   2j                  d?d        n`d2v r\2j                  d      |$d><   nGt
        j                  r&2j                  d(2j                  dd             |$d><   nt        d@dA dBC      dD2v rt!        2dD   E      2dD<   d(2v rzt#        2t$              rj2d(   }|D ]`  }'|'j'                  dFi       }(|(j'                  dGd       })|)*t        j(                  |)      }*dH|*v r|*dH   d'u r|*j                  dHd        |*|(dG<   b dIt*        t,           f 23fdJ}+d },@t.        D -cg c]  }-|-j0                   c}-v r t2        j5                   t/              K      },t7         L      }.|.t7         dML      }. |+|.xs g N       d6k(  r=t        j8                         j;                   2|$t#        t<              rnd'O      }$nMdPk(  rxt        j>                         j;                   2|$t#        t<              rnd'O      }$t        j>                         j;                   2|$t#        t<              rnd'O      }$nЉd8k(  r=t        j@                         j;                  2|$ t#        t<              rnd'Q      }$nd7k(  r=t        jB                         j;                  2|$ t#        t<              rnd'Q      }$nLdRk(  r-t        jD                         j;                  2|$ nd'Q      }$ndSk(  r=t        jF                         j;                  2|$ t#        t<              rnd'Q      }$n؉dTk(  r=t        jH                         j;                  2|$ t#        t<              rnd'Q      }$ndUk(  r=t        jJ                         j;                  2|$ t#        t<              rnd'Q      }$nTdVk(  r=t        jL                         j;                  2|$ t#        t<              rnd'Q      }$nd,k(  r=t        jN                         j;                  2|$ t#        t<              rnd'Q      }$nЉdk(  r t
        jP                  v sZ t
        jR                  v sH t
        jT                  v s6 t
        jV                  v s$ t
        jX                  v s t
        jZ                  v r=t        j\                         j;                  2|$ t#        t<              rnd'Q      }$n"dWk(  r=t        j^                         j;                  2|$ t#        t<              rnd'Q      }$
ndk(  s	dk(  rAdW v r=t        j\                         j;                  2|$ t#        t<              rnd'Q      }$
nt
        j`                  jc                   L      r=t        j`                         j;                   2|$t#        t<              rnd'O      }$
n7dk(  rO t
        jd                  v r=t        jf                         j;                  2|$ t#        t<              rnd'Q      }$	ndk(  r t
        jh                  v r~d4 v r=t        jj                         j;                   2|$t#        t<              rnd'O      }$	nt        jl                         j;                   2|$t#        t<              rnd'O      }$	nNdk(  rO t
        jn                  v r=t        jp                         j;                  2|$ t#        t<              rnd'Q      }$ndk(  r=t        jr                         j;                  2|$ t#        t<              rnd'Q      }$ndk(  r	t        jt                         jw                         }/|/t
        jx                  v r>t        jt                         j;                   2|$t#        t<              rnd'|X      }$n?d6 v rddY3v r9 j	                  dZ      r%t        jz                         j;                  2|$      }$nt        j|                         j;                  2|$      }$n|,|,j;                  2|$ t#        t<              rnd'Q      }$nd[k(  r=t        j~                         j;                   2|$t#        t<              rnd'O      }$ngd)k(  r=t        j                         j;                  2|$ t#        t<              rnd'Q      }$n%d9k(  r=t        j                         j;                   2|$t#        t<              rnd'O      }$nd\k(  r=t        j                         j;                  2|$ t#        t<              rnd'Q      }$nd]k(  r=t        j                         j;                  2|$ t#        t<              rnd'Q      }$n_d^k(  r=t        j                         j;                  2|$ t#        t<              rnd'Q      }$nd_k(  r-|,+|,j;                  2|$ t#        t<              rnd'Q      }$nd5k(  sd4k(  r=t        jl                         j;                  2|$ t#        t<              rnd'Q      }$nd`k(  r=t        jj                         j;                  2|$ t#        t<              rnd'Q      }$nbdak(  r=t        j                         j;                  2|$ t#        t<              rnd'Q      }$n d.k(  r=t        j                         j;                   2|$t#        t<              rnd'O      }$nމd/k(  r=t        j                         j;                  2|$ t#        t<              rnd'Q      }$nd0k(  r(t        j                         j;                   2|$b      }$nod1k(  sdck(  r=t        j                         j;                  2|$ t#        t<              rnd'Q      }$n(ddk(  r=t        j                         j;                  2|$ t#        t<              rnd'Q      }$nd2k(  r=t        j                         j;                  2|$ t#        t<              rnd'Q      }$ndek(  r=t        j                         j;                  2|$ t#        t<              rnd'Q      }$nbdfk(  r=t        j                         j;                  2|$ t#        t<              rnd'Q      }$n d-k(  r=t        j                         j;                  2|$ t#        t<              rnd'Q      }$nމd3k(  r=t        j                         j;                  2|$ t#        t<              rnd'Q      }$nd:k(  r=t        j                         j;                  2|$ t#        t<              rnd'Q      }$nZdk(  rt        j                         j;                  2|$ t#        t<              rnd'Q      }$3j                         D ]3  }0t        j                         j                  |0      s't        dg|0        n҉dhk(  r=t        j                         j;                  2|$ t#        t<              rnd'Q      }$ndMk(  r=t        j                         j;                  2|$ t#        t<              rnd'Q      }$nNd
k(  rt        j                         j                   i      r<t        j                         j;                  2|$ t#        t<              rnd'Q      }$nt        j                  djj                  |t
        j                  t        dk                   |xs/ t
        j                  xs t        dk      xs t
        j                  }t        j                         j;                  2|$ |t#        t<              rnd'l      }$n;t        j                         j;                  2|$ t#        t<              rnd'Q      }$g dmt
        j                  z   v rt        dn|&      d'u r3j                  dni       }13j                         D ]  }"|"|&j                         vs3|"   |1|"<    |$j                  dni        i |$dn   |1|$dn<   t        |$dn   o      |$dn<   n03j                         D ]  }"|"|&j                         vs3|"   |$|"<    t        dp|$        |$S c c}#}"w c c}-w )qNr   r  r5  r  r_  r  r  r  	providersr4  r  watsonx	functionsfunction_callr  top_pr  r   r   stopr^  max_completion_tokens
modalities
predictionaudiopresence_penaltyfrequency_penalty
logit_biasr   r   )r   r2  r1  seedr  r  max_retrieslogprobstop_logprobsextra_headersapi_versionparallel_tool_callsr  r  r   r   r2  r  r  r  r   r  Fr  ollamard  anyscaler  r  r  cerebrasxai	ai21_chat
volcenginedeepseek	codestralr  rf  cohere_chatre  ollama_chatrg  r  formatTfunctions_unsupported_modelr  r  z%Function calling is not supported by rZ  r  r1  r3  r  r  r  r  c           	         t        j                  d d        t        j                  d        t        j                  d        i }j                         D ]0  }|| vs|dk(  s
|dk(  s|dk(  r|dk(  rd	k(  r#|d
k(  r)|   ||<   2 |rVt        j
                  du s,du r(|j                         D ]  }j                  |d         y t        d d| d d      y )Nz
LiteLLM completion() model= z; provider = z(
LiteLLM: Params passed to completion() z4
LiteLLM: Non-Default params passed to completion() r   r   r   r  rm   r  Tr  r  r  r  r  )rn   r   r   r  r   r  r   r   )	r  r  r  r2  r  r   r  r  r  s	      r   r  z-get_optional_params.<locals>._check_valid_arg
  sA   ,UG=AT@UV	
 	7G	
 	CDVCWX	
  #((*A((;!'7"71=8Q& -?q,A&q) + ""d*'K4,?+002A&**1d3 3 - #233QRdQeeqrwqx  yc  d  r   r  r=  r  r  )r   r  r   r  anthropic_textr  r  maritalkr  	predibasehuggingfacegemini)r   r  r   r  r   aws_bedrock_clientzanthropic.claude-3
cloudflarerj  petals	deepinfra
perplexitytext-completion-codestralr  )r   r  r   ri  r  hosted_vllmvllmzLiteLLM now defaults to Watsonx's `/text/chat` endpoint. Please use the `watsonx_text` provider instead, to call the `/text/generation` endpoint. Param: watsonx_textr  zoAzure optional params - api_version: api_version={}, litellm.api_version={}, os.environ['AZURE_API_VERSION']={}AZURE_API_VERSION)r  r   r   r  r  )r  r4  rd  
extra_body)r3  z Final returned optional params: )ar  r   r   rx  ry  r   !common_cloud_provider_auth_paramsAzureOpenAIConfigmap_special_auth_paramsAmazonBedrockGlobalConfigVertexAIConfigIBMWatsonXAIConfigr  r  add_function_to_promptr   r?  r   r   r   r   r^   r   rJ   r   r  get_provider_chat_configr    AnthropicConfigr  r  AnthropicTextConfigCohereConfigCohereChatConfigTritonConfigMaritalkConfigReplicateConfigPredibaseConfigHuggingfaceConfigTogetherAIConfigrt  ru  rr  rs  vertex_language_modelsvertex_vision_modelsVertexGeminiConfigGoogleAIStudioGeminiConfigVertexAIAnthropicConfigis_supported_modelvertex_llama3_modelsVertexAILlama3Configvertex_mistral_modelsCodestralTextCompletionConfigMistralConfigvertex_ai_ai21_modelsVertexAIAi21ConfigSagemakerConfigAmazonConverseConfig_get_base_modelbedrock_converse_modelsAmazonAnthropicClaude3ConfigAmazonAnthropicConfigCloudflareChatConfigOllamaConfigOllamaChatConfigNLPCloudConfigPetalsConfigDeepInfraConfigDatabricksConfigNvidiaNimConfigCerebrasConfigXAIChatConfigAI21ChatConfigFireworksAIConfigVolcEngineConfigHostedVLLMChatConfig
VLLMConfigGroqChatConfigOpenAIConfigOpenrouterConfigIBMWatsonXChatConfigr  is_watsonx_text_paramr,  AzureOpenAIO1Configis_o1_modelrn   r   r!  r  r4   AZURE_DEFAULT_API_VERSIONOpenAILikeChatConfigrm  r!   r   )4r   r  r  r  r  r  r   r   r  r^  r	  r
  r  r  r  r  r  r   r2  r1  r  r  r  r  r  r  r  r  r  r  r  r   r   r  r  r  r   common_auth_dictr  r  tool_functionr  new_parametersr  r  r   r  r`  r   r3  r  r  s4   `    `            `          `                    @@r   get_optional_paramsrt  
  sx   L HMMOM"&&x0N$$&1<<9,1D1S/!&9[&HLL##{2#'77a ' O@@.{;;	 ')%779QQ#0/ R O !I-113KK'4o L    ;."&66%446NN#0/ O O !I-%88:RR#0/ S O!T!! 	t! 		!
 	T! 	$! 	$! 	! 	d! 	 ! 	d! 	d! 	! 	D! 	T!  	d!!" 	#!$ !#"&A!NL "''))DAqL**]"]"--Z^#^A&&"Q?UV 	
1)& 	))00((  8+#'??#w.#{2#z1#}4#v-#|3#z1#u,#{2#|3#z1#{2#y0#{2#}4#x/#y0#}4#|3#7+N+NN"h.,2) . 00*..w7 $$AB '**%t !$66*..{; $$AB ..ASAWAW/33KFB => - #CDWCXXYZ 
 ..0M./@A1
,- $$D* #7+   !HHZ4M&**<>J%!%z!:*n<&'=>%G"&&'=tD.<l+ #49 # #J -1O&+>'3C'38|C , 0HH,/B"C I 
 3)< 6X
 &6&<"=k)!113EE1+ *z+t/L  F 	
 
 0	0!557II1+ *z+t/L  J 	
 "557II1+ *z+t/L  J 	
 
	( "..0BB1+ *z+t/L  C 	
 
	-!224FF1+ *z+t/L  G 	
 
	(!..0BB1+'2'>E	 C 
 

	*!002DD1+ *z+t/L  E 	
 
	+!113EE1+ *z+t/L  F 	
 
	+!113EE1+ *z+t/L  F 	
 
	-!335GG1+ *z+t/L  H 	
 
	-!224FF1+ *z+t/L  G 	
 
	++++G333G...G333G222G000!446HH1+ *z+t/L  I 	
 
	(!<<>PP1+ *z+t/L  Q 	
 
 0	0{*x5/@!446HH1+ *z+t/L  I 	
 
	(	(	;	;)< 
< 
 "99;MM1+ *z+t/L  N 	
 
	+9U9U0U!668JJ1+ *z+t/L  K 	
 
	+9V9V0V%%CCEWW#5 / #.:k43P   X 	O &335GG#5 / #.:k43P   H 	O 
	+9V9V0V!446HH1+ *z+t/L  I 	
 
	+!113EE1+ *z+t/L  F 	
 
		)113CCEJ
888%::<NN#5 / #.:k43P  ! O 
O E!#}4##$89<<>PP/A,; Q  $ #*"?"?"A"S"S'9$3 #T # (-??#5 / #.:k43P   @ 	O 
	,!668JJ1+ *z+t/L  K 	
 
	(!..0BB1+ *z+t/L  C 	
 
	-!224FF1+ *z+t/L  G 	
 
	+!002DD1+ *z+t/L  E 	
 
	(!..0BB1+ *z+t/L  C 	
 
	+!113EE1+ *z+t/L  F 	
 
	,1L);;1+ *z+t/L  < 	
 
		)-@K-O!//1CC1+ *z+t/L  D 	
 
 ;	;!??ASS1+ *z+t/L  T 	
 
	,!224FF1+ *z+t/L  G 	
 
	,!113EE1+ *z+t/L  F 	
 

	*!002DD1+ *z+t/L  E 	
 
	%!//1CC1+ D 

 
	+/Bf/L!002DD1+ *z+t/L  E 	
 
	.!335GG1+ *z+t/L  H 	
 
	,!224FF1+ *z+t/L  G 	
 
	-!668JJ1+ *z+t/L  K 	
 
	&!,,.@@1+ *z+t/L  A 	
 
	&!002DD1+ *z+t/L  E 	
 

	*!..0BB1+ *z+t/L  C 	
 
	,!224FF1+ *z+t/L  G 	
 
		)!668JJ1+ *z+t/L  K 	
 #'')E))+AA%H  p  qv  pw  x  *
 
	.!446HH1+ *z+t/L  I 	
 
	(!..0BB1+ *z+t/L  C 	
 
	'&&(4454A%99;MM#5 / #.:k43P   N 	O    B  I  I!4!4jAT6U  5&&5125 44	  &779KK#5 /' #.:k43P   L 
O "668JJ1+ *z+t/L  K 	
 	8

-
-.	. 7M 
 '**<<J"'')N//11$1!$4JqM * &&|R8-!,/--OL)
 -G*<8-OL)
 ##%A++--%21%5" & 4_4EFGWVCs   4AAN,$AN2c                     i dd dd dd dd dd dd dd dd d	d d
d dd dd dd dd dddd dd d d d d d d d}| j                         D ci c]  \  }}|dk7  r|dk7  r||v r|||   k7  r||  }}}|S c c}}w )Nr  r  r  r  r  r   r   r  r^  r  r  r  r   r   r2  r   r1  r  )r  r  r  r  r  r  r_  )r  r  r  r  r  s        r   get_non_default_paramsrv    sO   T 	t 		
 	T 	$ 	$ 	 	d 	D 	T 	d 	 	 	r  	4!" 	#$ /N8 "'')	)DAqL**^#^A&& 	
1)  	 	s   #A:max_parallel_requestsrpmtpmdefault_max_parallel_requestsc                 T    | | S ||S |t        |dz  dz        }|dk(  rd}|S ||S y)a  
    Returns the max parallel requests to send to a deployment.

    Used in semaphore for async requests on router.

    Parameters:
    - max_parallel_requests - Optional[int] - max_parallel_requests allowed for that deployment
    - rpm - Optional[int] - requests per minute allowed for that deployment
    - tpm - Optional[int] - tokens per minute allowed for that deployment
    - default_max_parallel_requests - Optional[int] - default_max_parallel_requests allowed for any deployment

    Returns:
    - int or None (if all params are None)

    Order:
    max_parallel_requests > rpm > tpm / 6 (azure formula) > default max_parallel_requests

    Azure RPM formula:
    6 rpm per 1000 TPM
    https://learn.microsoft.com/en-us/azure/ai-services/openai/quotas-limits


    Nrl     r   rm   )r  )rw  rx  ry  rz  calculated_rpms        r   calculate_max_parallel_requestsr~    sS    : ($$	
	S4Z!^,QN	&	2,,r   healthy_deploymentsc                     t        d | D        d       }|(| D cg c]  }|d   j                  d      |k(  r| }}|S | S c c}w )Nc              3   :   K   | ]  }d |d   v r
|d   d      yw)orderr   Nr  )r   
deployments     r   r   z2_get_order_filtered_deployments.<locals>.<genexpr>  s1      	
1
*%566 '(11s   )defaultr   r  )minr   )r  	min_orderr  filtered_deploymentss       r   _get_order_filtered_deploymentsr    st    	
1	

 I  2 
1
*+//8IE 1 	  
 $# 
s    Ar   c                 @   | dk(  rX|j                   xs* t        j                   xs t        d      xs t        d      }|t        |t              r|S |j                  S | dk(  r|j
                  }||S |j                  S | dk(  r|j                  }||S |j                  S )z=
    Return the region for a model, for a given provider
    r  VERTEXAI_LOCATIONVERTEX_LOCATIONr5  r  )vertex_locationr   r4   r   r   aws_region_namewatsonx_region_nameregion_name)r2  r   vertex_ai_locationr  r  s        r   _get_model_regionr    s     k) ** -&&--.- +,	 	 )j9KS.Q%% %%% 
		)(88&""
 %%%	 
		),@@*&&%%%r   c                    t        j                  | j                  |       \  }}}}t        ||       }|/t	        j
                  dj                  | j                               y|dk(  rEt        j                         j                         }t        j                         j                         }n|dk(  rEt        j                         j                         }t        j                         j                         }n|dk(  rEt        j                         j                         }t        j                         j                         }nN|dk(  rEt        j                         j                         }t        j                         j                         }ng }g }|D ]  }||j                         v s y	 |D ]  }||j                         v s y
 y)z
    Infer if a model is in the EU or US region

    Returns:
    - str (region) - "eu" or "us"
    - None (if region not found)
    )r   r   )r2  r   Nz'Cannot infer model region for model: {}r4  r  r5  r  euus)r   r   r   r  rn   r   r!  r5  get_eu_regionsget_us_regionsr8  r7  r9  r  )r   r   r2  rB  model_region
eu_regions
us_regionsregions           r   _infer_model_regionr    s    (/'?'?"">($E1 %/L 5<<^=Q=QR	
 g%..0??A
..0??A
		+++-<<>
++-<<>
			)668GGI
668GGI
			)//1@@B
//1@@B


\''))  \''))  r   c                 L    | j                   dk(  ryt        |       }||dk(  ryy)8
    Return true/false if a deployment is in the EU
    r  Tr   Fr  r  r   r  s     r   _is_region_eur  ?  4     !!T) 'nELLD$8r   c                 L    | j                   dk(  ryt        |       }||dk(  ryy)z8
    Return true/false if a deployment is in the US
    r  Tr  Fr  r  s     r   _is_region_usr  M  r  r   allowed_model_regionc                 $    | j                   |k(  ryy)r  TF)r  )r   r  s     r   is_region_allowedr  [  s     !!%99r   r  c           	         d| j                   v rt        | j                  t              rt        | j                  t              r| j                   j                  dd      }t        j                         j                  || j                  | j                  | j                  xs t        j                  d|xs d      }|j                  dd      }|S y)	zM
    Pass the litellm params for an azure model, and get back the region
    r4  zazure/r   
   chat)r   r  r   r  timeoutr  zx-ms-regionN)r   r   r  r   r   r.  r   AzureChatCompletionget_headersr  ro  r   )r   r  _modelr5  r  s        r   get_model_regionr  f  s     	>'''~--s3~..4%%--h; 446BB"**#,,&22Wg6W6W C 
 !)]D Ar   c                 d    	 | j                  d      }t        |      d d }|S # t        $ r Y yw xY w)Nr   r   r   )r   r   r   )r   	_messagess     r   get_first_chars_messagesr    s=    JJz*		N4C(	 s    # 	//c                 H    dj                  d | D              }t        |      S )Nr   c              3   B   K   | ]  }|j                         r|  y wr   )isspace)r   chars     r   r   z$_count_characters.<locals>.<genexpr>  s     HTTDTr+  )r   r   )r  filtered_texts     r   _count_charactersr    s!    GGHTHHM}r   response_objc                 4   | j                   }d}|D ]  }t        |t              r1|j                  j                  *||j                  j                  z  }Dt        |t
              sU|j                  j                  l||j                  j                  z  } |S )Nr   )r:  r   rB   r   r   rR   delta)r  _choicesresponse_strchoices       r   get_response_stringr    s    7C7K7KHLfg&~~%%1 6 66 01||##/ 4 44  r   llm_providerdynamic_api_keyc                    |xs t         j                  }| dk(  s| dk(  r#|xs t         j                  xs t        d      }|S | dk(  s| dk(  r#|xs t         j                  xs t        d      }|S | dk(  r#|xs t         j
                  xs t        d      }|S | d	k(  r#|xs t         j                  xs t        d
      }|S | dk(  r#|xs t         j                  xs t        d      }|S | dk(  s| dk(  r#|xs t         j                  xs t        d      }|S | dk(  r#|xs t         j                  xs t        d      }|S | dk(  r#|xs t         j                  xs t        d      }|S | dk(  r#|xs t         j                  xs t        d      }|S | dk(  r.|xs* t         j                  xs t        d      xs t        d      }|S )Nr  rd  OPENAI_API_KEYrf  r$  ANTHROPIC_API_KEYri  AI211_API_KEYrk  ALEPH_ALPHA_API_KEYbasetenBASETEN_API_KEYre  r  COHERE_API_KEYr'  HUGGINGFACE_API_KEYrj  NLP_CLOUD_API_KEYr  REPLICATE_API_KEYr  TOGETHERAI_API_KEYTOGETHER_AI_TOKEN)r   r  
openai_keyr4   anthropic_keyai21_keyaleph_alpha_keybaseten_key
cohere_keyhuggingface_keynlp_cloud_keyreplicate_keytogetherai_api_key)r  r  r  s      r   get_api_keyr    s   0Gx<3K#KOW//O:>N3OJ NG 
	$8H(HUW22UjAT6UD NA 
	LW--LO1L> N; 
	&Sw..S*=R2S 	8 N1 
	"QW00QJ?P4Q. N+ 
	!\]%BOW//O:>N3O( N% 
	&Sw..S*=R2S 	" N 
	$UW22UjAT6U N 
	$UW22UjAT6U N 
	& /))/.// -.	 	 Nr   c                      dd l } ddl m } t        | d      r |j                  | j                        S  |j                         S )Nr   )rs  UTC)rs  r  rt  r  utcnow)dtrs  s     r   get_utc_datetimer    s6    !r5x||BFF##x  r   c                 :   d }	 | t         j                  v rVdt         j                  |    v rt         j                  |    d   S dt         j                  |    v rt         j                  |    d   S t        |       \  } }}}|dk(  r ||       }|S | t         j                  v rWdt         j                  |    v rt         j                  |    d   S dt         j                  |    v rt         j                  |    d   S yt               # t        $ r t        d|  d	      w xY w)
aY  
    Get the maximum number of output tokens allowed for a given model.

    Parameters:
    model (str): The name of the model.

    Returns:
        int: The maximum number of tokens allowed for the given model.

    Raises:
        Exception: If the model is not mapped yet.

    Example:
        >>> get_max_tokens("gpt-4")
        8192
    c                     d|  d}	 t         j                  j                  |      }|j                          |j	                         }|j                  d      }||S y # t
        $ r Y y w xY wNzhttps://huggingface.co/z/raw/main/config.jsonmax_position_embeddingsr   module_level_clientr   raise_for_statusr  r   
model_name
config_urlr5  config_jsonr  s        r   _get_max_position_embeddingsz4get_max_tokens.<locals>._get_max_position_embeddings  sv    .zl:OP
	2266zBH%%' #--/K&1oo6O&P#&2.. 		   AA 	A)(A)max_output_tokensr^  r  r'  r  NzModel zr isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json)r   r[  r   r   )r   r  r2  rB  r^  s        r   get_max_tokensr    s8   $&
G&&&"g&8&8&??))%01DEE!3!3E!::))%0>>+;%+H("Aq-/5GJG&&&"g&8&8&??))%01DEE!3!3E!::))%0>>  + 
UG  N  O
 	

s)   <D *D - D <D *D 7
D Dc                 0    t        j                  dd|       S )Nz-\d+$r   resubr  s    r   _strip_stable_vertex_versionr    s    66(B
++r   c                 H    t        j                         j                  |       S r   )r   rT  rU  r  s    r   _strip_bedrock_regionr    s    '')99*EEr   r  c                 0    t        j                  dd|       S )a  
    Strips the organization, custom suffix, and ID from an OpenAI fine-tuned model name.

    input: ft:gpt-3.5-turbo:my-org:custom_suffix:id
    output: ft:gpt-3.5-turbo

    Args:
    model_name (str): The full model name

    Returns:
    str: The stripped model name
    z(:[^:]+){3}$r   r  r  s    r   !_strip_openai_finetune_model_namer    s     66/2z22r   c                     |r|dk(  rt        |       }|S |r|dk(  s|dk(  rt        |       }|S |r|dk(  rt        |       }|S t        |       }|S )Nr5  r  r  r(  r  )r  r  r  )r   r2  strip_bedrock_regionstrip_versionstrip_finetunes        r   _strip_model_namer  )  sl    2i?4F##	{*.AX.M4F	"5"E4F:eLr   c                 (    t         j                  |    S r   )r   r[  r  s    r   _get_model_info_from_model_costr  :  s    c""r   rf  c                     |r[d| v rW| d   |k7  rO|dk(  r| d   j                  d      ry|dk(  r| d   j                  d      ry|dk(  r| d   j                  d      ryyy)zG
    Check if the model info provider matches the custom provider.
    rc  r  Tr  r5  F)ry  rf  r2  s     r   _check_provider_matchr  >  s     j()*.AA+-*3

*[
!3"  N2z8

*^
$8%  I-*3

*Y
3  r   )	TypedDictc                   @    e Zd ZU eed<   eed<   eed<   eed<   eed<   y)'PotentialModelNamesAndCustomLLMProvidersplit_modelcombined_model_namestripped_model_namecombined_stripped_model_namer2  N)r9  
__module____qualname__r   __annotations__r  r   r   r  r  [  s     "%%r   r  c           
         |$	 t        |       \  }}}}| }t        | |      }|}n|rJ| j                  |dz         r6| j	                  d      d   }| }t        ||      }dj                  ||      }n>| }dj                  ||       }t        | |      }dj                  |t        | |            }t        ||||t        t        |            S # t        $ r | }Y w xY w)Nr  r=  rh  rm   z{}/{})r  r  r  r	  r2  )	r   r   r  ry  rp  r!  r  rd   r   )r   r2  r  rB  r  r  r	  s          r   _get_potential_model_namesr  c  s$    "	 5EE5R2K,a $/-@
 (;$	!1!1c!" kk#&q)#/3F
 (/~~!4(
$ %nn-@%H/-@
 (/~~E?RS(
$
 3//%A &9: ;  	 K	 s   C CCc                     d|  d}	 t         j                  j                  |      }|j                          |j	                         }|j                  d      }||S y # t
        $ r Y y w xY wr  r  r  s        r   r  r    sx    *:,6KLJ..22:>!!# mmo #.//2K"L".** r  c                 .   	 i t         j                  t         j                  }| |v r||    } ||dk(  rd}|U|dk(  rPd| z   t         j                  v rd| z   } n5| dz   t         j                  v r| dz   } n| dz   t         j
                  v r| dz   } t        | |      }|d   }|d   }|d	   }|d
   }|d   }|dk(  r%t        |       }t        | |dddddddddddd      S |dk(  s|dk(  r#t        j                         j                  |       S 	 d}	d}
d}|t         j                  v r|}
t        |
      }	t        |	|      sd}	|	/| t         j                  v r| }
t        |
      }	t        |	|      sd}	|	/|t         j                  v r|}
t        |
      }	t        |	|      sd}	|	/|t         j                  v r|}
t        |
      }	t        |	|      sd}	|	/|t         j                  v r|}
t        |
      }	t        |	|      sd}	|r(|t        v r t        j!                  | t#        |            }|	.|,t%        t&        t(           |j                  | |	            }	|
d}
|	|
t+        d      |dk(  rd|	d<   |	j-                  d      }|'t/        j0                  dj3                  | |             d}|	j-                  d      }|'t/        j0                  dj3                  | |             d}t        dCi d |
d!|	j-                  d!d      d"|	j-                  d"d      d#|	j-                  d#d      d|d$|	j-                  d$d      d%|	j-                  d%d      d&|	j-                  d&d      d'|	j-                  d'd      d(|	j-                  d(d      d)|	j-                  d)d      d*|	j-                  d*d      d|d+|	j-                  d+d      d,|	j-                  d,d      d-|	j-                  d-d      d.|	j-                  d.d      d/|	j-                  d/d      d0|	j-                  d0d      d1|	j-                  d1d      d2|	j-                  d2|      d3|	j-                  d3      d4|	j-                  d4d      d|	j-                  dd      d5|	j-                  d5d6      d7|	j-                  d7d6      d8|	j-                  d8d6      d9|	j-                  d9d6      d:|	j-                  d:d6      d;|	j-                  d;d6      d<|	j-                  d<d6      d=|	j-                  d=d6      d>|	j-                  d>d      d?|	j-                  d?d      d@|	j-                  d@d      S # t4        $ r/}dAt7        |      v r|t5        dBj3                  | |            d}~ww xY w)Dzy
    Helper for 'get_model_info'. Separated out to avoid infinite loop caused by returning 'supported_openai_param's
    Nr  r  zmeta/z@latestr=  r  r  r	  r  r2  r'  r  r   r  )r  r^  max_input_tokensr  r  r  rc  r  r9  r?  rF  supports_assistant_prefillrR  rN  r  r   r  r  r  )r   existing_model_infoprovider_specific_model_infoz|This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.jsonr&  Tr?  r  z`model={}, custom_llm_provider={} has no input_cost_per_token in model_cost_map. Defaulting to 0.r  zamodel={}, custom_llm_provider={} has no output_cost_per_token in model_cost_map. Defaulting to 0.r  r^  r  r  cache_creation_input_token_costcache_read_input_token_costinput_cost_per_character&input_cost_per_token_above_128k_tokensinput_cost_per_queryr  input_cost_per_audio_tokenoutput_cost_per_audio_tokenoutput_cost_per_character'output_cost_per_token_above_128k_tokens+output_cost_per_character_above_128k_tokensr  output_cost_per_imageoutput_vector_sizerc  r  r9  rT  FrF  r  rR  rL  rP  rN  rW  supports_native_streamingry  rx  OllamaErrorzThis model isn't mapped yet. model={}, custom_llm_provider={}. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json.r  )r   r  azure_embedding_modelsrL  rN  rQ  r  r  rN   rZ  rU  r[  r  r  rK   r  get_provider_model_inforJ   rd   r`   r\   r,  r   rn   r   r!  r   r   )r   r2  r  potential_model_namesr  r  r	  r  r^  _model_infor  r  _input_cost_per_token_output_cost_per_tokenr  s                  r   rJ  rJ    sb   `
M**Mg.L.LM
Ju%E*/BFV/V"-*/Bk/Q'">">>%"g&C&CC	)"g&C&CC	) :-@!
 44IJ34IJ'<*(
$ ,M:34IJ-/5GJ %!%"&%&&'!.)-)-*.+/(,#'   !H,0C}0T'')88?? 59K!%C:>O"g&8&88)=#F,*@S #'K"u0B0B'B=#F,*@S #'K#0G4F4FF2=#F,*@S #'K"':g>P>P'P)=#F,*@S #'K"{g6H6H'H!=#F,*@S #'K"':o'M"7"O"O,7J*K #P # "'B"TN#22# 3  ;8C"ck  S 
 #k1:>675@__&6! %,$$v}}2
 )*%6Aoo'7" &-$$w~~2
 *+&  FF&??<>F "-1CT!JF #.//2Et"L	F
 &;F 1<5t1F -8OO14-F *5.*F 8C<d8F$ &1__5KT%R%F& '2oo6Mt&T'F( ,7??0$,)F. '=/F0 -8OO14-1F6 +6///+7F< 9D=t9=FB =HOOA4=CFH (37OQU'VIFJ '2oo6Mt&TKFL $/??3G#NMFN "-&(;"OFT !__V,UFV *5.*WF\ *5.*]Fb !,0A5 IcFd +6///+eFj ,7??0%,kFp )4-u)qFv &1__5KU%SwFx '2oo6Mu&UyFz $/??3G#O{F| 0;4e0}FB +6///+CFH  OOE40IFJ  OOE40KF FN  
CF"G n  u  u*
 	

s%   CU ,U 
QU 	V%*VVc                 l    t        j                  | |      }t        | |      }t        di |d|i}|S )a  
    Get a dict for the maximum tokens (context window), input_cost_per_token, output_cost_per_token  for a given model.

    Parameters:
    - model (str): The name of the model.
    - custom_llm_provider (str | null): the provider used for the model. If provided, used to check if the litellm model info is for that provider.

    Returns:
        dict: A dictionary containing the following information:
            key: Required[str] # the key in litellm.model_cost which is returned
            max_tokens: Required[Optional[int]]
            max_input_tokens: Required[Optional[int]]
            max_output_tokens: Required[Optional[int]]
            input_cost_per_token: Required[float]
            input_cost_per_character: Optional[float]  # only for vertex ai models
            input_cost_per_token_above_128k_tokens: Optional[float]  # only for vertex ai models
            input_cost_per_character_above_128k_tokens: Optional[
                float
            ]  # only for vertex ai models
            input_cost_per_query: Optional[float] # only for rerank models
            input_cost_per_image: Optional[float]  # only for vertex ai models
            input_cost_per_audio_token: Optional[float]
            input_cost_per_audio_per_second: Optional[float]  # only for vertex ai models
            input_cost_per_video_per_second: Optional[float]  # only for vertex ai models
            output_cost_per_token: Required[float]
            output_cost_per_audio_token: Optional[float]
            output_cost_per_character: Optional[float]  # only for vertex ai models
            output_cost_per_token_above_128k_tokens: Optional[
                float
            ]  # only for vertex ai models
            output_cost_per_character_above_128k_tokens: Optional[
                float
            ]  # only for vertex ai models
            output_cost_per_image: Optional[float]
            output_vector_size: Optional[int]
            output_cost_per_video_per_second: Optional[float]  # only for vertex ai models
            output_cost_per_audio_per_second: Optional[float]  # only for vertex ai models
            litellm_provider: Required[str]
            mode: Required[
                Literal[
                    "completion", "embedding", "image_generation", "chat", "audio_transcription"
                ]
            ]
            supported_openai_params: Required[Optional[List[str]]]
            supports_system_messages: Optional[bool]
            supports_response_schema: Optional[bool]
            supports_vision: Optional[bool]
            supports_function_calling: Optional[bool]
            supports_prompt_caching: Optional[bool]
            supports_audio_input: Optional[bool]
            supports_audio_output: Optional[bool]
            supports_pdf_input: Optional[bool]
    Raises:
        Exception: If the model is not mapped yet.

    Example:
        >>> get_model_info("gpt-4")
        {
            "max_tokens": 8192,
            "input_cost_per_token": 0.00003,
            "output_cost_per_token": 0.00006,
            "litellm_provider": "openai",
            "mode": "chat",
            "supported_openai_params": ["temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty"]
        }
    r=  r  r  )r   r    rJ  rM   )r   r2  r  r&  returned_model_infos        r   rU  rU    sS    F &AA)< )/K
 $ 
/F r   python_type_namec                     t         j                  dt        j                  dt        j                  dt        j                  dt
        j                  dt        j                  dddi}|j                  | d      S )	zConverts standard python types to json schema types

    Parameters
    ----------
    python_type_name : str
        __name__ of type

    Returns
    -------
    str
        a standard JSON schema type, "string" if not recognized.
    r  r!  r"  r#  r  r  NoneTyper$  )r   r9  r  floatr  r   r   r   )r+  python_to_json_schema_typess     r   json_schema_typer0    s]     	hiywxF# '**+;XFFr   c                 z   	 ddl }ddlm} ddlm} | j                  } |j                  |       } ||      }dj                  |d   D cg c]  }|j                          c}      }	i }
g } |j                  |       j                  }|j                         D ]Z  \  }}t        |d      r t        |j                  j                        }nd}d}d}|d   D ]  }|j                   |k(  st        |d	      rQ|j"                  }d
|v r|j%                  d      d   }n!d|v r	 t'        t)         ||                  }d}t        |      }dj                  |j*                  D cg c]  }|j                          c}      } |||d}t-        |j                         D cg c]  \  }}t/        |t&              s||f c}}      |
|<   |j0                  |j2                  k(  sJ|j5                  |       ] ||	d|
dd}|r||d   d<   |S # t
        $ r}|d}~ww xY wc c}w # t
        $ r Y w xY wc c}w c c}}w )ae  Using type hints and numpy-styled docstring,
    produce a dictionnary usable for OpenAI function calling

    Parameters
    ----------
    input_function : function
        A function with a numpy-style docstring

    Returns
    -------
    dictionnary
        A dictionnary to add to the list passed to `functions` parameter of `litellm.completion`
    r   N)literal_eval)NumpyDocStringr  Summary
annotation
Parametersr   optionalr  {r  )r   r  r  r  r   r  r  r  r  r  r  )r   astr2  numpydoc.docscraper3  r   r9  getdocr   strip	signaturer  rx  r  r0  r5  r  r   rp  r   r   descr   r   r  emptyr   )input_functionr   r2  r3  r  r  	docstringnumpydocsr  r  r  
param_info
param_namer   
param_typeparam_description
param_enum
param_data
param_dictr  r  r"  s                          r   function_to_dictrM    sV   $5 ""D~.Ii(H))0CD0C1QWWY0CDEK JO""">2==J'--/
E5,')%*:*:*C*CDJJ 
 #<0J*,:v.!+J!Z/%/%5%5c%:1%=

*!),T,z2J-K)LJ)1J "2*!=J$(II*//.R/Qqwwy/.R$S!! 1& ,

 "& * 0 0 2I 21jC6HaV 2I"

:
 ==EKK'"":.O 0V "$
F +:|Z(MI   E:  ) ! ! /S JsA   H 
HH#H2H76H7	HHH#	H/.H/c                 f    t        j                  |       }|j                  |      }t        |      S )Npath)httpxURL	copy_withr   )original_urlnew_pathr  modified_urls       r   
modify_urlrW  X  s+    
))L
!C==h=/L|r   	num_callsr  c                 f   d}d}|r|}|r|}t        |      D cg c]  }d|dg
 }	}t        j                         }
	 t        j                  | |	|||       t        j                         }||
z
  }|ddd dS c c}w # t        $ r*}t        j                         }||
z
  }|dd|dcY d }~S d }~ww xY w)	NzHey, how's it goingr   r   r  r   )r   r   r2  r   r  success)total_response_time
calls_madestatusrH  failed)rangetimer   batch_completionr   )r   r2  r   r   rX  r  test_prompt
test_callsrB  r   r   r#  response_timer  s                 r   load_test_modelrf  ^  s     (KJ
DI*DUVDUq&[9:DUHVJ
   3'	
 99; :-#0	
 	
 W$  
99; :-#0	
 	

s"   A89A= =	B0B+%B0+B0c                     | dk(  r"t        j                         j                         S | dk(  r"t        j                         j                         S | dk(  r"t        j                         j                         S g S )z,Return the fields required for each providerr  r  azure_ai)r   r_  get_required_paramsrZ  AzureAIStudioConfigr2  s    r   get_provider_fieldsrl    sk     l*'')==??		(##%99;;	
	***,@@BB 	r   c                  F   t               j                         D  ci c]  \  } }| |d nt        |       }} }i }i }t        j                  dd       }|r|j                  d      ng }|j                         D ]h  \  } }|/t        j                         || <   t        j                         || <   7t        j                  |      || <   t        j                  |      || <   j |D ]0  }t        j                         ||<   t        j                         ||<   2 ||fS c c}} w )Nrb  NO_PROXYr  )proxy)	r   rx  r   osgetenvrp  rQ  HTTPTransportAsyncHTTPTransport)r  r  proxiessync_proxy_mountsasync_proxy_mountsno_proxyno_proxy_urlsro  s           r   !create_proxy_transport_and_mountsry    s)    017799HC 	S[Ten49  
  yyT*H+3HNN3'Mmmo
U=%*%8%8%:c"&+&>&>&@s#%*%8%8u%Ec"&+&>&>U&Ks# & !&!4!4!6#"'":":"<3  0001s   Dc                    d}g }| ||dS 	 t        |       \  }}}}|r|dk(  r)dt        j                  v rd}	n|j	                  d       	n|dk(  rOd	t        j                  v r(d
t        j                  v rdt        j                  v rd}	nO|j                  g d       	n:|dk(  r)dt        j                  v rd}	n|j	                  d       	n|dk(  r)dt        j                  v rd}n|j	                  d       n|dk(  r)dt        j                  v rd}n|j	                  d       n|dk(  r)dt        j                  v rd}n|j	                  d       n|dk(  r=dt        j                  v rdt        j                  v rd}nU|j                  ddg       n@|dk(  r)dt        j                  v rd}n%|j	                  d       n|dk(  r)dt        j                  v rd}n|j	                  d       n|dk(  r)dt        j                  v rd}n|j	                  d       n|dk(  r)dt        j                  v rd}n|j	                  d       n|d k(  r)d!t        j                  v rd}nm|j	                  d!       nZ|d"k(  r)d#t        j                  v rd}n?|j	                  d#       n,|d$k(  s|d%k(  rLd&t        j                  v rd't        j                  v rd}n|j	                  d&       |j	                  d'       n|d(v r)d)t        j                  v rd}n|j	                  d)       n|d*k(  r)d+t        j                  v rd}n|j	                  d+       n{|d,k(  r)d-t        j                  v rd}n`|j	                  d-       nM|d.k(  r)d/t        j                  v rd}n2|j	                  d/       n|d0k(  r)d1t        j                  v rd}n|j	                  d1       n|d2k(  r)d3t        j                  v rd}n|j	                  d3       n|d4k(  r)d5t        j                  v rd}n|j	                  d5       n|d6k(  r)d7t        j                  v rd}nz|j	                  d7       ng|d8k(  r)dt        j                  v rd}nL|j	                  d       n9|d9k(  r)d:t        j                  v rd}n|j	                  d:       n|d;k(  s|d<k(  r)d=t        j                  v rd}n|j	                  d=       n|d>k(  r)d?t        j                  v rd}n|j	                  d?       n|d@k(  r)dAt        j                  v rd}n|j	                  dA       n||dBk(  r)dCt        j                  v rd}na|j	                  dC       nN|dDk(  r)dEt        j                  v rd}n3|j	                  dE       n |dFk(  r)dGt        j                  v rd}n|j	                  dG       n|dHk(  r_dIt        j                  v s6dJt        j                  v s$dKt        j                  v sdLt        j                  v rd}n|j	                  dI       n|dMk(  rdNt        j                  v r(dOt        j                  v sdPt        j                  v rd}nN|j	                  dN       |j	                  dP       n*| t        j                  v s6| t        j                  v s$| t        j                  v s| t        j                  v r)dt        j                  v rd}n|j	                  d       n| t        j                  v r)dt        j                  v rd}n|j	                  d       n~| t        j                  v r)dt        j                  v rd}nV|j	                  d       nC| t        j                  v r)dt        j                  v rd}n|j	                  d       n| t        j                  v r)dt        j                  v rd}n|j	                  d       n| t        j                  v s'| t        j                   v s| t        j"                  d   v r=dt        j                  v rdt        j                  v rd}nl|j                  ddg       nW| t        j$                  v r)dt        j                  v rd}n/|j	                  d       n| t        j&                  v r'dt        j                  v rd}n|j	                  d       n| t        j(                  v r'dt        j                  v rd}n|j	                  d       n| t        j*                  v r'dt        j                  v rd}n|j	                  d       nq| t        j,                  v r'd!t        j                  v rd}nJ|j	                  d!       n8| t        j.                  v r&d#t        j                  v rd}n|j	                  d#       |/g }|D ]&  }dQ|j1                         vs|j	                  |       ( |}|/g }|D ]&  }dR|j1                         vs|j	                  |       ( |}t3        |      dSk(  rd}||dS # t        $ r d}Y 
Dw xY w)Ta  
    Checks if the environment variables are valid for the given model.

    Args:
        model (Optional[str]): The name of the model. Defaults to None.
        api_key (Optional[str]): If the user passed in an api key, of their own.

    Returns:
        dict: A dictionary containing the following keys:
            - keys_in_environment (bool): True if all the required keys are present in the environment, False otherwise.
            - missing_keys (List[str]): A list of missing keys in the environment.
    FN)keys_in_environmentmissing_keysr  r  r  Tr4  AZURE_API_BASEr2  AZURE_API_KEY)r}  r2  r~  rf  r  re  r  r  r  rg  OPENROUTER_API_KEYr  VERTEXAI_PROJECTr  r'  r  ri  AI21_API_KEYr  r  rk  r  r  r  rj  r  r5  r  AWS_ACCESS_KEY_IDAWS_SECRET_ACCESS_KEY)r  r   OLLAMA_API_BASEr  ANYSCALE_API_KEYr,  DEEPINFRA_API_KEYr(  GEMINI_API_KEYr  GROQ_API_KEYr  NVIDIA_NIM_API_KEYr  CEREBRAS_API_KEYr  XAI_API_KEYr  r  VOLCENGINE_API_KEYr  r.  CODESTRAL_API_KEYr  DEEPSEEK_API_KEYr  MISTRAL_API_KEYpalmPALM_API_KEYr-  PERPLEXITYAI_API_KEYr  VOYAGE_API_KEYr  FIREWORKS_AI_API_KEYFIREWORKS_API_KEYFIREWORKSAI_API_KEYFIREWORKS_AI_TOKENr*  CLOUDFLARE_API_KEYCLOUDFLARE_ACCOUNT_IDCLOUDFLARE_API_BASEr  r   r   )r   r   rp  environr   extendr   r  ro  open_ai_embedding_modelsopenai_image_generation_modelsr  r  replicate_modelsrq  rt  rr  models_by_providerhuggingface_modelsrv  together_ai_modelsrx  baseten_modelsrw  r  r   )	r   r  r   r{  r|  rB  r2  new_missing_keysr  s	            r   validate_environmentr    s?	   "   L}#6(
 	

#'7e'D$1 (*2::-&*###$45 G+ BJJ.'2::5#rzz1&*###L !K/"bjj0&*###$78 H,2::-&*###$45 K/"bjj0&*###$78 L0#rzz1&*###$89 K/!RZZ/4G2::4U&*###%79L$MN M1$

2&*###$9: F*+&*###N3 M1#rzz1&*###$89 M1$

2&*###$9: I- BJJ.&*###$56 K/"bjj0&*###$78 I-1D1S#rzz1+rzz9&*###$78##$;< $== BJJ.&*###$56 J.!RZZ/&*###$67 K/"bjj0&*###$78 H,2::-&*###$45 F*+&*###N3 L0#rzz1&*###$89 J.!RZZ/&*###$67 E)

*&*###M2 K/+&*###N3 L0#rzz1&*###$89;."&AA"bjj0&*###$78 J.!RZZ/&*###$67 I- BJJ.&*###$56 F*+&*###N3 L0%3&*###$:; H,2::-&*###$45 N2&"**4&"**4(BJJ6'2::5&*###$:; L0#rzz1'2::5(BJJ6&*###$89##$9: W;;;>>>888>>>2::-&*###$45g..."bjj0&*###$78g+++2::-&*###$45g..."bjj0&*###$78g///#rzz1&*###$89 W///22222;??!RZZ/4G2::4U&*###%79L$MNg000$

2&*###$9:g)))+&*###N3g000#rzz1&*###$89g000$

2&*###$9:g,,, BJJ.&*###$56g..."bjj0&*###$78C		+ '',   (C, '',   (
<A"#6UU}  #"#s   i i#"i#c                  ,    t        j                  | i |S r   )r   r   )r  r   s     r   acreater    s    ///r   c                    dj                  d |D              }d}d| v r)	 dd l}ddlm}m}m}  |       }|j                  |      }|S t        t        j                  |            }|S # t        $ r t        d       Y Zw xY w)Nr  c              3   &   K   | ]	  }|d      yw)r   Nr  )r   r   s     r   r   z*prompt_token_calculator.<locals>.<genexpr>  s     ?h7GI&hs   r   claudez:Anthropic import failed please run `pip install anthropic`)	AI_PROMPTHUMAN_PROMPT	Anthropic)
r   rf  r   r  r  r  count_tokensr   r   r  )	r   r   r  r  rf  r  r  r  anthropic_objs	            r   prompt_token_calculatorr    s    88?h??DJ5	T 	A@!"//5
  ./
  	TRS	Ts   A( (A?>A?c                     	 | t         j                  v s| t         j                  v r t        j                  j                  |        y dddg}t        j                  | |       y # t        $ r t        d| d      w xY w)Nr   zHello WorldrZ  r   r   r   )r   r   r  )	r   r  ro  r  modelsretriever   r   rw   r  s     r   valid_modelr    su    H W;;;>>>MM""5)!'MBCHUX> HbBGGHs   AA$ A$ $A<c                 v    dddg}	 t        j                  | ||d       y# t        $ r Y yt        $ r Y yw xY w)aW  
    Checks if a given API key is valid for a specific model by making a litellm.completion call with max_tokens=10

    Args:
        model (str): The name of the model to check the API key against.
        api_key (str): The API key to be checked.

    Returns:
        bool: True if the API key is valid for the model, False otherwise.
    r   zHey, how's it going?rZ  r  )r   r   r  r^  TF)r   r   rv   r   )r   r  r   s      r   check_valid_keyr    sP      ,BCDH(G	
   s   " 	888r  c                 4    | dk(  ry| dk(  ry| dk(  ry| dk\  ryy)a  
    Retries on 408, 409, 429 and 500 errors.

    Any client error in the 400-499 range that isn't explicitly handled (such as 400 Bad Request, 401 Unauthorized, 403 Forbidden, 404 Not Found, etc.) would not trigger a retry.

    Reimplementation of openai's should retry logic, since that one can't be imported.
    https://github.com/openai/openai-python/blob/af67cfab4210d8e497c05390ce14f39105c77519/src/openai/_base_client.py#L639
    i  Ti  i  r  Fr  )r  s    r   _should_retryr  .  s9     c c c cr   c                     | yt        | t              r| S t        j                  j	                  |       st        d|        dt        j                  |       | j                  dddS )z
    Re-implementation of openai's 'type_to_response_format_param' function

    Used for converting pydantic object to api schema.
    Nz#Unsupported response_format type - r2  T)r4  r  r  )r   r2  )	r   r   r   r=  r>  rB  r   to_strict_json_schemar9  r3  s    r   r?  r?  K  su     /4(
   22?C=o=NOPP 55oF#,,
 r   response_headersc                 N   	 ddl }| | j                  d      }	 t        |      }|S d}|S # t        $ r_ |j                  j                  |      }|d}n;|j                  j                  |      }t        |t        j                         z
        }Y |S w xY w# t        $ r d}Y yw xY w)z
    Reimplementation of openai's calculate retry after, since that one can't be imported.
    https://github.com/openai/openai-python/blob/af67cfab4210d8e497c05390ce14f39105c77519/src/openai/_base_client.py#L631
    r   Nzretry-afterr  )emailr   r  r   r  parsedate_tz	mktime_tzra  )r  r  retry_headerretry_afterretry_date_tuple
retry_dates         r   &_get_retry_after_from_exception_headerr  i  s     '+//>L@!,/  K  @#(;;#;#;L#I #+"$K!&!6!67G!HJ"%j499;&>"?K @  s7   B + B B A$BB BB B$#B$remaining_retriesr  min_timeoutc                     t        |      }|d|cxk  rdk  r|S  d}d}|| z
  }t        |t        d|      z  |      }ddt        j                         z  z
  }	||	z  }
|
|k\  r|
S |S )Nr   <   g      ?g       @g       @rm   g      ?)r  r  powrandom)r  r  r  r  r  initial_retry_delaymax_retry_delay
nb_retriessleep_secondsjitterr  s              r   _calculate_retry_afterr    s     99IJK 1{#8b#8 $9 O00J +c#z.BBOTM ''Ff$G,7=+=r   rolesinitial_prompt_valuefinal_prompt_valuec                 n    t        |       d   } |||dt        j                  | <   t        j                  S )a  
    Register a prompt template to follow your custom format for a given model

    Args:
        model (str): The name of the model.
        roles (dict): A dictionary mapping roles to their respective prompt values.
        initial_prompt_value (str, optional): The initial prompt value. Defaults to "".
        final_prompt_value (str, optional): The final prompt value. Defaults to "".

    Returns:
        dict: The updated custom prompt dictionary.
    Example usage:
    ```
    import litellm
    litellm.register_prompt_template(
            model="llama-2",
        initial_prompt_value="You are a good assistant" # [OPTIONAL]
            roles={
            "system": {
                "pre_message": "[INST] <<SYS>>
", # [OPTIONAL]
                "post_message": "
<</SYS>>
 [/INST]
" # [OPTIONAL]
            },
            "user": {
                "pre_message": "[INST] ", # [OPTIONAL]
                "post_message": " [/INST]" # [OPTIONAL]
            },
            "assistant": {
                "pre_message": "
" # [OPTIONAL]
                "post_message": "
" # [OPTIONAL]
            }
        }
        final_prompt_value="Now answer as best you can:" # [OPTIONAL]
    )
    ```
    r  r   )r  r  r  )r   r   r  )r   r  r  r  s       r   register_prompt_templater    s=    R 5)!,E 40)Gu%
 %%%r   c                   R    e Zd Z	 	 ddee   dee   fdZd Zd Zde	fdZ
d	 Zd
 Zy)TextCompletionStreamWrapperNr   r2  c                 <    || _         || _        || _        || _        y r   )completion_streamr   r   r2  )selfr  r   r   r2  s        r   __init__z$TextCompletionStreamWrapper.__init__  s#     "3
,#6 r   c                     | S r   r  r  s    r   __iter__z$TextCompletionStreamWrapper.__iter__      r   c                     | S r   r  r  s    r   	__aiter__z%TextCompletionStreamWrapper.__aiter__  r  r   r  c           	         	 t               }|j                  dd       |d<   d|d<   |j                  dd       |d<   |j                  dd       |d<   t               }t        |t              rt
        |d   d   d   d	   |d
<   |d   d   d   |d<   |d   d   d   |d<   |g|d<   | j                  r3| j                  j                  dd      du r|j                  dd       |d<   |S # t
        $ r}t        d| dt        |             d }~ww xY w)Nr   r	  r  createdr   r:  r   r  r   r  r  finish_reasoninclude_usageFTusagez=Error occurred converting to text completion object - chunk: z	; Error: )rT   r   rS   r   rB   r   r   r   )r  r  r5  text_choicesr  s        r   !convert_to_text_completion_objectz=TextCompletionStreamWrapper.convert_to_text_completion_object  sE   	-/H"YYtT2HTN!2HX"'))It"<HY %		'4 8HW&=Lw  #(#3A#6w#?	#JL $))$4Q$7$@L!,1),<Q,?,PL)#/.HY ##''++OUCtK$)IIgt$<!O 	OPUwV_`cde`f_gh 	s   C!C$ $	D-DDc                    t                	 | j                  D ]$  }|dk(  s|t        | j                  |      }|c S  t        # t        $ r t        t        $ r-}t        | j                  | j                  xs d|i i       d }~ww xY w)NNoner  r   )r   r2  original_exceptioncompletion_kwargsextra_kwargs)rT   r  r   r  StopIterationr   r   r2  )r  r  processed_chunkr  s       r   __next__z$TextCompletionStreamWrapper.__next__  s     	//F?em#O"&"H"Hu"H"U&&	 0
   	  	 jj$($<$<$B#$"$ 	s   1A A B	(BB	c                    K   	 | j                   2 3 d {   }|dk(  s|t        | j                  |      }|c S 7 '6 t        # t        $ r t        w xY ww)Nr  r  )r  r   r  r  StopAsyncIteration)r  r  r  s      r   	__anext__z%TextCompletionStreamWrapper.__anext__  si     	%#55 'eF?em#O"&"H"Hu"H"U&&	'5
   	%$$	%s5   AA ><>#A A>A AANN)r9  r
  r  r`   r   r   r  r  r  rO   r  r  r  r  r   r   r  r    sK    
 *.-1
7 !	
7
 &c]
7} <*	%r   r  c              #     K   t        |t        j                        r|t        dt	        |      d      D ]  }t        d|||dz          }||| j                  d   _        npg }t        |      D ]Y  }t        j                  j                  |t        j                  j                  d|||dz                }|j                  |       [ || _        |   y wNr   r  	assistantrZ  )r  r  r   r   MockExceptionr`  r   rE   r:  r  r  rR   r   	rE  mock_responser   r  icompletion_obj_all_choicesj_streaming_choices	            r   mock_completion_streaming_objr  )  s      -!6!671c-(!,Kq1q59QR9.<N""1%+L1X$+MM$B$B!----(-AE2J .  %C %! ##$56  &2N" -s   CCc                  K   t        |t        j                        r|t        dt	        |      d      D ]  }t        d|||dz          }||| j                  d   _        npg }t        |      D ]Y  }t        j                  j                  |t        j                  j                  d|||dz                }|j                  |       [ || _        |   y wr  r  r  s	            r   #async_mock_completion_streaming_objr  @  s      -!6!671c-(!,Kq1q59QR9.<N""1%+L1X$+MM$B$B!----(-AE2J .  %C %! ##$56  &2N" -s   CCc                     	 dd l } |j                          t        | d      5 }t        j                  |      }d d d        |S # 1 sw Y   S xY w# t
        $ r}|d }~ww xY w)Nr   r)rp  getcwdopenr  loadr   )config_pathrp  config_fileconfigr  s        r   read_config_argsr  X  s\    
		+s#{YY{+F $ 	 $  s3   !A A	A AA A 	A AA c                     | j                  di       }d }t               }i }t        j                         }| d   }| d   g|j                  dg       z   }d|v r|d= t	        t        j                               }|*t        j                         |z
  dk  r|D ]  }		 t        |	t              r=|	j                  dd       | d<   |	j                  dd       | d<   |	j                  d|      }	nB|	|v r>|j                  |	      r,t        j                         ||	   k\  r|j                  |	       n| j                  d      r| d= t        d|	        || d<   i | |} t        j                  di | d|	i}t        d	|        ||c S  |t        j                         |z
  dk  r|S # t        $ rA}
t        |
       |j                  |	       t        j                         d
z   ||	<   Y d }
~
Td }
~
ww xY w)Nr   r   r  -   r  r   +trying to make completion call with model: r   
response: r  r  )r   r   ra  r   r   ru  rv  r   r   remover   r   r   r   addr   nested_kwargsr5  rate_limited_modelsmodel_expiration_timesr   original_modelr  r   r   r  s              r   completion_with_fallbacksr  i  s   JJx,MH%JG_N!M$5$5k2$FFIm#+&$**,'O 
tyy{Z7"<E*4 ).		)T(BF9%).:t)DF:&!IIg~>E00 /2259 IIK+A%+HH+22! ! ::g&w KE7ST,;()# #--DDeD
8*56'#O (G  
tyy{Z7"<\ O  a #''.IIK"$ 'u- s    BF&*AF&&	G0/6G++G0c                    K   | j                  di       }d }t               }i }t        j                         }| d   }| d   g|j                  dg       z   }d|v r|d= d| v r| d= t	        t        j                               }|2t        j                         |z
  dk  r|D ]  }		 t        |	t              r=|	j                  dd       | d<   |	j                  dd       | d<   |	j                  d|      }	nB|	|v r>|j                  |	      r,t        j                         ||	   k\  r|j                  |	       n| j                  d      r| d= t        d|	        || d	<   i | |} t        j                  di | d|	i d {   }t        d
|        ||c S  |t        j                         |z
  dk  r|S 7 :# t        $ rD}
t        d|
        |j                  |	       t        j                         dz   ||	<   Y d }
~
ad }
~
ww xY ww)Nr   r   r  r   r  r  r   r  r   r  zerror: r  r  )r   r   ra  r   r   ru  rv  r   r   r  r   r   r   r   r  r  s              r   async_completion_with_fallbacksr    s;    JJx,MH%JG_N!M$5$5k2$FFIm#+&
 $**,'O 
tyy{Z7"<E*4 ).		)T(BF9%).:t)DF:&!IIg~>E00 /2259 IIK+A%+HH+22! ! ::g&w KE7ST,;()# ")!4!4!Kv!KU!KK
8*56'#O (G  
tyy{Z7"<\ O L
  sm,#''.IIK"$ 'u- sV   B!H	$BF92H	3A	F9<F7=F9!H	5H	7F99	H9H;H	HH	c                     d| d}t        |g|      }||kD  r%t        d       t        |||      }t        |g|      }|||z
  fS )Nr  rZ  zH`tokentrimmer`: Warning, system message exceeds token limit. Trimming...)get_token_countr   shorten_message_to_fit_limit)system_messager^  r   system_message_eventsystem_message_tokensnew_system_messages         r   process_system_messager$    si    $,H+-A,BEJz)V	
 : *e
 !01C0De L.C!CCCr   c                 z    | d d d   } g }| D ]+  }t        ||      }||z
  }|dk  r |S t        |||||      }- |S )Nr  r  )final_messagesr   available_tokensr^  r   )r  attempt_message_addition)r   r^  r   r&  r   used_tokensr'  s          r   process_messagesr*    sk    "~HN%ne<%3q   2)-!
  r   c                     |g| z   }t        ||      }||k  r|S d|vr!t        |||      }t        || ||      r|g| z   S | S )Nr   r   r  )r  r  can_add_message)r&  r   r'  r^  r   temp_messagestemp_message_tokensupdated_messages           r   r(  r(    sg     I.M)=Nj( 
	'6w@PRWX?NJN#$~55r   c                 ,    t        || gz   |      |k  ryyr.  )r  )r   r   r^  r   s       r   r-  r-  "  s    x7)+U3zAr   c                     t        ||       S )Nr  )r1  r,  s     r   r  r  (  s    ux88r   c                     |d|v r|dk  r| S | d   }	 t        | g|      }||k  r	 | S ||z  }t        t        |      |z        dz
  }t        d|      }|dz  }|d| }|| d }	|dz   |	z   }
|
| d<   |
}`)	z_
    Shorten a message to fit within a token limit by removing characters from the middle.
    Ngptr  r   rm   r   r   z..)r  r  r   r  )r   tokens_neededr   r   total_tokensratio
new_lengthhalf_length	left_half
right_halftrimmed_contents              r   r  r  ,  s     Ue^0Ci G
&y%8=( N ,.W-.2
J'
 AoL[)	k\]+
#d*Z7,	!# r   
trim_ratioreturn_response_tokensc                    t        j                  |       } 	 |Z|t        j                  v rFt        j                  |   j	                  dt        j                  |   d         }t        ||z        }n| S d}| D ]  }|d   dk(  s||rdndz  }||d   z  } g }t        |       D ]  }|d   d	k7  r n|j                  |        t        |      r| dt        |        } t        |xs d| 
      }	t        d|	 d|        |	|k  r| S t        d|  d|	 d|        d}
|r1t        |||      \  }
}|dk(  r|
gS | D cg c]  }|d   dk7  s| } }t        | ||      }|
r|
g|z   }t        |      dkD  r|j                  |       |r|t        ||      z
  }||fS |S c c}w # t        $ r9}t!        j"                  dj%                  t'        |                   | cY d}~S d}~ww xY w)a  
    Trim a list of messages to fit within a model's token limit.

    Args:
        messages: Input messages to be trimmed. Each message is a dictionary with 'role' and 'content'.
        model: The LiteLLM model being used (determines the token limit).
        trim_ratio: Target ratio of tokens to use after trimming. Default is 0.75, meaning it will trim messages so they use about 75% of the model's token limit.
        return_response_tokens: If True, also return the number of tokens left available for the response after trimming.
        max_tokens: Instead of specifying a model or trim_ratio, you can specify this directly.

    Returns:
        Trimmed messages and optionally the number of tokens available for response.
    Nr  r^  r   r  r  r  r   r  r  zCurrent tokens: z, max tokens: zNeed to trim input messages: z, current_tokensz, max_tokens: )r   r^  r   r   )r   r^  r   z'Got exception while token trimming - {})r   r   r   r[  r   r  r   r   r   r1  r   r$  r*  r  r  r   rn   rH  r!  r   )r   r   r=  r>  r^  max_tokens_for_modelr   r   tool_messagescurrent_tokensr!  r&  response_tokensr  s                 r   trim_messagesrD  R  sT   , }}X&HL***'.'9'9%'@'D'D&(:(:5(A,(O($ !!5
!BC

  Gv(*.$b@')"44   )Gv&(  ) *
 } 53}#5"56H&U[b8L((8zlST J&O 	+H:5EnEUUcdncop	
 04/E-*E0, * Q,-- 08WxG76?h;VxHW)*E

  23nDN}!!!-0 #(?>5+QQO!?22% X&    5<<SVD	
 	sV   AF  3F  BF  0F  7F  ;F	FAF  F  F   	G").GG"G"check_provider_endpointc                    	 t         j                  j                         }g }g }t        j                  D ]V  }|j                  dd      }|}|j                          d}|j                          d}||v s||v sF|j                  |       X |D ]  }t        j                  dt        |            }	|dk(  r|j                  d       :|	"| r |j                  |	j                                ^t        j                  j                  |g       }
|j                  |
        |S # t        $ r$}t!        j"                  d|        g cY d}~S d}~ww xY w)	z
    Returns a list of valid LLMs based on the set environment variables

    Args:
        check_provider_endpoint: If True, will check the provider's endpoint for valid models.

    Returns:
        A list of valid LLMs
    rB  r   _API_KEYNr  r4  z	Azure-LLMzError getting valid models: )rp  r  r  r   r   r.  upperr   r  r$  rJ   r  
get_modelsr  r   r   rn   r   )rE  environ_keysvalid_providersvalid_modelsr   env_provider_1env_provider_2expected_provider_key_1expected_provider_key_2r  models_for_providerr  s               r   get_valid_modelsrR    sX   'zz(--H%--c26N%N *8)=)=)?(@&I#)7)=)=)?(@&I#'<7*l:  &&x0 .  (H3KK%h/ L O
 7"##K0 ,1H##O$>$>$@A&-&@&@&D&DXr&R###$78 (  ;A3?@	s%   A7D" :B'D" "	E+E
E
Ec           	         	 d|v r|d   du r| j                   dk(  ry d|v r|d   du r| j                   dk(  ry d|v r|d   du r| j                   dk(  ry dj                  t        t        |            }dj                  d	 |j	                         D              }t        d
       t        d       |r"|r t        d| j                    d| d| d       nW|rt        d| j                    d| d       n8|rt        d| j                    d| d       nt        d| j                    d       t        d
       y # t        $ r Y y w xY w)Nr   Tr   r  r  r[  img_generationz, c              3   B   K   | ]  \  }}| d t        |         yw)=N)repr)r   r  r   s      r   r   z/print_args_passed_to_litellm.<locals>.<genexpr>  s$     W*#u#aU}5s   r  z[92mRequest to litellm:[0mz[92mlitellm.(z)[0mz()[0m)r9  r   maprW  rx  r   r   )r   r  r   args_str
kwargs_strs        r   rr  rr    sw   . V#}%-!**l:F"|$,!**k9'()T1!**.>>99St_-YYWWW
	
 	0	
 F"#4#=#=">azJ<W_` "#4#=#=">azR "#4#=#=">a
|8T ,->-G-G,H	RSd s"   D5 D5 D5 C!D5 5	E Ec                 x    	 d| j                  d      z   dz   |j                  d      z   }|S # t        $ r Y y w xY w)Nztime-z%H-%M-%S-%frB  r   )strftimer   r   )r   r  response_ids      r   get_logging_idr_    sO    j))-883>AQAQRVAWW 	  s   *- 	99c                 b    | y | *| j                  di       }||j                  dd       }||S y )Nrf  r`  r  )r   rf  r`  s      r   r  r  (  sH     \\,3
!#d;J%!!r   c                     | y | j                  di       }|4|j                  dd       }||S |j                  di       }t        |      S y )Nr   r`  r   r  )r   r  )model_call_detailsr   _base_modelr   s       r   _get_base_model_from_metadatard  8  sb    !'++,<bAN!$((t<"!%%j"598LLr   c                   6    e Zd Zd	dedefdZd Zd Zd Zd Z	y)
ModelResponseIteratorrE  convert_to_deltac                     |du rdt        d      | _        | j                  j                  d   j                  }|j                  d   j                  j
                  |_        d| _        y || _        d| _        y )NT)r   r   F)rO   rE  r:  r  r   r   is_done)r  rE  rg  _deltas       r   r  zModelResponseIterator.__init__G  sk    t#"/t"<D((00399F+33A6>>FFFN  #1Dr   c                     | S r   r  r  s    r   r  zModelResponseIterator.__iter__Q  r  r   c                 L    | j                   rt        d| _         | j                  S NT)ri  r  rE  r  s    r   r  zModelResponseIterator.__next__T  s"    <<"""r   c                     | S r   r  r  s    r   r  zModelResponseIterator.__aiter__[  r  r   c                 T   K   | j                   rt        d| _         | j                  S wrm  )ri  r  rE  r  s    r   r  zModelResponseIterator.__anext__^  s&     <<$$"""s   &(NF)
r9  r
  r  rO   r  r  r  r  r  r  r  r   r   rf  rf  F  s)    }  ##r   rf  c                   *    e Zd Zd Zd Zd Zd Zd Zy)ModelResponseListIteratorc                      || _         d| _        y Nr   )model_responsesr  )r  ru  s     r   r  z"ModelResponseListIterator.__init__f  s    .
r   c                     | S r   r  r  s    r   r  z"ModelResponseListIterator.__iter__k  r  r   c                     | j                   t        | j                        k\  rt        | j                  | j                      }| xj                   dz  c_         |S Nrm   )r  r   ru  r  r  rE  s     r   r  z"ModelResponseListIterator.__next__n  sF    ::T1122--djj9

a
r   c                     | S r   r  r  s    r   r  z#ModelResponseListIterator.__aiter__v  r  r   c                    K   | j                   t        | j                        k\  rt        | j                  | j                      }| xj                   dz  c_         |S wrx  )r  r   ru  r  ry  s     r   r  z#ModelResponseListIterator.__anext__y  sJ     ::T1122$$--djj9

a
r*  N)r9  r
  r  r  r  r  r  r  r  r   r   rr  rr  e  s    
r   rr  c                         e Zd Zd fdZ xZS )CustomModelResponseIteratorc                 "    t         |           y r   )superr  )r  	__class__s    r   r  z$CustomModelResponseIterator.__init__  s    r   )r,  N)r9  r
  r  r  __classcell__)r  s   @r   r}  r}    s     r   r}  r   c                     d| vry| d   t        | d   t              ry| d   D ]*  }|d   dk(  s|j                  d      |d   d   dk(  s* y y)z
    Returns true, if message is marked as needing to be cached.

    Used for anthropic/gemini context caching.

    Follows the anthropic format {"cache_control": {"type": "ephemeral"}}
    r   Fr   r  cache_control	ephemeralTr   )r   r   s     r   is_cached_messager    so     y!Z	0BC%H9%FOv%O,8(0K? & r   rE  c                     	 | j                  d      sy| j                  d      d   } t        j                  | d      }t        j                  |      j                  d      | k(  S # t        $ r Y yw xY w)Nzdata:Fr  rm   T)validatezutf-8)ry  rp  base64	b64decode	b64encoder  r   )rE  decoded_bytess     r   is_base64_encodedr    su    ||
 GGCLO ((T: .55g>!CC s   A& AA& &	A21A2c                 6    d| v r| j                  d      d   S | S )z3
    s: b64str OR data:image/png;base64,b64str
    r  rm   )rp  )rE  s    r   get_base64_strr    s"     axwws|AHr   c                 8    | D ]  }|j                  d       y y)zm
    Returns true, if messages has tool call blocks.

    Used for anthropic/bedrock message validation.
    r'  TFr  )r   r   s     r   has_tool_call_blocksr    s%     ;;|$0  r   c           
      <    t        dt        dddi d            gS )z
    Prevent Anthropic from raising error when tool_use block exists but no tools are provided.

    Relevent Issues: https://github.com/BerriAI/litellm/issues/5388, https://github.com/BerriAI/litellm/issues/5747
    r  
dummy_toolzThis is a dummy tool callr  r9  r:  )r   r  )r9   r:   rk  s    r   add_dummy_toolr    s2     	 9!7$"$
	
 r   )ChatCompletionAudioObjectChatCompletionImageObjectChatCompletionTextObjectChatCompletionUserMessageOpenAIMessageContentValidUserMessageContentTypesc                     t        | t              r| j                  d      S t        | t              r| S t	        dt        |        d      )z
    Converts a message to a dictionary if it's a Pydantic model.

    Args:
        message: The message, which may be a Pydantic model or a dictionary.

    Returns:
        dict: The converted message.
    T)exclude_nonezInvalid message type: z". Expected dict or Pydantic model.)r   r   
model_dumpr   rB  r   )r   s    r   convert_to_dictr    sO     '9%!!t!44	GT	"$T']O3UV
 	
r   c                     | D cg c])  }t        t        t        t        t        |                  + } }t	        |       S c c}w )zI
    Ensures all messages are valid OpenAI chat completion messages.
    rQ  )rd   r5   r  r   &validate_chat_completion_user_messages)r   r   s     r   !validate_chat_completion_messagesr    sG     IQHP1tD!}=>   28DD	s   .Ac           	      ~   t        |       D ]|  \  }}	 |d   dk(  rn|j                  d      }|[t        |t              r3t        |t              r:|D ]5  }t        |t
              s|j                  d      t        vs,t        d       ~ | S # t        $ r&}dt        |      v rt        d| d| d	      |d}~ww xY w)
a;  
    Ensures all user messages are valid OpenAI chat completion messages.

    Args:
        messages: List of message dictionaries
        message_content_type: Type to validate content against

    Returns:
        List[dict]: The validated messages

    Raises:
        ValueError: If any message is invalid
    r  r   r   Nr   zinvalid content typezInvalid user message=z
 at index zL. Please ensure all user messages are valid OpenAI chat completion messages.)r   r   r   r   r   r   r  r   )r   r  r   user_contentr%  r  s         r   r  r    s     H%Q	yF" uuY/+!,4 #L$7$0D)$5#'88F#3;W#W*34J*K$K %1 &( O  	%Q/+A3j  >J  K  	s(   +B%B%B=B	B<!B77B<c                     ddl m}m} | | S t        | t              r| S t        | t
              r3| j                  d      | j                  d      t        d|  d      | S t        d|  dt        |        d      )	z
    Confirm the tool choice is passed in the OpenAI format.

    Prevents user errors like: https://github.com/BerriAI/litellm/issues/7483
    r   )#ChatCompletionToolChoiceObjectParam$ChatCompletionToolChoiceStringValuesr   r  z!Invalid tool choice, tool_choice=z3. Please ensure tool_choice follows the OpenAI specz. Got=zW. Expecting str, or dict. Please ensure tool_choice follows the OpenAI tool_choice spec)	litellm.types.llms.openair  r  r   r   r   r   r   r   )r  r  r  s      r   $validate_chat_completion_tool_choicer  *  s    
 	K	%	K	&??6"*kooj.I.Q3K=@st  

+K=tK?P>Q  Ri  	j r   c                       e Zd Zedededefd       Zedededefd       Z	ededede
fd       Zedededee   fd       Zedededefd       Zedee   dedee   fd	       Zy
)r  r   r   r,  c                    |t         j                  k(  r4t        j                  j	                  |       rt        j
                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                   |k(  rt        j"                         S t        j                   j$                  |k(  rt        j&                         S t        j                   j(                  |k(  rt        j*                         S t        j                   j,                  |k(  rt        j.                         S t        j                   j0                  |k(  rt        j2                         S t        j                   j4                  |k(  rd| v 	rt        j6                         S t        j                   j8                  |k(  rt        j:                         S t        j                   j<                  |k(  rt        j>                         S t        j                   j@                  |k(  rt        jB                         S t        j                   jD                  |k(  rt        jF                         S t        j                   jH                  |k(  rt        jJ                         S t        j                   jL                  |k(  rt        jN                         S t        j                   jP                  |k(  rt        jR                         S t        j                   jT                  |k(  rt        jV                         S t        j                   jX                  |k(  rt        jZ                         S t        j                   j\                  |k(  sWt        j                   j^                  |k(  s:t        j                   j`                  |k(  st        j                   jb                  |k(  rt        jd                         S t        j                   jf                  |k(  rt        jh                         S t        j                   jj                  |k(  rt        jl                         S t        j                   jn                  |k(  rt        jp                         S t        j                   jr                  |k(  rt        jt                         S t        j                   jv                  |k(  rt        jx                         S t        j                   jz                  |k(  rt        j|                         S t        j                   j~                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  st        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rLt        j                         j                  |       rt        j                         S t        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   jj                  |k(  rt        jl                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  st        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j8                  |k(  rt        j:                         S t        j                   j0                  |k(  rt        j2                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         j                  |       }|t        j                  v r	 t        j                         S d| v rt        j                         S d| v rt        j                         S d| v rt        j                         S d| v rt        j                         S d| v rt        j                         S t        j                         S )zC
        Returns the provider config for a given provider.
        r  r  amazonmetari  re  r  )}rJ   OPENAIr   openAIO1Configis_model_o1_reasoning_modelOpenAIO1ConfigDEEPSEEKDeepSeekChatConfigGROQrh  
DATABRICKSr_  XAIrb  TEXT_COMPLETION_OPENAIOpenAITextCompletionConfigCOHERE_CHATr?  COHEREr>  CLARIFAIClarifaiConfig	ANTHROPICr<  ANTHROPIC_TEXTr=  	VERTEX_AIrJ  
CLOUDFLARErY  SAGEMAKER_CHATSagemakerChatConfig	SAGEMAKERrS  rA  rd  
FRIENDLIAIFriendliaiChatConfigWATSONXrk  WATSONX_TEXTr9  EMPOWEREmpowerChatConfigGITHUBGithubChatConfigCUSTOMCUSTOM_OPENAIOPENAI_LIKELITELLM_PROXYrp  AIOHTTP_OPENAIAiohttpOpenAIChatConfigHOSTED_VLLMrf  	LM_STUDIOLMStudioChatConfig	GALADRIELGaladrielChatConfig	REPLICATErB  HUGGINGFACErD  TOGETHER_AIrE  
OPENROUTERrj  GEMINIrI  AI21	AI21_CHATrc  AZURErm  rn  r5  AZURE_AIrj  
AZURE_TEXTAzureOpenAITextConfig	NLP_CLOUDr\  	OOBABOOGAOobaboogaConfigOLLAMA_CHATr[  	DEEPINFRAr^  
PERPLEXITYPerplexityChatConfigMISTRAL	CODESTRALrP  
NVIDIA_NIMr`  CEREBRASra  
VOLCENGINEre  TEXT_COMPLETION_CODESTRALrO  	SAMBANOVASambanovaConfigMARITALKrA  VLLMrg  OLLAMArZ  r@  rC  TRITONr@  PETALSr]  BEDROCKrT  rU  rV  AmazonTitanConfigAmazonLlamaConfigAmazonAI21ConfigAmazonCohereConfigAmazonMistralConfigOpenAIGPTConfig)r   r   r`  s      r   r;  z.ProviderConfigManager.get_provider_chat_configG  s 	    +++&&BBBO))++!!**h6--//!!&&(2))++!!,,8++--!!%%1((**!!88HD5577!!--9++--!!((H4''))!!**h6))++!!++x7**,,!!00H<..00!!++x75 6688!!,,8//11!!00H<..00!!++x7**,,!!..(:,,..!!,,8//11!!))X5//11!!..(:--//!!))X5,,..!!((H4++--  ''83##11X=##//8;##11X=//11!!00H<2244!!--9//11!!++x7--//!!++x7..00!!++x7**,,!!--9,,..!!--9++--!!,,8++--!!((H45577  %%1##--9))++!!''83**,88u8E2244,,..!!**h6..00!!,,80022!!--9//11!!++x7))++!!++x7**,,!!--9++--!!++x7**,,!!,,8//11  ((H4##--9((**!!,,8**,,!!**h6))++!!,,8++--!!;;xG88::!!++x7**,,!!**h6))++!!,,8//11!!00H<..00!!&&(2%%''!!((H4''))!!++x7**,,!!((H4''))!!((H4''))!!))X5 557GGNJW<<< &&(( U"0022500225//11U"1133e#2244&&((r   c                 Z   t         j                  j                  |k(  rt        j                         S t         j                  j                  |k(  rt        j
                         S t         j                  j                  |k(  rt        j                         S t        d|j                   d      )Nz	Provider z" does not support embedding config)
r   rJ   VOYAGEr  r  r  r  IBMWatsonXEmbeddingConfigr,  r   r  s     r   get_provider_embedding_configz3ProviderConfigManager.get_provider_embedding_config  s    
 &&(20022!!((H40022!!))X544669X^^$44VWXXr   c                 P   t         j                  j                  |k(  rt        j                         S t         j                  j                  |k(  rt        j
                         S t         j                  j                  |k(  rt        j                         S t        j                         S r   )r   rJ   r  CohereRerankConfigr  AzureAIRerankConfigINFINITYInfinityRerankConfigr  s     r   get_provider_rerank_configz0ProviderConfigManager.get_provider_rerank_config  s|    
 &&(2--//!!**h6..00!!**h6//11))++r   c                     t         j                  j                  |k(  rt        j                         S t         j                  j                  |k(  rt        j
                         S y r   )r   rJ   rA  #FireworksAIAudioTranscriptionConfigDEEPGRAM DeepgramAudioTranscriptionConfigr  s     r   r  z=ProviderConfigManager.get_provider_audio_transcription_config  sM    
 ,,8>>@@!!**h6;;==r   c                     t         j                  |k(  rt        j                         S t         j                  |k(  rt        j
                         S t        j                         S r   )rJ   rA  r   FireworksAITextCompletionConfigr  TogetherAITextCompletionConfigr  r  s     r   #get_provider_text_completion_configz9ProviderConfigManager.get_provider_text_completion_config  sM    
 $$0::<<%%199;;1133r   c                     t         j                  |k(  rt        j                         S t         j                  |k(  rt        j
                         S y r   )rJ   rA  r   rd  r  LiteLLMProxyChatConfigr  s     r   r$  z-ProviderConfigManager.get_provider_model_info  sA    
 $$0,,..''831133r   N)r9  r
  r  staticmethodr   rJ   ri   r;  rk   r  rl   r  r`   rg   r  rj   r  rh   r$  r  r   r   r  r  F  s-   S)S)*S)	S) S)j 
Y
Y
Y 

Y 
Y 
,
,
, 

, 
,  
.	/  444 
"4 4 	}		 
"	#	 	r   r  service_type)r   
prometheusc                    t        t        | j                  di       xs i       }t        t        t           | j                  d      xs |j                  d            }t
        j                  ry|dk(  rt
        j                  ry|S )z
    Used for enforcing `disable_end_user_cost_tracking` param.

    service_type: "litellm_logging" or "prometheus" - used to allow prometheus only disable cost tracking.
    r   user_api_key_end_user_idNr  )rd   r   r   r`   r   r   disable_end_user_cost_tracking.disable_end_user_cost_tracking_prometheus_only)r   r  	_metadataend_user_ids       r   !get_end_user_id_for_cost_trackingr    s|     T>--j"=CDI56 	5==34K
 --$BBr   c                     	 ||y|| j                  |      s|dz   | z   } t        ||| d      }|dk\  S # t        $ r"}t        j                  d|        Y d}~yd}~ww xY w)z
    Returns true if the prompt is valid for prompt caching.

    OpenAI + Anthropic providers have a minimum token count of 1024 for prompt caching.
    NFrh  T)r   r  r   r  i   z)Error in is_prompt_caching_valid_prompt: )ry  r1  r   rn   r   )r   r   r  r2  token_countr  s         r   is_prompt_caching_valid_promptr  4  s    *53C3C4
 (#-5E#*.	
 d"" HLMs   6 .6 	A!AA!srt_or_vtt_contentc                     d}t        j                  ||       }|syg }|D ]<  }t        t        |      \  }}}}|dz  |dz  z   |z   |dz  z   }	|j	                  |	       > |rt        |      S dS )a  
    Extracts the total duration (in seconds) from SRT or VTT content.

    Args:
        srt_or_vtt_content (str): The content of an SRT or VTT file as a string.

    Returns:
        Optional[float]: The total duration in seconds, or None if no timestamps are found.
    z"(\d{2}):(\d{2}):(\d{2})[.,](\d{3})Ni  r  g     @@)r  findallrY  r  r   r  )
r  timestamp_pattern
timestamps	durationsmatchhoursminutessecondsmillisecondsr  s
             r    extract_duration_from_srt_or_vttr'  R  s     >-/ABJ I03C-ww|3g=v@UU' 
 '3y>0D0r   ending_pathc                 X   t        j                  |       }|j                  i       }|j                  j	                  d      }|j                  d      }|j                  d      D cg c]  }|s|	 }}|j                  d      D cg c]  }|s|	 }}g }	t        t        |            D ]#  }
||
d |dt        |      |
z
   k(  s|d|
 |z   }	 n ||z   }	ddj                  |	      z   }|j                  |      }t        |j                  |j                              S c c}w c c}w )z
    Adds an ending path to an API base URL while preventing duplicate path segments.

    Args:
        api_base: Base URL string
        ending_path: Path to append to the base URL

    Returns:
        Modified URL string with proper path handling
    )paramsrh  NrO  )rQ  rR  rS  rP  rstriplstriprp  r`  r   r   r   r*  )r   r(  rT  base_url	base_pathend_pathrE  base_segmentsend_segmentsfinal_segmentsr  modified_pathrV  s                r   _add_path_to_api_baser4  q  s<    99X&L%%R%0H!!((-I!!#&H !* 4: 41Q 4M:'~~c282!aA2L8 N3}%&-Es=/AA/E FF*2A.=N ' '5 #((>22M%%=%9L |%%\-@-@%ABB% ;8s   'D"/D"D'D'c                     t         j                  }|t        z   }| j                         D ci c]  \  }}||vs|| }}}|S c c}}w r   )r   OPENAI_CHAT_COMPLETION_PARAMSrW   rx  )r   openai_paramsr  r  r  r  s         r   !get_non_default_completion_paramsr8    sV    99M"%77N'A1N+B1   s
   AA)Fr   rp  r   )r   r   N)Nzgpt-3.5-turbo-0613NFNNFF)mainN)r   NNNFNNF)$NiX  FNFFFFNNNNNNNNNNNNNNNNNNNNNNNNNNNN)NNNNNNN)	NNNNNNNNN)NNNr   NN)NNNNNFNNNNNNNNNNNr   NNNNNNNNNNNNN)r   r   r   r   r   )NNNrt  )r   r   )Ng      ?FN)r   r  (  r;  r%  r  binasciir   rs  hashlibr   io	itertoolsr  loggingrp  r  r  struct
subprocesssystextwrap	threadingra  r  ru  dataclassesr   r   	functoolsr   r   	importlibr   r   os.pathr	   r
   r   aiohttpdotenvrQ  r  r  r   httpx._utilsr   
openai.libr   r   *openai.types.chat.completion_create_paramsr   pydanticr   r   
tokenizersr   r   litellm._service_loggerlitellm.litellm_core_utils,litellm.litellm_core_utils.audio_utils.utils/litellm.litellm_core_utils.json_validation_rulelitellm.caching.cachingr   litellm.caching.caching_handlerr   r   "litellm.integrations.custom_loggerr   'litellm.litellm_core_utils.core_helpersr   r   +litellm.litellm_core_utils.default_encodingr   2litellm.litellm_core_utils.exception_mapping_utilsr   r   r   1litellm.litellm_core_utils.get_llm_provider_logicr   r   6litellm.litellm_core_utils.get_supported_openai_paramsr    ,litellm.litellm_core_utils.llm_request_utilsr!   Flitellm.litellm_core_utils.llm_response_utils.convert_dict_to_responser"   r#   r$   r%   r&   :litellm.litellm_core_utils.llm_response_utils.get_api_baser'   Blitellm.litellm_core_utils.llm_response_utils.get_formatted_promptr(   9litellm.litellm_core_utils.llm_response_utils.get_headersr)   *litellm.litellm_core_utils.redact_messagesr*   r+    litellm.litellm_core_utils.rulesr,   ,litellm.litellm_core_utils.streaming_handlerr-   (litellm.litellm_core_utils.token_counterr.   r/   &litellm.llms.custom_httpx.http_handlerr0   r1   *litellm.router_utils.get_retry_from_policyr2   r3   litellm.secret_managers.mainr4   r  r5   r6   r7   r8   r9   r:   r;   litellm.types.rerankr<   litellm.types.utilsr=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   	open_textfr
  	json_datadumpsr  importlib.metadataconcurrent.futuresrX   typingrY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   OriginalError8litellm.llms.base_llm.audio_transcription.transformationrg    litellm.llms.base_llm.base_utilsrh   )litellm.llms.base_llm.chat.transformationri   /litellm.llms.base_llm.completion.transformationrj   .litellm.llms.base_llm.embedding.transformationrk   +litellm.llms.base_llm.rerank.transformationrl   _loggingrn   caching.cachingro   rp   rq   rr   rs   r  rt   ru   rv   rw   rx   ry   rz   r{   r|   r}   r~   r   r   r   proxy._typesr   r   types.llms.openair   r   r   types.routerr   MAX_THREADSr  sentry_sdk_instancecapture_exceptionr   posthog	slack_appalerts_channelheliconeLoggerathinaLoggerpromptLayerLoggerlangsmithLoggerlogfireLoggerweightsBiasesLoggercustomLoggerlangFuseLoggeropenMeterLogger
lagoLoggerdataDogLoggerprometheusLoggerdynamoLoggers3LoggergenericAPILoggergreenscaleLoggerlunaryLoggeraispendLoggersupabaseClientr   r   r  r   r   r   last_fetched_atlast_fetched_at_keysr  r   r   r  r)  r  r   ro  r  r  r  r  r  r   r  r  r  r  r	  r  r1  r7  r9  r?  rF  r<  rL  rN  rP  rR  rT  rW  rY  r`  r  r  r  r  r  r  r  r  r  r  rt  rv  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rJ  rU  r0  rM  rW  rf  rl  ry  r  r  r  r  r  r  r?  Headersr  r.  r  r  r  r  r  r  r  r  r$  r*  r(  r-  r  r  rD  rR  rr  r_  r  rd  rf  rr  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r'  r4  r8  r  r   r   <module>r     s:           	    	  	         ( &   * *       0 * E       ! 3 6 - U ; A 
 T  T 3 L Q 4   0 )      : Y+-G		!I
 $**Y'  1     0 > @ T N H $     " B 
 )  +6  
	  
   %'xS	" '/1 HT#s(^, 1(*Xd38n% * 2 3: /0(	EYYx
%

 #'
0q	l "TN 
B JN
1
1"*+E"F
1 3
C 
 
6 ( PT T#Y x~   $
#(59@D 	49jtnj 3-j 4.	j
 D012j <=j $j $,D>j\ CGEE2:3-E6E3 E2F"2 '+,0#,159@D49stns 5d3i(
)s tn	s
 $D>s D012s <=s $,D>s 	sl	 	 	C hsm PT , 6:  %-c] 	 H 6:%-c]	.S x} 3 SW B (3- SW c  QU  6:%-c]	 6:%-c]	.3 Xc] d > 6:

%-c]
	

c 
2d d t JuS$Y/ J\ "
)-"#  $&*#')-'+?C"&#'+IJc]J "#J smJ tnJ8 9J: sm;J< C==J> !?J@ tnAJB /7tnCJD $EJF }GJH tnIJZT )-GOPT~	$ # %)!%JN)-"&OOsmO SMO c]	O
 #O &d73D+E&FGO "#O $Of  !%))--1_C=_}_ c]_ c]	_
 3-_ C=_ 3-_ "#_ %TN_J %) $"&-1WW 3-W c]	W
 W $W %TNWt.*7?S	7J	* 

	
		
15ED DD t,-.EDN&$ &4 &R(#C=(	#( 
#( $,C=	(
 c](Vd  *&&.<&c]&6, ,8DV;W ,^. T . T ":=	"*23-c]4T c C C m  )c )HSM )X!:
# :
(3- :
z, ,F F3# 3# 3 S x}  "# # #d # SW 4 i **%-c]*,*ZS Xc] 0 6:f
f
%-c]f
f
RP# PHSM PY PfGs G4Xv  "(
(
(
 (
 	(

 (
 (
VS T-5H  1:  !"zVC=zVc]zV smzV 
	zVz	0$H3  .s :eDOT$9:;d^> 15u}}-J 15	>>> u}}-> 	>
 5#:>: !# 	/&/&/& /& 	/&dO% O%f >B-5c]0 >B-5c]0T ";|?DD"*&9   P  #(cC=c c !	cL1d 1tCy 1h/dtnc] # #> 8( 
/ D 0  &c c 	4(8#9 	d 	 5L0M *   
U9d?3 
 
(	E5E0F 	E"T:J5K "J%c	*+eD#I8Q Ql >O9: c]: 6:)-	t,-. D012 "#	
 
<1 1% 18 #CC #Cc #Cc #CLd t A@ s   i!!i+