
    gQ             D       4   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlmZmZmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZ d dl Z d dl!Z!d dl"Z"d dl#Z#d dl$m%Z% d dl&m'Z' d dl(Z(d d	l(m)Z)m*Z*m+Z+m,Z,m-Z- d d
l.m/Z/ d dl0m1Z1 d dl2m3Z3m4Z4 d dl5m)Z6 d dl7m8Z8m9Z9 d dl:m;Z; d dl<m=Z=m>Z> d dl?m@Z@ d dlAmBZB d dlCmDZD d dlEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZ ddl[m\Z\ ddl]m^Z^m_Z_m`Z` ddlambZb ddlcmdZdmeZemfZfmgZgmhZhmiZi ddljmkZk ddllmmZmmnZnmoZo ddlpmqZq ddlrmsZs ddltmuZumvZv ddlwmxZx dd lymzZz dd!l{m|Z| dd"l}m~Z~mZ dd#lmZ dd$lmZ dd%lmZ dd&lmZ dd'lmZ dd(lmZ dd)lmZmZ dd*lmZ dd+lmZ dd,lmZmZ dd-lmZ dd.lmZ dd/lmZ dd&lmZ dd0lmZ dd1lmZ dd2lmZ dd3lmZ dd4lmZ dd5lmZ dd&lmZ dd6lmZ dd/lmZ dd7lmZ dd8lmZ dd9lmZ dd:lmZ dd;lmZ dd<lmZ dd=lmZ dd>lmZ dd?lmZ dd@lmZ ddAlmZ dd&lmZ ddBlmZ ddClmZ ddDlmZmZmZmZmZmZ ddElmZmZmZmZmZmZmZ  e#j                  dF      Zd dGlEmZmZmZmZmZmZmZmZmZ  e       Z e       Z e       Z e       Z e       Z e|       Z eq       Z eu       Z ex       Z ez       Z es       Z e       Z e       Z e       Z e       Z e~       Z e       Z e       Z e       Z eū       Z e       Z e       Z e       Z eë       Z eǫ       Z  e       Z e       Z e˫       Z e       Z e       Z e       Z e       Z e       Z e       Z	 G dH dI      Z
 G dJ dK      Z G dL dM      Z G dN dO      Ze*g dddddddddddddddddddddddddddddddf dPedQedRee   dSee   dTeeeef      dUee   dVee   dWee   dXee   dYee   dZee   d[ee   d\eee      d]ee   d^ee   d_ee   d`ee   daee   dbee   dceeeee%   f      ddee   deee   dfee   dgee   dhee   diee   djee   dkee   dlee   dmee   dnee   doeeeFf   f@dp       Zdq Z	 ddreeeef   dPedsee   fdtZduee   dTeeeee!j0                  f      dPefdvZ	 	 	 	 	 	 	 	 ddPedQedXee   dWee   dreeeef   dxee   duee   dTeeeee!j0                  f      fdyZe*g dddddddddddddddddddddddddddddddf dPedQedTeeeee!j0                  f      dUee   dVee   dWee   dXee   dYee   d[ee   dZee   d\eee      d]ee   d^ee   d_ee   d`ee   daee   dbee   dceeeee%   f      ddee   deee   dfeeeef      dhee   diee   dgee   dnee   dRee   dSee   djee   dkee   dlee   dmee   doeeeFf   f@dz       Zd{ Zd| Ze*doefd}       Ze*g ddd~dddddwddddfdee   dee   dee   dkee   dlee   dee   dedbee   doefd       Ze*doeeef   fd       Ze*	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddeeeeeeeeee   f      f      f   dPee   dee   dee   d`ee   daeeeef      dhee   dZee   dWee   d_ee   deeeee   f      dXee   dYee   dee   dUee   dVee   dbee   dee   dkee   dlee   dmee   dsee   f,d       Z dedoeee%ef      fdZ!dedoeee%ef      fdZ"	 ddedPee   dlee   fdZ#e*	 	 	 ddedPee   dlee   dsee   fd       Z$e*doefd       Z%e*	 	 	 	 	 	 	 	 	 	 	 	 ddedPee   dWee   dee   dcee   dee   dee   dbee   dlee   dee   dkee   doefd       Z&e*doefd       Z'e*	 	 	 	 	 	 	 	 	 	 	 	 ddPededee   dee   dceed      deeed         dUee   dbee   dlee   dee   dkee   dee   doefd       Z(e*doefd       Z)e*	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddPededeeeef      dlee   dee   dkee   dee   dee   dee   dee   dTeeee!j0                  f      dcee   dee   dee   dsee   dee   doef"d       Z*dPedsededoefdZ+	 	 	 ddedeed      dee   dee   fdZ,d Z-d Z.	 ddedQee   doefdZ/	 ddedQee   doeeeef      fdZ0y)    N)futures)FIRST_COMPLETEDThreadPoolExecutorwait)deepcopy)partial)
AnyCallableDictListLiteralMappingOptionalTypeUnioncast)	BaseModel)overload)Loggingclientexception_typeget_litellm_paramsget_optional_params)CustomLogger)get_audio_file_for_health_check)_create_health_check_response_filter_model_params)r   )mock_embeddingmock_image_generation)get_content_from_model_response)AsyncHTTPHandlerHTTPHandler)_realtime_health_check)get_secret_str)GenericLiteLLMParams)CustomStreamWrapperUsageasync_completion_with_fallbacks#async_mock_completion_streaming_objcompletion_with_fallbacks convert_to_model_response_objectcreate_pretrained_tokenizercreate_tokenizerget_api_keyget_llm_provider!get_non_default_completion_paramsget_optional_params_embeddingsget_optional_params_image_gen!get_optional_params_transcription
get_secretmock_completion_streaming_objread_config_argssupports_httpx_timeouttoken_counter!validate_chat_completion_messages$validate_chat_completion_tool_choice   )verbose_logger)disable_cacheenable_cacheupdate_cache)get_completion_messages)custom_promptfunction_call_promptmap_system_message_pt	ollama_ptprompt_factory stringify_json_tool_call_content)ChunkProcessor)basetenmaritalkollama_chat)AnthropicChatCompletion)AzureAudioTranscription)AzureChatCompletion_check_dynamic_azure_params)AzureOpenAIO1ChatCompletion)AzureTextCompletion)AzureAIEmbedding)BedrockConverseLLM
BedrockLLM)BedrockEmbedding)BedrockImageGeneration)CodestralTextCompletion)handler)BaseLLMAIOHTTPHandler)BaseLLMHTTPHandler)	CustomLLMcustom_chat_llm_router)DatabricksChatCompletion)DatabricksEmbeddingHandler)aleph_alphapalm)GroqChatCompletion)Huggingface)
completion)	oobabooga)OpenAITextCompletion)OpenAIChatCompletion)OpenAIAudioTranscription)OpenAILikeChatHandler)OpenAILikeEmbeddingHandler)PredibaseChatCompletion)SagemakerChatHandler)SagemakerLLM)vertex_ai_non_gemini)	VertexLLM)GoogleBatchEmbeddings)VertexImageGeneration)VertexMultimodalEmbedding)VertexTextToSpeechAPI)VertexAIPartnerModels)VertexEmbedding)VertexAIModelGardenModels)WatsonXChatHandler)IBMWatsonXMixin)ChatCompletionAssistantMessageChatCompletionAudioParamChatCompletionModality$ChatCompletionPredictionContentParamChatCompletionUserMessageHttpxBinaryResponseContent)AdapterCompletionStreamWrapperChatCompletionMessageToolCallCompletionTokensDetails	FileTypesHiddenParamsPromptTokensDetailsall_litellm_paramscl100k_base)	ChoicesEmbeddingResponseImageResponseMessageModelResponseTextChoicesTextCompletionResponseTextCompletionStreamWrapperTranscriptionResponsec                   t    e Zd Zddddej                  dddee   dee   dee   dee   dee	eef      f
d	Z
y)
LiteLLMNX  )api_keyorganizationbase_urltimeoutmax_retriesdefault_headersr   r   r   r   r   c                Z    t               | _        t        | j                  d       | _        y )N
router_obj)localsparamsChatchat)selfr   r   r   r   r   r   s          A/var/www/openai/venv/lib/python3.12/site-packages/litellm/main.py__init__zLiteLLM.__init__   s     h6	    )__name__
__module____qualname__litellmnum_retriesr   strfloatintr   r    r   r   r   r      sq     &*"&#&%,%8%87;7 sm	7
 3-7 %7 c]7 "'#s("347r   r   c                       e Zd Zdee   fdZy)r   r   c                     || _         | j                   j                  dd      du r8| j                   j                  d       t        | j                   |      | _        y t        | j                   |      | _        y )NacompletionFTr   )r   getpopAsyncCompletionscompletionsCompletionsr   r   r   s      r   r   zChat.__init__   s\    ;;??=%0D8KKOOM*EU
FD  +4;;:NDr   N)r   r   r   r   r	   r   r   r   r   r   r      s    O8C= Or   r   c                   &    e Zd Zdee   fdZddZy)r   r   c                      || _         || _        y Nr   r   r   s      r   r   zCompletions.__init__       $r   Nc                 0   |j                         D ]  \  }}|| j                  |<    |xs | j                  j                  d      }| j                  + | j                  j                  d||d| j                  }|S t	        d||d| j                  }|S Nmodelr   messagesr   )itemsr   r   r   rb   r   r   r   kwargskvresponses          r   createzCompletions.create  s    LLNDAqDKKN #11??&1t11 h26++H
  "PPDKKPHr   r   r   r   r   r   r	   r   r   r   r   r   r   r          %8C= %
r   r   c                   &    e Zd Zdee   fdZddZy)r   r   c                      || _         || _        y r   r   r   s      r   r   zAsyncCompletions.__init__  r   r   Nc                 `  K   |j                         D ]  \  }}|| j                  |<    |xs | j                  j                  d      }| j                  3 | j                  j                  d||d| j                   d {   }|S t	        d||d| j                   d {   }|S 7 (7 wr   )r   r   r   r   r   r   s          r   r   zAsyncCompletions.create  s     LLNDAqDKKN #11??&8T__88 h26++ H
  )WuxW4;;WWH Xs$   A?B.B*!B.#B,$B.,B.r   r   r   r   r   r   r     r   r   r   r   r   	functionsfunction_callr   temperaturetop_pnstreamstream_options
max_tokensmax_completion_tokens
modalities
predictionaudiopresence_penaltyfrequency_penalty
logit_biasuserresponse_formatseedtoolstool_choiceparallel_tool_callslogprobstop_logprobsr   api_versionr   
model_listextra_headersreturnc!                   K   |!j                  dd      }"t        j                         }#|!j                  dd      }$i d| d|d|d|d|d	|d
|d|d|d|	d|
d|d|d|d|d|d|i d|d|d|d|d|d|d|d|d|d|d|d |d!|d"|d#|d$| d%d&}%|$"t        | |%j                  d d      '      \  }&}$}&}&|"xs t        j
                  }"|"(t        d0i |%d(d|"ii d{   }'|'t        d)      |'S 	 t        t        fi |%|!}(t        j                         })t        |)j                  |(      }*|#j                  d|*       d{   }+t        |+t              st        |+t               rt        |+t              rt!        d0i |+}'|+}'n"t        j"                  |+      r|+ d{   }'n|+}'|$d*k(  s|$d+k(  rGt        |'t$              r7t	        j&                         j)                  |'t	        j                          ,      }'t        |'t*              r|'j-                  |#-       |'S 7 @7 7 # t        $ r},|$xs d.}$t/        | |$|,|%|!/      d},~,ww xY ww)1a  
    Asynchronously executes a litellm.completion() call for any of litellm supported llms (example gpt-4, gpt-3.5-turbo, claude-2, command-nightly)

    Parameters:
        model (str): The name of the language model to use for text completion. see all supported LLMs: https://docs.litellm.ai/docs/providers/
        messages (List): A list of message objects representing the conversation context (default is an empty list).

        OPTIONAL PARAMS
        functions (List, optional): A list of functions to apply to the conversation messages (default is an empty list).
        function_call (str, optional): The name of the function to call within the conversation (default is an empty string).
        temperature (float, optional): The temperature parameter for controlling the randomness of the output (default is 1.0).
        top_p (float, optional): The top-p parameter for nucleus sampling (default is 1.0).
        n (int, optional): The number of completions to generate (default is 1).
        stream (bool, optional): If True, return a streaming response (default is False).
        stream_options (dict, optional): A dictionary containing options for the streaming response. Only use this if stream is True.
        stop(string/list, optional): - Up to 4 sequences where the LLM API will stop generating further tokens.
        max_tokens (integer, optional): The maximum number of tokens in the generated completion (default is infinity).
        max_completion_tokens (integer, optional): An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens.
        modalities (List[ChatCompletionModality], optional): Output types that you would like the model to generate for this request. You can use `["text", "audio"]`
        prediction (ChatCompletionPredictionContentParam, optional): Configuration for a Predicted Output, which can greatly improve response times when large parts of the model response are known ahead of time. This is most common when you are regenerating a file with only minor changes to most of the content.
        audio (ChatCompletionAudioParam, optional): Parameters for audio output. Required when audio output is requested with modalities: ["audio"]
        presence_penalty (float, optional): It is used to penalize new tokens based on their existence in the text so far.
        frequency_penalty: It is used to penalize new tokens based on their frequency in the text so far.
        logit_bias (dict, optional): Used to modify the probability of specific tokens appearing in the completion.
        user (str, optional):  A unique identifier representing your end-user. This can help the LLM provider to monitor and detect abuse.
        metadata (dict, optional): Pass in additional metadata to tag your completion calls - eg. prompt version, details, etc.
        api_base (str, optional): Base URL for the API (default is None).
        api_version (str, optional): API version (default is None).
        api_key (str, optional): API key (default is None).
        model_list (list, optional): List of api base, version, keys
        timeout (float, optional): The maximum execution time in seconds for the completion request.

        LITELLM Specific Params
        mock_response (str, optional): If provided, return a mock completion response for testing or debugging purposes (default is None).
        custom_llm_provider (str, optional): Used for Non-OpenAI LLMs, Example usage for bedrock, set model="amazon.titan-tg1-large" and custom_llm_provider="bedrock"
    Returns:
        ModelResponse: A response object containing the generated completion and associated metadata.

    Notes:
        - This function is an asynchronous version of the `completion` function.
        - The `completion` function is called using `run_in_executor` to execute synchronously in the event loop.
        - If `stream` is True, the function returns an async generator that yields completion lines.
    	fallbacksNcustom_llm_providerr   r   r   r   r   r   r   r   r   r   stopr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   deployment_idr   r   r   r   r   r   Tr   api_baser   z]No response from fallbacks. Got none. Turn on `litellm.set_verbose=True` to see more details.text-completion-openaitext-completion-codestralresponse_objectmodel_response_object)loopopenair   r   original_exceptioncompletion_kwargsextra_kwargsr   )r   asyncioget_event_loopr/   r   model_fallbacksr(   	Exceptionr   rb   contextvarscopy_contextrunrun_in_executor
isinstancedictr   iscoroutiner   OpenAITextCompletionConfig%convert_to_chat_model_response_objectr&   set_logging_event_loopr   )-r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   _r   funcctxfunc_with_contextinit_responsees-                                                r   r   r   "  s    h 

;-I!!#D **%:DA##H# 	Y# 		#
 	7# 	{# 	# 	Q# 	&# 	.# 	# 	j# 	 !6# 	j# 	j#  	!#" 	,##$ 	.%#& 	j'#( 	)#* 	?+#, 	-#. 	/#0 	{1#2 	23#4 	H5#6 	7#8 	9#: 	H;#< 	{=#> 	7?#@ 	jA#B 	C#D 	tE#H "'7"3"7"7
D"I(
$1 4W44I8 

)4i(@
 
 o  )
zA%6A&A &&(#CGGT2"2249JKKmT*j=/
 -.(9=9$H  /**H$H  #;;"&AA#9:99;aa (&-&;&;&= b H h 34++ ,  S
" L +"  
1=X 3 /
 	

s]   C1I!3H34I!AH: H6AH: 6H87A;H: 2I!6H: 8H: :	IIII!c                   K   	 t        d|         t        j                  |       r
|  d {   } | 2 3 d {   }t        d|        | 7 #7 6 y # t        $ r}|xs d}t	        |||      d }~ww xY ww)Nz'received response in _async_streaming: zline in async streaming: r   )r   r   r   )print_verboser   r  r   r   )r   r   r   argsliner
  s         r   _async_streamingr    s     
?zJKx(%~H" 	$5dV<=J &	(  
1=X 3 
 	

sR   A;(A AA AAAA AA A;	A8A33A88A;mock_responser   c           
          t        | t              ryt        | t        j                        r| t	        j
                  t        | dd      t        | dt        |             t        | d|xs d      |t        j                  dd      	      t        | t              r,| d
k(  r't	        j                  dt        | d|xs d      |      t        | t              r,| dk(  r't	        j                  dt        | d|xs d      |      t        | t              r,| dk(  r't	        j                  dt        | d|xs d      |      t        | t              rA| j                  d      r/t	        j
                  d| d|t        j                  dd      	      y y )Nstatus_code  textllm_providerr   POSTzhttps://api.openai.com/v1/)methodurl)r  messager  r   requestzlitellm.RateLimitErrorzthis is a mock rate limit errorr  r  r   z"litellm.ContextWindowExceededErrorz,this is a mock context window exceeded errorzlitellm.InternalServerErrorz$this is a mock internal server errorz Exception: content_filter_policyi  azure)r   r   r   APIErrorr   MockExceptiongetattrr   httpxRequestRateLimitErrorContextWindowExceededErrorInternalServerError
startswithr  r   r   s      r   !_handle_mock_potential_exceptionsr'    s   
 -+mV__5##}cBM63}3EF ~/B/Nh MM5QR
 	
 
M3	'M=U,U$$5 ~/B/Nh 
 	
 	=#&AA00B ~/B/Nh 
 	
 	=#&::)): ~/B/Nh 
 	
 
M3	'M,D,D*- ##! MM5QR
 	
-	'r   mock_timeoutc                 v   | du r|t        |t              rt        j                  |       ntt        |t              rt        j                  t        |             nEt        |t
        j                        r+|j                  t        j                  |j                         t        j                  dd|      y y )NTzThis is a mock timeout errorr   r  )	r   r   timesleepr   r   Timeoutconnectr   r(  r   r   s      r   _handle_mock_timeoutr/  '  s    
 t 3gu%JJw%JJuW~&/GOO4OJJw'oo2!
 	
 !4r   Fmock_tool_callsc
           	         	 |d}t        ||	|        ||j                  |d       t        || |       t        t        t
        t        f   |      }t        |t
              r*|j                  d      rt        j                  dd	| d
      }|
j                  dd      }|t        j                  |       t        |t              rt        d#i |S t        |      }|du rL|
j                  dd      du rt        t!        ||| |      | d|      S t        t#        ||| |      | d|      S t        |t        j                        r||||j$                  d   j&                  _        njg }t+        |      D ]S  }t        j,                  j/                  |t        j,                  j1                  |d            }|j3                  |       U ||_        t5        t        j                               |_        | |_        |r5|D cg c]  }t;        d#i | c}|j$                  d   j&                  _        t?        |dtA        ddd             	 t        j,                  jC                  |       \  }}}}||jD                  d<   ||jI                  |dd !       |S c c}w # tF        $ r Y (w xY w# tF        $ r;}t        |tJ        jL                        r|tG        d"jO                  |            d}~ww xY w)$aS  
    Generate a mock completion response for testing or debugging purposes.

    This is a helper function that simulates the response structure of the OpenAI completion API.

    Parameters:
        model (str): The name of the language model for which the mock response is generated.
        messages (List): A list of message objects representing the conversation context.
        stream (bool, optional): If True, returns a mock streaming response (default is False).
        mock_response (str, optional): The content of the mock response (default is "This is a mock request").
        mock_timeout (bool, optional): If True, the mock response will be a timeout error (default is False).
        timeout (float, optional): The timeout value to use for the mock response (default is None).
        **kwargs: Additional keyword arguments that can be used but are not required.

    Returns:
        litellm.ModelResponse: A ModelResponse simulating a completion response with the specified model, messages, and mock response.

    Raises:
        Exception: If an error occurs during the generation of the mock completion response.
    Note:
        - This function is intended for testing or debugging purposes to generate mock completion responses.
        - If 'stream' is True, it returns a response that mimics the behavior of a streaming completion.
    NThis is a mock requestr.  zmock-key)inputr   r&  zException: mock_streaming_errorz&This is a mock error raised mid-stream	anthropici  )r  r  r   r  
mock_delayr   Tr   F)r  r   r   r   completion_streamr   r   logging_objr   	assistant)contentrole)indexr  usage
         prompt_tokenscompletion_tokenstotal_tokensr   r   zmy-secret-keyzmy-original-responser3  r   original_responsez$Mock completion response failed - {}r   )(r/  pre_callr'  r   r   r   r   r   r%  r   r  r   r*  r+  r   r&   r)   r5   choicesr  r;  rangeutilsr   r   appendr   createdr   r~   
tool_callssetattrr'   r/   _hidden_paramsr   	post_callr   r  format)r   r   r   r   r  r0  r(  loggingr   r   r   
time_delaymodel_response_all_choicesi_choice	tool_callr  r
  s                      r   mock_completionr[  :  s)   HjJ 4M,uU "  
 	*' 3	
 #t)m
 mS)m.F.F-/
 $11@(	M ZZd3
!JJz"mT* 1=11&f5T>zz-/47*&I&m5TU'  (0 '  '"?"-uPQ# $,#  mW%:%:;98EN""1%--5L1X!--//#MM11 -K 2  0  ##G,  &2N"!$TYY[!1$ "1<!0I .:	:!0<N""1%--8
 	brJ	
	+2==+I+IPU+I+V(A"AqCVN))*?@
 '"8  
 1<  		  Ja)G>EEaHIIJsa   CJ/ ?J/ J/ .CJ/ J8J/ 4J  J/ J/  	J,)J/ +J,,J/ /	K386K..K3c!                 X   | t        d      t        |      }t        |      }t               }"|!j	                  dd      }#|!j	                  dd      }$|!j	                  dd      }%t        t        t           |!j	                  dd            }&|!j	                  d	d
      }'|!j	                  dd      }(|!j	                  dd      })|!j	                  dd      }*|!j	                  dd      }+|!j	                  dd      },|!j	                  dd      }-|!j	                  dd      }.|!j	                  dd      }/|!j	                  dd      }0|!j	                  dd      xs |}1|!j	                  dd      }2|!j	                  dd      }3|!j	                  dd      }4|1i }1||1j                  |       |!j	                  dd      }5|!j	                  dd      }6|!j	                  dd      }7|!j	                  dd      }8|!j	                  dd      }9|!j	                  dd      }:|!j	                  dd      };|!j	                  d d      }<|!j	                  d!d      }=|!j	                  d"d      }>|!j	                  d#d      }?|!j	                  d$d      }@|!j	                  d%d      }A|!j	                  d&d      }B|!j	                  d'd      }C|!j	                  d(d      }D|!j	                  d)d      }E|!j	                  d*d      }F|!j	                  d+d      }G|!j	                  d,d      }H|!j	                  d-d      }I|!j	                  d.d      }J|!j	                  d/d      }K|!j	                  d0d      }Lt        t        t           |!j	                  d1d            }Mt        t        t           |!j	                  d2d            }Nt        ||2xs d|3|43      }t        |!4      }Oi }Pt        |+t              rM|+j                  | |OMN5      \  } }}Q	 ||}#|5|5}6|+}R|0xs t         j"                  }0|0t%        dYi |"S | 3| D Scg c]  }S|Sd6   | k(  sSd7    }T}St!        j&                  dYd8|Ti|"S t         j(                  r%| t         j(                  v rt         j(                  |    } t+               }Ut-        |Ud9t!        j.                                |!j	                  d:d      d;u rd:}*||} d:}*t1        | |*|#|<      \  } }*}V}#U:t3        Ud=      r.|*Uj4                  d<   |!j	                  d>d      |Uj4                  d?<   |xs |!j	                  d@d
      xs d
}t        |t6        j8                        rt;        |*      s|j<                  xs d
}n%t        |t6        j8                        st?        |      }|:#|;!t!        j@                  |* dA|  |:|;|*dBi       n$|<"|=}=t!        j@                  |* dA|  |<|=|*dCi       i }W|>s|?s@sAsBr6| i i}W|>r|>W|    d"<   |?r|?W|    d#<   @r@W|    d$<   ArAW|    d%<   BrBW|    d&<   E t        Et              rEdu rtC        |      }VV}tE        dYi dD|dE|dF|dG|dH|dI|dJ|dK|dL|
dM|	dN|dO|dP|dQ|dR|dS|dT|dU| d|*dV|dW|dX|dY|d|6dZ|d[|d\|d]|d^|O}Qt         jF                  r0Qj	                  d_d      rQjI                  d_      }XtK        ||X`      }tM        dYi d.Jda|d	|'d|(d|)d|*d|#db|!j	                  dbd      dct         j(                  dd|,d|-d|.d|/d'CdeLd |<d|:d!|=d|;d|7d,|!j	                  d,      df|!j	                  df      d|!j	                  d      d*Fdg|!j	                  dg      d(DdhWdi|!j	                  di      d+Gdj|!j	                  dj      d1Md2N}PRjO                  | |Q|P|*k       |$s|%s|&r9|!jI                  dd       tQ        | ||||$|%RJ|!j	                  dld      |*|&|m      S |*d:k(  r\d}YKCt        KtR        jT                        st        KtR        jV                        rtY        d\|iKn      }Yt[        do      xs d:}Z|#xs t         j\                  xs t[        dp      }#|xs/ t         j^                  xs t[        dq      xs t         j`                  }|xs< t         jb                  xs* t         jd                  xs t[        dr      xs t[        ds      }Qj	                  dti       jI                  dud      xs t[        dv      }[|1xs t         jf                  }1||Qdw<   t!        jh                         jk                  | x      rlt         jh                  jm                         }\|\jo                         D ]  \  }]}^|]Qvs^Q]<    tp        js                  | ||1||#|Y[Utt        QP|(RJ|K|*y      }_nkt         jv                  jm                         }\|\jo                         D ]  \  }]}^|]Qvs^Q]<    tx        js                  | ||1||#|ZY[Utt        QP|(RJ|Kz      }_Qj	                  dId      rRj{                  ||_|1||#d{|       _S |*d}k(  rvt[        do      xs d:}Z|#xs t         j\                  xs t[        dp      }#|xs t         j^                  xs t[        dq      }|xs< t         jb                  xs* t         jd                  xs t[        dr      xs t[        ds      }Qj	                  dti       jI                  dud      xs t[        dv      }[|1xs t         jf                  }1||Qdw<   t         jv                  jm                         }\|\jo                         D ]  \  }]}^|]Qvs^Q]<    t|        js                  | ||1||#|Z[Utt        QP|(RJ|K~      }_|Qj	                  dId      sJd;u rRj{                  ||_|1||#d{|       _S |*dk(  r|#xs t         j\                  xs t[        d      }#|xs/ t         jb                  xs t         j~                  xs t[        d      }|1xs t         jf                  }1||Qdw<   d| v rt        |      }	 t        js                  | ||1U||#JRQP|K|*t        |      }_Qj	                  dId      rRj{                  ||_d|1i|       _S |*dk(  s/d| v s+d| v s'|*t         j                  v r|!j	                  d,      d;u rdtR        _E        |#xs! t         j\                  xs t[        d      xs d}#dtR        _/        |xs/ t         jb                  xs t         j~                  xs t[        d      }|1xs t         jf                  }1||Qdw<   t         j                  jm                         }\|\jo                         D ]  \  }]}^|]Qvs^Q]<    t         j                  rt         j                  tR        _G        t        |      dkD  r&d|d   v rt        |d   d   t              r	|d   d   }an"dj                  |D bcg c]  }b|bd   	 c}b      }at        js                  | |Utt        ||*|#JKRQP|(|      }c|Qj	                  dId      du r-Jdu r)Hdu r%t!        j                         j                  cU      }cQj	                  dId      sJd;u rRj{                  ||cd|1i|       c}_|_S |*dk(  r|#xs! t         j\                  xs t[        d      xs d}#|xs/ t         jb                  xs t         j                  xs t[        d      }|1xs t         jf                  }1t         j                  jm                         }\|\jo                         D ]  \  }]}^|]Qvs^Q]<    t        js                  | ||1Utt        ||#JRQP|(|WK|*t              }_|_S |*dk(  r|#xs! t         j\                  xs t[        d      xs d}#|xs/ t         jb                  xs t         j~                  xs t[        d      }|1xs t         jf                  }1||Qdw<   t        js                  | ||1U||#JRQP|K|*t        |      }_|_S | t         j                  v sS|*dk(  sN|*dk(  sI|*dk(  sD|*dk(  s?|*dk(  s:|*dk(  s5|*dk(  s0|*dk(  s+|*dk(  s&|*dk(  s!|*dk(  s|*dk(  s|*t         j                  v sd| v r9|#xs! t         j\                  xs t[        d      xs d}#|9xs! t         j                  xs t[        d      xs d}9|9tR        _G        |xs/ t         jb                  xs t         j~                  xs t[        d      }|1xs t         jf                  }1||Qdw<   t         j                  jm                         }\|\jo                         D ]  \  }]}^|]Qvs^Q]<    	 t        js                  | ||1Utt        ||#JRQP|(|WK|9|*      }_Qj	                  dId      rRj{                  ||_d|1i|       _S d| v s|*dk(  s| t         j                  v r|xs< t         j                  xs* t         jb                  xs t[        d      xs t[        d      }d|#xs! t         j\                  xs t[        d      xs d}#Wxs t         j                  }Wt        | ||#Utt        QP|(t        dR|WJ|1      }U|Qj	                  dId      d;u rRj{                  |dU       U}_|_S d| v s|*dk(  s| t         j                  v rd}e|xs< t         j                  xs* t         jb                  xs t[        d      xs t[        d      }e|#xs! t         j\                  xs t[        d      xs d}#t!        j                         j                  | |#      }#t        js                  | |d;|J|#UQPd||1t        eR      }_|_S |*dk(  r|xsC t         j                  xs1 t         jb                  xs t        j                  j	                  d      }Wxs t         j                  }W|#xs. t         j\                  xs t[        d      xs t[        d      xs d}#|#|#j                  d      s|#dz  }#t        js                  | ||J|#UQPd||1t        |R      }_|_S |*dk(  r|xsC t         j                  xs1 t         jb                  xs t        j                  j	                  d      }Wxs t         j                  }W|#xs. t         j\                  xs t[        d      xs t[        d      xs d}#|#|#j                  d      s|#dz  }#t        js                  | ||#Jt         j                  Utt        QP|(t        |R|1|K|*      }_|Qj	                  dId      sJd;u rRj{                  ||_       _}_|_S |*dk(  r|xs/ t         j                  xs t[        d      xs t         jb                  }f|#xs! t         j\                  xs t[        d      xs d}#t        | ||#Utt        QP|(t        fR      }_dI|Qv rQdI   d;u rt        _| dR      }_Qj	                  dId      sJd;u rRj{                  ||_       _}_|_S |*dk(  r|xs< t         j                  xs* t[        dë      xs t[        dī      xs t         jb                  }g|#xs! t         j\                  xs t[        dū      xs d}#t        jr                  | ||#Utt        QP|(t        t         j                  gRǫ      }UdI|Qv rQdI   d;u rt        U| dR      }_|_S U}_|_S |*dk(  r|xs< t         j                  xs* t[        dɫ      xs t[        dʫ      xs t         jb                  }h|#xs! t         j\                  xs t[        d˫      xs d}#|1xs t         jf                  xs i }1|1i }1||1j                  |       t        js                  | ||J|#UQPd||1t        hRKͫ      }_|_S |*dk(  r|xs< t         j                  xs* t        dɫ      xs t        dʫ      xs t         jb                  }h|#xs! t         j\                  xs t        d˫      xs d}#|1xs t         jf                  xs i }1|1i }1||1j                  |       t        js                  | ||J|#UQPd||1t        hR      }_|_S |*dk(  r|xs/ t         j                  xs t[        dѫ      xs t         jb                  }i|#xs! t         j\                  xs t[        dҫ      xs d}#t        js                  | ||#Utt        QP|(t        iRdWԫ      }U|U}_|_S |*dk(  rd}*|xsd t         j                  xsR t        j                  j	                  d֫      xs1 t        j                  j	                  d׫      xs t         jb                  }j|1xs t         jf                  }kWxs t         j                  }Wt        js                  | ||#kxs i Utt        QP|(t        jJRW|Kث      }UdI|Qv rQdI   d;u rJdu rt        U| dR      }_|_S U}_|_S |*dk(  rJd}*t        jr                  | |U|#tt        QPd|(t        Rګ      }UdI|Qv rQdI   d;u rt        U| dR      }_|_S U}_|_S |*dk(  r|#xs' t         j\                  xs t        j                  dܫ      }#|xs/ t         jb                  xs t         j                  xs t[        dݫ      }|1xs t         jf                  }1	 t        js                  | ||1Utt        ||#JRQP|(|WKt        d۬ޫ      }_Qj	                  dId      rRj{                  ||_d|1i|       _S |*dk(  rZ|#xs t         j\                  xs d}#|xs< t         jb                  xs* t         j                  xs t[        d      xs t[        d      }t[        d      xs d}lt[        d      xs d}ml|md}n|1xs t         jf                  }o|ornj                  o       n}1t         j                  jm                         }\|\jo                         D ]3  \  }]}^|]dtk(  rdtQv rQ]   j                  ^       $^Q]<   *]Qvs/^Q]<   5 | |dQ}pt        js                  | ||1||#Utt        |QP|(RJ|d߬      }_|Rj{                  |tR        jb                  |_       |_S |*dk(  sd| v s| t         j                  v r	 	 _S |*dk(  rt        d      |*dk(  s|*dk(  r(QjI                  dd      xs1 QjI                  dd      xs t         j                  xs t[        d      }qQjI                  dd      xs1 QjI                  dd      xs t         j                  xs t[        d      }rQjI                  dd      xs QjI                  dd      xs t[        d      }s|xs* t[        d      xs t[        d      xs t         jb                  }tt        Q      }ut        js                  | |Utt        |uP|(t        rqs|tRJ||*K|#|      }_|_S |*dk(  rQjI                  dd      xs1 QjI                  dd      xs t         j                  xs t[        d      }qQjI                  dd      xs1 QjI                  dd      xs t         j                  xs t[        d      }rQjI                  dd      xs QjI                  dd      xs t[        d      }st        Q      }u| j                  d      sD| j                  d      s3| j                  d      s"| j                  d      s| j                  d      r1t        js                  | |Utt        uP|(t        |#rqsRJ|1W|K       }Und| v sPj	                  d*      8dPd*   v r1t        js                  | |Utt        uP|(t        rqsdRJ||*K|#|      }Und| v r1t        js                  | |Utt        uP|(t        |#rqsRJ|1W|K       }UnKt        jr                  | |Utt        uP|(t        rqsRJ      }UdIQv rQdI   d;u rJdu rt        U| dR      }_|_S U}_|_S |*dk(  rQjI                  dd      xs3 QjI                  dd      xs t         j                  xs t[        d      }v|vt        d      |#xsG QjI                  dd      xs3 QjI                  dd      xs t         j\                  xs t[        d      }#|xs0 t         jb                  xs t         j                  xs t[        d	      }t        js                  | |Utt        QP|(t        RJ|#W|v|
      }wdI|Qv rQdI   d;u rJdu rwS w}_|_S |*dk(  r|#xs> QjI                  dd      xs* QjI                  dd      xs t         j\                  xs d}#|xs t         jb                  xs t[        d      }t!        j                   |      }xt        js                  | ||xtt        QP|(t        RJ|#W||      }wdI|Qv rQdI   d;u rJdu rwS w}_|_S |*dk(  r.t        js                  | |Utt        QPW|(t        RJ      }U|U}_|_S |*dk(  r/t        js                  | |Utt        QPWD|(t        RJ      }U|U}_|_S |*dk(  rWxs t         j                  }WdQv rȐt	        j
                  d       QjI                  d      }y|yj                         j                         }z|zj                  rzj                  Qd<   zj                  rzj                  Qd<   zj                  rzj                  Qd<   d>QvsQd>   yj                  j                  Qd><   t!        j                         j                  |       }F|Ft         j                  v s| j                  d      rD| j!                  dd      } t"        js                  | |WUtt        QP|(t        R||JK|#      }_nC| j!                  dd      } t$        js                  | |WUtt        QP|(t        R||JK|#      }_Qj	                  dId      rRj{                  |d_       _}_|_S |*dk(  r2t&        js                  | ||1Utt        ||#JRQP|(|WKt        dޫ      }_|_S |*dk(  rf|xs= QjI                  d d      xs( t        d!      xs t        d"      xs t        d#      }|#xsl QjI                  d$|QjI                  d|QjI                  dd                  xs6 t        d%      xs( t        d&      xs t        d'      xs t        d(      }#QjI                  d)|QjI                  d*d            }{d}||{Z{j	                  d$|#      }#|{j	                  d |{j	                  da|            }|{j	                  d+|{j	                  d,d            }|||Qd+<   t        js                  | ||J|#UQPd||1t        |RKͫ      }_|_S |*d-k(  r^Wxs t         j                  }Wt)        jr                  | ||WUtt        QP|(t        R.
      }UdI|Qv rQdI   d;u rt        U| d-R      }_|_S U}_|_S |*d/k(  rRt         j\                  xs |#xs t[        d0      xs d1}#t        js                  | ||J|#UQPd/||1t        |RKͫ      }_|_S |*d2k(  rt         j\                  xs |#xs t[        d0      xs d1}#|xsE t         j*                  xs2 t        j                  j	                  d3      xs t         jb                  }t-        j.                  |#|| |QRJUt        4	      }}|Jd;u sQj	                  dId      d;u r}S }}_|_S |*d5k(  r=t         j\                  xs |#}#t        js                  | ||J|#UQP|*||1t        |R      }_|_S |*d6k(  r|xs1 t         j0                  xs t         jb                  xs t[        d7      }t[        d8      }~|#xs( t         j\                  xs t[        d9      xs d:~ d;}#Wxs t         j                  }Wt        js                  | ||J|#UQPd6||1t        |R      }_|_S |*d<k(  st         j\                  d=k(  rd<}*|xsE t         j2                  xs2 t        j                  j	                  d>      xs t         jb                  }t5        jr                  | |Utt        QP|(t        |R?
      }Ut7        j8                  |U      sdIQv rQdI   d;u rt        U| d<R      }_|_S U}_|_S |*d@k(  s| t         j:                  v r}|#xs t         j\                  }#d@}*QjI                  dId      }t=        jr                  | ||#Utt        |QP|(t        RKA      }U|d;u r"UdB   d   dC   d   }t        || d@R      }_|_S U}_|_S |*dDk(  rt         j\                  xs |#xs d}|dk(  rt        dE      	 dj                  |D bcg c]  }b|bd   	 c}b      }at         j>                  jA                  | |ag|
|||!j	                  dFdG      dHdIJ      }|jC                         }	 |dK   d   dL   d   }|UjD                  d   jF                  _        tK        tM        jL                               |U_        | |U_        |U}_|_S |*t         jR                  v rd}t         jT                  D ]  }|dM   |*k(  sdN   } t        dO|"       tW        J|P      }|1xs t         jf                  }1 dYi dU| d^|d|1dQUdRtt        da|d|#d.JdSRdTQd7Pd|(dU|dhWd/KdVt        }_|d;u rt        _| |*RW      S _S t        dO|"       c c}Sw # t        $ r'}`Rj{                  ||t        `      d|1i|       |`d}`~`ww xY wc c}bw # t        $ r'}`Rj{                  ||t        `      d|1i|       |`d}`~`ww xY w# t        $ r'}`Rj{                  ||t        `      d|1i|       |`d}`~`ww xY wc c}bw # t        $ r}`tY        | |*`|"|!X      d}`~`ww xY w(Z  a  
    Perform a completion() using any of litellm supported llms (example gpt-4, gpt-3.5-turbo, claude-2, command-nightly)
    Parameters:
        model (str): The name of the language model to use for text completion. see all supported LLMs: https://docs.litellm.ai/docs/providers/
        messages (List): A list of message objects representing the conversation context (default is an empty list).

        OPTIONAL PARAMS
        functions (List, optional): A list of functions to apply to the conversation messages (default is an empty list).
        function_call (str, optional): The name of the function to call within the conversation (default is an empty string).
        temperature (float, optional): The temperature parameter for controlling the randomness of the output (default is 1.0).
        top_p (float, optional): The top-p parameter for nucleus sampling (default is 1.0).
        n (int, optional): The number of completions to generate (default is 1).
        stream (bool, optional): If True, return a streaming response (default is False).
        stream_options (dict, optional): A dictionary containing options for the streaming response. Only set this when you set stream: true.
        stop(string/list, optional): - Up to 4 sequences where the LLM API will stop generating further tokens.
        max_tokens (integer, optional): The maximum number of tokens in the generated completion (default is infinity).
        max_completion_tokens (integer, optional): An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens.
        modalities (List[ChatCompletionModality], optional): Output types that you would like the model to generate for this request.. You can use `["text", "audio"]`
        prediction (ChatCompletionPredictionContentParam, optional): Configuration for a Predicted Output, which can greatly improve response times when large parts of the model response are known ahead of time. This is most common when you are regenerating a file with only minor changes to most of the content.
        audio (ChatCompletionAudioParam, optional): Parameters for audio output. Required when audio output is requested with modalities: ["audio"]
        presence_penalty (float, optional): It is used to penalize new tokens based on their existence in the text so far.
        frequency_penalty: It is used to penalize new tokens based on their frequency in the text so far.
        logit_bias (dict, optional): Used to modify the probability of specific tokens appearing in the completion.
        user (str, optional):  A unique identifier representing your end-user. This can help the LLM provider to monitor and detect abuse.
        logprobs (bool, optional): Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the content of message
        top_logprobs (int, optional): An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. logprobs must be set to true if this parameter is used.
        metadata (dict, optional): Pass in additional metadata to tag your completion calls - eg. prompt version, details, etc.
        api_base (str, optional): Base URL for the API (default is None).
        api_version (str, optional): API version (default is None).
        api_key (str, optional): API key (default is None).
        model_list (list, optional): List of api base, version, keys
        extra_headers (dict, optional): Additional headers to include in the request.

        LITELLM Specific Params
        mock_response (str, optional): If provided, return a mock completion response for testing or debugging purposes (default is None).
        custom_llm_provider (str, optional): Used for Non-OpenAI LLMs, Example usage for bedrock, set model="amazon.titan-tg1-large" and custom_llm_provider="bedrock"
        max_retries (int, optional): The number of retries to attempt (default is 0).
    Returns:
        ModelResponse: A response object containing the generated completion and associated metadata.

    Note:
        - This function is used to perform completions() using the specified language model.
        - It supports various optional parameters for customizing the completion behavior.
        - If 'mock_response' is provided, a mock completion response is returned for testing or debugging.
    Nzmodel param not passed in.)r   )r   r   r  r0  r(  force_timeoutr   	logger_fnverboseFr   litellm_logging_objidmetadata
model_infoproxy_server_requestr   headersensure_alternating_rolesuser_continue_messageassistant_continue_messager   r   cooldown_timecontext_window_fallback_dictr   input_cost_per_tokenoutput_cost_per_tokeninput_cost_per_secondoutput_cost_per_secondinitial_prompt_valuerolesfinal_prompt_value	bos_token	eos_tokenpreset_cache_keyhf_model_namesupports_system_message
base_model(disable_add_transform_inline_image_blocktext_completionatext_completionr   r   zno-log	prompt_idprompt_variables)r   rf  rg  rh  r   )r   r   non_default_paramsr{  r|  
model_namelitellm_paramsdeploymentsr>  r  Tr   r   r   r   rQ  aws_region_nameregion_namerequest_timeout/rk  rl  litellm_providerrm  rn  r  r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   functions_unsupported_model)r   r   r   litellm_call_idmodel_alias_mapcompletion_call_idno_logazure_ad_token_providerlitellm_trace_idcustom_prompt_dictlitellm_metadatadrop_paramsr   r   optional_paramsr  r   r5  )
r   r   r  r0  rT  r   r5  r   r(  r   )azure_client_paramsazure_clientAZURE_API_TYPEAZURE_API_BASEAZURE_API_VERSIONAZURE_OPENAI_API_KEYAZURE_API_KEY
extra_bodyazure_ad_tokenAZURE_AD_TOKENr   rF  )r   r   re  r   r   r   dynamic_paramsr  rV  r  r  r  r^  r9  r   r   r   r   )r   r   re  r   r   r   api_typer  r  rV  r  r  r  r^  r9  r   r   r   )re  r   r   )r3  r   rH  additional_args
azure_text)r   r   re  r   r   r   r  r  rV  r  r  r  r^  r9  r   r   r   azure_aiAZURE_AI_API_BASEAZURE_AI_API_KEYz	command-r)r   r   re  rV  r   r   r   r9  r  r  r   r   r   encodingr   r   zft:babbage-002zft:davinci-002r   OPENAI_API_BASEhttps://api.openai.com/v1OPENAI_API_KEYr   r;   )r   r   rV  r  r   r   r   r   r   r9  r  r  r^  r   r   groqGROQ_API_BASEzhttps://api.groq.com/openai/v1GROQ_API_KEY)r   r   re  rV  r  r   r   r   r9  r  r  r^  r   r  r   r   r  aiohttp_openaicustom_openai	deepinfra
perplexity
nvidia_nimcerebras	sambanova
volcenginedeepseekanyscalemistraltogether_aizft:gpt-3.5-turboOPENAI_ORGANIZATION)r   r   re  rV  r  r   r   r   r9  r  r  r^  r   r  r   r   r   	replicateREPLICATE_API_KEYREPLICATE_API_TOKENREPLICATE_API_BASEzhttps://api.replicate.com/v1)r   r   r   rV  r  r  r  r^  r  r   r9  r  r   re  rG  clarifaiCLARIFAI_API_KEYCLARIFAI_API_TOKENCLARIFAI_API_BASEzhttps://api.clarifai.com/v2)r   r   fake_streamr   r   r   rV  r  r  r   r   re  r  r   r9  anthropic_textANTHROPIC_API_KEYANTHROPIC_API_BASEANTHROPIC_BASE_URLz%https://api.anthropic.com/v1/completez/v1/complete)r   r   r   r   r   rV  r  r  r   r   re  r  r   r9  r4  z%https://api.anthropic.com/v1/messagesz/v1/messages)r   r   r   r   r  rV  r  r  r  r^  r  r   r9  re  r   r   r   	nlp_cloudNLP_CLOUD_API_KEYNLP_CLOUD_API_BASEzhttps://api.nlpcloud.io/v1/gpu/)r   r   r   rV  r  r  r  r^  r  r   r9  )r   r9  r^   ALEPH_ALPHA_API_KEYALEPHALPHA_API_KEYALEPH_ALPHA_API_BASEz$https://api.aleph-alpha.com/complete)r   r   r   rV  r  r  r  r^  r  default_max_tokens_to_sampler   r9  cohereCOHERE_API_KEY
CO_API_KEYCOHERE_API_BASEz!https://api.cohere.ai/v1/generate)r   r   r   r   r   rV  r  r  r   r   re  r  r   r9  r   cohere_chatzhttps://api.cohere.ai/v1/chatrI   MARITALK_API_KEYMARITALK_API_BASEzhttps://chat.maritaca.ai/api)r   r   r   rV  r  r  r  r^  r  r   r9  r   r  huggingfaceHF_TOKENHUGGINGFACE_API_KEY)r   r   r   re  rV  r  r  r  r^  r  r   r   r9  r  r   r   rc   )r   r   rV  r   r  r  r  r   r^  r  r9  
databricksDATABRICKS_API_BASEDATABRICKS_API_KEY)r   r   re  rV  r  r   r   r   r9  r  r  r^  r   r  r   r  r   
openrouterzhttps://openrouter.ai/api/v1OPENROUTER_API_KEY
OR_API_KEYOR_SITE_URLzhttps://litellm.aiOR_APP_NAMEliteLLM)zHTTP-RefererzX-Titler   )r   r   re  r   r   rV  r  r  r  r^  r9  r   r   r   togethercomputerr_   zPalm was decommisioned on October 2024. Please use the `gemini/` route for Gemini Google AI Studio Models. Announcement: https://ai.google.dev/palm_docs/palm?hl=envertex_ai_betageminivertex_projectvertex_ai_projectVERTEXAI_PROJECTvertex_locationvertex_ai_locationVERTEXAI_LOCATIONvertex_credentialsvertex_ai_credentialsVERTEXAI_CREDENTIALSGEMINI_API_KEYPALM_API_KEY)r   r   rV  r  r  r  r^  r  r  r  r  gemini_api_keyr9  r   r   r   r   r   r   	vertex_aizmeta/	codestraljambaclaude)r   r   rV  r  r  r  r^  r  r   r  r  r  r9  r   re  r  r   r   )r   r   rV  r  r  r  r^  r  r  r  r  r9  r   	predibase	tenant_idpredibase_tenant_idPREDIBASE_TENANT_IDzMissing Predibase Tenant ID - Required for making the request. Set dynamically (e.g. `completion(..tenant_id=<MY-ID>)`) or in env - `PREDIBASE_TENANT_ID`.r   PREDIBASE_API_BASEPREDIBASE_API_KEY)r   r   rV  r  r  r  r^  r  r9  r   r   r  r   r  r   r   z/https://codestral.mistral.ai/v1/fim/completionsCODESTRAL_API_KEYr6  )r   r   rV  r  r  r  r^  r  r9  r   r   r  r   r   sagemaker_chat)r   r   rV  r  r  r  r  r^  r  r9  r   	sagemaker)r   r   rV  r  r  r  r  ru  r^  r  r9  r   bedrockaws_bedrock_clientz'aws_bedrock_client' is a deprecated param. Please move to another auth method - https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication.aws_access_key_idaws_secret_access_keyaws_session_tokenz	converse/ )r   r   r  rV  r  r  r  r^  r  r9  r   r   r   r   r   zinvoke/watsonxwatsonx_textapikeyWATSONX_APIKEYWATSONX_API_KEY
WX_API_KEYr  WATSONX_API_BASEWATSONX_URLWX_URLWML_URLwx_credentialswatsonx_credentialstokenwatsonx_tokenvllm)
r   r   r  rV  r  r  r  r^  r  r9  ollamaOLLAMA_API_BASEhttp://localhost:11434rJ   OLLAMA_API_KEY)	r   r   r   r   r  r9  r   rV  r  triton
cloudflareCLOUDFLARE_API_KEYCLOUDFLARE_ACCOUNT_IDCLOUDFLARE_API_BASEz.https://api.cloudflare.com/client/v4/accounts/z/ai/run/rH   zhttps://app.baseten.coBASETEN_API_KEY)
r   r   rV  r  r  r  r^  r  r   r9  petals)r   r   r   rV  r  r  r  r^  r  r9  r   rJ  r  customzGapi_base not set. Set api_base or litellm.api_base for custom endpointstop_k(   )promptr   r   r   r   )r   r   )jsondataoutputprovidercustom_handler8Unable to map your input to a model. Check your input - )async_fnr   
custom_llmrV  r  r9  r  r   r  r7  r   r   )
ValueErrorr9   r:   r   r   r   r   boolupdater   r   r@   r0   r   LiteLLMLoggingObjget_chat_completion_promptr   r   r*   batch_completion_modelsr  r   rP  r'   r/   hasattrrQ  r   r,  r7   readr   register_modelrC   r   add_function_to_promptr   rB   r   update_environment_variablesr[  r   AzureOpenAIAsyncAzureOpenAIrN   r4   r   r   AZURE_DEFAULT_API_VERSIONr   	azure_keyre  AzureOpenAIO1Configis_o1_model
get_configr   azure_o1_chat_completionsrb   r  AzureOpenAIConfigazure_chat_completionsrR  azure_text_completions
openai_keyrF   base_llm_http_handlerr  r   +openai_text_completion_compatible_providersr  r  r   lenlistjoinopenai_text_completionsr  groq_keyGroqChatConfiggroq_chat_completionsbase_llm_aiohttp_handleropen_ai_chat_completion_modelsopenai_compatible_providersOpenAIConfigopenai_chat_completionsreplicate_modelsreplicate_keyr  replicate_chat_completionclarifai_modelsclarifai_keyClarifaiConfig_convert_model_to_urlanthropic_keyosenvironendswithanthropic_chat_completionsnlp_cloud_keynlp_cloud_chat_completionr&   aleph_alpha_keyr^   r   
cohere_keyr$   maritalk_keyopenai_like_chat_completionhuggingface_keyr  rc   getenvdatabricks_keydatabricks_chat_completionsopenrouter_keyOpenrouterConfigtogether_ai_modelsr  r  r   vertex_chat_completionr%  %vertex_partner_models_chat_completion#vertex_model_garden_chat_completionrl   r  predibase_keypredibase_chat_completionsr   codestral_text_completionssagemaker_chat_completionsagemaker_llmr<   warning_get_credentialsget_frozen_credentials
access_key
secret_keyr  metar  AmazonConverseConfig_get_base_modelbedrock_converse_modelsreplace bedrock_converse_chat_completionbedrock_chat_completionwatsonx_chat_completionvllm_handler
ollama_keyrJ   get_ollama_responsecloudflare_api_keybaseten_keyrH   inspectisgeneratorpetals_modelspetals_handlermodule_level_clientpostr#  rJ  r  r;  r   r*  rN  r   _custom_providerscustom_provider_mapr[   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r   r  r0  r(  r]  r^  r_  r   r`  ra  rb  rc  rd  r   re  rf  rg  rh  r   r   ri  rj  r   rk  rl  rm  rn  ro  rp  rq  rr  rs  rt  ru  rv  rw  rx  ry  rz  r   r   r  r{  r|  r~  r  r  rT  mr  rV  dynamic_api_keyr  r  r  r  r  configr   r   r   r
  r"  r  	_responserQ  rT  r\  r^  r_  r`  rb  
hf_headersopenrouter_site_urlopenrouter_app_nameopenrouter_headers_headersr$  r  r  r  r  
new_paramsr  _model_responsetext_completion_model_responser   credsr  r  	generator
account_idr  resp_stringr  respresponse_jsonstring_responser'  item
handler_fns                                                                                                                                           r   rb   rb     s7   p }5660(CH6;OK8Dzz*d+HJJ5Mjj!2D9O

>4(HILJJ4M

;-IjjE*G **%:DA **%:DA	D$	Bzz*d+HL$/J!::&<dC

;-IjjD):]G/5zz"D0 BHB LR::$dL  }%**tK **]D1KJJ5M#)::.Ld#S ::nd3L!::&<dC"JJ'>E"JJ'>E#ZZ(@$G!::&<dCJJw%E$8$?

;-I

;-Izz"4d;JJ5M$jj)BDIL$/J/5zz2D0, jj!2E:Ozz"4e<**]E2KZZ$'FZZ%(FXc]FJJ{D$ABIHTNFJJ7I4,PQ&!9!BU3#=	H ;&IN %'89i>S::!#5#!1 ;  	)xO 
H"%K%8!8!8	 ,4t44!-7-71\?e;S"#Z   22S{SdSS""u0G0G'G++E '9JJw&$.")$!E")@P 3	A
="OX %'.BR*SCVN))*?@;A::!4<N))-8
 FVZZ(93?F3gu}}-6L7
 ll)cGGU]]3GnG  +0E0Q""*+1UG40D1F,?7 "-%;"""*+1UG41F2H,?7   !"'#DX"5)*@A5:"5)'2!BT"5)*>?9B"5)+69B"5)+6 $/2D9'50,h?H&%G-  
 
' 
 $ 
 	 

  
  
 * 
  
 " 
 #8 
 " 
 " 
  
 . 
 0 
  "! 
" # 
& ' 
( !4) 
* ,+ 
, - 
. / 
0 $1 
2 $3 
4 5 
6 &7 
8 $9 
: !4; 
<  ? 
D ))o.A.A)4/
 +:*=*=-+' ,!-HH
 , !
#!
!
 (!
  	!

 !
 !4!
 !
 #JJ'8$?!
 $33!
  "!
 !
 "!
 "6!
 .!
 !
  #8!!
" "6#!
$ $:%!
& #8'!
( ()!
* #JJ'89+!
, %+JJ/H$I-!
. #)**-D"E/!
0 "1!
2 $ZZ(:;3!
4 (5!
6  27!
8 $ZZ(:;9!
: 6^;!
< 

=1=!
>  ?!
@ .A!
D 	,,+) 3 	- 	
 O|JJ~t,"+ /'!::lD9$7)  ') #N!66#5#56ff&=&=>!<)6(D!'"
 ""23>wHS7#3#3SzBR7SH  5&&5125 44	   /??/$$/ 45/ o.  -00rBFF $ .,-  0G(3@0**,88u8E 44??A"LLNDAq0-.*	 + 5??%##% +#1#1#1"/$3#1' ' +#!(;% @ , !22==?"LLNDAq0-.*	 + 2<<%##% +%#1#1#1"/$3#1' ' +#!% = * ""8U3!!"#&.#*'2$,%	 " 	`6 M6 !L0!"23>wHS7#3#3SzBR7SH Uw22UjAT6U 
  /??/$$/ 45/ o.  -00rBFF $ .,-  0G(3@0 ..99;F1_,)*OA&	 ' .88!!'!--+ /-##'# 9 H( ""8U3{d7J!!"#&.#*'2$,%	 " 	X4 E4 !J. 3##312   2??2%%2 01	  0G(3@0 e#;XN0;;%##1#% + '$3#1#!(;%! < 6 ""8U3!!"#&.%.$8	 " X2 K2  #;;5(5("BBC

,-5&FO  /##//0/ /	  "&F  0??0%%0 ./	  0G(3@0 77BBDF1_,)*OA&	 '
 ##&-&:&:# H!!,x{95t< "!Y/X"NX'79#5X"NO 0::!-+$7!'# /-# ; I$  ##He4=5(#u, $>>@ff$-^ g 	 ""8U3{d7J!!"#&/%.$8	 "  !HZ/ Y/ !F* 4##4o.4 4	   .??.##. n-	  0G ++668F1_,)*OA&	 ' -77!-+!'# /-##5$7!# 8 Hd. - !$44  /##//0/ /	   0??0%%0 ./	  0G(3@0/::!-!'# /-$7! ; HV- s, W;;;"o5"k1"l2"l2"j0"k1"l2"j0"j0"i/"h."m3"g&I&II!U*
  /##//0/ /	   ''34 	  #/F  0??0%%0 ./	  0G(3@0 ))446F1_,)*OA&	 '2==%##1"/#% + '$3#1'#'9!!-(;# > : ""8U3!!"#&.%.$8	 " H* y) 5 "k1000  5((5??5 125 34   2##2232 2	  "4!Qw7Q7Q6!!-+ /-#!%##5'N" ""8U3t;!!")&4 "  &HZ( W( %"j0///L 4''4??4 014 23   1##1121 1	  --/EEeXVH,77 !'!- /-$.!$# 8 Hn' M' !$44 7((7??7 ::>>"56	  "4!Qw7Q7Q ;##;23; 23; ;  #H,=,=n,MN*,77!'!- /-$4!# 8 Hf& G& !K/ 7((7??7 ::>>"56	  "4!Qw7Q7Q  ;##;23; 23; ;  #H,=,=n,MN*1<<!!'#*#=#=-+ /-#!#$7# = H& ""8U3{d7J!!"#&. " 
  Hh$ g$ !K/ #((#12# ??	   5##5235 5	  1!!-+ /-#!%#H ?*x/HD/P.(3 '	 ""8U3{d7J!!"#&. "   HJ# I# !M1 #**#34# 23# ??   :##:45: :	  )33!!-+ /-#!-4-?-?'#N ?*x/HD/P."(5 '	  %Hx! w! !H, #%%#./# l+# ??   7##7/07 7	  66BG(}-,77!'!- /-$,!"# 8 HH! g  !M1 #%%#!"23# ",/# ??   3##3!"343 3	  66BG(}-,77!'!- /-$1!"# 8 Hx Y !J. #''#01# ??	   2##2122 2	  9CC!!-+ /-#!$#$.#5 D N  &HZ Y !M1"/ #**#::>>*-# ::>>"78# ??  !3GOOJ!3!Qw7Q7Q(33!!"(b-+ /-#!''##5! 4 N& O+#H-55( /"(5 '	  %HB A !K/"-&11!-!+ /-#!#N ?*x/HD/P."(3 '	  %HP O !L0 4##49923   4??4))4 23	  0G6AA%##1"/#% + '$3#1'#'9!%(4# B : ""8U3!!"#&.%.$8	 " j _ !L0U7#3#3U7UH  ,??,)), 23, l+  #-]";"S?S",]";"Hy !4."
 1'//H"))(3(G --88:F1$#6'*11!4-.*o-)*OA& ' #LOLD /99!!-+ /-##'$0 : H" (  l e  =0"e+333 V U !F* v  !$448Kx8W##$4d; 2"&&':DA2))2 01	   ##$5t< 3"&&';TB3**3 12	   ##$8$? 6"&&'>E645   #./#n-# ??	  "/2J-88!-+ *-#! 20#5-#'$7!+' 9 HV k !K/##$4d; 2"&&':DA2))2 01	   ##$5t< 3"&&';TB3**3 12	   ##$8$? 6"&&'>E645  "/2J  )##I.##K0##G,##H-!F!Q!Q%#1"/$.#1'%%$6#4'9 ' +#'9#!% "R "( U"""<0<| <<!7!B!B%#1"/$.#1'%$6#4'9#' ' +#(;!%"/' "C "* U"!D!O!O%#1"/$.#1'%%$6#4'9 ' +#'9#!% "P "* "6!@!@%#1"/$.#1'%$6#4'9 ' +"" /'1T9#u,2&,7$+	 H $O%Ht s !K/##K6 5"&&'<dC5..5 34	     q 
  4"&&z484"&&z484 ##4 23   3??3((3 12	  9CC!-+ /-#!#'!#5# D O& O+#H-55(&&&HJ I !$?? E"&&z48E"&&z48E ##E E  SSJ?R4SG-4-K-K.* 9CC!=+ /-#!#'!#5 D O$ O+#H-55(&&&H| { !$446AA!-+ /-#5#!#' B N &HX W !K/*55!-+ /-#5+#!#' 6 N  &Hr q !I-!3!Qw7Q7Q#6&& n &5%8%89M%N"*;;=TTV##;@;K;KO$78##?D?O?OO$;<;;;@;;O$78%_<&'89A +//;; $$56 !557GGNJW<<<@P@PA k26;FF%'9#1"/$3#1'% '"/# +!% G $ i42==%'9#1"/$3#1'% '"/# +!% > $ ""8U3!!" &. "   HT S !I-.99!-+!'# /-##5!$-# : HP k !N2 0"&&x60!"230 ""340 ",/   
-"&&#''"O$7$7
D$I
- ""45
- "-0
- "(+
- "),  -00 ##)4N $(E))--eX>(,,n00GD '**"&&'  +0(,77!'!- /-$2!# 8 HN
 m	 !F*!3!Qw7Q7Q)44!#5-+ /-#!#N O+0IT0Q /"(. '	   &Ht s !H,   ,,/0, ,	  -77!'!- /-$,!# 8 Hd A !M1   ,,/0, ,	   #%%#::>>"23# ??	  $77!! /#'-!
I d"o&9&9(E&Jd&R   HF C !H,''38H,77!'!- /-$7!# 8 H~ _ !L0 4--4??4 23	  $$;<J Y##Y34Y DJ<xX	  "4!Qw7Q7Q,77!'!- /-$0!# 8 H~ ]  9,#;;"+ #&&#::>>"34# ??	  %//!-+ /-#!##N "">2O+0IT0Q /"(1 '	  %HX W !H,9N9N0N37#3#3H"*$((59F+66!!-+ /-#!#N ~,Y7:9EiP.(0 '	  %H^ ] !H,""4h4"C{cRi ] " XXJgwy1JKF..33"#)(&0'2!&!'GR!8	 4 D !IIKM ,F3A6x@CO8GN""1%--5%(%5N"#(N %Hj g  7#<#<< 37N33
#'::%)*:%;N 4 % NtfU 
 0$VJ 0G " !    .	
 ,   " ( $ !0  . $   $6   "!H$ ~*&.(; '	   J4&Q s?H  !!"#&)!f%.$8	 "  J #OF  !!"#&)!f%.$8	 "  ^  !!"#&)!f%.$8	 "  V Kn  
 3 "
 	

s  0+Bq Bq "Bn%0Bn%7Bq O	Bq FBq 2A+Bq ABq =DBq A!Bq 9BBq >(Bn* &*Bq C5Bq A3Bq :BoBBq &B)Bq 6Bq B!Bq A)E Bq F*Bq F2*Bo" G*Bq HC Bq K(CBq N8CBq RDBq V CBq Y#B<Bq \ Bq \$CBq _'CBq b)B
Bq d4C'Bq hBq h A
Bq i+Bq i/A7Bq k'.Bp l*Bq m DBq qABq r Bq r?EBq xH Bq @#Bq @'DBq EBq EB?Bq HBq H	3Bq H=4Bq I2G"Bq Q7Bq RE,Bq W:ABq YBq YABq Z6B4Bq ]+Bq ]/ABq ^2B0Bq a#CBq d&Bq d*BBq f<Bq g ABq hBqhB:Bq k2Bq k>BBq nBq nBq n*	Bon3"BooBooBq o"	Bpo+"BppBppBq p	Bqp"Bq q BqqBq q	Bq-qBq(q(Bq-c                     	 ddl }|j                  dd      }d|d<   d|d<   |j                  dd      }|j                  d	t              }|d
k(  r5|j	                  |j                  dd      |j                  |      d      }n"|j	                  |j                  |      d      } ||g| i |S # t        $ r}t        d|       d}~ww xY w)z8
    Executes a litellm.completion() with 3 retries
    r   N?tenacity import failed please run `pip install tenacity`. Errorr      r   retry_strategyconstant_retryoriginal_functionexponential_backoff_retryr;   r?  
multipliermaxTr   r   reraiser   r  tenacityr   r   rb   Retryingwait_exponentialstop_after_attemptr  r   r  r
  r   r  r  retryers           r   completion_with_retriesr    s   
 **]A.KF=F=MSZZ*NN 

#6
C44##**aR*@,,[9 $ 
 ##,,[94 $ 
 $6t6v66/  
MaSQ
 	

s   B2 2	C;C		Cc                    K   	 ddl }|j                  dd      }d|d<   d|d<   |j                  dd      }|j                  d	t              }|d
k(  r5|j	                  |j                  dd      |j                  |      d      }n"|j	                  |j                  |      d      } ||g| i | d{   S # t        $ r}t        d|       d}~ww xY w7 #w)z{
    [DEPRECATED]. Use 'acompletion' or router.acompletion instead!
    Executes a litellm.completion() with 3 retries
    r   Nr  r   r  r   r  r  r  r  r;   r?  r  Tr  r  r  r  s           r   acompletion_with_retriesr    s
    

 **]A.KF=F=ZZ 02BCN

#6
C44##**aR*@,,[9 $ 
 ##,,[94 $ 
 *<T<V<<<)  
MaSQ
 	

( =s4   CB< B/C7C8C<	CCCCc                  L  K   t        j                         }t        |       dkD  r| d   n|d   }d|d<   d}	 t        t        g| i |}t        j                         }t        |j                  |      }t        ||j                  dd            \  }}}}d}	|dk(  s|d	k(  s|d
k(  s|dk(  s|dk(  s|dk(  s|dk(  s}|dk(  sx|dk(  ss|dk(  sn|dk(  si|dk(  sd|dk(  s_|dk(  sZ|dk(  sU|dk(  sP|dk(  sK|dk(  sF|dk(  sA|dk(  s<|dk(  s7|dk(  s2|dk(  s-|dk(  s(|d k(  s#|d!k(  s|d"k(  s|d#k(  s|d$k(  s|d%k(  s
|d&k(  s|dk(  ri|j                  d|       d{   }
t        |
t              rt        d+i |
}	nMt        |
t              r|
}	n:t        j                  |
      r%|
 d{   }	n|j                  d|       d{   }	|	+t        |	t              rt        |	d'      r||	j                   d(<   |	t#        d)      |	S 7 7 ]7 D# t$        $ r}|xs d}t'        |||| |*      d}~ww xY ww),ay  
    Asynchronously calls the `embedding` function with the given arguments and keyword arguments.

    Parameters:
    - `args` (tuple): Positional arguments to be passed to the `embedding` function.
    - `kwargs` (dict): Keyword arguments to be passed to the `embedding` function.

    Returns:
    - `response` (Any): The response returned by the `embedding` function.
    r   r   T
aembeddingNr   r   r   r  
xinferencevoyager  r  r  r  r  r  r  r  r  r  r  	ai21_chatr  r  fireworks_air  r  r  r  r  r  r  r  r  r  openai_likejina_airQ  r   zCUnable to get Embedding Response. Please pass a valid llm_provider.r   r   )r   r   rD  r   	embeddingr   r   r   r/   r   r   r   r   r   r  r1  rQ  r+  r   r   )r  r   r   r   r   r  r  r  r  r   r	  r
  s               r   r  r    s     !!#D4y1}DG&/EF<N
y24262 &&(#CGGT2'7&**Z">(
$1 158+"g-"l2"h."i/"o5"h."j0"l2"k1"l2"f,"l2"j0"k1"k1"l2"j0"n4"h."k1"h."l2"i/"h."m3"i/"j0"m3"m3"i/"h. #'"6"6t=N"OOM-.,=}=M+<=($$]3!.. "11$8IJJH 8%67"23=PH##$9:U  + P / K  
1=X 3 "
 	

s`   4H$DG= G7AG= G9G= 6G;7?G= 6H$7G= 9G= ;G= =	H!HH!!H$r   
dimensionsencoding_formatr   r  cachingc                 \   |j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  d      }|j                  dd      }|j                  d	d      }|j                  d
d      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      } |j                  dd      }!g d}"ddgt        z   }#|"|#z   }$|j                         D %&ci c]  \  }%}&|%|$vs|%|& }'}%}&t	        | |||      \  } }}(}|(|(}t        d~| |
|||d|'})|t        | |      S |"| t        j                  | d|  |||di       | &|!xs d}!t        j                  | d|  | |!|di       	 d}*|}+|+j                  | |
|)||||||||di |d|       |du s|dk(  rt        d       xs d}|xs t        j                  xs t        d!      }|xs/ t        j                  xs t        d"      xs t        j                  }|)j                  d#d      xs t        d$      },|xs/ t        j                  xs t        j                  xs t        d%      }|t!        d&      t"        j%                  | |||||,|+|t'               |)||||xs |'      }*ny| t        j(                  v s|d(k(  s
|d)k(  s|d*k(  r|xs! t        j                  xs t        d+      xs d,}t        j*                  xs t        d-      xs dt,        _        |xs/ t        j                  xs t        j.                  xs t        d.      }|||)d<   d(}d}t0        j%                  | ||||+|t'               |)|||/      }*n|d0k(  r}|xs t        j                  xs t3        d1      }|xs/ t        j                  xs t        j4                  xs t3        d2      }t6        j%                  | ||||+|t'               |)||3
      }*n|d4k(  s|d5k(  s
|d6k(  s|d7k(  r{|xs t        j                  xs t        d8      }|/t        j                  xs t        j8                  xs t        d9      }t:        j%                  | ||||+|t'               |)||3
      }*n|d:k(  s|d;k(  r|xs< t        j<                  xs* t3        d<      xs t3        d=      xs t        j                  }-|t?        |t@              r|}ni }tC        j$                  | ||)tD        |-||+t'               |||>      }*n|d?k(  r`|xs/ t        jF                  xs t3        d@      xs t        j                  }tH        j%                  | |tD        |||+t'               |)||A
      }*n|dBk(  rJt?        |tJ              r|g}.n|}.tL        jO                  | |.tD        |+|)t'               |||i |tP        |C      }*nM|dDk(  r8|t!        dE      tR        j%                  | |||||+|t'               |)||i F      }*n|dGk(  rU|xs t        dH      xs t        j                  }/tT        jW                  | |tD        |+|)t'               ddd|tP        dG|/I      }*n|dJk(  rf|)j                  dKd      xs> |)j                  dLd      xs* t        jX                  xs t        dM      xs t        dN      }0|)j                  dOd      xs> |)j                  dPd      xs* t        jZ                  xs t        dQ      xs t        dR      }1|)j                  dSd      xs, |)j                  dTd      xs t        dU      xs t        dV      }2dW|)v sdX|)v s| t\        j^                  v r3t\        ja                  | |tD        |+|)t'               |0|1|2|tP        dJY      }*ntb        j%                  | |tD        |+|)t'               |0|1|2dJ||tP        |Z      }*nJ|d[k(  r+te        j$                  | |tD        ||+|)t'               |\      }*n|d]k(  rt        j                  xs |xs t        d^      xs d_}t?        |tJ              r|g}tg        d` |D              st        jh                  da| | d]b      |du rtj        jl                  ntj        jn                  }3 |3|| |tD        |+|)t'               c      }*no|ddk(  r.tp        j%                  | |tD        |+|)t'               tP        e      }*n<|dfk(  rJ|xs t        j                  xs t        dg      }t0        j%                  | ||||+|t'               |)||3
      }*n|dhk(  rJ|xs t        j                  xs t        di      }t0        j%                  | ||||+|t'               |)||3
      }*n|djk(  r+tR        j%                  | |||||+|t'               |)||i F      }*nn|dkk(  rYts        jt                  |)||l      }4|4dm   }|4dn   }do|4v r|4do   |)do<   tR        j%                  | |||||+|t'               |)i ||p      }*n|dqk(  rr|xs! t        j                  xs t        dr      xs ds}|xs! t        j                  xs t        dt      xs du}t0        j%                  | ||||+|t'               |)||3
      }*n|dvk(  r||xs t        j                  xs t        dw      }|xs/ t        j                  xs t        j.                  xs t        dx      }tv        j%                  | ||||+|t'               |)||3
      }*nty               }5t!        dy|5       |*t{        |*dz      r||*j|                  d{<   |*ty               }5t!        dy|5       |*S c c}&}%w # t~        $ r1}6|j                  ||tK        |6      |       t        | |6||}      d}6~6ww xY w)aw  
    Embedding function that calls an API to generate embeddings for the given input.

    Parameters:
    - model: The embedding model to use.
    - input: The input for which embeddings are to be generated.
    - encoding_format: Optional[str] The format to return the embeddings in. Can be either `float` or `base64`
    - dimensions: The number of dimensions the resulting output embeddings should have. Only supported in text-embedding-3 and later models.
    - timeout: The timeout value for the API call, default 10 mins
    - litellm_call_id: The call ID for litellm logging.
    - litellm_logging_obj: The litellm logging object.
    - logger_fn: The logger function.
    - api_base: Optional. The base URL for the API.
    - api_version: Optional. The version of the API.
    - api_key: Optional. The API key to use.
    - api_type: Optional. The type of the API.
    - caching: A boolean indicating whether to enable caching.
    - custom_llm_provider: The custom llm provider.

    Returns:
    - response: The response received from the API call.

    Raises:
    - exception_type: If an exception occurs during the API call.
    r  Nr   rpmtpmr   r`  ri  r  max_parallel_requestsrc  rb  rd  r  r   re  rk  rl  rm  rn  )r   r  r  r   r   r   r   r   r   r   r   r   r  r  )r   r   r  r  r   r   r  r  r  g        r  )r   r  r  r^  rd  rc  rb  r  rt  stream_responseri  r  Tr  r  r  r  r  r  zPNo API Base provided for Azure OpenAI LLM provider. Set 'AZURE_API_BASE' in .env)r   r3  r   r   r   r  r9  r   rV  r  r   r  r   re  r   r  r  r  r  r  r  )r   r3  r   r   r9  r   rV  r  r   r  r   r  r  r  )
r   r3  r   r   r9  r   rV  r  r   r  r  r  hosted_vllm	lm_studioOPENAI_LIKE_API_BASEOPENAI_LIKE_API_KEYr  r  r  r  )r   r3  r  r  r   re  r9  rV  r  r   r   r  r  )
r   r3  r  r   r   r9  rV  r  r   r  r  )r   r3  r  r9  r  rV  r   r   r  r  r   r  r   r  z7api_base is required for triton. Please pass `api_base`)r   r3  r   r   r   r9  r   rV  r  r   r  r  r  r  )r   r3  r  r9  r  rV  r  r  r  r  r  r   r   r  r  r  r  VERTEX_PROJECTr  r  r  VERTEX_LOCATIONr  r  r  VERTEX_CREDENTIALSimagevideo)r   r3  r  r9  r  rV  r  r  r  r  r  r   )r   r3  r  r9  r  rV  r  r  r  r   r   r  r  r   rc   )r   r3  r  r   r9  r  rV  r   r  r  r  c              3   <   K   | ]  }t        |t                y wr   )r   r   ).0r  s     r   	<genexpr>zembedding.<locals>.<genexpr>  s     ?z$,s   z+Invalid input for ollama embeddings. input=r  r   r  )r   r   promptsr  r9  r  rV  r  )r   r3  r  r9  r  rV  r  r  MISTRAL_API_KEYr  FIREWORKS_AI_API_KEYr  r  )r  r   r   r   r   r  )r   r3  r   r   r   r9  r   rV  r  r  r   r  r  XINFERENCE_API_KEYzstub-xinference-keyXINFERENCE_API_BASEzhttp://127.0.0.1:9997/v1r  r  r  z*No valid embedding model args passed in - rQ  r   rG  )r   r   r   r   r   )Br   r   r   r   r/   r1   r   r   r3  r5  r$   r   r   r8  r   r9  r+  r?  r  r   open_ai_embedding_modelsr   r   rA  rO  r4   rd  databricks_embeddingopenai_like_keyopenai_like_embeddingr_  r   r   cohere_embedr  rb  r  r   bedrock_embedding
embeddingsr  rB  google_batch_embeddingsbatch_embeddingsr  r  vertex_multimodal_embedding%SUPPORTED_MULTIMODAL_EMBEDDING_MODELSmultimodal_embeddingvertex_embeddingrc   allBadRequestErrorr  ollama_aembeddingsollama_embeddingsrp  rv   get_watsonx_credentialsazure_ai_embeddingr   r1  rQ  r   rR  r   )7r   r3  r  r  r   r   r   r   r  r  r   r   r  r^  r   r  r   r  r  r   r`  ri  r  r  rc  rb  rd  r  r   re  rk  rl  rm  rn  openai_paramsr  default_paramsr   r   r~  r  r  r   rT  r  r_  transformed_inputr  r  r  r  ollama_embeddings_fncredentialsr  r
  s7                                                          r   r  r  7  s   Z JJw%EZZ$'F
**UD
!C
**UD
!C**]D1K-3ZZ8M-NJJ5M+1::ot+LM"JJ'>EL$/Jzz*d+H!::&<dCL$/JJJ5MjjD)G!::&<dC"JJ'>E"JJ'>E#ZZ(@$GM  	 	N
 #^3N'A1N+B1   =M/	=9E "!4 '/ O  EGG',A,M&'q0,@-B(;3	
 (!7!>3&'q0-B.D(;3	
Q
04.,,+"#2&(<($($(#%!. !4# 	- 	
& D=/7:%&67B7HW7#3#3W~FV7WH  5&&5!"565 44	  -00 $ 2 01 
  3??3$$3 "/2	   f 
 .77!'-#02 /%'0= 8 H" W555"h."m3"l2  /##/!"34/ /	  $$ !"78   4??4%%4 ""23	  (3@0HK /88!#02 /%' 9 H !L0X7#3#3XzBW7XH  4??4))4 23	  ,55!#02 /% 6 H  =0"i/"m3"k1 VG,,V?U0V 
 OO =..=%&;<  -66!#02 /% 7 H !H,0C}0T #%%#./# l+# ??  (Zt-L'#-- /!"#02%H !M1 #**#34# ??	  #,,!!#02 /% - H !I-%%%*G!$)!(33'!# /02%!!++ 4 H !H, M  -66$7!#02 /%! 7 H !H,N>*:;Nw  /??!# /02# $#'%+$,& @ H  !K/##$4d; 4"&&':DA4))4 ""454 ""23   ##$5t< 5"&&';TB5**5 ""565 ""34   ##$8$? 8"&&'>E8!"898 ""67	  ?*o-.TTU 7KK% '$3#4#6#4$6'9)"/(3 L  ,55% '$3#4#6#4$6'9(3#)"/# 6   !K/ **!!# /02	H !H,   ,,!"34, ,	  %%???--I%Q!)  % ))-- !
 ,!!# /02H !K/$..!# /02+ / H !I-UUNCT4UG.88!#02 /% 9 H !N2T7??Tn=S.T  /88!#02 /% 9 H !H,,66$7!#02 /%! 7 H !I-)AA /8K "),G":.H+%+6w+?(,66$7!#02 /!% 7 H !L0 )??)!"67) )	   .##.!"78. .	  /88!#02 /% 9 H !J. 7##7!"56   6??6%%6 ""45	  *33!#02 /% 4 H 8DI$PQQGH6F$G=PH##$9:8DI$PQQef  
%%!!f 	& 	
   3	
 	

s%   =o+
o+go1 1	p+:,p&&p+c                    K   t        j                         }t        |       dkD  r| d   n|d   }d|d<   d}	 t        t        g| i |}t        j                         }t        |j                  |      }t        ||j                  dd            \  }}}}|dk(  s{|d	k(  sv|d
k(  sq|dk(  sl|dk(  sg|dk(  sb|dk(  s]|dk(  sX|dk(  sS|dk(  sN|dk(  sI|dk(  sD|dk(  s?|dk(  s:|dk(  s5|dk(  s0|dk(  s+|dk(  s&|dk(  s!|dk(  s|dk(  s|dk(  s|t        j                  v r:|j                  d|       d{   }	t        j                  |	      r%|	 d{   }	n|j                  d|       d{   }	|j                  dd      du s t        |	t              st        |	t               rt        t#        |	|||        ||!      S t        |	t$              r|	S t        j                  |	      r
|	 d{   }	t%               }
t        j&                  j(                  j+                  |
|	|"      }
|
S 7 7 7 7 B# t,        $ r}|xs d}t/        |||| |#      d}~ww xY ww)$zP
    Implemented to handle async streaming for the text completion endpoint
    r   r   Tr   Nr   r   r   r  r  r  r  r  r  r  r  r  r  r  r  r  ai21r  r   r  r   r  r  r  r   F)r   r   r   r  )r8  r   r   )text_completion_responser   r   r   )r   r   rD  r   ry  r   r   r   r/   r   r   rM  r   r  r   r   r&   r  r   rL  LiteLLMResponseObjectHandlerconvert_chat_to_text_completionr   r   )r  r   r   r   r   r  r  r  r  r   r  r
  s               r   rz  rz    s     !!#D4y1}DG&/E F=Q
888 &&(#CGGT2'7&**Z">(
$1
  8+"g-"l2"o5"j0"i/"l2"k1"l2"f,"l2"j0"k1"k1"f,"l2"&AA"j0"&>>"m3"h."k1"g&I&II "11$8IJJH""8,!)> "11$8IJJHJJx'4/($?@($78."2%(;	# $7	 	 ($:;$$X.!)>'='?$'.}}'Q'Q'q'q)A!$7 (r ($
 ,+E K) K* *  
1=X 3 "
 	

s   4IC4H* +H",H* 
H$H* %H&&AH* 8I9H* 
IH* %H(&;H* !I"H* $H* &H* (H* *	I3I		IIr"  best_ofechor   suffixc           
        )*+ dd l }	 dv r*d   }t        |t              r|j                  d       t	               *i )||)d<   ||)d<   ||)d<   ||)d<   ||)d<   ||)d<   ||)d	<   |	|	)d
<   |
|
)d<   ||)d<   ||)d<   ||)d<   ||)d<   ||)d<   ||)d<   ||)d<   ||)d<   ||)d<   ||)d<   t        ||      \  }}}}|dk(  r	|du r	dvrdd<   t        | t              rdd l}t        j                  d      +t        |       dkD  rt        | d   t              r| D cg c]  }d  }})*+fd}|j                  j                         5 } t        |       D !"cg c]  \  }!}"| j                  ||!|"       }#}!}"t        |j                  j                  |#            D ]  \  }!}$|$j!                         ||!<    |*_        d d d        *S g }%t        | t              r?t        |       dkD  r1t        | d   t              r| D ]  }&d|&d}'|%j%                  |'        nt        | t              rd| dg}%nv|dk(  s|dk(  s|d k(  s
|d!k(  s|d"k(  rNt        | t              r>t        |       dkD  r0t        | d   t              rt'        j(                  d#$       d| dg}%nt+        d%|  d&      j                  d'd        |.|dk(  r)|t,        j.                  vrd(|z   )j                  dd        t1        d)      dd*<   t3        |%d+)}(j5                  d,d-      du r|(S |du s$j5                  dd-      du st        |(t6              rt9        |(||.      }(|(S t        |(t8              r|(S t        |(t              r|(S t,        j:                  j<                  j?                  |(*/      **S c c}w c c}"}!w # 1 sw Y   *S xY w)0Nr   enginer  r  r   r   r   r   r   r   r   r   r   r  r   r   r   r   r   r   r   r   r   r   r  Ttop_n_tokensr  ztext-davinci-003c                     
j                  |      }i }t        |dd|}|j                  dd       	d<   d	d<   |j                  dd       	d<   |j                  dd       	d<   |d   d	   S )
Nr  )r   r"  r   ra  ry  objectrN  r   rJ  r   )decodery  r   )rX  individual_promptdecoded_prompt
all_paramsr   r  r   r   r  r  	tokenizers        r   process_promptz'text_completion.<locals>.process_prompt  s    %.%5%56G%HN!>F!>o!>J7F  $-$%8
 %8H 6>\\$5M,T29J,X6:B,,yRV:W,Y78@Wd8S,W5#I.q11r   )r<  r;  r   r  r  r   r   z\List of lists being passed. If this is for tokens, then it might not work across all models.)msgzVUnmapped prompt format. Your prompt is neither a list of strings nor a string. prompt=z;. File an issue - https://github.com/BerriAI/litellm/issuesr"  ztext-completion-openai/z;model is not set. Set either via 'model' or 'engine' param.ry  r   r   F)r8  r   r   r   )r   r  ) copyr   r   r   r   r/   rE  concurrent.futurestiktokenencoding_for_modelrD  r   r   	enumeratesubmitas_completedresultrJ  rM  r<   rq  r   r   rL  r+  rb   r   r&   r   rL  r  r  ),r"  r   r  r  r   r   r   r   r   r   r   r   r   r  r   r   r   r   r   r   r   r   r  r   r  _engine_modelr  
concurrentx	responsesr  executorrX  r  completed_futuresfuturer   pr  r   r  r  r  s,    `                    ``                 @@@r   ry  ry  1  s   j 4 6"=Z5E

857&(O%,	""&$/@+,(2%&.
#(2%} #.>*+"&$*!!,:()$*!)4&#( "&&.
#)4&%,	"&1D-. >N/>:F( m+4<V+)*~& fd#% 334FGI6{Q:fQi#>+126aT6	22 2"  ''::< 5>f4E)4E0A0 !;LM4E & ) &/"**778IJ&	6 (.}}	!& 8A,4 = 0/ H&$CK!O
6!9c8RA%!4GOOG$  
FC	 #78  8+"g-"l2"&AA"&>>vt$K!Ovay$'n	
 $78dekdl  mh  i
 	
 JJxx'???-6E 5t<}VWW $F 
  	
 H zz-'4/$::h&$.h 34.&) 3	
 	H9	: (23 	22RR%= 	S 	
  $#a 3() = 0/s%   ?	O.O=OA	OOO$
adapter_idc                   K   	 d}t         j                  D ]  }|d   | k(  s|d   } |)t        dj                  | t         j                              |j	                  |      }t        di | d{   }d}t        |t              r|j                  |      }t        |t              r|j                  |      }|S 7 L# t        $ r}|d}~ww xY ww)	zD
    Implemented to handle async calls for adapter_completion()
    Nra  adapterHNo matching adapter given. Received 'adapter_id'={}, litellm.adapters={}r}  r   r8  r   )r   adaptersr+  rS  !translate_completion_input_paramsr   r   r   "translate_completion_output_paramsr&   ,translate_completion_output_params_streamingr   )r  r   translation_objr  
new_kwargsr   translated_responser
  s           r   aadapter_completionr&  ?  s     26$$DDzZ'"&y/ % "Zaa 0 0  %FFfFU
DOD]R\D]>]  	 h."1"T"T! #U # h 34LL&. M    #" ?^   sA   CC AC 4C5AC  CC 	CCCCc                    d }t         j                  D ]  }|d   | k(  s|d   } |)t        dj                  | t         j                              |j	                  |      }t        di |}d }t        |t              r|j                  |      }|S t        |t              st        j                  |      r|j                  |      }|S )Nra  r  r  r}  r  r  r   )r   r  r+  rS  r   rb   r   r   r!  r&   r  r  r"  )r  r   r#  r  r$  r   r%  s          r   adapter_completionr(  h  s     /3O  :#"9oO ! V]]G,,
 	
 !BB&BQJ:D:Rz:RH  (M*-PP Q 
  
H1	2g6I6I(6SHH"* I  	 r   r3  c                 :   |xs/ t         j                  xs t         j                  xs t        d      }|j	                  dd       }|t        j                  |      }||j                  j                  | |      }|S |j                  j                  |       }|S )Nr  r   )r   r3  r   r3  )	r   r   rA  r$   r   r   OpenAImoderationsr   )r3  r   r   r   openai_clientr   s         r   
moderationr/    s    
 	 	,??	,	, *+	  JJx.M
  ,,33%u3M O !,,33%3@Or   c                 L  K   ddl m} |xs/ t        j                  xs t        j                  xs t        d      }|j                  dd       }|t        ||      st        j                  d|      }n|}t        di |}	 t        j                  |xs d||j                  |j                        \  }}	}
}|'|j                  j                  | |	       d {   }|S |j                  j                  | 
       d {   }|S # t        j                  $ r Y dw xY w7 E7 !w)Nr   )AsyncOpenAIr  r   T)is_asyncr   r  r  r*  r+  r   )r   r1  r   r   rA  r$   r   r   rO  _get_openai_clientr%   r/   r   r  r-  r   )r3  r   r   r   r   r1  r.  _openai_clientr  _custom_llm_provider_dynamic_api_key_dynamic_api_baser   s                r   amoderationr8    sF     # 	 	,??	,	, *+	  JJx.MJ}k$J '>&P&P 'Q '

 '*4V4O$$kr$7(11'//	 	I#%57H '33::e:TT O (33:::GGO ""  UGsH   A?D$6D 8"D$D %D$ D"D$DD$DD$"D$c                    K   t        j                         }t        |       dkD  r| d   n|d   }d|d<   d}	 t        t        g| i |}t        j                         }t        |j                  |      }t        ||j                  dd            \  }}}}|j                  d|       d{   }	t        |	t              st        |	t              rt        |	t              rt        d
i |	}	|	}
|
S t        j                  |	      r|	 d{   }
|
S |j                  d|       d{   }
|
S 7 7 $7 
# t        $ r}|xs d}t!        |||| |	      d}~ww xY ww)a  
    Asynchronously calls the `image_generation` function with the given arguments and keyword arguments.

    Parameters:
    - `args` (tuple): Positional arguments to be passed to the `image_generation` function.
    - `kwargs` (dict): Keyword arguments to be passed to the `image_generation` function.

    Returns:
    - `response` (Any): The response returned by the `image_generation` function.
    r   r   Taimg_generationNr   r   r   r   r   )r   r   rD  r   image_generationr   r   r   r/   r   r   r   r   r   r  r   r   r  r   r   r   r   r  r  r  r  r	  r   r
  s               r   aimage_generationr=    sw     !!#D4y1}DG&/E $F"
'9$9&9 &&(#CGGT2'7&**Z">(
$1
 #2249JKKmT*j=/
 -. - > >$H    /**H  "11$8IJJH L + K 
1=X 3 "
 	

sx   4EA4D2 +D,,AD2 .E/D2 	D.
D2 ED2 %D0&D2 +E,D2 .D2 0D2 2	E;EEEqualitysizestylec                 	   	 t               }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  d	i       }|j                  d
      }|j                  dd      }|j                  dd      }|j                  dd      xs i }||j                  |       t        j                  j                         }||t        |||
      \  }}}}
nd}d}||j                  d<   g d}t        }||z   }|j                         D  ci c]  \  }} ||vs||  }!}} t        d7||||||||d|!}"|}#|#j                  |||"|d|||||di d	|       |t        ||      S |dk(  rt        d      xs d}$|
xs t        j                  xs t        d      }
|xs t        j                  xs t        d      }|	xs< t        j                   xs* t        j"                  xs t        d      xs t        d      }	|"j%                  dd      xs t        d      }%d|	d }&|&j                         D ]  \  }} ||vs| ||<    t&        j)                  || ||	|
||"|||||!      }|S |dk(  r!t*        j)                  || ||	|
||"|||"
      }|S |d#k(  r+|t-        d$      t.        j)                  || |||"||%      }|S |d&k(  r|"j%                  d'd      xs1 |"j%                  d(d      xs t        j0                  xs t        d)      }'|"j%                  d*d      xs1 |"j%                  d+d      xs t        j2                  xs t        d,      }(|"j%                  d-d      xs |"j%                  d.d      xs t        d/      })t4        j)                  || |||"||'|(|)|0
      }|S |t        j6                  v rd}*t        j8                  D ]  }+|+d1   |k(  s|+d2   }* |*t;        d3|       |d4u r2d},|t=        |t>              r|},|*jA                  || |	|
||"|||,5	      }|S d}-|t=        |tB              r|}-|*j)                  || |	|
||"|||-5	      }|S c c} }w # t,        $ r}.tE        |||.t               |6      d}.~.ww xY w)8zv
    Maps the https://api.openai.com/v1/images/generations endpoint.

    Currently supports just Azure + OpenAI.
    r:  Fr  Nr^  r  rd  rc  rb  r`  r   r   re  r  zdall-e-2r   r   )r   r  r   r   r   r   r   r   r   r   r   r   r>  r?  r@  )r   r   r>  r   r?  r@  r   r   )	r   r  r  r^  rd  rc  rb  rt  r  r  r  r  r  r  r  r  r  r  r  zapplication/json;)zContent-Typezapi-key)r   r"  r   r   r   r9  r  rV  r   r:  r   re  )
r   r"  r   r   r   r9  r  rV  r:  r   r  z!Model needs to be set for bedrock)r   r"  r   r9  r  rV  r:  r  r  r  r  r  r  r  r  r  r  )
r   r"  r   r9  r  rV  r  r  r  r:  r&  r'  r(  T)	r   r"  r   r   rV  r  r9  r   r   r   r   )#r   r   r-  r   rL  r   r/   rQ  r   r   r2   r5  r   r$   r   r   r   r9  r   r?  r;  rO  r   bedrock_image_generationr  r  vertex_image_generationr  r  r+  r   r!   r=  r"   r   )/r"  r   r   r>  r   r?  r@  r   r   r   r   r   r   r   r  r:  r  r^  r  rd  rc  rb  r`  r   r   re  rV  r  r  r  r  r   r   r~  r  rT  r  r  r   r  r  r  r'  r  async_custom_clientcustom_clientr
  s/                                                  r   r;  r;    sP   ,p
x **%6> **%6=JJ{D1	'-zz/4'H%zz*@$GZZd3
::j"-17<Q1RHd+

?D9

9d39r$NN=)(/(C(C(E 3 ?DT$7!EAE& E"*16%%g.
" ,&7#\\^
+TQq/FAqD^ 	 
 8 

+ 3

 !

 /,,+"#2&(<($$(#%
 !4 	- 	
" $(uMRR')%&67B7HW7#3#3W~FV7WH  7&&7!"56   3??3$$3 ""893 "/2  -00 $ 2 01 
 !4"O (--/1G#!"GAJ 0 4DD!/ /-' / E Nn S !H,4EE!/ /- / F NP y !I-} CDD5FF/ /- / G Nr a !K/##$4d; 6"&&':DA6))6 ""45	   ##$5t< 7"&&';TB7**7 ""56	   ##$8$? :"&&'>E:!"89 
 5EE/ /-0 2#5 / F N| c  7#<#<< 37N33
#'::%)*:%;N 4 % NtfU 
 $&BF#%*V=M*N*0' "0!A!A!#%#1$3 3#. "B 
": # 8<%*V[*I$*M "0!@!@!#%#1$3 3#( "A 
" u
v  
 3 $h
 	

sd   ER R R%AR -CR )R ,%R /R C"R %/R AR "1R R 	S #R;;S c                    K   t        j                         }t        |       dkD  r| d   n|d   }d|d<   d}	 t        t        g| i |}t        j                         }t        |j                  |      }t        ||j                  dd            \  }}}}|j                  d|       d{   }	t        |	t              rt        d
i |	}
|
S t        |	t              r|	}
|
S t        j                  |	      r|	 d{   }
|
S |j                  d|       d{   }
|
S 7 r7 $7 
# t        $ r}|xs d}t!        |||| |	      d}~ww xY ww)a
    Calls openai + azure whisper endpoints.

    Allows router to load balance between them
    r   r   TatranscriptionNr   r   r   r   r   )r   r   rD  r   transcriptionr   r   r   r/   r   r   r   r   r   r  r   r   r<  s               r   rH  rH    sr     !!#D4y1}DG&/E#F 
}6t6v6 &&(#CGGT2'7&**Z">(
$1
 #2249JKKmT*,=}=H  '<=$H    /**H  "11$8IJJH L + K 
1=X 3 "
 	

s   4EA4D$ +D, D$ ED$  E!D$ ;D <D$ ED$ D"D$ ED$  D$ "D$ $	E-EEEfilelanguage)r#  r  srtverbose_jsonvtttimestamp_granularities)wordsegmentr   c                 P   |j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  d      }|j                  d	d      }|j                  d
g        |j                  dd      }|j                  dd      }|rt        |      |j                  d<   |t        j
                  }t        j                  j                         }t        | ||
      \  } }}}
||}	t        | |||||||      }|j                  | |i ||||di d||       d}|dk(  r|
xs t        j                  xs t        d      }
|xs t        j                  xs t        d      }|j                  dd      xs t        d      }|	xs/ t        j                  xs t        j                   xs t        d      }	||d	<   t"        j%                  | ||||||||
|	|||      }n|dk(  s
|dk(  s|dk(  r|
xs! t        j                  xs t'        d      xs d}
t        j(                  xs t'        d      xs dt        _        |	xs/ t        j                  xs t        j*                  xs t'        d      }	t,        j%                  | |||||||||
|	      }nK|d k(  rFt.        j%                  | |||||"t1        |t2              st1        |t4              r|nd||||
|	d i !      }|t7        d"      |S )#rG  r  Nrd  rc  rb  rH  Fr`  r   tagsr  r   r  )r   rK  r"  r   rO  r   r   r  r  rd  rc  rb  rt  r  r  r  r  r  r  r  r  )r   
audio_filer  rV  rH  r   r   r9  r   r   r   r  r   r   r  r  r  r  r  r  )r   rU  r  rV  rH  r   r   r9  r   r   r   deepgram)r   rU  r  rV  rH  r   r   r   r9  r   r   r   re  z8Unmapped provider passed in. Unable to get the response.)r   r   r   model_call_detailsr   DEFAULT_MAX_RETRIESr   rL  r   r/   r3   r5  r   r$   r   r   r9  azure_audio_transcriptionsaudio_transcriptionsr4   r   rA  openai_audio_transcriptionsrB  r   r"   r!   r+  )r   rJ  rK  r"  r   rO  r   r   r   r   r   r   r   r   r   r  rd  rc  rb  rH  r`  r   r  r   rV  r  r  r   r  s                                r   rI  rI  H  s   6 jj!2D9O!::&<dCL$/Jzz*d+HZZ 0%8NZZ 0%8N-3ZZ8M-NJJ5M
JJvr**]D1K 	

8T"  ;>v;..x800]]88:N<L)<x=9E "!7' 7/	O 44.$8$  $!
 
 0 5   15Hg%Sw//S>BR3S U7..U.AT2U 	  $4d; 
~@

  //  / o.	 	 ,9(-BB+))+#)# C 
  	x'&(.0  +++,+ +	 	    /0 	 bW__b0B0BbjQaFb.CC+))+# D 
 

	*(==+)) %v{3!&*:;	  #+ *+ > 
. STTOr   c                  X  K   t        j                         }t        |       dkD  r| d   n|d   }d|d<   |j                  dd      }	 t	        t
        g| i |}t        j                         }t	        |j                  |      }t        ||j                  dd            \  }}}}|j                  d|       d{   }	t        j                  |	      r|	 d{   }
|
S |j                  d|       d{   }
|
S 7 A7 $7 
# t        $ r}|xs d	}t        |||| |
      d}~ww xY ww)z%
    Calls openai tts endpoints.
    r   r   Taspeechr   Nr   r   r   r   )r   r   rD  r   r   speechr   r   r   r/   r   r  r   r   r<  s               r   r]  r]    sB    
 !!#D4y1}DG&/EF9 **%:DA
v/// &&(#CGGT2'7&**Z">(
$1
 #2249JKK}-**H  "11$8IJJH L* K 
1=X 3 "
 	

sm   AD*A4D ;C=<D C?D  D*!D 6D7D <D*=D ?D D 	D'D""D''D*voicer   projectrb  speedre  r]  c                 8   |j                  dd       }|j                  dd       }|j                  dd       }|j                  dd       }|j                  dd       }t        | ||      \  } }}}|j                  dg        i }|||d<   |||d	<   |
t        j                  }
|"t        j
                  xs t        j                  }|j                  d
d       }|j                  | |i ||||	d i d||       d }|dk(  r|t        |t              st        j                  d| |      |xs! t        j                  xs t        d      xs d}|xs/ t        j                  xs t        j                  xs t        d      }|xs! t        j                   xs t        d      xs d }|xs! t        j"                  xs t        d      xs d }|xs t        j$                  }t&        j)                  | |||||||||
||      }n|dk(  r|t        |t              st        j                  d| |      |xs t        j                  xs t        d      }|xs t        j*                  xs t        d      }|xs< t        j                  xs* t        j,                  xs t        d      xs t        d      }|j                  di       j                  dd       xs t        d      }|r||d<   t.        j)                  | ||||||||||
||      }n|d k(  s|d!k(  rt1        d)i |}|j                  xs d"}|j2                  xs t        j2                  xs t5        d#      }|j6                  xs t        j6                  xs t5        d$      }|j8                  xs t5        d%      }|+t        |t:              st        j                  d&| | |      t<        j)                  |||||
|| |||||'      }|)t?        d(jA                  |t        jB                              |S )*Nr   r  rd  r   rc  r  rS  r   ra  r`  rT  r  r   z;'voice' is required to be passed as a string for OpenAI TTSr  r  r  r  r  OPENAI_PROJECT)r   r3  r_  r  r   r   r   r`  r   r   r   r]  r  z:'voice' is required to be passed as a string for Azure TTSr  r  r  r  r  r  r  )r   r3  r_  r  r   r   r   r  r   r   r   r   r]  r  r  r  r  r  r  zN'voice' is required to be passed as a dict for Vertex AI TTS, passed in voice=)	_is_asyncr  r  r  r   r   r   r3  r_  r  r   r9  z@Unable to map the custom llm provider={} to a known provider={}.r   )"r   r/   r   r   r  r   r   rX  r5  r   r   r  r   r4   r   rA  r   r`  re  rO  audio_speechr   r9  r?  r%   r  r$   r  r  r   vertex_text_to_speechr   rS  provider_list) r   r3  r_  r   r   r   r   r`  r   rb  r   r   ra  r   re  r   r]  r   r   r  rd  r   rc  r  r  r9  r   r  generic_optional_paramsr  r  r  s                                    r   r^  r^  +  s   * ::fd#D%+ZZ0A4%HO!::&<dCJJ5ML$/J<L)<x=9E JJvrO"-<)*#( ))))GV-G-G**2D9K,,.$8$  $!
 
 0 -  6:Hh&=E3!7))U0   +++,+ +	 	  ,,!!, *+	 	  ##/0 	 	  *+ 	 	 ,W__*77+%# 8 
 
	'=E3!7))T0 
 Ow//O:>N3O![W%8%8[JGZ<[  ++  + 01+ /* 	 )8(;(;L"(M(Q(Qd)
 )

 	 /<OO,)66+#)%# 7 
 
	+/BFV/V"6"@"@*339r#22 2%%201 	 $33 3&&312 	 $66 645 	
 Zt%<))hinhop0 
 )551,.+# 6 
 NUU#W%:%:
 	

 Or   model_paramsc                    K   ddl m}  ||      }d }|t        j                  v r-t        j                  |   }t	        j
                  |       |d d }||d<   ||d<   d|d<   t        d	i | d {    i }|S 7 w)
Nr   )*pick_cheapest_chat_model_from_llm_provider)r      r   r   r;   r   r   ),litellm.litellm_core_utils.llm_request_utilsrk  r   models_by_providerrandomshuffler   )r   r   ri  rk  cheapest_modelfallback_modelsmodelsr   s           r   ahealth_check_wildcard_modelsrt    s     
 @/N '+Og888++,?@v !**L /L!"L

%
%%%HO &s   A.A;0A91	A;mode)	r   rb   r  re  audio_transcriptionr;  batchrerankrealtimec           
      |   K   	  j                  dd      t        d      t        j                  v r$|"t        j                     j                  d      }t	              \  }}t        j                  v r$|"t        j                     j                  d      }ddi d<   |xs d	}d
v rt                d{   S  fd fd fd fd fd fd fd fdd}||v r> ||           d{   }t        |di       j                  di       xs i }t        |      S t        d| d      7 7 A# t        $ re}t        j                         }	t        |	t              r|	dd }	|ddt        |       d|	 icY d}~S t        |      dz   dz   |	z   }
d|
icY d}~S d}~ww xY ww)a   
    Support health checks for different providers. Return remaining rate limit, etc.

    Returns:
        {
            "x-ratelimit-remaining-requests": int,
            "x-ratelimit-remaining-tokens": int,
            "x-ms-region": str,
        }
    r   Nzmodel not setru  rF  zno-cacheTcacher   *)r   r   ri  c                  .    t        j                  di  S )Nr   )r   r   ri  s   r   <lambda>zahealth_check.<locals>.<lambda>G  s    G//?,?r   c                  P    t        j                  di t               dxs diS )Nr"  testr   )r   rz  r   ri  r"  s   r   r  zahealth_check.<locals>.<lambda>H  s*    '":": #&|4#'#r   c                  R    t        j                  di t              d xs dgiS )Nr3  r  r   )r   r  r   )r3  ri  s   r   r  zahealth_check.<locals>.<lambda>L  s,    !3!3 "&|4"'x"r   c                  R    t        j                  di t               xs dddS )Nr  alloy)r3  r_  r   )r   r]  r   r  s   r   r  zahealth_check.<locals>.<lambda>P  s+    GOO %&|4%&%r   c                  X    t        j                  di t               dt               iS )NrJ  r   )r   rH  r   r   r~  s   r   r  zahealth_check.<locals>.<lambda>U  s(    7+A+A ,&|4,46,r   c                  H    t        j                  di t               diS )Nr"  r   )r   r=  r   r  s   r   r  zahealth_check.<locals>.<lambda>Y  s%    (A(A )&|4))r   c                  T    t        j                  di t               xs ddgdS )Nr  zmy sample text)query	documentsr   )r   arerankr   r  s   r   r  zahealth_check.<locals>.<lambda>]  s-    goo &|4l+,r   c            
          t         j                  dd       j                  dd       j                  dd             S )Nr   r   r   )r   r   r   r   r   )r#   r   )r   r   ri  s   r   r  zahealth_check.<locals>.<lambda>b  sA     6$7%))*d;$((D9(,,]DA!r   )r   rb   r  re  rv  r;  rx  ry  rQ  re  zMode zI not supported. See modes here: https://docs.litellm.ai/docs/proxy/healthi  errorzerror:zz. Missing `mode`. Set the `mode` for the model - https://docs.litellm.ai/docs/proxy/health#embedding-models  
stacktrace: zQ
Have you set 'mode' - https://docs.litellm.ai/docs/proxy/health#embedding-modelsz
stack trace: )r   r   r   
model_costr/   rt  r  r   	traceback
format_excr   r   )ri  ru  r"  r3  r  mode_handlersr  _response_headersr
  stack_traceerror_to_returnr   r   s   ` ``       @@r   ahealth_checkr    s&    8W*+//>=O,,G&&&4<%%e,008D+;%+H("AqG&&&4<%%e,008D !
W ~v%<6$7)   @
$!
9#
J = 1mD133I 	#3R8<<YKQr  11BCCvfg gX 4  ***,k3'%et,K<6#a&  *e  fq  er  s 
 Fbc   	 ))!*sw   F<B>E EE F<;E E	/E 7F<8E 	E 	F9=F4F9F<F4.F9/F<4F99F<c                     	 t        j                  |        t        j                  rt	        |        y y # t
        $ r Y y w xY wr   )r<   debugr   set_verboseprintr   )print_statements    r   r  r    s=    _-/"  s   04 	A A c                      t         j                  &t        t         j                        }t        di | |S t	        d      )Nz`No config path set, please set a config path using `litellm.config_path = 'path/to/config.json'`r   )r   config_pathr6   rb   r+  )r   config_argss     r   config_completionr    sA    &&w':':;2F2k22n
 	
r   chunksc           
         | d   d   }| d   d   }| d   d   }| d   d   }| d   j                  dd       }| d   d   d   d	   }| d   d   d   d
   }|||||d d||dgd d d dd}	g }
| D ]R  }|d   }|D ]F  }|t        |d      s|j                  d      %|j                  d      }|
j                  |       H T dj                  |
      }||	d   d   d<   t	        |      dkD  rn	 	 t        ||      |	d   d<   t        ||d      |	d   d<   |	d   d   |	d   d   z   |	d   d<   t        di |	S # t        $ r t        d       d|	d   d<   Y Tw xY w)Nr   ra  r   rN  r   system_fingerprintrJ  finish_reasonr   )r  r=  r   r  rB  )ra  r   rN  r   r  rJ  r>  r  r  r   r>  rC  z1token_counter failed, assuming prompt tokens is 0T)r   r  count_response_tokensrD  rE  r   )	r   r1  rM  rF  rD  r8   r   r  r   )r  r   ra  r   rN  r   r  r  r   r   content_listchunkrJ  choicerY  combined_contents                   r   $stream_chunk_builder_text_completionr    s    
4BAYx FQi	"G1IgE';TB2Jy)!,_=Mbz)$Q'
3H 0 $!.	
 "!% 
H( L	"F"FF+JJv&2 **V,##G,   ww|, &6HY6"
q /-:(.
/* .;".HW)* 	/*Xg->?R-SS Wn% "-H-- 	/ 	IJ-./*	/s   ,D5 5EEc                    	 | t        j                  dddd      | sy t        | |      }|j                  } t	        |       dk(  ry t        | d   d   d   t         j                  j                        rt        | |      S | d   d   }|j                  |       }| D cg c]0  }t	        |d         dkD  rd	|d   d   d
   v r|d   d   d
   d	   |2 }}t	        |      dkD  rP|j                  |      }	t        t        |j                  d         }
d |
j                  _        |	|
j                  _        | D cg c]0  }t	        |d         dkD  rd|d   d   d
   v r|d   d   d
   d   |2 }}t	        |      dkD  rNt        t        |j                  d         }
d |
j                  _        |j#                  |      |
j                  _        | D cg c]0  }t	        |d         dkD  rd|d   d   d
   v r|d   d   d
   d   |2 }}t	        |      dkD  r|j'                  |      |d   d   d   d<   | D cg c]0  }t	        |d         dkD  rd|d   d   d
   v r|d   d   d
   d   |2 }}t	        |      dkD  r=t        t        |j                  d         }
|j)                  |      |
j                  _        t-        |      }|j/                  | |||      }t1        |d|       |S c c}w c c}w c c}w c c}w # t2        $ rK}t5        j6                  dj9                  t;        |                   t        j                  dddd      d }~ww xY w)Nr  z=Error building chunks for logging/streaming usage calculationr  )r  r  r  r   r   rJ  )r  r   r   rO  deltar   r;  r  r   )r  r   completion_outputr   r>  zAlitellm.main.py::stream_chunk_builder() - Exception occurred - {})r   r  rG   r  rD  r   rL  r   r  build_base_responseget_combined_tool_contentr   r   rJ  r  r;  rO  "get_combined_function_call_contentr   get_combined_contentget_combined_audio_contentr   r    calculate_usagerP  r   r<   	exceptionrS  r   )r  r   
start_timeend_time	processorr   r   r  tool_call_chunkstool_calls_listrY  function_call_chunkscontent_chunksaudio_chunksr  r>  r
  s                    r   stream_chunk_builderr    s   j
>""W	  "684	!! v;!1Ii #W]]%>%>
 8  q	'"008  
5#$q(i 0 3G <<i #G,\:F	  	 
  1$'AABRSO7H$4$4Q$78G&*GOO#)8GOO&   
5#$q(5#3A#6w#??i #G,_=I	  	  
 #$q(7H$4$4Q$78G&*GOO#<<=QR OO)  
5#$q(U9-a099i #G,Y7C	  	 
 ~"..~> Y"9-i8  
5#$q(5+A.w77i #G,W5A	  	 
 |q 7H$4$4Q$78G$-$H$H$VGOO!;HE))/	 * 
 	'5)C
 


0  
  OVVA	

 S	
 	

sb   K9 &K9 9K9 K9 5K%A$K9 75K*,A"K9 5K/1K9 45K4)A;K9 %K9 9	MAMMr   )FNr2  NFNNN)NNNNNNNNNNNNNNNNNNNNN)NN)NNN)NNNNNNNr   NNNN)NNNNNNr   NNNNN)NNNNNNNNNNNNNNN)r   NN(1  r   r   datetimer  r#  rX  ro  sysr*  r  uuidr  r   r	  r   r   r   r  r   	functoolsr   typingr	   r
   r   r   r   r   r   r   r   r   dotenvr   r   r
  pydanticr   typing_extensionsr   r   r   r   r   r   r   "litellm.integrations.custom_loggerr   ,litellm.litellm_core_utils.audio_utils.utilsr   -litellm.litellm_core_utils.health_check_utilsr   r   *litellm.litellm_core_utils.litellm_loggingr.  )litellm.litellm_core_utils.mock_functionsr   r   8litellm.litellm_core_utils.prompt_templates.common_utilsr    &litellm.llms.custom_httpx.http_handlerr!   r"   litellm.realtime_api.mainr#   litellm.secret_managers.mainr$   litellm.types.routerr%   litellm.utilsr&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   _loggingr<   caching.cachingr=   r>   r?   0litellm_core_utils.prompt_templates.common_utilsr@   +litellm_core_utils.prompt_templates.factoryrA   rB   rC   rD   rE   rF   0litellm_core_utils.streaming_chunk_builder_utilsrG   llmsrH   rI   rJ   llms.anthropic.chatrK   llms.azure.audio_transcriptionsrL   llms.azure.azurerM   rN   llms.azure.chat.o1_handlerrO   llms.azure.completion.handlerrP   llms.azure_ai.embedrQ   llms.bedrock.chatrR   rS   llms.bedrock.embed.embeddingrT    llms.bedrock.image.image_handlerrU   !llms.codestral.completion.handlerrV   llms.cohere.embedrW   r  !llms.custom_httpx.aiohttp_handlerrX   "llms.custom_httpx.llm_http_handlerrY   llms.custom_llmrZ   r[   llms.databricks.chat.handlerr\   llms.databricks.embed.handlerr]   llms.deprecated_providersr^   r_   llms.groq.chat.handlerr`   llms.huggingface.chat.handlerra   llms.nlp_cloud.chat.handlerrb   r]  llms.ollama.completionr  llms.oobabooga.chatrc   llms.openai.completion.handlerrd   llms.openai.openaire   "llms.openai.transcriptions.handlerrf   llms.openai_like.chat.handlerrg   "llms.openai_like.embedding.handlerrh   llms.petals.completionr  llms.predibase.chat.handlerri   llms.replicate.chat.handlerrR  llms.sagemaker.chat.handlerrj   !llms.sagemaker.completion.handlerrk   llms.vertex_airl   8llms.vertex_ai.gemini.vertex_and_google_ai_studio_geminirm   <llms.vertex_ai.gemini_embeddings.batch_embed_content_handlerrn   8llms.vertex_ai.image_generation.image_generation_handlerro   6llms.vertex_ai.multimodal_embeddings.embedding_handlerrp   4llms.vertex_ai.text_to_speech.text_to_speech_handlerrq   ,llms.vertex_ai.vertex_ai_partner_models.mainrr   2llms.vertex_ai.vertex_embeddings.embedding_handlerrs   'llms.vertex_ai.vertex_model_garden.mainrt   llms.vllm.completionr~  llms.watsonx.chat.handlerru   llms.watsonx.common_utilsrv   types.llms.openairw   rx   ry   rz   r{   r|   types.utilsr}   r~   r   r   r   r   r   get_encodingr  r   r   r   r   r   r   r   r   r   rO  rG  r[  re  rJ  r  r[  r?  r=  r@  rY  r  rm  rn  r|  r{  r  rB  ri  r  r  rC  r  rj  rk  rf  rp  r}  r  ra  r  rB  rK  ro  r   r   r   r   r   r   r   r,  r   rE  r   r  r   r'  r,  r/  r[  r  r  r  r  rz  ry  r&  r(  r/  r8  r=  r;  rH  rI  r]  r^  rt  r  r  r  r  r  r   r   r   <module>r     s        	  
     H H          &   < X T Q < 7 5     0 % F F U  M 0 0 8 D N C > 1 = : D F 6 D B > B E 8 6 6 P 5 * @ 4 H @ J = @ P = ; 0 O X O O N 9 9 6    !8  /
 
 
 /0 .0 68 68 *, %' 46 ,. 79 ,. 46 m46 46 $, #5#7  $& 13 " "$ 79 /1 /1 (=(? %&?&A #-/ ,. 24 35 13 *, 02 02 7 7	O 	O $ $   $#'+/#'!!%)	 $+/9=AE04(,)-!%>B !%*.#"&"!%!!%$(Iu
u
 u
 ~	u

 C=u
 eE3J'(u
 %u
 E?u
 }u
 TNu
 TNu
 u
 $C=u
 456u
  =>!u
" ,-#u
$ uo%u
&  'u
( )u
* 3-+u
. eD$y/$9:;/u
0 3-1u
2 D>3u
4 #5u
6 "$7u
8 tn9u
: 3-;u
@ smAu
B #Cu
D c]Eu
F Gu
H D>Iu
N =--.Ou
 u
p
( *.8
i-.8
8
 "#8
v
4.
eE3567
 
, #1I&*#(:>NJNJNJ TNNJ }	NJ
 i-.NJ d^NJ 4.NJ eE3567NJb  :>#'!!%)	+/ $9=AE04(,)-!%>B .2#"&*.$( $#'"!%!!%KG#
G#
 G#
 eE3567	G#

 %G#
 E?G#
 }G#
 TNG#
 TNG#
 $C=G#
 G#
 456G#
 =>G#
 ,-G#
  uo!G#
"  #G#
$ %G#
& 3-'G#
* eD$y/$9:;+G#
, 3--G#
. D>/G#
0 %T	*+1G#
2 tn3G#
4 3-5G#
6 "$7G#
: D>;G#
> ~?G#
@ C=AG#
D smEG#
F #GG#
H c]IG#
J KG#
P =--.QG#
 G#
TF7@=> ^
): ^
 ^
B   $%)"!%!"!U

 	U


 c]U

 smU

 #U

 c]U

 smU

 U

 3-U

$ %U

 U

r \

!#>>?\
 \
~ 
   	 	 	 	 	 	 	 	 	!%) 	#'! 	"!%!!%)-aG$T%T%T#Y"7889::G$ C=	G$
 G$ G$  G$ S#XG$" #G$( )G$. /G$4 5G$: c49n;G$@ TNAG$B TNCG$D EG$J %KG$L E?MG$N OG$V smWG$X #YG$Z c][G$\ ]G$` "#aG$ G$Z&&eI==>?&R  eI==>? N GK}6>sm0   !)-	,,C=, c], "#	, ,` 2
 2
 2
j   !%)!"!%E
E
C=E
 }E
 c]	E

 c]E
 3-E
 C=E
 3-E
 c]E
 smE
 #E
 E
 E
V +
-B +
 +
\ 
 #  	JN!%!"!%!%%vv
v sm	v
 SMv <=v &d73D+E&FGv #v 3-v c]v smv  #!v" ##v( )v vr %
&@ %
 %
P  )-!"!%"&!!%#59%)")-"#III E#t)$%I c]	I
 smI #I 3-I c]I #I tnI eE5==012I c]I C=I d^I  "#!I" d^#I&  'I I^%(8<	L 	  !s*s*

	
s* SMs*  D>!s*p
 .2G.G.$TNG.G.V NRm
m
$TNm
eM#99:;m
r   