
    g                    b   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlmZmZmZmZmZmZmZmZmZmZ d dlZd dlZd dlmZ d dlmZ d dlmZ d dlZd dl Zd dl!Zd dlm"Z" d dl#m$Z$ d d	l%m&Z&m'Z'm(Z( d d
l)m*Z* d dl+m,Z, d dl-m.Z/ d dl0m1Z1 d dl2m3Z3 d dl4m5Z5 d dl6m7Z7 d dl8m9Z9 d dl:m;Z; d dl<m=Z= d dl>m?Z? d dl@mAZAmBZB d dlCmDZD d dlEmFZF d dlGmHZHmIZImJZJmKZKmLZL d dlMmNZNmOZOmPZP d dlQmRZS d dlTmUZUmVZV d dlWmXZX d dlYmZZZm[Z[ d dl\m]Z]m^Z^ d dl_m`Z`maZambZbmcZc d d ldmeZemfZfmgZgmhZhmiZimjZjmkZkmlZlmmZmmnZnmoZompZpmqZqmrZrmsZsmtZtmuZumvZv d d!lwmxZx d d"lymzZz d d#lym{Z| d d$lym}Z} d d%l~mZmZmZmZmZmZmZmZmZmZ d&d'lmZ er	d d(lmZ eZneZ G d) d*ej                        Z G d+ d,      Zy)-    Ndefaultdict)
TYPE_CHECKINGAnyCallableDictListLiteralOptionalTupleUnioncast)AsyncOpenAI)	BaseModel)overload)get_secret_str)verbose_router_logger)	DualCacheInMemoryCache
RedisCache)CustomLogger)!_get_parent_otel_span_from_kwargs)Logging)RouterBudgetLimiting)LeastBusyLoggingHandler)LowestCostLoggingHandler)LowestLatencyLoggingHandler)LowestTPMLoggingHandler)LowestTPMLoggingHandler_v2)simple_shuffle)get_deployments_for_tag)"_get_router_metadata_variable_namereplace_model_in_jsonl)InitalizeOpenAISDKClient)CooldownCache)DEFAULT_COOLDOWN_TIME_SECONDS_async_get_cooldown_deployments/_async_get_cooldown_deployments_with_debug_info_get_cooldown_deployments_set_cooldown_deployments)#_check_non_standard_fallback_formatget_fallback_model_grouprun_async_fallback)!get_num_retries_from_retry_policy)#async_raise_no_deployment_exceptionsend_llm_exception_alert)PromptCachingDeploymentCheck)0increment_deployment_failures_for_current_minute1increment_deployment_successes_for_current_minute)FlowItem	Scheduler)AllMessageValuesBatch
FileObject	FileTypes)#CONFIGURABLE_CLIENTSIDE_AUTH_PARAMSVALID_LITELLM_ENVIRONMENTSAlertingConfigAllowedFailsPolicyAssistantsTypedDictCustomRoutingStrategyBase
DeploymentDeploymentTypedDictLiteLLM_ParamsModelGroupInfoOptionalPreCallChecksRetryPolicyRouterCacheEnumRouterGeneralSettingsRouterModelGroupAliasItemRouterRateLimitErrorRouterRateLimitErrorBasicRoutingStrategy)ServiceTypes)GenericBudgetConfigType)	ModelInfo)StandardLoggingPayload)
CustomStreamWrapperEmbeddingResponseModelResponseRulesfunction_setupget_llm_provider!get_non_default_completion_params
get_secretget_utc_datetimeis_region_allowed   )PatternMatchRouter)Spanc                       e Zd ZdZy)RoutingArgs<   N)__name__
__module____qualname__ttl     C/var/www/openai/venv/lib/python3.12/site-packages/litellm/router.pyr^   r^      s    
Cre   r^   c            Q       T   e Zd ZU g Zeed<   dZee   ed<   dZ	e
ed<   dZdZee   ed<   dZee   ed<   dddddddi ddddddddddd	dg g g i ddd
di ddddddi dd e       f'deeee   eeeef      f      dee   dee   dee   dee
   dee   dee   dedeee      de
dee   dee
   dee
   dee
   dee   dee   dee
   deded   deee      d ed!ed"ed#eeeeeef   f      d$ed%ed&e
d'eeeef      d(eeef   d)ee
   d*ee   d+ee   d,ee   d-ed.   d/ee   d0ed1ee   d2ee    d3ee   d4dfPd5Z!d6 Z"d7ee   fd8Z#d/ee   fd9Z$d-ee%ef   d0efd:Z&d;efd<Z'd=ed>eeeef      d4ee(e)f   fd?Z*d=ed>eeeef      d4ee(e)f   fd@Z+e,d=ed>ee-   dAedB   d4e)fdC       Z.e,	 dd=ed>ee-   dAed   d4e(fdD       Z.e,	 dd=ed>ee-   dAeedB   ed   f   d4ee)e(f   fdE       Z.	 dd=ed>ee-   dAefdFZ.d=ed>eeeef      d4ee(e)f   fdGZ/d=edHed4dfdIZ0dHed4dfdJZ1d;edHed4dfdKZ2d;edHefdLZ3dHedMed4eeee
f      fdNZ4dOee   d>eeeeef      eeeeef         f   fdPZ5d=ed>eee-      fdQZ6e,d=ed>eeeef      dAedB   d4e)fdR       Z7e,	 dd=ed>eeeef      dAed   d4e(fdS       Z7	 dd=ed>eeeef      dAefdTZ7e,	 dd=ed>ee-   dUe
dAed   d4e(f
dV       Z8e,d=ed>ee-   dUe
dAedB   d4e)f
dW       Z8	 dd=ed>ee-   dUe
fdXZ8d=edUe
dYe9dZe:ed[f   dHeeef   f
d\Z;d=ed4efd]Z<d=ed>ee-   dHeeef   fd^Z=d_ed=efd`Z>d_ed=efdaZ?d_ed=efdbZ@d_ed=efdcZAddeBd=efdeZCddeBd=efdfZDd=edgedhefdiZEd=efdjZFd=efdkZGd=efdlZH	 	 	 dd=ed_edmee   dnee   doee   f
dpZI	 	 	 dd=ed_edmee   dnee   doee   f
dqZJd=ed_efdrZK	 	 	 ddsed=edmee   dnee   doee   f
dtZLdsed=efduZM	 dd=edgeeef   doee   d4eNfdvZOdgeeef   d=efdwZP	 dd=edgeeef   doee   d4eNfdxZQdgeeef   d=efdyZRd=ed4eSfdzZTd=ed4eSfd{ZUd=ed4eVfd|ZWd=ed4eVfd}ZXd4eVfd~ZYd=efdZZdYe9fdZ[	 ddYe9ded   fdZ\	 	 ddYe9deed      dee]   fdZ^d Z_	 	 	 	 ddHedee   d ee   d!ee   d"ee   f
dZ`d ZadYefdZb	 ddHedee   fdZc	 	 	 	 	 ddeddee   dee   d!ee   d"ee   dee   fdZed Zf	 dd eeeee   f      dee   d4eee      fdZg	 	 ddedde
de
dee   dee   d4ee
ef   fdZhd Zid4ee   fdZjd4efdZkdee   fdZldHededd4efdZmdedeen   d4e
fdZo	 ddedeee
f   dee   d4efdZpd4efdZqd=ede(dHed4efdZrd=edeen   fdZsd=edeen   d4e:ee   ee   f   fdZtd;efdZu	 dd;edeen   deev   fdZw	 	 dd=edee   d>eee-      deen   dee   deev   fdZxdedefdZydedededed4eez   f
dZ{d;ezd4efdZ|de}fdZ~d;ezd4ezfdZd;ezd4eez   fdZd;ezd4eez   fdZded4eez   fdZded4eez   fdZded4eez   fdZe,	 dd;ededdd4efd       Ze,d;ddeded4efd       Z	 dd;ee   dedee   d4efdÄZded4ee   fdĄZded4ee   fdńZdeded4ee   fdǄZded4ee   fdȄZded4e:ee
   ee
   f   fdɄZded4eee
f   fdʄZ	 ddedee   d4efd˄Zddee   d4ee   fd̈́Z	 ddedee   d4ee   fdτZd4ee   fdЄZ	 ddee   d4eee      fdфZ	 ddee   dee   d4eeee   f   fdӄZded4efdԄZdՄ Zdք ZddׄZ	 dd=eded>eeeef      dee   fd؄Zd=ed4ee   fdلZd=ed4efdڄZ	 	 	 dd=ed>eeeeef         dgeeeef      dee   d4e:eeeef   f   f
d܄Z	 	 	 	 dd=ed>eeeeef         dgeeeef      dee   dee   f
d݄Z	 	 	 	 dd=ed>eeeeef         dgeeeef      dee   dee   f
dބZdee   dee   d4ee   fdZ	 ddeen   fdZ	 ddeddee   fdZdedfdZd ZdefdZd Zd Zy)Routermodel_namesFcache_responsesi  default_cache_time_secondsNleastbusy_loggerlowesttpm_loggerINFOr   simple-shuffle
model_listassistants_config	redis_url
redis_host
redis_portredis_passwordcache_kwargscaching_groups
client_ttlpolling_intervaldefault_prioritynum_retriesmax_fallbackstimeoutdefault_litellm_paramsdefault_max_parallel_requestsset_verbosedebug_level)DEBUGrn   default_fallbacks	fallbackscontext_window_fallbackscontent_policy_fallbacksmodel_group_aliasenable_pre_call_checksenable_tag_filteringretry_afterretry_policymodel_group_retry_policyallowed_failsallowed_fails_policycooldown_timedisable_cooldownsrouting_strategy)ro   
least-busyusage-based-routinglatency-based-routingcost-based-routingusage-based-routing-v2optional_pre_call_checksrouting_strategy_argsprovider_budget_configalerting_configrouter_general_settingsreturnc(                 T   ddl m}( || _        || _        || _        || _        dt        _        | j                  du rQ|dk(  r$t        j                  t        j                         n(|dk(  r#t        j                  t        j                         |'xs
 t               | _        || _        g | _        i | _        d})d}*i }+|
| _        ||E|Cd})|||+d	<   |||+d
<   |t'        |      |+d<   |||+d<   |+j)                  |       t+        d)i |+}*|r7t        j,                   t        j.                  d)d|)i|+t        _        || _        t3        |*t5                     | _        t7        ||*      | _        || _        d| _        || _        g | _         tC               | _"        |\tG        jH                  |      }| jK                  |       | jL                  | _'        |D ]  },d|,d   v sd| j"                  |,d   d   <   ! ng | _&        ||| _(        nt        jP                  | _(        | xs tR        | _*        tW        | j,                  | jT                        | _,        |!| _-        t5               | _.        ||| _/        n;t        j^                  t        j^                  | _/        nt`        jb                  | _/        ||| _2        n;t        jd                  t        jd                  | _2        nt        jf                  | _2        |xs t        jh                  | _5        || _6        |"| _7        |xs t        jp                  }-| js                  |-       |-| _8        |t        jt                  H|xs t        jt                  }-| jp                  | jp                  jw                  d|-i       n
d|-ig| _8        |xs t        jx                  | _<        |xs t        jz                  }.| js                  |.       |.| _=        t}        t~              | _@        t}        t~              | _A        t}        t~              | _B        g | _C        |xs i | _D        |xs i }t        j                  ||       | _F        || _G        | j                  j                  d|       | j                  j                  dd       | j                  j                  di       j)                  d|	i       i | _I        	 | j                  |"|$       d| _K        t        t        j                  t              r*t        j                  jw                  | j                         n)t        j                  jw                  | j                         t        t        j                  t              r*t        j                  jw                  | j                         n| j                  gt        _P        t        t        j                  t              r*t        j                  jw                  | j                         n| j                  gt        _R        t        t        j                  t              r*t        j                  jw                  | j                         n| j                  gt        _T        t        j                  d| jn                   d| j                   d| jp                   d| jz                   d| jx                   d | j,                  j                   d!        |(       | _X        |$| _Y        |%| _Z        d| _[        t        j                  || j                  "      r|#|#jw                  d#       nd#g}#d| _^        |vt        |t              rt        d)i || _^        nt        |t              r|| _^        t        j                  d$j                  | j                  j                  d%                   || _d        d| _e        |vt        |t              rt        d)i || _e        nt        |t              r|| _e        t        j                  d&j                  | j                  j                  d%                   |&| _g        |#| j                  |#       | j                  | j                          | j                          | j                  t        j                  d'(      | _l        y)*a  
        Initialize the Router class with the given parameters for caching, reliability, and routing strategy.

        Args:
            model_list (Optional[list]): List of models to be used. Defaults to None.
            redis_url (Optional[str]): URL of the Redis server. Defaults to None.
            redis_host (Optional[str]): Hostname of the Redis server. Defaults to None.
            redis_port (Optional[int]): Port of the Redis server. Defaults to None.
            redis_password (Optional[str]): Password of the Redis server. Defaults to None.
            cache_responses (Optional[bool]): Flag to enable caching of responses. Defaults to False.
            cache_kwargs (dict): Additional kwargs to pass to RedisCache. Defaults to {}.
            caching_groups (Optional[List[tuple]]): List of model groups for caching across model groups. Defaults to None.
            client_ttl (int): Time-to-live for cached clients in seconds. Defaults to 3600.
            polling_interval: (Optional[float]): frequency of polling queue. Only for '.scheduler_acompletion()'. Default is 3ms.
            default_priority: (Optional[int]): the default priority for a request. Only for '.scheduler_acompletion()'. Default is None.
            num_retries (Optional[int]): Number of retries for failed requests. Defaults to 2.
            timeout (Optional[float]): Timeout for requests. Defaults to None.
            default_litellm_params (dict): Default parameters for Router.chat.completion.create. Defaults to {}.
            set_verbose (bool): Flag to set verbose mode. Defaults to False.
            debug_level (Literal["DEBUG", "INFO"]): Debug level for logging. Defaults to "INFO".
            fallbacks (List): List of fallback options. Defaults to [].
            context_window_fallbacks (List): List of context window fallback options. Defaults to [].
            enable_pre_call_checks (boolean): Filter out deployments which are outside context window limits for a given prompt
            model_group_alias (Optional[dict]): Alias for model groups. Defaults to {}.
            retry_after (int): Minimum time to wait before retrying a failed request. Defaults to 0.
            allowed_fails (Optional[int]): Number of allowed fails before adding to cooldown. Defaults to None.
            cooldown_time (float): Time to cooldown a deployment after failure in seconds. Defaults to 1.
            routing_strategy (Literal["simple-shuffle", "least-busy", "usage-based-routing", "latency-based-routing", "cost-based-routing"]): Routing strategy. Defaults to "simple-shuffle".
            routing_strategy_args (dict): Additional args for latency-based routing. Defaults to {}.
            alerting_config (AlertingConfig): Slack alerting configuration. Defaults to None.
            provider_budget_config (ProviderBudgetConfig): Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None.
        Returns:
            Router: An instance of the litellm.Router class.

        Example Usage:
        ```python
        from litellm import Router
        model_list = [
        {
            "model_name": "azure-gpt-3.5-turbo", # model alias
            "litellm_params": { # params for litellm completion/embedding call
                "model": "azure/<your-deployment-name-1>",
                "api_key": <your-api-key>,
                "api_version": <your-api-version>,
                "api_base": <your-api-base>
            },
        },
        {
            "model_name": "azure-gpt-3.5-turbo", # model alias
            "litellm_params": { # params for litellm completion/embedding call
                "model": "azure/<your-deployment-name-2>",
                "api_key": <your-api-key>,
                "api_version": <your-api-version>,
                "api_base": <your-api-base>
            },
        },
        {
            "model_name": "openai-gpt-3.5-turbo", # model alias
            "litellm_params": { # params for litellm completion/embedding call
                "model": "gpt-3.5-turbo",
                "api_key": <your-api-key>,
            },
        ]

        router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": "openai-gpt-3.5-turbo"}])
        ```
        r   )ServiceLoggingTrn   r   localNredisurlhostportpasswordtype)redis_cachein_memory_cache)ry   r   modellitellm_params)cachedefault_cooldown_time)fallback_param*)params
router_objr}   max_retriesmetadatarw   r   r   z)Intialized router with Routing strategy: z"

Routing enable_pre_call_checks: z

Routing fallbacks: z

Routing content fallbacks: z$

Routing context window fallbacks: z

Router Redis Caching=
)rp   r   router_budget_limitingz+[32mRouter Custom Retry Policy Set:
{}[0mexclude_nonez3[32mRouter Custom Allowed Fails Policy Set:
{}[0m
moderation)	call_typerd   )mlitellm._service_loggerr   r   r   r   r   litellmsuppress_debug_infor   setLevelloggingrn   r   rG   r   rq   deployment_namesdeployment_latency_maprx   strupdater   r   Cacherj   r   r   r5   	schedulerrz   default_deploymentr   provider_default_deployment_idsr[   pattern_routercopydeepcopyset_model_listrp   healthy_deploymentsr   r&   r   r%   cooldown_cacher   failed_callsr{   openaiDEFAULT_MAX_RETRIESr|   ROUTER_MAX_FALLBACKSrequest_timeoutr}   r   r   r   validate_fallbacksr   appendr   r   r   inttotal_calls
fail_callssuccess_callsprevious_modelsr   Chatchatr~   
setdefaultdeployment_statsrouting_strategy_initaccess_groups
isinstance_async_success_callbacklistdeployment_callback_on_successsuccess_callback#sync_deployment_callback_on_success_async_failure_callback$async_deployment_callback_on_failurefailure_callbackdeployment_callback_on_failuredebugr   service_logger_objr   r   router_budget_loggerr   !should_init_router_budget_limiterr   dictrE   infoformat
model_dumpr   r   r=   r   add_optional_pre_call_checks_initialize_alertinginitialize_assistants_endpointfactory_functionamoderation)/selfrp   rq   rr   rs   rt   ru   rj   rv   rw   rx   ry   rz   r{   r|   r}   r~   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   
cache_typer   cache_configm
_fallbacks_content_policy_fallbackss/                                                  rf   __init__zRouter.__init__   s   b 	;&&&<#$8!&*#t#f$%..w||<'%..w}}=#>'<'> 	$ "3 	 ')#  	 ')$ Z%;
@V J$&/U#%'1V$%'*:V$)+9Z( -$4|4K}}$ ' N: N N#2D #]_


 #-;
 !1"&-J*:<,02!z2J
+-1__D$a 011PQD//2B0CG0LM  
  O $!.D!(!6!6D*K.K+**D4F4F
 "3O 	 "*D  ,&22D%99D$!.D"".!(!6!6D!(!=!=D9'"9"9& 0 3'"3"3
z:#(G,E,E,Q*Gg.G.GJ~~)%%sJ&78#&
"3!4 %H(H(H 	%
 %H(H(H 	" 	/HI(A%(3)
 (3(
 +6+
  	 # 	
 "8!=2LL(>4P	 '=###..y'B##..}a@##..z2>EE~.	
 ')	 	""-"7 	# 	
 "g55t<++2243V3VW++2243V3VWg..5$$++D,T,TU(,(P(P'QG$g55t<++2299
 99/G+ g..5$$++D,O,OP(,(K(K'LG$##78M8M7N O//3/J/J.K L""&..!1 2**.*G*G)H I1151N1N0O P$$(JJ$:$:#;2?	
 #1"2%:"&<#DH!AA!$:U:U
 (3(//0HI,D+E(37#,-$/$?,$?!L+6$0!!&&DKK%%00d0C % 	% CG!+.5,>,VAU,V)02DE,@)!&&LSS--88d8K :I#/--.FG+%%'++-00< 1 
re   c                 D   | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j
                        | _        | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j                        | _	        y N)
r   r   acreate_assistantsadelete_assistantaget_assistantsacreate_threadaget_threada_add_messageaget_messagesarun_threadr   s    rf   r   z%Router.initialize_assistants_endpoint3  s    "&"7"78R8R"S!%!6!6w7P7P!Q#44W5L5LM"33G4J4JK001D1DE!2273H3HI!2273H3HI001D1DEre   r   c           	          |y|D ]J  }t        |t              st        d| d      t        |      dk7  s1t        d| dt        |       d       y)z3
        Validate the fallbacks parameter.
        NzItem 'z' is not a dictionary.rZ   zDictionary 'z%' must have exactly one key, but has z keys.)r   r   
ValueErrorlen)r   r   fallback_dicts      rf   r   zRouter.validate_fallbacks>  sl     !+MmT2 6-8N!OPP=!Q& "=/1VWZ[hWiVjjpq 	 ,re   c                     |y|D ]s  }d }|dk(  rt        | j                        }n1|dk(  r,t        | j                  | j                  | j                        }|Ut
        j                  j                  |       u y y )Nprompt_caching)r   r   )
dual_cacher   rp   )r1   r   r   r   rp   r   	callbacksr   )r   r   pre_call_check	_callbacks       rf   r   z#Router.add_optional_pre_call_checksM  s{     $/":48	!%55 <4:: NI#'?? 4#'::/3/J/J#'??!I
 (%%,,Y7 #; 0re   c                 .   t        j                  d|        |t        j                  j                  k(  s|t        j                  k(  rt        | j                  | j                        | _        t        t        j                  t              r*t        j                  j                  | j                         n| j                  gt        _        t        t        j                  t              r*t        j                  j                  | j                         y y |t        j                  j                  k(  s|t        j                  k(  rpt!        | j                  | j                  |      | _        t        t        j                  t              r*t        j                  j                  | j"                         y y |t        j$                  j                  k(  s|t        j$                  k(  rpt'        | j                  | j                  |      | _        t        t        j                  t              r*t        j                  j                  | j(                         y y |t        j*                  j                  k(  s|t        j*                  k(  rpt-        | j                  | j                  |      | _        t        t        j                  t              r*t        j                  j                  | j.                         y y |t        j0                  j                  k(  s|t        j0                  k(  rpt3        | j                  | j                  i       | _        t        t        j                  t              r*t        j                  j                  | j4                         y y y )NzRouting strategy: )router_cacherp   )r  rp   routing_args)r   r   rK   
LEAST_BUSYvaluer   r   rp   rl   r   r   input_callbackr   r   r  USAGE_BASED_ROUTINGr   rm   USAGE_BASED_ROUTING_V2r   lowesttpm_logger_v2LATENCY_BASEDr   lowestlatency_logger
COST_BASEDr   lowestcost_logger)r   r   r   s      rf   r   zRouter.routing_strategy_init^  s    	""%78H7I#JK : : @ @@?#=#==$;!ZZDOO%D! '00$7&&--d.C.CD*.*?*?)@&'++T2!!(()>)>? 3  C C I II?#F#FF$;!ZZ??2%D!
 '++T2!!(()>)>? 3  F F L LL?#I#II'A!ZZ??2(D$
 '++T2!!(()A)AB 3  = = C CC?#@#@@(C!ZZ??2)D%
 '++T2!!(()B)BC 3  : : @ @@?#=#==%=!ZZ??&D"
 '++T2!!(()?)?@ 3 re   
deploymentc                     	 t        j                  |      }|d   }d|v r|d   dd dz   |d<   |S # t        $ r(}t        j                  dt        |              |d}~ww xY w)z
        returns a copy of the deployment with the api key masked

        Only returns 2 characters of the api key and masks the rest with * (10 *).
        r   api_keyN   z
**********z+Error occurred while printing deployment - )r   r   	Exceptionr   r   r   )r   r  _deployment_copyr   es        rf   print_deploymentzRouter.print_deployment  s}    
	#}}Z8#34D#ENN*,:9,Ebq,IH,Ty)## 	!''=c!fXF G		s   -0 	A!#AA!r   messagesc                     	 t        j                  d| d       ||d<   ||d<   | j                  |d<   | j                  ||        | j                  di |}|S # t
        $ r}|d}~ww xY w)	z
        Example usage:
        response = router.completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hey, how's it going?"}]
        zrouter.completion(model=z,..)r   r&  original_functionr   kwargsNrd   )r   r   _completion_update_kwargs_before_fallbacksfunction_with_fallbacksr"  )r   r   r&  r*  responser$  s         rf   
completionzRouter.completion  s    
	!''*B5'(NO#F7O!)F:*.*:*:F&'00uV0L3t33=f=HO 	G	s   AA 	A+$A&&A+c           	         d }	 | j                  |||j                  dd             }| j                  ||       |d   j                         }|d   }| j	                  ||      }|j                  dd       }||||j                  k7  rd }	n|}	|| j                         vr| j                  |       t        j                  di i ||| j                  |	d|}
t        j                  d	| d
       t        |
t              r.| j!                  ||
|      }|rt        j"                  d|d      |
S # t$        $ r,}t        j                  d	| dt'        |       d       |d }~ww xY w)Nspecific_deploymentr   r&  r1  r  r*  r   r   r   r  r&  cachingclientzlitellm.completion(model=)[32m 200 OK[0mr   r.  r*  Response output was blocked. messager   llm_provider)[31m Exception [0mrd   )get_available_deploymentpop_update_kwargs_with_deploymentr   _get_clientgetr   get_model_ids routing_strategy_pre_call_checksr   r/  rj   r   r   r   rR   "_should_raise_content_policy_errorContentPolicyViolationErrorr"  r   )r   r   r&  r*  
model_namer  datapotential_model_clientdynamic_api_keymodel_clientr.  _should_raiser$  s                rf   r+  zRouter._completion  s    
<	66!$*JJ/Dd$K 7 J
 //:f/U./446DgJ%)%5%5%f &6 &" %jjD9O+*6#'='E'EE#5 D..00555L))  (#33*	
 H "&&+J<7NO
 (M2 $ G G(6 !H ! !!== >#%'  O 	!&&+J<7KCPQF8SZ[ G		s   D1D6 6	E+?'E&&E+streamTc                    K   y wr   rd   r   r   r&  rP  r*  s        rf   acompletionzRouter.acompletion
        	   c                    K   y wr   rd   rR  s        rf   rS  zRouter.acompletion  rT  rU  c                    K   y wr   rd   rR  s        rf   rS  zRouter.acompletion  rT  rU  c                 &  K   	 ||d<   ||d<   ||d<   | j                   |d<   | j                  ||       |j                  d      xs | j                  }t	        j                         }| j                  |      }|r| j                  |||       d {   S |+t        |t              r | j                  di | d {   }n | j                  di | d {   }t	        j                         }	|	|z
  }
t        j                  | j                  j                  t        j                   |
d||	t#        |      	             |S 7 7 7 n# t$        $ r;}t        j                  t'        | |t)        j*                         |
             |d }~ww xY ww)Nr   r&  rP  r(  r)  priority)r   r&  r*  rS  servicedurationr   
start_timeend_timeparent_otel_spanlitellm_router_instancerequest_kwargserror_traceback_stroriginal_exceptionrd   )_acompletionr,  rE  rz   time_is_prompt_management_model_prompt_management_factoryr   r   schedule_acompletionasync_function_with_fallbacksasynciocreate_taskr   async_service_success_hookrL   ROUTERr   r"  r0   	traceback
format_exc)r   r   r&  rP  r*  request_priorityr]  rg  r.  r^  	_durationr$  s               rf   rS  zRouter.acompletion  s    +	#F7O!)F:%F8*.*;*;F&'00uV0L%zz*5N9N9NJ*.*J*J5*Q'*!<<%! =   
  +
;KS0Q!:!:!:!DV!DD!C!C!C!Mf!MMyy{H :-I''BB(//&+)%%Fv%N C 	 O- EM  		(,0#)(1(<(<(>'(	 G		sl   FBE
 EE
 F'E
 >E?E
 EA)E
 FE
 E
 E
 
	F6F		FFc                   K   d}	 t        j                  d| d|        t        |      }t        j                         }| j	                  |||j                  dd      |       d{   }t        j                         }||z
  }	t        j                  | j                  j                  t        j                  |	d||t        |                   | j                  ||       | j                  ||	       |d
   j                         }
|
d   }| j                  ||	      }| j                   |xx   dz  cc<   t#        j$                  di i |
|| j&                  |d|}|j)                  dd      }| j+                  ||d      }|]t-        |t        j.                        rC|4 d{    	 | j1                  |||       d{    | d{   }ddd      d{    n&| j1                  |||       d{    | d{   }t-        t2              r.| j5                  |||      }|rt#        j6                  d|d      | j8                  |xx   dz  cc<   t        j:                  d| d       | j                  |||       |S 7 )7 7 7 7 # 1 d{  7  sw Y   xY w7 7 # t<        $ rE}t        j:                  d| dt?        |       d       || j@                  |xx   dz  cc<   |d}~ww xY ww)z
        - Get an available deployment
        - call it with a semaphore over the call
        - semaphore specific to it's rpm
        - in the semaphore,  make a check against it's local rpm before running
        NzInside _acompletion()- model: 
; kwargs: r1  )r   r&  r1  rb  async_get_available_deploymentrZ  r  r_  r3  r   r   rZ   r5  litellm_logging_objmax_parallel_requestsr  r*  client_type)r  logging_objr_  r9  r:  r;  r<  zlitellm.acompletion(model=r8  )r  r.  r_  r?  r@  rd   )!r   r   r   rf  ru  rB  rk  rl  r   rm  rL   rn  _track_deployment_metricsrC  r   _get_async_openai_model_clientr   r   rS  rj   rE  rD  r   	Semaphore&async_routing_strategy_pre_call_checksrR   rH  rI  r   r   r"  r   r   )r   r   r&  r*  rJ  r_  r]  r  r^  rr  rK  rN  	_responser{  rpm_semaphorer.  rO  r$  s                     rf   re  zRouter._acompletionS  s     
q	!''0z&J  AHJ#BB!$*JJ/Dd$K%	  C   J yy{H :-I''BB(//&>)%%Fv%N C 	 **%8H +  //:f/U./446DgJ>>% ? L Z(A-(++  (#33*	
 I 5;JJ%t5K !,,%3 - M
 (Zw00. )= EE#-$/)9 F   
 &/H )== AA) +%5 B    "+? (M2 $ G G(6 !H ! !!== >#%'  z*a/*!&&,ZL8OP **%!!1 +  OIr )

  / )=== +4  	!&&,ZL8LSQRVHT[\ %
+q0+G	s   LA"J5 (J)EJ5 *J+J5 .JJ	JJJJ5 !J"J5 >J1?	J5 J3	BJ5 LJ5 J5 JJJ5 J."J%#J.*J5 3J5 5	L>A K>>LLr*  c                     |j                  d| j                        |d<   |j                  dt        t	        j
                                      |j                  di       j                  d|i       y)zm
        Adds/updates to kwargs:
        - num_retries
        - litellm_trace_id
        - metadata
        r{   litellm_trace_idr   model_groupN)rE  r{   r   r   uuiduuid4r   )r   r   r*  s      rf   r,  z&Router._update_kwargs_before_fallbacks  s[     !'

=$:J:J K},c$**,.?@*b)00-1GHre   c                     | j                   j                         D ]+  \  }}||vr||||<   |dk(  s||   j                  |       - y)z@
        Adds default litellm params to kwargs, if set.
        Nr   )r~   itemsr   )r   r*  kvs       rf   *_update_kwargs_with_default_litellm_paramsz1Router._update_kwargs_with_default_litellm_params  sN     //557DAqAMq	jq	  # 8re   c                 8   |j                  di       j                  |d   d   |j                  di       |j                  di       j                  d      d       |j                  di       |d<   | j                  ||d         |d<   | j	                  |	       y
)z
        2 jobs:
        - Adds selected deployment, model_info and api_base to kwargs["metadata"] (used for logging)
        - Adds default litellm params to kwargs, if set.
        r   r   r   
model_infoapi_baser  r  r  )r*  rK  r}   r*  N)r   r   rE  _get_timeoutr  )r   r  r*  s      rf   rC  z%Router._update_kwargs_with_deployment  s     	*b)00()9:7C(nn\2>&NN+;R@DDZP	
  *~~lB?| --
+; < . 
y 	77v7Fre   c                     | j                  ||d      }|j                  dd      }||||j                  k7  rd}|S |}|S )a  
        Helper to get AsyncOpenAI or AsyncAzureOpenAI client that was created for the deployment

        The same OpenAI client is re-used to optimize latency / performance in production

        If dynamic api key is provided:
            Do not re-use the client. Pass model_client=None. The OpenAI/ AzureOpenAI client will be recreated in the handler for the llm provider
        asyncry  r   N)rD  rE  r   )r   r  r*  rL  rM  rN  s         rf   r}  z%Router._get_async_openai_model_client  sh     "&!1!1!&g "2 "

 !**Y5'&2#9#A#AAL  2Lre   rK  c                     |j                  dd      xsf |j                  dd      xsR |j                  dd      xs> |j                  dd      xs* | j                  xs | j                  j                  dd      }|S )z6Helper to get timeout from kwargs or deployment paramsr}   Nr   )rE  r}   r~   )r   r*  rK  r}   s       rf   r  zRouter._get_timeout  s     JJy$' 	@zz+T2	@xx4	@
 xx!4	@ ||	@ **..y$? 	 re   modelsc                    K   dt         dt        t           f fd}dt         dt        t           dt        f fd}t	        |t
              rQt        d |D              r?g }|D ]  }|j                   |d||d|        t        j                  |  d{   }|S t	        |t
              rt        d	 |D              rg }t        |      D ]'  \  }	}
|D ]  }|j                   |d||	|
d
|        ) t        j                  |  d{   }t        t        |            D cg c]  }g  }}|D ]A  }t	        |t              r||d      j                  |d          .|d   j                  |       C |S yy7 7 qc c}w w)a6  
        Async Batch Completion. Used for 2 scenarios:
        1. Batch Process 1 request to N models on litellm.Router. Pass messages as List[Dict[str, str]] to use this
        2. Batch Process N requests to M models on litellm.Router. Pass messages as List[List[Dict[str, str]]] to use this

        Example Request for 1 request to N models:
        ```
            response = await router.abatch_completion(
                models=["gpt-3.5-turbo", "groq-llama"],
                messages=[
                    {"role": "user", "content": "is litellm becoming a better product ?"}
                ],
                max_tokens=15,
            )
        ```


        Example Request for N requests to M models:
        ```
            response = await router.abatch_completion(
                models=["gpt-3.5-turbo", "groq-llama"],
                messages=[
                    [{"role": "user", "content": "is litellm becoming a better product ?"}],
                    [{"role": "user", "content": "who is this"}],
                ],
            )
        ```
        r   r&  c                 |   K   	  j                   d| |d| d{   S 7 # t        $ r}|cY d}~S d}~ww xY wwzs
            Wrapper around self.async_completion that catches exceptions and returns them as a result
            r   r&  Nrd   rS  r"  r   r&  r*  r$  r   s       rf   _async_completion_no_exceptionszARouter.abatch_completion.<locals>._async_completion_no_exceptionsJ  D     -T--WEHWPVWWWW 0   <$ "$ <$ 	949<9<idxc                    K   	  j                   d| |d| d{   |fS 7 # t        $ r}||fcY d}~S d}~ww xY wwr  r  )r   r&  r  r*  r$  r   s        rf   *_async_completion_no_exceptions_return_idxzLRouter.abatch_completion.<locals>._async_completion_no_exceptions_return_idxU  sS     *$**TTVTT T  #vs4   A & $& A & 	=8=A =A c              3   <   K   | ]  }t        |t                y wr   )r   r   .0r   s     rf   	<genexpr>z+Router.abatch_completion.<locals>.<genexpr>h  s     -T8ajD.A8   r  Nc              3   <   K   | ]  }t        |t                y wr   )r   r   r  s     rf   r  z+Router.abatch_completion.<locals>.<genexpr>o  s     /VX
1d0CXr  )r   r  r&  rZ   r   rd   )r   r	   r6   r   r   r   allr   rk  gather	enumerateranger	  tuple)r   r  r&  r*  r  r  _tasksr   r.  r  r=  	responses_final_responsess   `             rf   abatch_completionzRouter.abatch_completion&  s    H				"&'7"8				+,	 	& h%#-T8-T*TF=gET\g`fgh   %^^V44HO$'C/VX/V,VF )( 3W#EMMB "'S7FL $ !4 &nnf55I<A#h-<P/Q<Pq<PO/Q%h.#HQK077D#A&--h7	 &
 #"# -W' 5 6/Qs8   BFE;A4FE=F&	E?/AF=F?Fc           	          K   dt         dt        t           f fd}g }|D ]  }|j                   |d||d|        t	        j
                  |  d{   }|S 7 w)a  
        Async Batch Completion - Batch Process multiple Messages to one model_group on litellm.Router

        Use this for sending multiple requests to 1 model

        Args:
            model (List[str]): model group
            messages (List[List[Dict[str, str]]]): list of messages. Each element in the list is one request
            **kwargs: additional kwargs
        Usage:
            response = await self.abatch_completion_one_model_multiple_requests(
                model="gpt-3.5-turbo",
                messages=[
                    [{"role": "user", "content": "hello"}, {"role": "user", "content": "tell me something funny"}],
                    [{"role": "user", "content": "hello good mornign"}],
                ]
            )
        r   r&  c                 |   K   	  j                   d| |d| d{   S 7 # t        $ r}|cY d}~S d}~ww xY wwr  r  r  s       rf   r  z]Router.abatch_completion_one_model_multiple_requests.<locals>._async_completion_no_exceptions  r  r  r  Nrd   )r   r	   r6   r   rk  r  )r   r   r&  r*  r  r  message_requestr.  s   `       rf   -abatch_completion_one_model_multiple_requestsz4Router.abatch_completion_one_model_multiple_requests  sv     ,				"&'7"8		 'OMM/ /=C  ( !00 1s   AAAAc                    K   y wr   rd   rR  s        rf   "abatch_completion_fastest_responsez)Router.abatch_completion_fastest_response  rT  rU  c                    K   y wr   rd   rR  s        rf   r  z)Router.abatch_completion_fastest_response  rT  rU  c                    K   |j                  d      D cg c]  }|j                          }}dt        dt        t        t        t        f      dt
        dt        dt        t        t        t        f   f
 fd}g dt        j                  ffd	}|D ]2  }t        j                   |d|||d
|      }	j                  |	       4 r_t        j                  t        j                          d{   \  }
|
D ]'  } ||       d{   }|d|j"                  d<   |c S  r_t        d      c c}w 7 F7 0w)z
        model - List of comma-separated model names. E.g. model="gpt-4, gpt-3.5-turbo"

        Returns fastest response from list of model names. OpenAI-compatible endpoint.
        ,r   r&  rP  r*  r   c                    K   	  j                   d| ||d| d{   S 7 # t        j                  $ r& t        j                  dj                  |               t        $ r}|cY d}~S d}~ww xY ww)zn
            Wrapper around self.acompletion that catches exceptions and returns them as a result
            r   r&  rP  Nz4Received 'task.cancel'. Cancelling call w/ model={}.rd   )rS  rk  CancelledErrorr   r   r   r"  )r   r&  rP  r*  r$  r   s        rf   r  zRRouter.abatch_completion_fastest_response.<locals>._async_completion_no_exceptions  sw     -T--fEHU[f_effff)) %++JQQRWX  s>   A4% #% A4% A A1%A,&A1'A4,A11A4taskc                   K   	 |  d {   }t        |t        t        f      r@t        j                  d       D ]  }|j                           |	 j                  |        S 	 	 j                  |        y 7 n# t        $ r Y S w xY w# t        $ r Y /w xY w# t        $ r Y y w xY w# 	 j                  |        w # t        $ r Y w w xY wxY ww)Nz=Received successful response. Cancelling other LLM API calls.)	r   rR   rP   r   r   cancelremoveKeyErrorr"  )r  resulttpending_taskss      rf   check_responsezARouter.abatch_completion_fastest_response.<locals>.check_response  s     #f}6I&JK)//W +
 +!!((. L!((. $         !((. s   CB
 A9AB
 A;$C%B( 'B 8C9B
 ;	BCBC
	BB( BB( 	B%"C$B%%C(C*B<;C<	CCCCCr  )return_whenNT!fastest_response_batch_completionzAll tasks failedrd   )splitstripr   r	   r   boolr   r   rR   rP   r"  rk  Taskrl  r   waitFIRST_COMPLETED_hidden_params)r   r   r&  rP  r*  r   r  r  r  r  donecompleted_taskr  r  s   `            @rf   r  z)Router.abatch_completion_fastest_response  sV     &+[[%56%5!'')%56		"&tCH~"6	@D	PS	="5y@A	  	w|| 	, E&&/ (6EKD
   &  (/7+B+B) #D- #'-n==%QUF))*MN!M #'	  *++ 7h# >s:   D?D6CD?4D;5D?D=D?D?+D?=D?rY  c                    K   y wr   rd   r   r   r&  rY  rP  r*  s         rf   ri  zRouter.schedule_acompletion  rT  rU  c                    K   y wr   rd   r  s         rf   ri  zRouter.schedule_acompletion  rT  rU  c                   K   t        |      }t        t        j                               }t	        ||d      }| j
                  j                  |       d {    t        j                         | j                  z   }	t        j                         }
| j
                  j                  }d}|
|	k  r| j                  ||       d {   \  }}| j
                  j                  |j                  |j                  |       d {   }|rn7t        j                  |       d {    t        j                         }
|
|	k  r|r]	  | j                   d|||d| d {   }|j"                  j%                  di        |j"                  d   j'                  d	d
i       |S t-        j.                  d|d      7 `7 7 7 7 c# t(        $ r}t+        |d|       |d }~ww xY ww)Nzgpt-3.5-turborY  
request_idrJ  requestFr   r_  idrJ  health_deploymentsr  additional_headers%x-litellm-request-prioritization-usedTrY  %Request timed out while polling queuer   r<  rd   )r   r   r  r  r4   r   add_requestrf  r}   ry   _async_get_healthy_deploymentspollr  rJ  rk  sleeprS  r  r   r   r"  setattrr   Timeout)r   r   r&  rY  rP  r*  r_  _request_iditemr^  	curr_timepoll_intervalmake_request_healthy_deploymentsr  r  r$  s                    rf   ri  zRouter.schedule_acompletion   s     =VD$**,'"&
 nn(((666 99;-IIK	77(",0,O,O.> -P - '# ! "&!4!4????#7 "5 " L
 mmM222 IIK	 (" "2$"2"2 #(6#EK# 	 ((334H"M(()=>EE<dC ! 
 //?% K 	7' 3
  :x0s   AG#F8A,G#F;<G#F=G#"F?#G#G#G GAG G#;G#=G#?G#G 	G GG  G#r(  args.c                   K   t        |      }t        t        j                               }t	        |||      }| j
                  j                  |       d {    t        j                         | j                  z   }	t        j                         }
| j
                  j                  }d}|
|	k  r| j                  ||       d {   \  }}| j
                  j                  |j                  |j                  |       d {   }|rn7t        j                  |       d {    t        j                         }
|
|	k  r|ri	  ||i | d {   }t!        |j"                  t$              r<|j"                  j'                  di        |j"                  d   j)                  ddi       |S t/        j0                  d
|d      7 l7 7 7 7 ~# t*        $ r}t-        |d	|       |d }~ww xY ww)Nr  r  Fr  r  r  r  TrY  r  r   r<  )r   r   r  r  r4   r   r  rf  r}   ry   r  r  r  rJ  rk  r  r   r  r   r   r   r"  r  r   r  )r   r   rY  r(  r  r*  r_  r  r  r^  r  r  r  r  r  r  r$  s                    rf   _schedule_factoryzRouter._schedule_factory^  s     =VD$**,'"
 nn(((666 99;-IIK	77(",0,O,O.> -P - '# ! "&!4!4????#7 "5 " L
 mmM222 IIK	 (" 
"3T"DV"DD	i66=,,778LbQ,,-ABII@$G ! 
 //?% I 	7' 3
 E  :x0s   AG0GA,G0G<G0G
G0"G#G0G0G GAG +G0G0
G0G0G 	G-G((G--G0c                     | j                  |      }|yt        |      dk7  ry|d   d   j                  dd       }|yd|v r'|j                  d      d   }|t        j
                  v ryy)	NrJ  FrZ   r   r   r   /T)get_model_listr	  rE  r  r   )_known_custom_logger_compatible_callbacks)r   r   rp   litellm_modelsplit_litellm_models        rf   rg  z"Router._is_prompt_management_model  s    ((E(:
z?a"1&67;;GTJ -"/"5"5c":1"="g&W&WWre   c                   K   |j                  dd       }|$t        di dt               t               d|\  }}t	        t
        |      }| j                  |dddg|j                  dd             }|d	   j                  d
d       }|j                  d      xs |d	   j                  dd       }|j                  d      xs |d	   j                  dd       }|t        |t              st        d| dt        |             |*t        |t              st        d| dt        |             |j                  ||t        |      ||      \  }}}	i ||	}||d
<   ||d<   ||d<   ||d<   ||d<   | j                  |      }
|
t!        |
      dk(  r.|j                  d       t#        j$                  di | d {   S  | j&                  di | d {   S 7 7 w)Nrw  rS  )r(  	rules_objr]  userpromptrolecontentr1  r2  r   r   	prompt_idprompt_variablesz*Prompt ID is not set or not a string. Got=z, type=z2Prompt variables is set but not a dictionary. Got=r  )r   r&  non_default_paramsr  r  r&  r  r   r(  rd   )rE  rT   rS   rX   r   LiteLLMLoggingrA  rB  r   r   r  r   r   get_chat_completion_promptrV   r  r	  r   rS  rj  )r   r   r&  r*  litellm_logging_objectprompt_management_deploymentr  r  r  optional_params_model_lists              rf   rh  z!Router._prompt_management_factory  sg     "(,A4!H!)-; .)6!&"2"4 	.*"F "&n6L!M'+'D'D%(;< &

+@$ G (E (
$ 55EFJJT
 JJ{+ !/K0

#k4
  	 "::
 
)*:;??
 	 Jy#$><YKwtT]N_`  '
;KT0RDEUDVV]^bcs^t]uv 
 #==#!#DF#S#!1 >  	)x /F.o.w%z(>$%'{%5!"))U);#k"2a"7JJ*+ ,,6v6667T77A&AAA 7As$   F<G!>G?G!GG!G!r  c                    	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   |j                  di       j	                  d|i        | j
                  di |}|S # t        $ r}|d }~ww xY w)Nr   r  r(  r{   r   r  rd   )_image_generationrE  r{   r   r   r-  r"  r   r  r   r*  r.  r$  s         rf   image_generationzRouter.image_generation  s    
	#F7O%F8*.*@*@F&'$*JJ}d>N>N$OF=!j"-44mU5KL3t33=f=HO 	G	   A.A1 1	B:A<<Bc           	         d}	 t        j                  d| d|        | j                  |dddg|j                  dd             }| j	                  ||	       |d
   j                         }| j                  ||	      }| j                  |xx   dz  cc<   | j                  |       t        j                  di i ||| j                  |d|}| j                  |xx   dz  cc<   t        j                  d| d       |S # t        $ rE}	t        j                  d| dt        |	       d       || j                   |xx   dz  cc<   |	d }	~	ww xY w)Nr;  #Inside _image_generation()- model: rt  r  r  r  r1  r2  r3  r   rZ   r4  r  r6  r7  zlitellm.image_generation(model=r8  r?  r@  rd   )r   r   rA  rB  rC  r   r}  r   rG  r   r
  rj   r   r   r"  r   r   )
r   r  r   r*  rJ  r  rK  rN  r.  r$  s
             rf   r  zRouter._image_generation  s   
*	!''5eWJvhO 66#)h?@$*JJ/Dd$K 7 J
 //:f/U./446D>>% ? L
 Z(A-( 11Z1H// $#33*	
 H z*a/*!&&1*=TU O 	!&&1*=QRUVWRXQYY`a %
+q0+G	s   C?D 	EA EEc           	      l  K   	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   | j                  ||        | j                  di | d {   }|S 7 # t
        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)Nr   r  r(  r{   r)  r`  rd   )_aimage_generationrE  r{   r,  rj  r"  rk  rl  r0   ro  rp  r	  s         rf   aimage_generationzRouter.aimage_generation,  s     	#F7O%F8*.*A*AF&'$*JJ}d>N>N$OF=!00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B4A A- $A+%A- *B4+A- -	B166B,,B11B4c           	      4  K   |}	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd              d {   }| j                  ||       |d	   j                         }|d
   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   dz  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   dz  cc<   |d }~ww xY ww)Nr  rt  r  r  r  r1  r2  r3  r   r   rZ   r  rx  ry  rv  z litellm.aimage_generation(model=r8  r?  r@  rd   )r   r   r   ru  rB  rC  r   r}  r   r   r  rj   rD  r   rk  r~  r  r   r   r"  r   r   )r   r  r   r*  rJ  r_  r  rK  rN  r.  r  r$  s               rf   r  zRouter._aimage_generationA  s    
C	!''5eWJvhO  AH#BB#)h?@$*JJ/Dd$K  C   J
 //:f/U./446DgJ>>% ? L
 Z(A-(00 $#33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&2:,>UV OoH )
  . )=== *  	!&&2:,>RSVWXSYRZZab %
+q0+G	s   HAG F#B8G F&G F.,F(-	F.6F*7F.;G F,G "G#	G ,G-5G "H#G &G (F.*F.,G .G 4F75G <G G 	HA HHHfilec           	      .  K   	 ||d<   ||d<   | j                   |d<   | j                  ||        | j                  di | d{   }|S 7 # t        $ r;}t	        j
                  t        | |t        j                         |             |d}~ww xY ww)a  
        Example Usage:

        ```
        from litellm import Router
        client = Router(model_list = [
            {
                "model_name": "whisper",
                "litellm_params": {
                    "model": "whisper-1",
                },
            },
        ])

        audio_file = open("speech.mp3", "rb")
        transcript = await client.atranscription(
        model="whisper",
        file=audio_file
        )

        ```
        r   r  r(  r)  Nr`  rd   )	_atranscriptionr,  rj  r"  rk  rl  r0   ro  rp  )r   r  r   r*  r.  r$  s         rf   atranscriptionzRouter.atranscription  s     .	#F7O!F6N*.*>*>F&'00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		;   BAA AA BA 	B6BBBc           	      *  K   |}	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd              d {   }| j                  ||       |d	   j                         }| j                  ||      }| j                  |xx   d
z  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   d
z  cc<   t        j$                  d| d       |	S 7 G7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   d
z  cc<   |d }~ww xY ww)Nz!Inside _atranscription()- model: rt  r  r  r  r1  r2  r3  r   rZ   )r  r6  r7  rx  ry  rv  zlitellm.atranscription(model=r8  r?  r@  rd   )r   r   r   ru  rB  rC  r   r}  r   r   r  rj   rD  r   rk  r~  r  r   r   r"  r   r   )r   r  r   r*  rJ  r_  r  rK  rN  r.  r  r$  s               rf   r  zRouter._atranscription  s    
A	!''3E7*VHM  AH#BB#)h?@$*JJ/Dd$K  C   J //:f/U./446D>>% ? L
 Z(A-(--  #33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&/
|;RS OkD )
  . )=== *  	!&&/
|;OPSTUPVxW^_ %
+q0+G	s   HAG FB3G F!G F)'F#(	F)1F%2F)6G F'G F>	G 'G (5G HG !G #F)%F)'G )F;/F20F;7G  G 	HA HHHinputvoicec           	        K   	 ||d<   ||d<   | j                  |dddg|j                  dd             d{   }| j                  ||	       |d
   j                         }|d    | j                  j                         D ])  \  }}||vr|||<   |dk(  s||   j                  |       + | j                  ||d      }	|j                  dd      }
|
|	|
|	j                  k7  rd}n|	}t        j                  di i |d|i| d{   }|S 7 7 # t        $ r;}t        j                  t        | |t!        j"                         |             |d}~ww xY ww)a  
        Example Usage:

        ```
        from litellm import Router
        client = Router(model_list = [
            {
                "model_name": "tts",
                "litellm_params": {
                    "model": "tts-1",
                },
            },
        ])

        async with client.aspeech(
            model="tts",
            voice="alloy",
            input="the quick brown fox jumped over the lazy dogs",
            api_base=None,
            api_key=None,
            organization=None,
            project=None,
            max_retries=1,
            timeout=600,
            client=None,
            optional_params={},
        ) as response:
            response.stream_to_file(speech_file_path)

        ```
        r  r  r  r  r  r1  Nr2  r)  r   r   r   r  ry  r   r7  r`  rd   )ru  rB  r,  r   r~   r  r   rD  rE  r   r   aspeechr"  rk  rl  r0   ro  rp  )r   r   r  r  r*  r  rK  r  r  rL  rM  rN  r.  r$  s                 rf   r  zRouter.aspeech  s    @3	#F7O#F7O#BB#)h?@$*JJ/Dd$K  C   J
 00uV0L./446DM3399;1VO !F1I*_1I$$Q' < &*%5%5%f' &6 &" %jjD9O+*6#'='E'EE#5$__ l  H OK<  		(,0#)(1(<(<(>'(	 G		sR   E5D DAD A4D DD ED D 	E!6EEEc           	      6  K   	 ||d<   t         |d<   | j                  |d<   | j                  ||        | j                  di | d {   }|S 7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY wwNr   r  r(  r)  r`  rd   )
r  _arerankr,  rj  r"  rk  rl  r0   ro  rp  r   r   r*  r.  r$  s        rf   arerankzRouter.arerankL  s     	#F7O#F7O*.--F&'00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   BAA 	A
A BA 	B6BBBc           	        K   d }	 t        j                  d| d|        | j                  ||j                  dd              d {   }| j	                  ||       |d   j                         }|d   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i || j                  |d	| d {   }| j                  |xx   dz  cc<   t        j                  d
| d       |S 7 7 8# t        $ rE}t        j                  d
| dt        |       d       || j                  |xx   dz  cc<   |d }~ww xY ww)NzInside _rerank()- model: rt  r1  )r   r1  r3  r   r   rZ   )r6  r7  zlitellm.arerank(model=r8  r?  r@  rd   )r   r   ru  rB  rC  r   r}  r   r   r!  rj   r   r   r"  r   r   )	r   r   r*  rJ  r  rK  rN  r.  r$  s	            rf   r  zRouter._areranka  s    
&	!''+E7*VHE  $BB$*JJ/Dd$K  C   J //:f/U./446DgJ>>% ? L Z(A-($__ #33* 	 H z*a/*!&&(4KL O7  	!&&(4HQPWX %
+q0+G	sO   EAD DBD D5D ED D 	EA EEEc                   K   dddg}	 |j                  d| j                        |d<   | j                  ||       | j                  |||j	                  dd              d {   }|d   j                         }| j                  j                         D ])  \  }}||vr|||<   |d	k(  s||   j                  |       + t        j                  di i |d
| j                  i| d {   S 7 7 # t        $ rO}| j                  dkD  r9||d<   ||d<   | j                  |d<    | j                  di | d {  7  cY d }~S |d }~ww xY ww)Nr  z
dummy-textr  r{   r)  r1  r2  r   r   r6  r   r   r&  r(  rd   )rE  r{   r,  ru  rB  r   r~   r  r   r   
_arealtimerj   r"  async_function_with_retries)	r   r   r*  r&  r  rK  r  r  r$  s	            rf   r$  zRouter._arealtime  sr    #=>	$*JJ}d>N>N$OF=!00uV0L  $BB!$*JJ/Dd$K  C   J ./446D3399;1VO !F1I*_1I$$Q' < !++b.a.ay$BVBV.aZ`.abbb c 	!#"'w%-z".2oo*+=T==GGGGG	ss   EAC4 #C0$AC4 +A C4 +C2,C4 /E0C4 2C4 4	E==E:D=;E?E EEEEis_retryis_fallbackis_asyncc                    d|dg}	 ||d<   ||d<   |j                  d| j                        |d<   |j                  di       j                  d|i       | j	                  |||j                  dd       	      }|d
   j                         }	| j                  j                         D ])  \  }
}|
|vr|||
<   |
dk(  s||
   j                  |       + t        j                  di i |	|| j                  d|S # t        $ r}|d }~ww xY w)Nr  r  r   r  r{   r   r  r1  r2  r   )r  r6  rd   )rE  r{   r   r   rA  rB  r   r~   r  r   text_completionrj   r"  )r   r   r  r&  r'  r(  r*  r&  r  rK  r  r  r$  s                rf   r*  zRouter.text_completion  s1    $78	#F7O%F8$*JJ}d>N>N$OF=!j"-44mU5KL 66!$*JJ/Dd$K 7 J ./446D3399;1VO !F1I*_1I$$Q' < **s-r-rSWSgSg-rkq-rss 	G	s   B2C9 ;=C9 9	D	DD	c           	        K   |j                  dd       9| j                  ||j                  d      | j                  ||f|       d {   S 	 ||d<   ||d<   | j                  |d<   | j                  ||        | j                  di | d {   }|S 7 M7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)	NrY  )r   rY  r(  r  r*  r   r  r(  r)  r`  rd   )rE  r  rB  atext_completion_atext_completionr,  rj  r"  rk  rl  r0   ro  rp  )	r   r   r  r&  r'  r(  r*  r.  r$  s	            rf   r,  zRouter.atext_completion  s      ::j$'3//J/"&"7"7V_ 0   	#F7O%F8*.*@*@F&'00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		sI   AC"	B
C"AB BB C"B 	C$6CCC"c           	      0  K   	 t        j                  d| d|        t        |      }| j                  |d|dg|j	                  dd              d {   }| j                  ||       |d   j                         }|d	   }| j                  ||      }| j                  |xx   d
z  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   d
z  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   d
z  cc<   |d }~ww xY ww)N#Inside _atext_completion()- model: rt  r  r  r1  r2  r3  r   r   rZ   r  rx  ry  rv  zlitellm.atext_completion(model=r8  r?  r@  rd   )r   r   r   ru  rB  rC  r   r}  r   r   r,  rj   rD  r   rk  r~  r  r   r   r"  r   r   )r   r   r  r*  r_  r  rK  rJ  rN  r.  r  r$  s               rf   r-  zRouter._atext_completion  s    B	!''5eWJvhO  AH#BB#)f=>$*JJ/Dd$K  C   J
 //:f/U./446DgJ>>% ? L Z(A-(// $#33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&1*=TU OmF )
  . )=== *  	!&&1%8LSQRVHT[\  &!+&G	   HAG F!B8G F$G F,*F&+	F,4F(5F,9G F*G  G!	G *G+5G  H!G $G &F,(F,*G ,F>2F53F>:G G 	HA HHH
adapter_idc           	        K   	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   |j                  di       j	                  d|i        | j
                  di | d {   }|S 7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)	Nr   r1  r(  r{   r   r  r`  rd   )_aadapter_completionrE  r{   r   r   rj  r"  rk  rl  r0   ro  rp  )	r   r1  r   r&  r'  r(  r*  r.  r$  s	            rf   aadapter_completionzRouter.aadapter_completion:  s     	#F7O#-F< *.*C*CF&'$*JJ}d>N>N$OF=!j"-44mU5KL?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   CA0A= 4A;5A= :C;A= =	C6B<<CCc           	      0  K   	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd              d {   }| j                  ||       |d	   j                         }|d
   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   dz  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   dz  cc<   |d }~ww xY ww)Nz&Inside _aadapter_completion()- model: rt  r  zdefault textr  r1  r2  r3  r   r   rZ   )r1  r6  r7  rx  ry  rv  z"litellm.aadapter_completion(model=r8  r?  r@  rd   )r   r   r   ru  rB  rC  r   r}  r   r   r4  rj   rD  r   rk  r~  r  r   r   r"  r   r   )r   r1  r   r*  r_  r  rK  rJ  rN  r.  r  r$  s               rf   r3  zRouter._aadapter_completionW  s    B	!''8z&R  AH#BB#)nEF$*JJ/Dd$K  C   J
 //:f/U./446DgJ>>% ? L Z(A-(22 ",#33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&4ZL@WX OmF )
  . )=== *  	!&&4UG;OPSTUPVxW^_  &!+&G	r0  c                    	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   |j                  di       j	                  d|i        | j
                  di |}|S # t        $ r}|d }~ww xY w)Nr   r  r(  r{   r   r  rd   )
_embeddingrE  r{   r   r   r-  r"  r   r   r  r(  r*  r.  r$  s          rf   	embeddingzRouter.embedding  s    		#F7O#F7O*.//F&'$*JJ}d>N>N$OF=!j"-44mU5KL3t33=f=HO 	G	r  c           	         d }	 t        j                  d| d|        | j                  |||j                  dd             }| j	                  ||       |d   j                         }|d   }| j                  ||d	      }|j                  d
d       }||||j                  k7  rd }	n|}	| j                  |xx   dz  cc<   | j                  |       t        j                  di i ||| j                  |	d|}
| j                  |xx   dz  cc<   t        j                  d| d       |
S # t         $ rE}t        j                  d| dt#        |       d       || j$                  |xx   dz  cc<   |d }~ww xY w)NzInside embedding()- model: rt  r1  r   r  r1  r3  r   r   syncry  r   rZ   r4  r  r6  r7  zlitellm.embedding(model=r8  r?  r@  rd   )r   r   rA  rB  rC  r   rD  rE  r   r   rG  r   r9  rj   r   r   r"  r   r   )r   r  r   r*  rJ  r  rK  rL  rM  rN  r.  r$  s               rf   r7  zRouter._embedding  s   
4	!''-eWJvhG 66$*JJ/Dd$K 7 J
 //:f/U./446DgJ%)%5%5%f& &6 &" %jjD9O+*6#'='E'EE#5Z(A-( 11Z1H(( "#33*	
 H z*a/*!&&*:,6MN O 	!&&*:,6J3q6(RYZ %
+q0+G	s   D+D0 0	E>9A E99E>c           	      .  K   	 ||d<   ||d<   | j                   |d<   | j                  ||        | j                  di | d {   }|S 7 # t        $ r;}t	        j
                  t        | |t        j                         |             |d }~ww xY wwr  )	_aembeddingr,  rj  r"  rk  rl  r0   ro  rp  r8  s          rf   
aembeddingzRouter.aembedding  s     	#F7O#F7O*.*:*:F&'00uV0L?T??I&IIHO J 		(,0#)(1(<(<(>'(	 G		r  c           	      ,  K   d }	 t        j                  d| d|        t        |      }| j                  |||j	                  dd              d {   }| j                  ||       |d   j                         }|d   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i ||| j                  |d	|}	| j                  ||d
      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   dz  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   dz  cc<   |d }~ww xY ww)NzInside _aembedding()- model: rt  r1  r;  r3  r   r   rZ   r=  rx  ry  rv  zlitellm.aembedding(model=r8  r?  r@  rd   )r   r   r   ru  rB  rC  r   r}  r   r   r@  rj   rD  r   rk  r~  r  r   r   r"  r   r   )r   r  r   r*  rJ  r_  r  rK  rN  r.  r  r$  s               rf   r?  zRouter._aembedding  s    
A	!''/wjI  AH#BB$*JJ/Dd$K  C   J
 //:f/U./446DgJ>>% ? L
 Z(A-()) "#33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&+J<7NO OkD )
  . )=== *  	!&&+J<7KCPQF8SZ[ %
+q0+G	s   HAG FB8G F"G F*(F$)	F*2F&3F*7G F(G F?	G (G)5G HG "G $F*&F*(G *F<0F31F<8G G 	HA HHHc           	      b  K   	 ||d<   | j                   |d<   |j                  d| j                        |d<   | j                  ||        | j                  di | d {   }|S 7 # t
        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY wwNr   r(  r{   r)  r`  rd   )_acreate_filerE  r{   r,  rj  r"  rk  rl  r0   ro  rp  r   s        rf   acreate_filezRouter.acreate_fileE	  s     
	#F7O*.*<*<F&'$*JJ}d>N>N$OF=!00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		;   B/AA( A& A( %B/&A( (	B,16B''B,,B/c                   K   	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd              d {   }| j                  ||       |d	   j                         }|d
   }| j                  ||      }| j                  |xx   dz  cc<   t        |d
         \  }}	}
}
t        |d   |      |d<   t        j                  di i ||	| j                  |d|}| j                  ||d      }|\t        |t         j"                        rB|4 d {    	 | j%                  ||       d {    | d {   }d d d       d {    n%| j%                  ||       d {    | d {   }| j&                  |xx   dz  cc<   t        j(                  d| d       |S 7 s7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t*        $ rH}t        j,                  d| d| dt/        |       d       || j0                  |xx   dz  cc<   |d }~ww xY ww)Nr/  rt  r  files-api-fake-textr  r1  r2  r3  r   r   rZ   r   r  )file_contentnew_model_namecustom_llm_providerr6  r7  rx  ry  rv  litellm.acreate_file(model=r8  , r?  r@  rd   )r   r   r   ru  rB  rC  r   r}  r   rU   r#   r   rE  rj   rD  r   rk  r~  r  r   r   r"  	exceptionr   r   )r   r   r*  r_  r  rK  rJ  rN  stripped_modelrM  r  r.  r  r$  s                 rf   rD  zRouter._acreate_file]	  s    
J	!''5eWJvhO  AH#BB#)6KLM$*JJ/Dd$K  C   J
 //:f/U./446DgJ>>% ? L Z(A-( 9I7m95N/A 4#F^NF6N ++ +>#33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&-j\9PQ O}V )
  . )=== *  	!++-eWBvh>RSVWXSYRZZab  &!+&G	s   I AG, GCG, 5G6G, 9GG	GGG G, +G,G, G(	G, G*5G, I G, G, GGG, G%GG%!G, *G, ,	H=5AH88H==I c           	      b  K   	 ||d<   | j                   |d<   |j                  d| j                        |d<   | j                  ||        | j                  di | d {   }|S 7 # t
        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY wwrC  )_acreate_batchrE  r{   r,  rj  r"  rk  rl  r0   ro  rp  r   s        rf   acreate_batchzRouter.acreate_batch	  s     
	#F7O*.*=*=F&'$*JJ}d>N>N$OF=!00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		rF  c                 T  K   	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd              d {   }t        d	      }|j                  |i       j                  |d
   d   |j                  di       |j                  d
i       j                  d      d       |j                  di       |d<   |d
   j                         }|d   }| j                  ||       | j                  ||      }| j                  |xx   dz  cc<   t        |d         \  }	}
}	}	t        j                  di i ||
| j                   |d|}| j#                  ||d      }|\t%        |t&        j(                        rB|4 d {    	 | j+                  ||       d {    | d {   }d d d       d {    n%| j+                  ||       d {    | d {   }| j,                  |xx   dz  cc<   t        j.                  d| d       |S 7 7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t0        $ rH}t        j2                  d| d| dt5        |       d       || j6                  |xx   dz  cc<   |d }~ww xY ww)Nz Inside _acreate_batch()- model: rt  r  rH  r  r1  r2  rS  )function_namer   r   r  r  r  r3  rZ   rI  rL  rx  ry  rv  rN  r8  zlitellm._acreate_batch(model=rO  r?  r@  rd   )r   r   r   ru  rB  r"   r   r   rE  r   rC  r}  r   rU   r   rT  rj   rD  r   rk  r~  r  r   r   r"  rP  r   r   )r   r   r*  r_  r  metadata_variable_namerK  rJ  rN  r  rM  r.  r  r$  s                 rf   rS  zRouter._acreate_batch	  s(    
P	!''25'F8L  AH#BB#)6KLM$*JJ/Dd$K  C   J
 &H.&" 4b9@@",-=">w"G",..r"B */? D H H T $.>>,#CF< ./446DgJ//:f/U>>% ? L Z(A-( ,<$w-+P(A"Aq,, +>#33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&-j\9PQ OIb )
  . )=== *  	!++/wb@TUXYZU[T\\cd  &!+&G	s   J(AI H0EI H3I !H;9H5:	H;H7H;I H9I /I0	I 9I:5I /J(0I 3I 5H;7H;9I ;III	I I 	J%AJ  J%%J(c           	        K   	 | j                         }|t        d      g fd}t        j                  |D cg c]
  } ||       c}ddi d{   }|D ]  }t	        |t
              s|c S  rd   t        dj                              c c}w 7 E# t        $ r;}t        j                  t        | t        j                         |             |d}~ww xY ww)	z
        Iterate through all models in a model group to check for batch

        Future Improvement - cache the result.
        NRouter not yet initialized.c                   K   	 t        | d   d         \  }}}}t        j                        }|j                  dd        t	        j
                  dd|i| d {   S 7 # t        $ r}j                  |       Y d }~y d }~ww xY ww)Nr   r   rI  rM  rd   )rU   r   r   rB  r   aretrieve_batchr"  r   )rJ  r  rM  
new_kwargsr$  r*  receieved_exceptionss        rf   try_retrieve_batchz2Router.aretrieve_batch.<locals>.try_retrieve_batch.
  s       4D()9:7C40A*Aq "&v!6JNN#8$?!(!8!8 ",?"CM"    !  (//2 sA   BAA$ A"A$ !B"A$ $	B-B>BBBreturn_exceptionsTr   z7Unable to find batch in any model. Received errors - {}r`  )r  r"  rk  r  r   r7   r   rl  r0   ro  rp  )	r   r*  filtered_model_listr^  r   resultsr  r$  r]  s	    `      @rf   r[  zRouter.aretrieve_batch
  s    6	"&"5"5"7"* =>>#%  " $NN9LM9L$U+9LM"& G "fe,!M "
 $*1-- IPP(  N(  		(,0#)(1(<(<(>'(	 G		sK   C&8B B
B BB 2B 5C&6(B 	C#(6CC##C&c                   K   | j                  |      }|t        d      dt        ffd}t        j                  |D cg c]
  } ||       c}  d{   }dg dddd}|D ]g  }||d	   t        |d	      rt        |d	      |d	<   t        |d
      |d
<   |d   j                  |j                         t        |dd      du scd|d<   i |S c c}w 7 w)zQ
        Return all the batches across all deployments of a model group.
        r  NrY  r   c                 x   K   	 t        j                  di i | d    d {   S 7 # t        $ r Y y w xY ww)Nr   rd   )r   alist_batchesr"  r)  s    rf   r^  z0Router.alist_batches.<locals>.try_retrieve_batchk
  sT     $22 ;/0;F;     s(   :+ )+ :+ 	7:7:r   F)objectrK  first_idlast_idhas_morerf  rg  rK  rh  T)	r  r"  rA   rk  r  hasattrgetattrextendrK  )r   r   r*  r`  r^  ra  final_resultsr  s     `     rf   rd  zRouter.alist_batches^
  s     #11U1C&9::	,? 	  5HI5HE '5HI
 

 
 F! ,49T07
0KM*-+269+Ei(f%,,V[[9 6:u5=04M*-  / J
s*   ?CCCCC-AC
Cc                    K   |j                  d      r=| j                  |d         r(| j                  |d          d {   }|d   d   |d<    |di | d {   S 7 7 w)Nr   r  rI  r   rd   )rE  r  ru  )r   r(  r*  r  s       rf   )_pass_through_moderation_endpoint_factoryz0Router._pass_through_moderation_endpoint_factory
  s|     
 ::g4#6#6&/#6#R#BBWo  C   J ))9:7CF7O&0000	 1s#   ?A%A!A%A#A%#A%r   )
assistantsr   c                 T     	 	 ddt         t        d      dt         d   f fd}|S )NrM  r   azurer7  r   c                    K   dk(  r j                   d| |d| d {   S dk(  r j                  ddi| d {   S y 7 &7 w)Nro  )r(  rM  r7  r   r(  rd   ))_pass_through_assistants_endpoint_factoryrn  )rM  r7  r*  r   r(  r   s      rf   new_functionz-Router.factory_function.<locals>.new_function
  s     
 L(KTKK &7(;! 	   l*KTKK &7   +s!   AA	 AAAANN)r   r
   )r   r(  r   ru  s   ``` rf   r   zRouter.factory_function
  s:     IM.2	!)'2C*D!E	]+	& re   rM  rq  r7  c                    K   |E| j                   .| j                   d   }|j                  | j                   d          nt        d       |d||d| d{   S 7 w)z@Internal helper function to pass through the assistants endpointNrM  r   z'custom_llm_provider' must be set. Either via:
 `Router(assistants_config={'custom_llm_provider': ..})` 
or
 `router.arun_thread(custom_llm_provider=..)`)rM  r7  rd   )rq   r   r"  )r   r(  rM  r7  r*  s        rf   rt  z0Router._pass_through_assistants_endpoint_factory
  s      &%%1&*&<&<=R&S#d445EFG s  ' 
 3F
FL
 
 	
 
s   AAAAc                 	  K   |j                  d      }|j                  dd      }|j                  d| j                        }|j                  d| j                        }|j                  d| j                        }|j                  dd      }	 | j                  |||||	        | j                  |i |d|i d{   }	t        j                  d
|	        |	S 7 # t        $ r}
t        j                  dt        j                                 |
}d}|j                  d      }d}|du s||
| |d|}d|vr| j                  |d<   d|vrd|d<   	 t        j                  d       t        |      }|r/|j                  ||d       t!        |i | d{  7  }	|	cY d}
~
S t#        |
t$        j&                        r|F| j)                  ||      }|||j                  ||d       t!        |i | d{  7  }	|	cY d}
~
S dj+                  |||      }t        j                  dj+                  |             |
xj,                  dj+                  |      z  c_        nt#        |
t$        j.                        r|F| j)                  ||      }|||j                  ||d       t!        |i | d{  7  }	|	cY d}
~
S dj+                  |||      }t        j                  dj+                  |             |
xj,                  dj+                  |      z  c_        ||t        j                  d|        t1        |t3        t4        |            \  }}|
|||   d   }|Dt        j                  d| d|        t7        |d       r|xj,                  d| d| z  c_        ||j                  ||d       t!        |i | d{  7  }	|	cY d}
~
S n# t        $ r}t        j8                          t;        |      }t        j<                  d!j+                  t5        |      t        j                         t?        | |"       d{  7               t5        |      }Y d}~nd}~ww xY wt7        |d       rW|xj,                  d#j+                  ||      z  c_        tA        |      dkD  r$|xj,                  d$j+                  |      z  c_        |d}
~
ww xY ww)%z
        Try calling the function_with_retries
        If it fails after num_retries, fall back to another model group
        r   disable_fallbacksFr   r   r   mock_timeoutN)r*  r  r   r   r   zAsync Response: 	Tracebackr;  T)litellm_routerrd  r|   fallback_depthr   zTrying to fallback b/w models)r   )fallback_model_grouporiginal_model_group)r   r  zmodel={}. context_window_fallbacks={}. fallbacks={}.

Set 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbackszGot 'ContextWindowExceededError'. No context_window_fallback set. Defaulting                             to fallbacks, if available.{})msgz
{}zmodel={}. content_policy_fallback={}. fallbacks={}.

Set 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbackszGot 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting                             to fallbacks, if available.{}zinside model fallbacks: r   z7No fallback model group found for original model_group=z. Fallbacks=r=  zlitellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - {}
{}

Debug Information:
Cooldown Deployments={}ra  r_  z;
Received Model Group={}
Available Model Group Fallbacks={}z
Error doing the fallback: {})!rE  rB  r   r   r   _handle_mock_testing_fallbacksr%  r   r   r"  ro  rp  r|   r   r+   r   r-   r   r   ContextWindowExceededError(_get_fallback_model_group_from_fallbacksr   r=  rI  r,   r   r   ri  	print_excr   errorr(   r	  )r   r  r*  r  ry  r   r   r   rz  r.  r$  rd  r~  r  fallback_failure_exception_strinput_kwargsis_non_standard_fallback_formaterror_messagegeneric_fallback_idxnew_exceptionr_  s                        rf   rj  z$Router.async_function_with_fallbacks
  s    
 &,ZZ%8,2JJ7JE,R$*JJ{DNN$K	39::&(E(E4
  4:::&(E(E4
  zz.$7A	%//'#)A)A 0  >T==.: H "''*:8*(EFO	
  s	%!'')I4H4H4J3K(LM!"#' 28**W2E -/* D(,@,H #'&8 L l2040B0B_-|312-.OD%**+JK 3V'3/ 3 ''4=4H &8&&&    H
 $Oa!C!CD/; II*B,7 J  - 07"44$++8L8L *<!*** $ $  ( )q  )x  )x')A9) .22!;;A6 -< 		V]]=%AA	7#F#FG/; II*B,7 J  - 07"44$++8L8L *<!*** $ $  ( )p  )w  )w')A9) .22!;;A6 -< 		V]]=%AA	([-D)//2J9+0VW0&/(,S+(> ?(*> -40</89M/Ns/S,+3-22UVaUbbnoxnyz ##5yA.66<st  tA  AM  NW  MX  ;Y  Y600 ''4H4H &8&&&    H
 $O D##%#DV#L %++ o  v  vM*!,,.M48-=  	 25]1C.D )95"**.m.t.t(/ * 56:&..8??:. %$gs	%s   B
S!/C <C=C S!C 
S'A,SAOFO S!S!&AO;G><OSS!B2O:J=;OSS!D
OOOSS!S	Q1(A&Q,QQ,'S,Q11A(SSS!r  c                 J   |j                  dd      }|j                  dd      }|j                  dd      }|"|du rt        j                  |dd| d| 	      |"|du rt        j                  |dd| d
| 	      |#|du rt        j                  |dd| d| 	      yy)a  
        Helper function to raise a litellm Error for mock testing purposes.

        Raises:
            litellm.InternalServerError: when `mock_testing_fallbacks=True` passed in request params
            litellm.ContextWindowExceededError: when `mock_testing_context_fallbacks=True` passed in request params
            litellm.ContentPolicyViolationError: when `mock_testing_content_policy_fallbacks=True` passed in request params
        mock_testing_fallbacksNmock_testing_context_fallbacks%mock_testing_content_policy_fallbacksTr;  #This is a mock exception for model=z#, to trigger a fallback. Fallbacks=r   r>  r=  zF, to trigger a fallback.                     Context_Window_Fallbacks=zF, to trigger a fallback.                     Context_Policy_Fallbacks=)rB  r   InternalServerErrorr  rI  )	r   r*  r  r   r   r   r  r  r  s	            rf   r  z%Router._handle_mock_testing_fallbacks  s
     "(,Dd!K)/,d*
& 17

3T1
- "-2HD2P--!=k]Jmnwmxy  +6.$644!=k] K..F-GI  2=5=55!=k] K..F-GI  > >re   c           
      *  K   t        j                  d| d|        |j                  d      }|j                  d| j                        }t	        |      }|j                  d| j
                        }|j                  d| j                        }|j                  d      }|j                  d      }	|j                  d	      xs i }
d
|
v rFt        |
d
   t              r3| j                  |
d
         }||
j                  dt        |      i       t        j                  d| d|	        	 | j                  ||        | j                  |g|i | d {   }|S 7 # t        $ r&}d }|}t!        |dd       }|t        |t"              r|}		 | j%                  |j                  d      xs d|       d {  7  \  }}| j'                  ||||||       | j(                  | j*                  &| j-                  ||j                  d            }||}	|	dkD  r| j/                  ||      }n | j1                  ||	|	||      }t3        j4                  |       d {  7   t7        |	      D ]  }	  | j                  |g|i | d {  7  }t9        j:                  |      r| d {  7  }|c cY d }~S # t        $ r}| j/                  ||      }|	|z
  }|j                  d      }| | j%                  ||       d {  7  \  }}ng }| j1                  |||	||      }t3        j4                  |       d {  7   Y d }~d }~ww xY w t=        |      t>        j@                  v rtC        |d|	       tC        |d|       |d }~ww xY ww)Nz+Inside async function with retries: args - z; kwargs - r(  r   r   r   r   r{   r   r  r  model_group_sizez/async function w/ retries: original_function - z, num_retries - )r  r*  r;  r  )r  r   all_deploymentsr   regular_fallbacksr   )rP  r  r   )r*  r$  )r$  remaining_retriesr{   r   r  r   )"r   r   rB  r   r   r   r   rE  r   r   r  r   r	  %_handle_mock_testing_rate_limit_error	make_callr"  rj  r   r  should_retry_this_errorr   r   r.   	log_retry_time_to_sleep_before_retryrk  r  r  inspectiscoroutinefunctionr   r   LITELLM_EXCEPTION_TYPESr  )r   r  r*  r(  r   r_  r   r   r  r{   	_metadatarp   r.  r$  current_attemptrd  deployment_num_retriesr  _all_deployments_retry_policy_retriesr   r  _modelr  _timeouts                            rf   r%  z"Router.async_function_with_retries  s
    ##9${6(S	
 #JJ':;JJ{DNN;	<VD#)::&(E(E$
  $*::&(E(E$
  &,ZZ%8jj/ !**Z06B	I%*Y}5Ms*S,,	-8P,QJ%  "4c*o!FG##=>O=PP`al`mn	
c	%66' 7  ,T^^,=OOOOHO P  [	%"O!"%,Qt%D"%1j&7 5 99 **W-3%5 :    3 "2 (($8 0)A"+)A )  !!-00< )-(N(N0fjj>Q )O )% )4"7KQv9KL ::$"-'$8 0 ; K --,,,#(#52%3T^^4E%W%WPV%WWWH22  *2>>#O  2!^^6Q^?F(3o(E%,2JJw,?F)"&"E"E&,1A #F #   0,a 02,#??,*;$/,@(8  @  H "--111+2 $6B &'7+J+JJ*M;G*M?K$$w[	%s   D(N+,E  EE  NE   
N*AN9F<:B%NI" N4J<JJ<-J0.J<4N6N7N<	MAML	
9M M
MNM;NNNc                    K   |j                  d      } ||i |}t        j                  |      st        j                  |      r
| d{   }| j	                  ||       d{   }|S 7 !7 w)z^
        Handler for making a call to the .completion()/.embeddings()/etc. functions.
        r   N)r.  r  )rE  r  r  isawaitableset_response_headers)r   r(  r  r*  r  r.  s         rf   r  zRouter.make_callS  sx      jj)$d5f5&&x0G4G4G4Q%~H22; 3 
 
  &
s$   A	A1A-A1&A/'A1/A1c                     |j                  dd      }|9|du r4t        j                  d|        t        j                  |dd| d      yy)	z
        Helper function to raise a mock litellm.RateLimitError error for testing purposes.

        Raises:
            litellm.RateLimitError error when `mock_testing_rate_limit_error=True` passed in request params
        mock_testing_rate_limit_errorNTzTlitellm.router.py::_mock_rate_limit_error() - Raising mock RateLimitError for model=r;  r  z , to trigger a rate limit error.r  )rB  r   r   r   RateLimitError)r   r*  r  r  s       rf   r  z,Router._handle_mock_testing_rate_limit_errorb  st     9?

+T9
% *5-5!&&fgrfst ((!=k]Jjk  6 6re   r  r   r  r  c                    d}|t        |t              rt        |      }d}|t        |t              rt        |      }t        |t        j                        r||t        |t        j
                        r||t        |t        j                        r|t        |t        j                        r|dk  r|t        |      dkD  r|t        |t        j                        r	 |dk  r||dk  r|y)au  
        1. raise an exception for ContextWindowExceededError if context_window_fallbacks is not None
        2. raise an exception for ContentPolicyViolationError if content_policy_fallbacks is not None

        2. raise an exception for RateLimitError if
            - there are no fallbacks
            - there are no healthy deployments in the same model group
        r   rZ   T)
r   r   r	  r   r  rI  NotFoundErrorr   r  AuthenticationError)	r   r  r   r  r   r   r  _num_healthy_deployments_num_all_deploymentss	            rf   r  zRouter.should_retry_this_error{  s    " $% *z:Mt/T'*+>'?$ &:ot+L#&#7  ug@@A(4K ugAAB(4KeW223KeV223(A-%1)*Q.eV778
 %) $q(Kre   c                      ddl m}  fd}	 t        j                         } |d      5 }|j	                  |      }|j                         cddd       S # 1 sw Y   yxY w# t        $ r
  |       cY S w xY w)z~
        Sync wrapper for async_function_with_fallbacks

        Wrapped to reduce code duplication and prevent bugs.
        r   )ThreadPoolExecutorc                  8   t        j                         } 	 t        j                  |        | j                   j                  i       | j                          t        j                  d       S # | j                          t        j                  d       w xY w)z9Run the coroutine in a new event loop within this thread.N)rk  new_event_loopset_event_looprun_until_completerj  close)new_loopr  r*  r   s    rf   run_in_new_loopz7Router.function_with_fallbacks.<locals>.run_in_new_loop  s}    --/H-&&x0226D66GG  &&t,  &&t,s   5A2 2'BrZ   )max_workersN)concurrent.futuresr  rk  get_running_loopsubmitr  RuntimeError)r   r  r*  r  r  r  executorfutures   ```     rf   r-  zRouter.function_with_fallbacks  sh     	:
	-	%((*A $2h!9}} 322  	%"$$	%s.   A& !A	A& A#A& #A& &A98A9c                 l    |yd}|D ])  }t        |j                               d   |k(  s#||   } |S  |S )a@  
        Returns the list of fallback models to use for a given model group

        If no fallback model group is found, returns None

        Example:
            fallbacks = [{"gpt-3.5-turbo": ["gpt-4"]}, {"gpt-4o": ["gpt-3.5-turbo"]}]
            model_group = "gpt-3.5-turbo"
            returns: ["gpt-4"]
        Nr   )r   keys)r   r   r  r~  r  s        rf   r  z/Router._get_fallback_model_group_from_fallbacks  sQ     48DDIIK #{2'+K'8$##	  $#re   r$  r  c                    |t        |      dk(  rn!|t        |t              rt        |      dkD  ryd}t        |d      r,t        |j                  d      r|j                  j
                  }t        |d      r|j                  }|%t        j                  |||| j                        }|S t        j                  ||| j                        }|S )	z
        Calculate back-off, then retry

        It should instantly retry only when:
            1. there are healthy deployments in the same model group
            2. there are fallbacks for the completion call
        NrZ   r   r.  headerslitellm_response_headers)r  r   response_headersmin_timeout)r  r   r  )
r	  r   r   ri  r.  r  r  r   _calculate_retry_afterr   )r   r$  r  r{   r   r  r  r}   s           rf   r  z"Router._time_to_sleep_before_retry  s    " &3+?1+D+.5'(1,481j!gajj)&D zz11101 99'44"3'!1 ,,	G  44"3' ,,G re   c                   K   	 |j                  dd      }|t        d      |d   j                  d      y|d   d   j                  dd      }|j                  dd      }|j                  dd      }||yt        |t              rt	        |      }t        |      }	|j                  d	d
      }
t               }|j                  d      }t        j                  j                  j                  |||      }| j                  j                  ||
|	t        j                  j                         d{    t        j                   j                  j                  |||      }| j                  j                  |d|	t        j                  j                         d{    t#        | |       |S 7 ~7 # t$        $ r7}t'        j(                  dj                  t	        |                   Y d}~yd}~ww xY ww)zG
        Track remaining tpm/rpm quota for model in model_list
        standard_logging_objectNzstandard_logging_object is Noner   r   r  r  model_idtotal_tokensr   %H-%Mr  current_minuter   keyr  r_  rc   rZ   ra  deployment_idzOlitellm.router.Router::deployment_callback_on_success(): Exception occured - {})rE  r  r   r   r   r   rX   strftimerF   TPMr  r   r   async_increment_cacher^   rc   RPMr3   r"  r   rP  )r   r*  completion_responser]  r^  r  deployment_namer  r  r_  r  dtr  tpm_keyrpm_keyr$  s                   rf   r   z%Router.deployment_callback_on_success)  s    F	HN

)4I# '. !BCC&'++J7?"()9"::"F"J"J $# 699-N,00TB&"*C(RB#DV#L &=&A&A.RS&T
 &'!#" *--33::. ;  jj66&%5#--	 7    *--33::. ;  jj66%5#--	 7    B,0"$
 /  	!++ahhF
 	sk   G83F5 G8A F5 8G89B9F5 2F13A*F5 F3F5 0G81F5 3F5 5	G5>-G0+G80G55G8c                    d}|d   j                  d      nc|d   d   j                  dd      }|d   j                  di       xs i }|j                  dd      }||yt        |t              rt        |      }|t	        | |      }|S y)z
        Tracks the number of successes for a deployment in the current minute (using in-memory cache)

        Returns:
        - key: str - The key used to increment the cache
        - None: if no key is found
        Nr   r   r  r  r  r  )rE  r   r   r   r3   )	r   r*  r  r]  r^  r  r  r  r  s	            rf   r   z*Router.sync_deployment_callback_on_success{  s     "#''
3; !12:>BB=RVWK 0155lBGM2Jd+B"bjB$W>C(, C Jre   c                 F   	 |j                  dd      }t        |dd      }|j                  di       j                  di       }t        j                  j                  j                  |      }|j                  di       j                  d| j                        }	|3t        j                  j                  |	      }	|	|	d
k  r| j                  }	t        |t              r1|j                  dd      }
t        | |
       t        | |||
|	      }|S y# t        $ r}|d}~ww xY w)a`  
        2 jobs:
        - Tracks the number of failures for a deployment in the current minute (using in-memory cache)
        - Puts the deployment in cooldown if it exceeds the allowed fails / minute

        Returns:
        - True if the deployment should be put in cooldown
        - False if the deployment should not be put in cooldown
        rP  Nstatus_coder;  r   r  )rd  r   )r  r   r  r  ra  exception_statusrd  r  time_to_cooldownF)rE  rj  r   litellm_core_utilsexception_mapping_utils_get_response_headersr   utils&_get_retry_after_from_exception_headerr   r   r2   r*   r"  )r   r*  r  r]  r^  rP  r  _model_infoexception_headers_time_to_cooldownr  r  r$  s                rf   r   z%Router.deployment_callback_on_failure  s<    ,	

;5I&y-D **%5r:>>|RPK ' : : R R h h#, !i ! !'

+;R @ D D!3!3! !, MMHH): I  " %,0AA0E(,(:(:%+t, +d ;@,0"/ 3,0%5'0,%6  	G	s   DD 	D DD r  c                 (  K   |d   d   j                  dd      }|d   d   j                  dd      }|d   j                  di       xs i }|j                  dd      }||yt        |t              rt        |      }t	        |      }	t               }
|
j                  d      }t        j                  j                  j                  |||	      }| j                  j                  |d
|	t        j                  j                         d{    y7 w)z3
        Update RPM usage for a deployment
        r   r   r  Nr  r  r  r  r  rZ   r  )rE  r   r   r   r   rX   r  rF   r  r  r   r   r  r^   rc   )r   r*  r  r]  r^  r  r  r  r  r_  r  r  r  s                rf   r   z+Router.async_deployment_callback_on_failure  s     !!12:>BB$
 -.z:>>}dS,-11,CIr
^^D$'"*C RB<VD

 "%%++22. 3 
 jj..-%%	 / 
 	
 	
s   DD
DDc                    	 t        |      j                  t        |      d}|j                         D ]T  \  }}|dvr|||<   |dk(  st	        |t
              s'i |d<   |d   j                         D ]  \  }}|dk7  s|||   |<    V t        | j                        dkD  r| j                  j                  d       | j                  j                  |       | j                  |d   d<   |S # t        $ r}|d}~ww xY w)z
        When a retry or fallback happens, log the details of the just failed model call - similar to Sentry breadcrumbing
        )exception_typeexception_string)r   r&  r(  r   r      r   N)r   r`   r   r  r   r   r	  r   rB  r   r"  )r   r*  r$  previous_modelr  r  
metadata_k
metadata_vs           rf   r  zRouter.log_retry  s   	 #'q'"2"2$'FN 	 II()N1%*_At)<13N:.282D2J2J2L.
J%)::<FN1-j9 3M  4''(1,$$((+  ''7484H4HF:01M 	G	s+   AC, 	C, #C, >A-C, ,	C<5C77C<r  r_  c                     |}| j                   j                  ||d      }|#d}| j                   j                  ||dd       |S |dz  }| j                   j                  ||d       |S )zf
        Update deployment rpm for that minute

        Returns:
        - int: request count
        T)r  r_  
local_onlyrZ   r_   )r  r  r  rc   )r  r  r  )r   	get_cache	set_cache)r   r  r_  r  request_counts        rf   _update_usagezRouter._update_usage   s      

,,*:t - 
  MJJ  =Tr !   QMJJ  =T !  re   r  r  exception_strc                 
   | j                  |      }|t        |      dk(  ry	 dg}||D ]  }||v s y t        |t              rt	        |      }|dk\  r|dk  r|dk(  ry|d	k(  ry|d
k(  ry|dk(  ryyy# t
        $ r Y yw xY w)aZ  
        A function to determine if a cooldown is required based on the exception status.

        Parameters:
            model_id (str) The id of the model in the model list
            exception_status (Union[str, int]): The status of the exception.

        Returns:
            bool: True if a cooldown is required, False otherwise.
        r  rZ   FAPIConnectionErrori  i  i  Ti  i  i  )get_model_groupr	  r   r   r   r"  )r   r  r  r  r  ignored_stringsignored_strings          rf   _is_cooldown_requiredzRouter._is_cooldown_required;  s    " **h*7"s;'71'<%	34O)&5N%6$ '6 *C0#&'7#8 3&+;c+A#s*%,%,%, !  		s-   A6 A6 +A6 "A6 (A6 .A6 6	BBc                 n    | j                   y| j                   D ]  }t        |t              sd|v s y y)NFr   T)r   r   r   )r   fallbacks     rf   _has_default_fallbackszRouter._has_default_fallbacksw  s6    >>!H(D)(? ' re   r.  c                 J   |j                   d   j                  dk7  ry|j                  d| j                        }|2d}|D ](  }t	        |j                               d   |k(  s#||   } n |y| j                         ryt        j                  dj                  ||             y)z
        Determines if a content policy error should be raised.

        Only raised if a fallback is available.

        Else, original response is returned.
        r   content_filterFr   NTzyContent Policy Error occurred. No available fallbacks. Returning original response. model={}, content_policy_fallbacks={})
choicesfinish_reasonrE  r   r   r  r  r   r   r   )r   r   r.  r*  r   r~  r  s          rf   rH  z)Router._should_raise_content_policy_error  s     A,,0@@#)::&(E(E$
 
 $/#' 0		$Q'50+/;( 1
 $/((*"" H  O  O/	

 re   c                     g }	 | j                  |      \  }}t        |t              rg S 	 t	        | |      }g }|D ]  }|d   d   |v r|j                  |         ||fS # t        $ r Y Aw xY w)NrI  r  r  r  )#_common_checks_available_deploymentr   r   r"  r)   r   r   r   r_  r  r  unhealthy_deploymentsr   r  s           rf   _get_healthy_deploymentszRouter._get_healthy_deployments  s    !#	"&"J"J #K #A *D1	 2
 !:$(;K!
 %'*J,'-1FF#**:6	 + #$444  		s   &A" "	A.-A.c                   K   g }	 | j                  |      \  }}t        |t              rg |fS 	 t	        | |       d{   }g }|D ]  }|d   d   |v r|j                  |         ||fS # t        $ r Y Iw xY w7 <w)z
        Returns Tuple of:
        - Tuple[List[Dict], List[Dict]]:
            1. healthy_deployments: list of healthy deployments
            2. all_deployments: list of all deployments
        rI  r  Nr  r  )r  r   r   r"  r'   r   r  s           rf   r  z%Router._async_get_healthy_deployments  s      "$	"&"J"J #K #A *D1+++ 2
 'F$(;K'
 !
 %'*J,'-1FF#**:6	 +
 #$444  		!
s3   A?(A. A? A=-A?.	A:7A?9A::A?c                 r    t         j                  D ]$  }t        |t              s|j	                  |       & y)a  
        Mimics 'async_routing_strategy_pre_call_checks'

        Ensures consistent update rpm implementation for 'usage-based-routing-v2'

        Returns:
        - None

        Raises:
        - Rate Limit Exception - If the deployment is over it's tpm/rpm limits
        N)r   r  r   r   r  )r   r  r  s      rf   rG  z'Router.routing_strategy_pre_call_checks  s,     !**I)\2((4 +re   r{  c           
      |  K   t         j                  D ].  }t        |t              s	 |j	                  ||       d{    0 y7 # t         j
                  $ r}|t        j                  |j                  |t        j                         t        j                                      t        j                  |j                  |t        j                         f      j                          t!        | |j"                  ||d   d   | j$                         |d}~wt&        $ r}|t        j                  |j                  |t        j                         t        j                                      t        j                  |j                  |t        j                         f      j                          |d}~ww xY ww)O  
        For usage-based-routing-v2, enables running rpm checks before the call is made, inside the semaphore.

        -> makes the calls concurrency-safe, when rpm limits are set for a deployment

        Returns:
        - None

        Raises:
        - Rate Limit Exception - If the deployment is over it's tpm/rpm limits
        NrP  traceback_exceptionr^  targetr  r  r  r  )r   r  r   r   async_pre_call_checkr  rk  rl  async_failure_handlerro  rp  rf  	threadingThreadfailure_handlerstartr*   r  r   r"  )r   r  r_  r{  r  r$  s         rf   r  z-Router.async_routing_strategy_pre_call_checks  s}    " !**I)\2(#88EUVVV + W-- ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'-04)*+,#-l#;D#A)-);); G  ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'GsF   $F<AAAF<AF9B<DF9"BF44F99F<rb  c           
        K   |}t         j                  D ]2  }t        |t              s	 |j	                  |||||       d{   }4 |S 7 # t
        $ r}	|t        j                  |j                  |	t        j                         t        j                                      t        j                  |j                  |	t        j                         f      j                          |	d}	~	ww xY ww)r  r   r   r&  rb  r_  Nr  r  )r   r  r   r   async_filter_deploymentsr"  rk  rl  r  ro  rp  rf  r  r  r  r  )
r   r   r   r&  r_  rb  r{  returned_healthy_deploymentsr  r$  s
             rf   !async_callback_filter_deploymentsz(Router.async_callback_filter_deployments'  s     ( (;$ **I)\2'@@"'0L%-+9-= A   1 +8 ,+1 ! ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'Gs;   &C1AAAC1A	C.BC))C..C1r   c                    |}|j                         D ]  \  }}t        |t              r||z  }n7t        |t              r|t	        j
                  |      z  }n|t        |      z  }t        |t              r||z  }it        |t              r|t	        j
                  |      z  }|t        |      z  } t        j                  |j                               }|j                         S )z
        Helper function to consistently generate the same id for a deployment

        - create a string from all the litellm params
        - hash
        - use hash as id
        )
r  r   r   r   jsondumpshashlibsha256encode	hexdigest)r   r  r   
concat_strr  r  hash_objects          rf   _generate_model_idzRouter._generate_model_idZ  s     !
"((*DAq!S!a
At$djjm+
c!f$
!S!a
At$djjm+
c!f$
 + nnZ%6%6%89$$&&re   deployment_info_model_name_litellm_paramsr  c           
         t        di ||t        di ||d}|j                  j                  }|j                  j                  |j                  j                  dz   |z   }t        j                  ||i       | j                  |      dur3t        j                  d|j                   d|j                  d	           y| j                  |      }|j                  d
      }| j                  j                  |       |S )a^  
        Create a deployment object and add it to the model list

        If the deployment is not active for the current environment, it is ignored

        Returns:
        - Deployment: The deployment object
        - None: If the deployment is not active for the current environment (if 'supported_environments' is set in litellm_params)
        )rJ  r   r  Nr  )
model_costr4  TzIgnoring deployment z% as it is not active for environment supported_environmentsr   rd   )r@   rB   r   r   rM  r   register_model$deployment_is_active_for_environmentr   warningrJ  r  _add_deploymentto_jsonrp   r   )r   r,  r-  r.  r  r  r   s          rf   _create_deploymentzRouter._create_deploymentv  s(       

")<O<"	

 !//55$$88D))==CkQ  	[	
 44
4KSWW!))&z'<'<&==bcmcxcx  zR  dS  cT  U ))Z)@
"""5u%re   c           	      P   |j                   d|j                   vs|j                   d   yt        d      }|t        d      |t        vrt        dt         d|       |j                   d   D ]"  }|t        vst        dt         d| d	|        ||j                   d   v ryy
)a  
        Function to check if a llm deployment is active for a given environment. Allows using the same config.yaml across multople environments

        Requires `LITELLM_ENVIRONMENT` to be set in .env. Valid values for environment:
            - development
            - staging
            - production

        Raises:
        - ValueError: If LITELLM_ENVIRONMENT is not set in .env or not one of the valid values
        - ValueError: If supported_environments is not set in model_info or not one of the valid values
        r1  TLITELLM_ENVIRONMENT)secret_namezPSet 'supported_environments' for model but not 'LITELLM_ENVIRONMENT' set in .envz#LITELLM_ENVIRONMENT must be one of z. but set as: z&supported_environments must be one of z for deployment: F)r  r   r  r;   )r   r  litellm_environment_envs       rf   r3  z+Router.deployment_is_active_for_environment  s     !!)'z/D/DD$$%=>F,9NO&b  &@@56P5QQ_`s_tu  ))*BCD55 <=W<XXfgkfll}  I  ~J  K  D *"7"78P"QQre   c                    t        j                  |      }g | _        |D ]
  }|j                  d      }|j                  d      }t	        |t
              rI|j                         D ]6  \  }}t	        |t              s|j                  d      s)t        |      ||<   8 |j                  di       }d|vr| j                  ||      }	|	|d<   |j                  dd       8t	        |d   t              r%|d   D ]  }
|
|d<   | j                  ||||        | j                  ||||        t        j                  d| j!                                 |D cg c]  }|d   	 c}| _        y c c}w )	NrJ  r   os.environ/r  r  organization)r,  r-  r.  r  z
Initialized Model List )r   r   rp   rB  r   r   r  r   
startswithrW   r+  rE  r   r7  r   r   get_model_namesri   )r   rp   original_model_listr   r-  r.  r  r  r  _idorgr   s               rf   r   zRouter.set_model_list  s   "mmJ7 )E))L1K#ii(89O/40+113DAq!!S)all=.I-7]* 4 !&		, ;K ;&--k?K$'D!"">48D/J +>:C69ON3++(-$/(7$/	 ,  ; ''$) +$3 +	 ( 9 )F 	##'(<(<(>'?@	
 6@@ZAlOZ@@s   E,c           	      |   dd l }| j                  j                  |j                  j                         |j                  j
                  (t        |dd       t        |d      |j                  _        |j                  j                  (t        |dd       t        |d      |j                  _        t        j                  |j                  j                  |j                  j                  dd             \  }}}}d|j                  v r{| j                  j                  |j                  |j                  d             |j                  j                   r/| j"                  j                  |j                  j                          |j                  j                  d	g       xs g }|D ]i  }|j                  d
i       }	dD ]P  }
|
|	v s|	|
   j%                  d      s|	|
   j'                  dd      }|j(                  j                  |d      |	|
<   R k |t        j*                  vrt-        d|       t/        j0                  | |j                  d             t        j2                  du ryt5        d       	 	 d|j                  j                  v rR|j                  j6                  <t        j8                  j;                  |j                  d       }||j                  _        |S |S # t,        $ rM}t=        j>                  djA                  |j                  j                  tC        |                   Y d }~|S d }~ww xY w)Nr   rpmtpmrM  r   rM  r   Tr   dataSources
parameters)endpointr  r>  r;  zUnsupported provider - ra  r   zAuto inferring regionrr  )r   modez1Unable to get the region for azure model - {}, {})"osr   r   r   r   rF  rj  rG  r   rU   rE  rJ  r   add_patternr6  r  r  r   r@  replaceenvironprovider_listr"  r$   
set_clientenable_preview_featuresprintregion_namer  get_model_regionr   r   r   r   )r   r  rN  r  rM  rM  r  data_sourcesdata_sourcer   	param_keyenv_nameregionr$  s                 rf   r5  zRouter._add_deployment  s    	$$Z%>%>%D%DE
 %%))1
E40<,3J,FJ%%) %%))1
E40<,3J,FJ%%) $$++11 * 9 9 = =%t!
	
$ *''' ++%%z'9'9t'9'L $$''44;;J<Q<Q<T<TU "0044]BGM2'K __\26F0	&6)+<+G+G+V%i088KH(*

x(DF9%	 1 ( g&;&;;56I5JKLL 	!++$(
0B0BPT0B0U	

 **d2)*z88>>>"11==E$]];;'1'@'@t < F =CJ--9 z  %++GNN"1177Q
 s   7A*K% %	L;.AL66L;c                    |j                   j                  | j                         v ry|j                  d      }| j                  j                  |       | j                  |       | j                  j                  |j                         |S )z
        Parameters:
        - deployment: Deployment - the deployment to be added to the Router

        Returns:
        - The added deployment
        - OR None (if deployment already exists)
        NTr   r4  )	r  r  rF  r6  rp   r   r5  ri   rJ  )r   r  _deployments      rf   add_deploymentzRouter.add_deploymenta  s       ##t'9'9';; !((d(;{+ 	
3 	
 5 56re   c                 v   |j                   j                  xs d}| j                  |      }|x|j                  |j                  k(  ryd}t	        | j
                        D ]'  \  }}|d   d   |j                   j                  k(  s&|}) || j
                  j                  |       | j                  |       |S )z
        Add or update deployment
        Parameters:
        - deployment: Deployment - the deployment to be added to the Router

        Returns:
        - The added/updated deployment
        r;  r  Nr  r  r4  )r  r  get_deploymentr   r  rp   rB  r_  )r   r  _deployment_model_id_deployment_on_routerremoval_idxr  r   s          rf   upsert_deploymentzRouter.upsert_deploymentz  s      *4477=26:6I6I) 7J 7
 !,((,A,P,PP *.K'8
U&t,
0E0E0H0HH"%K 9 &##K0 	z2re   r  c                     d}t        | j                        D ]  \  }}|d   d   |k(  s|} 	 || j                  j                  |      }|S y# t        $ r Y yw xY w)z
        Parameters:
        - id: str - the id of the deployment to be deleted

        Returns:
        - The deleted deployment
        - OR None (if deleted deployment not found)
        Nr  r  )r  rp   rB  r"  )r   r  deployment_idxr  r   r  s         rf   delete_deploymentzRouter.delete_deployment  so     0FCt$*!$ 1	)**>: 		s   A 	AAc                     | j                   D ]n  }d|v sd|d   v s||d   d   k(  st        |t              rt        di |c S t        |t              r|c S t	        dj                  t        |                   y)l
        Returns -> Deployment or None

        Raise Exception -> if model found in invalid format
        r  r  zModel invalid format - {}Nrd   rp   r   r   r@   r"  r   r   )r   r  r   s      rf   rb  zRouter.get_deployment  s     __Eu$|1D)Du\2488!%.)2E22#E:6$'(C(J(J4PU;(WXX % re   model_group_namec                     | j                   D ]^  }|d   |k(  st        |t              rt        di |c S t        |t              r|c S t	        dj                  t        |                   y)rk  rJ  zModel Name invalid - {}Nrd   rl  )r   rm  r   s      rf   "get_deployment_by_model_group_namez)Router.get_deployment_by_model_group_name  sg     __E\"&66eT*%...z2 L#$=$D$DT%[$QRR % re   received_model_namec                      y r   rd   r   r  rp  r  s       rf   get_router_model_infozRouter.get_router_model_info       	re   c                      y r   rd   rr  s       rf   rs  zRouter.get_router_model_info  rt  re   c                 
   |&| j                  |      }||j                  d      }|t        d      |j                  di       j                  dd      }|"|j                  di       j                  dd      }|}t	        j
                  |j                  di       j                  d	d
      t        di |j                  di             \  }}}	}	|dk(  r|t        j                  d       n|dk7  r|}| j                  j                  |      }
d|v rp|
n|
D ]i  }	 |j                  di       j                  d      |j                  di       j                  d      k(  r#|j                  di       j                  d	      } nk |j                  dj                  |            sdj                  ||      }n|}t	        j                  |      }|j                  di       }|j                  |       |S # t        $ r Y w xY w)a  
        For a given model id, return the model info (max tokens, input cost, output cost, etc.).

        Augment litellm info with additional params set in `model_info`.

        For azure models, ignore the `model:`. Only set max tokens, cost values if base_model is set.

        Returns
        - ModelInfo - If found -> typed dict with max tokens, input cost, etc.

        Raises:
        - ValueError -> If model is not mapped yet
        Nra  Tr   zDeployment not foundr  
base_modelr   r   r;  r   r   rr  zCould not identify azure model. Set azure 'base_model' for accurate max tokens, cost tracking, etc.- https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-modelsr   r  z{}/z{}/{}rI  rd   )rb  r   r  rE  r   rU   rB   r   r  r   router"  r@  r   get_model_infor   )r   r  rp  r  r^  rw  r   r  rM  r  potential_modelspotential_modelmodel_info_namer  user_model_infos                  rf   rs  zRouter.get_router_model_info  s*   & >--r-:K&(333F
344  ^^L"599,M
#(8"=AA,PTUJ -4,D,D..!126::7BG)QJNN;KR,PQ-
)#Q ')j.@!'' O !G+E#22889LMe| 0 <'7O	*..|R@DD '^^L"=AA$GH %4$7$78H"$M$Q$Q '%E "H (8 -@ AB%nn-@%HO#O++/B
 %..r:/*! % s   A$G66	HHc                 ^    | j                   D ]  }d|v sd|d   v s||d   d   k(  s|c S  y)z
        For a given model id, return the model info

        Returns
        - dict: the model in list with 'model_name', 'litellm_params', Optional['model_info']
        - None: could not find deployment in list
        r  r  Nrp   )r   r  r   s      rf   rz  zRouter.get_model_info/  sB     __Eu$|1D)D|,T22 L % re   c                 Z    | j                  |      }|y|d   }| j                  |      S )zT
        Return list of all models in the same model group as that model id
        r  NrJ  r  )rz  r  )r   r  r  rJ  s       rf   r  zRouter.get_model_group=  s>    
 ((B(/
-
""j"99re   user_facing_model_group_namec                    d}d}d}d}| j                  |      }|y|D ]  }d}	d|v r|d   |k(  rd}	n!d|v r| j                  j                  |      d}	|	s9t        di |d   }
|
j                  }d}||j                  dd      }|"|j                  di       j                  dd      }|"|j                  di       j                  dd      }d}||j                  d	d      }|"|j                  di       j                  d	d      }|"|j                  di       j                  d	d      }	 t        j                  |
j                  
      }d\  }}	 t        j                  |
j                  |
j                        \  }}}}|0t        j$                  ||      }|g }t'        |ddddd|d|d
      }|t)        d||gd|}n||j*                  vr|j*                  j-                  |       |j                  dd      -|d   (|j.                  |d   |j.                  kD  r
|d   |_        |j                  dd      -|d   (|j0                  |d   |j0                  kD  r
|d   |_        |j                  dd      (|j2                  |d   |j2                  kD  r
|d   |_        |j                  dd      (|j4                  |d   |j4                  kD  r
|d   |_        |j                  dd      	 |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   
|d   |_        |j                  dd      ||j                  d      }|j                  d	d      ||j                  d	      }|	|d}||z  }||d}||z  } ||||_        |||_         |||_        |S # t        $ r d}Y w xY w# t        j                  j                  $ r8}t        j                  dj!                  t#        |                   Y d}~d}~ww xY w)z
        For a given model group name, return the combined model info

        Returns:
        - ModelGroupInfo if able to construct a model group
        - None if error constructing model group info
        Nr  FrJ  Tr   rG  r  rF  rI  )r;  r;  rH  z.litellm.router.py::get_model_group_info() - {}r   r   )
r  
max_tokensmax_input_tokensmax_output_tokensinput_cost_per_tokenoutput_cost_per_tokenlitellm_providerrM  supported_openai_paramssupports_system_messages)r  	providersr  r  r  r  "supports_parallel_function_callingsupports_visionsupports_function_callingr  rd   )!r  r   ry  rB   #configurable_clientside_auth_paramsrE  r   rz  r   r"  rU   rM  
exceptionsBadRequestErrorr   r  r   r   get_supported_openai_paramsModelMapInforC   r  r   r  r  r  r  r  r  r  r  rG  rF  )r   r  r  model_group_info	total_tpm	total_rpmr  rp   r   is_matchr   _deployment_tpm_deployment_rpmr  r  r>  r  r$  r  s                      rf   _set_model_group_infozRouter._set_model_group_infoI  ss    6:#'	#'	SW+((K(@
EH%%*=*L%''--k:F+Fe4D.EFN BB 0 .2O&"'))E4"8&"')),<b"A"E"EeT"R&"'))L""="A"A%"N .2O&"'))E4"8&"')),<b"A"E"EeT"R&"'))L""="A"A%"N"$33.:N:NO
 +1'M<4;4L4L(..(6(J(J51|Q !*1*M*M'\+' +2.0+)##%)&*)**+%1,C-1
  '#1 $ <$Zd$   '7'A'AA$..55lCNN#5t<H"#56B(99A%&89*;;< 9CCU8V$5NN#6=I"#67C(::B%&9:*<<= :DDW9X$6>>"8$?K$99A!"89&;;< =G.=$9 >>"94@L$::B!"9:&<<= >H/>$: NN#GN "#GHDPJN$GNN#4d;G"#45=7;$4NN#>EQ"#>?4GAE$>NN#<dCO"#<=I?I1@$< >>%.:?V&0nnU&;O>>%.:?V&0nnU&;O*$ !I_,	*$ !I_,	A  B '$'0 $$'0 $ 3>7 !D  M  "!
" %%55 %++DKKCPQFS s*   0 O,0O>,O;:O;>Q-QQc                     || j                   v rT| j                   |   }t        |t              r|}nt        |t              r|d   du ry|d   }ny| j	                  ||      S | j	                  ||      S )z
        For a given model group name, return the combined model info

        Returns:
        - ModelGroupInfo if able to construct a model group
        - None if error constructing model group info or hidden model group
        hiddenTNr   )r  r  )r   r   r   r   r  )r   r  r  _router_model_groups       rf   get_model_group_infozRouter.get_model_group_info  s     $000))+6D$$&*#D$'>T)*.w-'--/-8 .   ))#+ * 
 	
re   c                   K   t               }|j                  d      }g }g }| j                  |      }|y|D ]  }|j                  di       j                  d      }|d   j                  d      }	||	=|j	                  t
        j                  j                  j                  ||	|	             |j	                  t
        j                  j                  j                  ||	|	              ||z   }
| j                  j                  |

       d{   }|y|dt        |       }|t        |      d }d}|!|D ]  }t        |t              s|d}||z  } d}|!|D ]  }t        |t              s|d}||z  } ||fS 7 qw)z
        Returns current tpm/rpm usage for model group

        Parameters:
        - model_group: str - the received model name from the user (can be a wildcard route).

        Returns:
        - usage: Tuple[tpm, rpm]
        r  r  Nrv  r  r  r   r   )r  r   r  )r  r   )rX   r  r  rE  r   rF   r  r  r   r  r   async_batch_get_cacher	  r   r   )r   r  r  r  tpm_keysrpm_keysrp   r   r  r  combined_tpm_rpm_keyscombined_tpm_rpm_valuestpm_usage_listrpm_usage_list	tpm_usager  	rpm_usages                    rf   get_model_group_usagezRouter.get_model_group_usage+  s     
 ! ((K(@
E %		, ; ? ? EB+01A+B+F+F,M z]2OO##))00'#1 1  OO##))00'#1 1   * !)8 3(,

(H(H& )I )
 #
 #*)@3x=)Q)@X)Q $(	%#a% ($%	NI	 $ $(	%#a% ($%	NI	 $
 )##3#
s   DFF<F$F5Fc                 0  K   | j                  |       d {   \  }}| j                  |      }||j                  |j                  }nd }||j                  |j                  }nd }i }|||xs dz
  |d<   ||d<   |||xs dz
  |d<   ||d<   |S 7 |w)Nr   x-ratelimit-remaining-tokenszx-ratelimit-limit-tokensx-ratelimit-remaining-requestszx-ratelimit-limit-requests)r  r  rG  rF  )r   r  current_tpmcurrent_rpmr  	tpm_limit	rpm_limitreturned_dicts           rf   get_remaining_model_group_usagez&Router.get_remaining_model_group_usaget  s     )-)C)CK)P#P [44[A',<,@,@,L(,,II',<,@,@,L(,,II <E q=M89 9BM45 >G q?M:; ;DM675 $Qs   BBA=Bc                 ~  K   t        |t              rt        |d      rt        |j                  t              r|j                  j                  di        ||j                  d   d<   |j                  d   }d|vr?d|vr;|9| j                  |       d{   }|j                         D ]  \  }}|	|||<    |S 7 &w)a  
        Add the most accurate rate limit headers for a given model response.

        ## TODO: add model group rate limit headers
        # - if healthy_deployments > 1, return model group rate limit headers
        # - else return the model's rate limit headers
        r  r  zx-litellm-model-groupr  r  N)r   r   ri  r  r   r   r  r  )r   r.  r  r  remaining_usageheaderr  s          rf   r  zRouter.set_response_headers  s      x+"23822D9##../CRH  ##$89' "*!8!89M!N /6HH4<NN+(,(L(L) # &5%:%:%<MFE(5:*62 &= #s   BB=B;B=2
B=rJ  c                     g }| j                   D ]G  }d|v sd|d   v s|d   d   }||d   |k(  r|j                  |       4|7|j                  |       I |S )z\
        if 'model_name' is none, returns all.

        Returns list of model id's.
        r  r  rJ  )rp   r   )r   rJ  idsr   r  s        rf   rF  zRouter.get_model_ids  so     __Eu$|1D)D<(.)eL.AZ.OJJrN'JJrN % 
re   model_aliasc                     g }| j                   D ]M  }||d   |k(  s|,t        j                  |      }||d<   |j                  |       =|j                  |       O |S )ze
        Return all deployments of a model name

        Used for accurate 'get_model_list'.
        rJ  )rp   r   r   r   )r   rJ  r  returned_modelsr   alias_models         rf   _get_all_deploymentszRouter._get_all_deployments  sk     68__E%%*=*K*"&--"6K0;K-#**;7#**51 % re   c                 h    | j                         }|g S g }|D ]  }|j                  |d           |S )zn
        Returns all possible model names for router.

        Includes model_group_alias models too.
        rJ  )r  r   )r   rp   ri   r   s       rf   rA  zRouter.get_model_names  sD     ((*
IAq/ re   c           
      \   t        | d      rg }|!|j                  | j                  |             t        | d      r| j                  j	                         D ]d  \  }}t        |t              r|}n*t        |t              rt        d
i |}|d   du r<|d   }nC|j                  | j                  ||             f t        |      d	k(  rE| j                  j                  |      }|(|j                  |D cg c]  }t        d
i | c}       ||| j                  z  }|S |S yc c}w )z>
        Includes router model_group_alias'es as well
        rp   Nr  r   r  Tr   )rJ  r  r   rd   )ri  rk  r  r   r  r   r   r   rH   r	  r   ry  rA   rp   )	r   rJ  r  r  model_value_router_model_name_model_valuepotential_wildcard_modelsr   s	            rf   r  zRouter.get_model_list  sL    4&9;O%&&t'@'@J'@'WXt01040F0F0L0L0N,K!+s32=*#K6'@'O;'O'1T9$1=g1F. #**11'9{ 2  1O& ?#q(,0,?,?,E,Ej,Q),8#**;TU;Ta,1q1;TU !4??2&&"" Vs   :D)model_access_groupc                 $   ddl m}  |t              }| j                  |      }|ri|D ]d  }|j	                  di       j	                  dg       D ]=  }| ||k(  s|d   }||   j                  |       %|d   }||   j                  |       ? f |S )a  
        If model_name is provided, only return access groups for that model.

        Parameters:
        - model_name: Optional[str] - the received model name from the user (can be a wildcard route). If set, will only return access groups for that model.
        - model_access_group: Optional[str] - the received model access group from the user. If set, will only return models for that access group.
        r   r   r  r  r   rJ  )collectionsr   r   r  rE  r   )r   rJ  r  r   r   rp   r   groups           rf   get_model_access_groupszRouter.get_model_access_groups  s     	,#D)((J(?
UU<488"ME)5 $66)*<J)%077
C%&|_
%e,33J? N   re   c                     | j                  |      }t        |      dk(  ry|j                  |g       }|D ]   }| j                  j	                  |        y y)zG
        Return True if model access group is a wildcard route
        )r  r   Fr  T)r  r	  rE  r   ry  )r   r  r   r  r   s        rf   )_is_model_access_group_for_wildcard_routez0Router._is_model_access_group_for_wildcard_route4  sm     441 5 
 }"""#5r:E""(((7C 
 re   c                     t        |       }i }g d}|D ]K  }||v r||   ||<   |dk(  s| j                  dk(  s%| j                  j                  j	                         ||<   M |S )a  
        Get router settings method, returns a dictionary of the settings and their values.
        For example get the set values for routing_strategy_args, routing_strategy, allowed_fails, cooldown_time, num_retries, timeout, max_retries, retry_after
        r   r   r   r   r{   r}   r   r   r   r   r   r   r   )varsr   r  r  r#  )r   	_all_vars_settings_to_returnvars_to_includevars        rf   get_settingszRouter.get_settingsK  s}    
 J	 
 #Ci+4S>#C(..))-DD+/+D+D+Q+Q+V+V+X#C( # #"re   c                    g d}g d}| j                         }|D ]  }||v rg||v rt        ||         }t        | ||       '|dk(  r1|d   ||   k7  r&| j                  ||   |j	                  di              t        | |||          nt        j                  dj                  |              t        j                  d| j                                 y)	z-
        Update the router settings.
        r  )r}   r{   r   r   r   r   r   r   zSetting {} is not allowedzUpdated Router settings: N)r  r   r  r   rE  r   r   r   )r   r*  _allowed_settings_int_settings_existing_router_settingsr  _casted_values          rf   update_settingszRouter.update_settingsj  s    


 %)$5$5$7!C''-'$'s$4MD#}5 1156HIVTW[X22-3C[28** 73 3  D#vc{3%++,G,N,Ns,ST' ( 	##&?@Q@Q@S?T$UVre   c                    |d   d   }t        |      }|dk(  r1dj                  |      }| j                  j                  |d|      }|S |dk(  r|j	                  d      du r]| d	}| j                  j                  |d|      }|6	 t        j                  | |
       | j                  j                  |d|      }|S | d}| j                  j                  |d|      }|6	 t        j                  | |
       | j                  j                  |d|      }|S |j	                  d      du r[| d}| j                  j                  ||      }|5	 t        j                  | |
       | j                  j                  ||      }|S | d}| j                  j                  ||      }|5	 t        j                  | |
       | j                  j                  ||      }|S )a  
        Returns the appropriate client based on the given deployment, kwargs, and client_type.

        Parameters:
            deployment (dict): The deployment dictionary containing the clients.
            kwargs (dict): The keyword arguments passed to the function.
            client_type (str): The type of client to return.

        Returns:
            The appropriate client based on the given client_type and kwargs.
        r  r  rx  z{}_max_parallel_requests_clientTr  r  r_  r  rP  _stream_async_clientrL  _async_client_stream_client)r  r_  _client)r   r   r   r  rE  r$   rS  )r   r  r*  rz  r  r_  	cache_keyr7  s           rf   rD  zRouter._get_client  sd    l+D1+LV+T119@@JIZZ))$AQ * F MG#zz(#t+'j(<=	--!dEU .  > -7704J "ZZ11%#')9 2 F
 'j6	--!dEU .  > -7704J "ZZ11%#')9 2 F
 zz(#t+'j7	--!4D .  > -7704J "ZZ11%8H 2 F 'j0	--!4D .  > -7704J "ZZ11%8H 2 F re   c           	      	   t        j                  d|        t        j                  |      }g }	 t	        j
                  |      }d}	d}
d}t        |      }t               }|j                  d      }| d| }| j                  j                  |d	|
      xs i }t        |      D ]  \  }}	 |j!                  di       j!                  dd      }|"|j!                  di       j!                  dd      }| j#                  ||      }|xs" |j!                  di       j!                  dd      }t%        |t&              rZ|j!                  dd      Ht%        |d   t(              r5||d   kD  r-|j+                  |       d	}	|
dj                  ||d   |      z  }
|j!                  di       }|j!                  di       j!                  dd      }| j                  j                  |d	|
      xs d}t%        |t&              r| j.                  dk7  rv|j!                  |d      ||<   t1        |||         }t%        |t&              rB|j!                  dd      0t%        |d   t(              r|d   |k  r|j+                  |       d	}|M|j!                  d      <|j!                  d      }|)t3        t5        d!i ||      s|j+                  |       +|/t        j6                  du sCt	        j8                  |t5        d!i |      \  }}}}t	        j:                  ||      }|t        j<                  j?                  |      }dg}|jA                         D ]8  \  }} ||vs||v st        j                  d|        |j+                  |       :  tC        |      tC        |      k(  r<	 |d	u rtE        |      |	d	u r't	        jF                  dj                  |
      |d       tC        |      dkD  r!tI        |      D ]  }|jK                  |        tC        |      dkD  rt        j<                  jM                  |      }|S # t        $ r9}t        j                  dj                  t        |                   |cY d}~S d}~ww xY w# t        $ r8}t        j,                  dj                  t        |                   Y d}~5d}~ww xY w)"a  
        Filter out model in model group, if:

        - model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models
        - filter models above rpm limits
        - if region given, filter out models not in that region / unknown region
        - [TODO] function call and model doesn't support function calling
        z2Starting Pre-call checks for deployments in model=)r&  zllitellm.router.py::_pre_call_checks: failed to count tokens. Returning initial list of deployments. Got - {}NFr;  r  z:rpm:Tr  r  rw  r   )r  rp  r   r  z%Model={}, Max Input Tokens={}, Got={}zAn error occurs - {}r  r   r   rF  allowed_model_region)r   r  rx  rH  )passed_paramsresponse_formatz1INVALID MODEL INDEX @ REQUEST KWARG FILTERING, k=rI  z~litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.
{}r<  rd   )'r   r   r   r   r   token_counterr"  r  r   r   r   rX   r  r   r  r  rE  rs  r   r   r   r   rP  r   maxrY   rB   drop_paramsrU   r  r  get_non_default_paramsr  r	  rJ   r  reversedrB  _get_order_filtered_deployments)!r   r   r   r&  rb  _returned_deploymentsinvalid_model_indicesinput_tokensr$  _context_window_error_potential_error_str_rate_limit_errorr_  r  r  r  model_group_cacher  r  rw  r  r.  r  current_request_cache_localcurrent_requestr  rM  r  r  r   special_paramsr  r  s!                                    rf   _pre_call_checkszRouter._pre_call_checks  se     	##@H	
 !%.A B "	)"00(CL !&!!<^L W-G5 01JJ  ?O !   	 	  ))>?OCW'^^L"=AA,PTU
%!+0@"!E!I!I$d"J "77)u 8 
 # jnn5Er&J&N&NT'
 z40"'94@L #:.@#A3G(:6H+II-44S904-,CJJ %z2D'E|,
 ! )nn-=rBO!~~lB7;;D"EH 

$$ TDT %   	 ( ,d3))-EE.?.C.CHa.P!(+"%/1B81L#
 5'++E48D #?5#93?+E2oE-44S9,0)  *"&&'=>J'5'9'9:P'Q$'3,'5'H'H-A .44S9  )g.A.AU.J3:3K3K0Q0Q40*Aq +2*M*M5H+' +2 *1)M)M&4 *N *& '8%8N 2 8 8 :1$;;^@S177"STUSV W 288= !;M  @\ $%-B)CC !D(/  '$.88 ^  e  e,  !#  $%) 56%))#. 7 $%)$+MM$Q$Q%%! %$G  	)!''~  F  FF
 )(	)l  W%//0F0M0McRSf0UVVWs6   Q /C(R	R.R	R	R	S-SSc                 t    || j                   vry| j                   |   }t        |t              r|}|S |d   }|S )z
        Get the model from the alias.

        Returns:
        - str, the litellm model name
        - None, if model is not in model group alias
        Nr   )r   r   r   )r   r   _items      rf   _get_model_from_aliaszRouter._get_model_from_alias  sK     ...&&u-eS!E  'NEre   c                 X    | j                   D cg c]  }|d   d   |k(  s| c}S c c}w )z6
        Get the deployment by litellm model.
        r   r   r  )r   r   r   s      rf    _get_deployment_by_litellm_modelz'Router._get_deployment_by_litellm_model  s2      ??T?aa0@.A'.Je.S?TTTs   ''r1  c                 r   |du r|| j                  |      fS || j                         v rY| j                  |      }|*|j                  j                  }||j                  d      fS t        d| d| j                         | j                  |      }||}|| j                  vrY| j                  j                  |      }|r||fS | j                  +t        j                  | j                        }	||	d   d<   ||	fS | j                  |	      }
t        |
      d
k(  r| j                  |      }
t!        j"                  d|
        t        |
      d
k(  r't%        j&                  dj)                  |      |d      t$        j*                  r%|t$        j*                  v rt$        j*                  |   }||
fS )aF  
        Common checks for 'get_available_deployment' across sync + async call.

        If 'healthy_deployments' returned is None, this means the user chose a specific deployment

        Returns
        - str, the litellm model name
        - List, if multiple models chosen
        - Dict, if specific model chosen
        TrI  ra  r   zBLiteLLM Router: Trying to call specific deployment, but Model ID :z6 does not exist in                     Model ID List: r   r   r  r   zinitial list of deployments: zBYou passed in model={}. There is no 'model_name' with this string r;  r<  )r  rF  rb  r   r   r   r  r  ri   r   get_deployments_by_patternr   r   r   r  r	  r   r   r   r  r   model_alias_map)r   r   r&  r  r1  r  deployment_model_model_from_aliaspattern_deploymentsupdated_deploymentr   s              rf   r  z*Router._common_checks_available_deployment  s   $ $&$??e?LLLd((**,,e,<J%#-#<#<#B#B ')>)>D)>)QQQTUZT[ \$$($6$6#79 
 !66U6C(%E((("&"5"5"P"P #Q # #111 &&2%)]]++&" AF"#34W=000 #7757I"#q("&"G"Ge"G"T##+,?+@A	
 "#q())\cc   ""u0G0G'G++E )))re   c           
      \  K   | j                   dk7  rR| j                   dk7  rC| j                   dk7  r4| j                   dk7  r%| j                   dk7  r| j                  |||||      S 	 t        |      }| j                  ||||      \  }}t	        |t
              r|S t        | |       d	{   }t        j                  d
|        t        j                  d|        | j                  ||      }| j                  |||t        t        t           |      nd	||       d	{   }| j                  r,|*| j                  |t        t        t            |      ||      }t#        | |||       d	{   }t%        |      dk(  rt'        | ||       d	{   }	|	t)        j(                         }
| j                   dk(  r4| j*                  (| j*                  j-                  ||||       d	{   }n| j                   dk(  r4| j.                  (| j.                  j-                  ||||       d	{   }n| j                   dk(  r5| j0                  )| j0                  j-                  |||||       d	{   }n`| j                   dk(  rt3        | ||      S | j                   dk(  r2| j4                  &| j4                  j-                  ||       d	{   }nd	}|t'        | ||       d	{   }	|	t        j6                  d| d| j9                  |       d|        t)        j(                         }||
z
  }t;        j<                  | j>                  jA                  tB        jD                  |d||
|             |S 7 7 _7 7 7 7 ]7 7 7 # tF        $ r}tI        jJ                         }|j|jM                  dd	      }|VtO        jP                  |jR                  ||f      jU                          t;        j<                  |jW                  ||             |d	}~ww xY ww)z
        Async implementation of 'get_available_deployments'.

        Allows all cache calls to be made async => 10x perf impact (8rps -> 100 rps).
        r   ro   r   r   r   )r   r&  r  r1  rb  r   r&  r  r1  r  Nzasync cooldown deployments: zcooldown_deployments: r   cooldown_deploymentsr  r   r   r&  rb  )llm_router_instancer   rb  r   r   )ra  r   r_  r  r   r&  r  )r  r   r&  r  rb  r  r   r   r  r   $get_available_deployment for model: , Selected deployment:  for model: z2<routing_strategy>.async_get_available_deployments)r[  r\  r   r_  r]  r^  rw  r  ),r   rA  r   r  r   r   r'   r   r   _filter_cooldown_deploymentsr!  r   r	   r6   r   r  r   r!   r	  r/   rf  r  async_get_available_deploymentsr  r  r    rl   r   r%  rk  rl  r   rm  rL   rn  r"  ro  rp  rE  r  r  r  r  r  )r   r   r&  r  r1  rb  r_  r   r  rP  r]  r  r^  rr  r$  r  r{  s                    rf   ru  z%Router.async_get_available_deployment  s     !!%==%%)99%%)==%%)@@%%500!$7- 1  Z	@P)-)Q)Q!$7	 *R *&E& -t4**)H(,?O* $  "''./C.DE "''*@AU@V(WX"&"C"C$7%9 #D #
 )-(N(N$7  + ./:-!1 )O 
) 
# **x/C&*&;&;(,T$Z9L(M%#1	 '< '# )@$(-$7	) # &'1,"E,0%5# 	
  J%%)AA,,8 22RR$),?!)#	 S    %%)==**6 00PP$),?!)#	 Q    %%)@@--9 33SS$),?!)#'5 T    &&*::%(,(;  %%5))5 //OO$),? P    "
!"E,0%5# 	
  !&&6ug=TUYUjUjkuUvTw  xD  EJ  DK  L yy{H :-I''BB(//&R%5)% C 	 }$
#(#&0  	"+"6"6"8),001FM*$$*::!45 eg''#99!=PQ G!	s   A"P,%4N P,N *N +A9N $N%AN 3N4#N N	AN /N0AN 2N3AN 6N7!N P,;N NN /N0BN ?P, N N N 	N N N N N N 	P)"BP$$P))P,c                    | j                  ||||      \  }}t        |t              r|S t        |      }t	        | |      }| j                  ||      }| j                  r|| j                  ||||      }t        |      dk(  rU| j                  |      }	| j                  j                  |	|      }
t	        | |      }t        ||
| j                  |	      | j                  d
k(  r*| j                  | j                  j                  ||      }n| j                  dk(  rt!        | ||      S | j                  dk(  r+| j"                  | j"                  j                  |||      }nx| j                  dk(  r,| j$                   | j$                  j                  ||||      }n=| j                  dk(  r,| j&                   | j&                  j                  ||||      }nd}|nt)        j*                  d| d       | j                  |      }	| j                  j                  |	|      }
t	        | |      }t        ||
| j                  |	      t)        j*                  d| d| j-                  |       d|        |S )zB
        Returns the deployment based on routing strategy
        r  r  r  Nr  r   r  )	model_idsr_  )r   r   r   cooldown_listr   r	  ro   r  r   )r  r   rb  r   r  r   r
  z, No deployment availabler  r  )r  r   r   r   r)   r  r   r  r	  rF  r   get_min_cooldownrI   r   rl   get_available_deploymentsr    r  rm   r  r   r   r%  )r   r   r&  r  r1  rb  r   r_  r  r  _cooldown_time_cooldown_listr  s                rf   rA  zRouter.get_available_deployment  s8    &*%M%M 3	 &N &
"" )40&&+L,
  9$(;K 
 #?? 3!5 @ 
 &&8+?"&"7"7$7!-	 #8 # "#q(**e*<I!00AA#6F B N 7(,?ON ','+'B'B,	    L0T5J5J5V..HH!7J I J ""&66 "$($7  !!%<<))522LL!$7- M J !!%::%%1..HH!$7!	 I J !!%==((411KK!$7!	 L J J!&&6ug=VW **e*<I!00AA#6F B N 7(,?ON ','+'B'B,	  	""25'9PQUQfQfgqQrPss  AF  @G  H	
 re   r  c                     g }t        j                  d|        |D ]   }|d   d   }||v s|j                  |       " |D ]  }|j                  |        |S )a  
        Filters out the deployments currently cooling down from the list of healthy deployments

        Args:
            healthy_deployments: List of healthy deployments
            cooldown_deployments: List of model_ids cooling down. cooldown_deployments is a list of model_id's cooling down, cooldown_deployments = ["16700539-b3cd-42f4-b426-6a12a1bb706a", "16700539-b3cd-42f4-b426-7899"]

        Returns:
            List of healthy deployments
        zcooldown deployments: r  r  )r   r   r   r  )r   r   r  deployments_to_remover  r  s         rf   r  z#Router._filter_cooldown_deploymentsQ  sr     !###&<=Q<R$ST-J&|4T:M 44%,,Z8 . 0J&&z2 0""re   c                     	 |j                  di       j                  dd      }||| j                  ||       yyy# t        $ r+}t        j                  dt        |              Y d}~yd}~ww xY w)z7
        Tracks successful requests rpm usage.
        r  r  Nz$Error in _track_deployment_metrics: )rE  r  r"  r   r  r   )r   r  r_  r.  r  r$  s         rf   r|  z Router._track_deployment_metricsl  s    
	Y!~~lB7;;D$GH '&& "2 (    	Y!''*NsSTvh(WXX	Ys   8= 	A1!A,,A1rP  c                 H    t        ||| j                  | j                        S )N)rP  r  r   r   )"_get_num_retries_from_retry_policyr   r   )r   rP  r  s      rf   r.   z(Router.get_num_retries_from_retry_policy~  s)     2#%)%B%B**	
 	
re   c                    | j                   }|yt        |t        j                        r|j                  |j                  S t        |t        j
                        r|j                  |j                  S t        |t        j                        r|j                  |j                  S t        |t        j                        r|j                  |j                  S t        |t        j                        r|j                  |j                  S yy)a  
        BadRequestErrorRetries: Optional[int] = None
        AuthenticationErrorRetries: Optional[int] = None
        TimeoutErrorRetries: Optional[int] = None
        RateLimitErrorRetries: Optional[int] = None
        ContentPolicyViolationErrorRetries: Optional[int] = None
        N)r   r   r   r  BadRequestErrorAllowedFailsr  AuthenticationErrorAllowedFailsr  TimeoutErrorAllowedFailsr  RateLimitErrorAllowedFailsrI  'ContentPolicyViolationErrorAllowedFails)r   rP  r   s      rf   get_allowed_fails_from_policyz$Router.get_allowed_fails_from_policy  s     >B=V=V' y'"9"9:$@@L'CCCy'"="=>$DDP'GGGy'//2$==I'@@@y'"8"89$??K'BBBy'"E"EF$LLX'OOO Y Gre   c                 J   ddl m} | j                  y | j                  } ||j                  dg|j                        }|| _        t        j                  j                  |       t        j                  j                  |j                         t        j                  d       y )Nr   )SlackAlertingslack)alerting_thresholdalertingdefault_webhook_urlz2[94m
Initialized Alerting for litellm.Router[0m
)1litellm.integrations.SlackAlerting.slack_alertingr#  r   r%  webhook_urlslack_alerting_loggerr   r  r   r   !response_taking_too_long_callbackr   r   )r   r#  router_alerting_config_slack_alerting_loggers       rf   r   zRouter._initialize_alerting  s    S'151E1E!.5HHY 6 B B"
 &<"  !78  ''"DD	
 	""H	
re   CustomRoutingStrategyc                 `    t        | d|j                         t        | d|j                         y)a  
        Sets get_available_deployment and async_get_available_deployment on an instanced of litellm.Router

        Use this to set your custom routing strategy

        Args:
            CustomRoutingStrategy: litellm.router.CustomRoutingStrategyBase
        rA  ru  N)r  rA  ru  )r   r.  s     rf   set_custom_routing_strategyz"Router.set_custom_routing_strategy  s4     	&!::	

 	,!@@	
re   c                 N    d t         _        | j                  j                          y r   )r   r   flush_cacher  s    rf   r2  zRouter.flush_cache  s    

 re   c                     g t         _        g t         _        g t         _        g t         _        d | _        | j                          y r   )r   r   r   r   r   r   r2  r  s    rf   resetzRouter.reset  s6    #% *,'#% *,' re   )F)FFF)T)ro  rv  )NNNNr   )NNNNN)NNF)NNFN)r`   ra   rb   ri   r	   __annotations__rj   r   r  rk   r   tenacityrl   r   rm   r   rG   r   rA   r   r   r   r>   r   r  floatr
   rH   rE   r=   rD   rM   r<   r   r   r   r   rK   r   r%  rR   rP   r/  r+  r   r6   rS  re  r,  r  rC  r}  r  r  r  r  ri  r   r   r  rg  rh  r
  r  r  r  r9   r  r  r  r!  r  r$  r*  r,  r-  r4  r3  rQ   r9  r7  r@  r?  r8   rE  rD  r7   rT  rS  r[  rd  rn  r   r   rt  rj  r  r%  r  r  r"  r  r-  r  r  r   r   r   r   r  r\   r  r  r  rH  r  r  rG  r  r  r!  r+  r@   r7  r3  r   r   r5  r_  rf  ri  rb  ro  r  rs  rz  r  rC   r  r  r  r  r  rF  r  rA  r  r  r  r  r  rD  r  r  r  r  ru  rA  r  r|  r.   r!  r   r?   r0  r2  r4  rd   re   rf   rh   rh      s   K&+OXd^+&11H:>h67>:>h67> ;?#'$($((,*/ ,0*.%) #' 7;!06 )+)+ ',%*     ,0 DH&(DH48 "#WZ
$*+T$sCx.-AAB
Z
 $$78Z
 C=Z
 SMZ
 SMZ
 !Z
 "$Z
 Z
 !K
Z
" #Z
& #5/'Z
( #3-)Z
, c]-Z
.  
/Z
4 %5Z
6 !)!
7Z
< (0}=Z
> ?Z
@ _-AZ
B $I
CZ
H IZ
J #'KZ
L #'MZ
N $eC!::;;<
OZ
T !%UZ
V #WZ
X YZ
Z +t#$
[Z
` #'#
aZ
f  
gZ
l '
mZ
r  
sZ
x $D>yZ
z "&
{Z
J #++@"AKZ
L  $MZ
N !))@ AOZ
P ".1QZ
R "*!"
SZ
X 
YZ
x	F$ 8(01F(G8"? %os&: ;?TX?B4 ($(c3h$8	}11	2&@@$(c3h$8@	}11	2@H $()9$:DKDM	 
 UZ$()9$:DKEN	 
 kp$()9$:DI'RV-Y`afYgJgDh	"M1	2  	22 '(2 	2h{{$(c3h$8{	}11	2{z	IS 	I$ 	I4 	I
$ 
$$ 
$G Gt GPT G( t 44 t uczAR8S "Z#S	Z# T#s(^,d4S#X3G.HHIZ#x++$(.>)?$@+^ $(c3h$8BI$-	  SX$(c3h$8BI%.	  	K,K, tCH~&K, 	K,b di$()9$:FISZ[`Sa	 
 $()9$:FISZ[_S`	  << '(< 	<|;; ; $	;
 CHo; S#X;z  $BBBB '(BB S#X	BBHs 3 , ,C ,\c # *Es E3 EN( (3 (TC) CC CJS3 Ss S3 Sj3 *(C (Tc H $)&+#(## # 4.	#
 d^# 4.#R $)&+#(## # 4.	#
 d^# 4.#JCS C# CR $)&+#(  4.	
 d^ 4.:CS C CR $)	 S$Y 4.	 
$6c4i 0 6 6x $(	 S$Y 4.	 
2CuS$Y'7 C CL 
	0OO 
	Ob 
	0UU 
	Un? 
?B//f
1#
1 :F# 56: EI(,	
#
 &g.?&@A
 %	
,R%n &*$(373711 c]1 D>	1
 #+4.1 #+4.1f}%~   :>)1#8 /3*.3737,0?? &d^? "$	?
 #+4.? #+4.? $D>?B%H &*$S$s)^,-$ c]$ 
$s)		$< /3*.// / 	/
 &d^/ "$/ 
sEz	/fPd! 
#!F< 
<| 
+3C= 
D     t  D 4<TN	> (,	::  S/:  }	:
 
:x ##$1#;?#	#J5c 5Xd^ 5.55,4TN5	tDz4:%	&5<54 5( 15	;; #4.; n-	;F *.041,1, "$Z1, 4 012	1,
 #4.1, !1, n-1,f'c '4 '800 0 	0
 0 
*	0d&z &d &P+A +AZb* b bH 8L 2 J  8J;O  DC HZ,@ .s x
/C " #	*	$ EI58>B	 
 58>A	  !	KTNK !K SM	K
 
KZ $ 
:# 
:(4. 
:A A >AA 	.	!A F
 
8P 
>G$G$	x}hsm+	,G$R cSVh > ;?""*23-"	"H c " =A,4SM	!	"(c   +/-"3--	$*+	,-` UY"3-DLSM	c49n	8"%	.#>0WdY@ *.}%}% "}% tCH~&	}%
 !}%~3 8C= &Uc Ud U 48,0.3L*L* 4S#X/0L* c4i()	L*
 &d^L* 
sE$*%%	&L*b 48,0.3)-uu 4S#X/0u c4i()	u
 &d^u !ut 48,0.3)-{{ 4S#X/0{ c4i()	{
 &d^{ !{z##':#EI#Y#	d#8 FJY,4TNY& BF
"
19#
&Py &PP
0
%>
.!re   rh   )rk  r   enumr%  r  r#  r   r  rf  ro  r  r  r   typingr   r   r   r   r	   r
   r   r   r   r   httpxr   r   pydanticr   typing_extensionsr   r   litellm.litellm_core_utils2litellm.litellm_core_utils.exception_mapping_utilsr   litellm._loggingr   litellm.caching.cachingr   r   r   "litellm.integrations.custom_loggerr   'litellm.litellm_core_utils.core_helpersr   *litellm.litellm_core_utils.litellm_loggingr   r  &litellm.router_strategy.budget_limiterr   "litellm.router_strategy.least_busyr   #litellm.router_strategy.lowest_costr   &litellm.router_strategy.lowest_latencyr   &litellm.router_strategy.lowest_tpm_rpmr   )litellm.router_strategy.lowest_tpm_rpm_v2r   &litellm.router_strategy.simple_shuffler    )litellm.router_strategy.tag_based_routingr!    litellm.router_utils.batch_utilsr"   r#   /litellm.router_utils.client_initalization_utilsr$   #litellm.router_utils.cooldown_cacher%   &litellm.router_utils.cooldown_handlersr&   r'   r(   r)   r*   ,litellm.router_utils.fallback_event_handlersr+   r,   r-   *litellm.router_utils.get_retry_from_policyr.   r  !litellm.router_utils.handle_errorr/   r0   Dlitellm.router_utils.pre_call_checks.prompt_caching_deployment_checkr1   >litellm.router_utils.router_callbacks.track_deployment_metricsr2   r3   litellm.schedulerr4   r5   litellm.types.llms.openair6   r7   r8   r9   litellm.types.routerr:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   litellm.types.servicesrL   litellm.types.utilsrM   rN   r  rO   litellm.utilsrP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   &router_utils.pattern_match_deploymentsr[   opentelemetry.tracer\   _SpanEnumr^   rh   rd   re   rf   <module>r_     s,              #       &  ! 9 " 2 H H ; U P G F H N J P A M U =  
 2 T T    ( 0 7 9 6   G1DD$)) [Y [Yre   