
    g                    5   U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlmZmZ d dlmZmZmZmZmZmZmZmZmZ er	d dlmZ eZneZddZee_         ej:                  de       g Ze e!d<   ejD                  jG                  d ejD                  jI                  d	             	 d dl%Z%d dl&Z&d dl'Z'd dl(Z(d dl)Z)d d
l*m+Z+ g dZ.d Z/d dl0m1Z1 d dl2m3Z3 d dl4Z4d dl4m5Z5 d dl6m7Z7m8Z8 d dl9m:Z:m;Z; d dl<m=Z= d dl>m?Z? d dl@mAZAmBZB d dlCmDZDmEZE d dlF d dlGmHZI d dlJmKZK d dlLmMZM d dlNmOZO d dlPmQZQ d dlRmSZSmTZTmUZU d dlVmWZWmXZX d dlYmHZZ d dl[mHZ\ d d l]m^Z^ d d!l_m`Z`maZambZb d d"lcmdZd d dlcmHZe d d#lfmgZgmhZh d d$limjZjmkZk d d%llmmZmmnZn d d&lompZp d d'lqmrZr d d(lsmtZt d dlumHZv d d)lumwZw d dlxmHZy d d*lzm{Z{m|Z| d d+l}m~Z~ d dlmHZ d d,lmZ d d-lmZ d d.lmZ d d/lmZ d d0lmZ d dlmHZ d dlmHZ d dlmHZ d d1lmZ d d2lmZmZmZ d dlmHZ d dlmHZ d dlmHZ d dlmHZ d d3lmZ d d4lmZ d dlmHZ d d5lmZ d dlmHZ d d6lmZ d dlmHZ d d7lmZ d dlmHZ d dlmHZ d d8lmZ d dlmHZ d d9lmZ d dlmHZ d d:lmZmZmZmZmZmZmZmZmZmZmZmZmZ d dlmHZ d dlmHZ d d;lmZ d d<lmZmZmZmZ d d=lmZmZmZ d d>lmZ d d?lmZ d d@lmZmZmZmZ d dAlmZ d dBlmZmZmZmZ d dClmZ d dDlmZ d dElmZmZ d dFlmZ d dDlmZ d dGlmZ d dHlmZ 	 d dIlmZ dKe4_        d dlZd dLlmZ d dMl'mZmZmZmZmZmZmZmZmZmZmZmZ d dNl mZ d dOlmZ d dPlmZ d dQlmZmZm	Z	m
Z
mZ d dRlmZ d dSlmZ d dTlmZ d dUlmZ 	 d dlmc mZ  ej.                  dVdW      Z eQ       at2        j5                         aee!dX<    ej.                  dY      Zee   e!dZ<    er       Ze
d[Z e!e!d\<   n e!e      Z  ej.                  d]      Z"ee   e!d^<   e"
d_Z#e$e!d`<   n e$e"      Z#e daZ%dbe% dcZ&e&ddz  Z&deZ't6        r ej.                  dfdg      ndgZ(t6        r ej.                  dhdie' dje&       n	dke' dje& Z)dl Z*dm Z+e3dnefdo       Z, e e        e       e(e)eee,p      Z-dq Z. ej.                  drds      dtk(  rt6        r
e.e-_/         G du dve0jb                        Z2e-jg                  e4      dwedxe4fdy       Z5 e       ZHdzgZ6	 ejD                  jo                  ejD                  jI                  e8            Z9ejD                  ju                  e9d{d|      Z;e-jy                  d} ee;dK~      d        ejz                  e;      D ]  Z>e>j                  d      se>dk7  sejD                  j                  e>      d    ZAejD                  ju                  e;eA      ZB ej                  eBdK       ejD                  ju                  e;e>      ZDejD                  ju                  eBd      ZE ej                  eDeE        edWk7  rO eGde d}        ej.                  d      eej                  d<   e-j                  d      dwefd       ZJe-j                  ee6dKdzgdzg       d dlmLZL daMdaNdaOdaPdaQdaRdKaSdZTdaUdaVee   e!d<   dKZWdaXdaYee5   e!d<   daZee    e!d<   i a[e\e!d<   i a]e\e!d<   dZ^dZ_da`ee   e!d<   daadabee   e!d<    e:e2j                  j                        Ze eee      Zfe4j                  j                  ef       daiee;   e!d<   dajdakdaldamdandaodapi aqg Zree!d<   dZsdatdauevd   e!d<   dawdaxdaydazda{ eO       Z|da}ee   e!d<   da~ee!d<   daee   e!d<    eeet6              adZdZdZdaeeD   e!d<   ddddddddi d	dedee   dee   dee   dee   dee   dee   deee$ef      dee   dee\   de\fdZdwefdZd ZdefdZddefdZd Zd Z	 ddee\ef   dedee   fdZ	 	 	 	 	 	 	 	 ddZdee   dee   dee   dee   dee$   dee   fdZd ZdÄ Z G dĄ dū      Z e       adƄ Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddǄZdȄ Zdede\fdɄZdede\fdʄZdede\fd˄Zdede\fd̄Zi fd͐e\fd΄Zdτ ZdЄ Z G dф dҫ      ZeHjA                  d eeW      gdgի      eHjA                  d eeW      gdgի       eeW      fdefdׄ              ZeHjE                  d eeW      gdgի      eHjE                  d eeW      gdgի      eHjE                  d eeW      gdgի      eHjE                  d eeW      gdgdddiiet       e&jF                  e&jH                  ee e#eee7      d eeW      fdwededee   defd                                   ZeHjE                  d eeW      gdgի      eHjE                  d eeW      gdgի      eHjE                  d eeW      gdgի      eHjE                  d eeW      gdgի      d eeW      fdwededee   defd                            ZeHjE                  d eeW      ge	dg      eHjE                  d eeW      ge	dg      eHjE                  d eeW      ge	dg      eHjE                  d eeW      ge	dg      d eeW      fdwededee   defd                            ZeHjE                  d eeW      ge	dg      eHjE                  d eeW      ge	dg       eeW      fdwededefd              ZeHjE                  d eeW      gdgի      eHjE                  d eeW      gdgի       eeW      fdwededefd              ZeHjE                  d eeW      gdgի      eHjE                  d eeW      gdgի       ed       eeW      fdwedededefd              Zd dl'mZmZmZ d dl4mZ e-j]                  d       e-j]                  d       eeX      fded͐efd              ZeHjA                  d eeW      gdgի      eHjA                  d eeW      gdgի       eeW      fdwededefd              ZeHjE                  d eeW      gdgի      eHjE                  d eeW      gdgի       eeW      fdwededefd              ZeHje                  d	 eeW      gdgի      eHje                  d
 eeW      gdgի       eeW      fdwedededefd              ZeHjE                  d eeW      gdgի      eHjE                  d eeW      gdgի       eeW      fdwededefd              ZeHjA                  d eeW      gdgի      eHjA                  d eeW      gdgի       eeW      fdwedededefd              ZeHjE                  d eeW      gdgի      eHjE                  d eeW      gdgի       eeW      fdwedededefd              ZeHjA                  d eeW      gdgի      eHjA                  d eeW      gdgի       eeW      fdwedededefd              ZeHjE                  d eeW      gdgի      eHjE                  d eeW      gdgի       eeW      fdwedededefd              ZeHjE                  d eeW      ge	dg      eHjE                  d eeW      ge	dg       eeW      fdwededefd              ZeHjE                  dd g eeW      ged!       eeW      fd"ededwedefd#       ZeHjE                  d$d%g eeW      ge&      dwefd'       ZeHjA                  d(d%g eeW      g)      d͐efd*       Zd+ededefd,Zd+ededefd-Zdee   dedefd.ZeHjE                  d/d0dg eeW      g1       eeW      fd+edefd2       ZeHjE                  d3d4dg eeW      g1       eeW      fd+edefd5       ZeHjA                  d6d7dg eeW      gd8       eeW       e'j                  dd9:      dfdedee   d;ee   fd<       ZeHjA                  d=d>dgd eeW      g?       eeW      dddfded@ee   dAee   dBee   fdC       ZeHjA                  dDdEdgd eeW      g?       eeW      dFddddfded@ee   dAee   dBee   dGee   dHee   fdI       ZeHjA                  dJdKdgd eeW      g?       eeW      dFddddfded@ee   dAee   dBee   dGee   dHee   fdL       ZeHjA                  dMdNdgd eeW      g?       eeW      dddddfded@ee   dAee   dBee   dGee   dHee   fdO       ZeHjA                  dPdg eeW      g)      eHjA                  dQdg eeW      g)       eeW      dfdedRee   fdS              Zde5dTee   dUee   dee   fdVZeHjA                  dWdg eeW      g)       eeW      dfdedUee   fdX       ZeHjE                  dYdZdg eeW      g1      d[efd\       ZeHjA                  d]d^dg eeW      gd8      d_        ZeHjA                  d`dadbg eeW      gd8       eeW      fdefdc       ZeHjE                  dddeg eeW      gdf      d eeW      fdwededee   defdg       ZАe-jA                  dhdegdi      dwefdj       ZeHjE                  dkdl      dwefdm       ZҐe-jA                  dndl      doefdp       ZӐe-jE                  dqdl      drefds       ZՐe-jA                  dtdl      du        ZeHjE                  dvdwg eeW      ged!       eeW      fdredefdx       ZeHjA                  dydwg eeW      ged!       eeW      fdzedefd{       ZeHjE                  d|dwg eeW      ged!       eeW      fdredefd}       ZeHjE                  d~dwg eeW      ged!       eeW      fdredefd       ZeHjE                  ddg eeW      gdf      defd       Z	 eHjE                  ddg eeW      gdf       eeW      fdredefd       ZeHjA                  ddg eeW      ged!       eeW      fdedefd       ZeHjA                  ddg eeW      gdf       eeW      fdevd   dedee   fd       ZeHjE                  ddg eeW      gdf       eeW      fdredefd       ZeHjA                  ddgd eeW      g      d        ZeHjA                  ddg eeW      gdf      defd       ZeHjA                  dd eeW      g      d        ZeHjA                  d eeW      g      dwefd       ZeHjA                  d eeW      g      d        Ze-j                  eH       e-j                  eZ       e-j                  e       e-j                  ev       e-j                  eƫ       e-j                  e       e-j                  eī       e-j                  e       e-j                  e       e-j                  e       e-j                  e       e-j                  e       e-j                  e       e-j                  e       e-j                  e       e-j                  e       e-j                  e\       e-j                  eI       e-j                  ey       e-j                  ee       e-j                  e       e-j                  e       e-j                  e       e-j                  e       y# e,$ rZ- e,de- d      dZ-[-ww xY w# e$ r dJZY w xY w# e$ r 	 d dlZn# e$ r Y nw xY wY w xY w# e$ r Y w xY w(      N)datetime	timedelta)	TYPE_CHECKINGAnyListOptionalTuplecastget_args
get_originget_type_hints)Spanc                 \    | d| d|j                    d|  d}||j                  |       y y )N:: 
)__name__write)messagecategoryfilenamelinenofilelinetraceback_infos          O/var/www/openai/venv/lib/python3.12/site-packages/litellm/proxy/proxy_server.pyshowwarningr   #   s>     z6("X->->,?r'"MN

>"     default)r   messagesz../..)AsyncIOSchedulerzMissing dependency z$. Run `pip install 'litellm[proxy]'`)z%'The thing I wish you improved is...'z'A feature I really want is...'z*'The worst thing about this product is...'z$'This product would be better if...'z 'I don't like how this works...'z&'It would help me if you could add...'z/'This feature doesn't meet my needs because...'z&'I get frustrated when the product...'c                     d} t        j                  t              }t                t        dd| z  z   dz          t        dd| z  z   dz          t        ddj	                  |      z          t        ddj	                  d      z          t        dd| z  z   dz          t        dd| z  z   dz          t                t        d	       t                t                t                t        d
       t                t                y )N<   z[1;37m#-z#[0m z[1;37mz# {:^59} #[0mz-https://github.com/BerriAI/litellm/issues/newz/ Thank you for using LiteLLM! - Krrish & IshaanzR[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new[0m)randomchoicelist_of_messagesprintformat)	box_widthr   s     r   generate_feedback_boxr,   K   s    I mm,-G	G	
sY
.
;<	
sY
.
;<	,,33G<
<=	

$
$%T
U	V 

sY
.
;<	
sY
.
;<	G	
;<	G	G	G	b 
G	Gr   )defaultdict)asynccontextmanager)Routerverbose_proxy_loggerverbose_router_logger)	DualCache
RedisCache)RejectedRequestError)SlackAlerting)!_get_parent_otel_span_from_kwargs get_litellm_metadata_from_kwargs)AsyncHTTPHandlerHTTPHandler)*router)log_db_metrics)check_response_size_is_safe)
JWTHandler)LicenseCheck)get_complete_model_listget_key_modelsget_team_models)user_api_key_authuser_api_key_auth_websocket)	html_form)get_logging_caching_headers3get_remaining_tokens_and_requests_from_request_datainitialize_callbacks_on_proxy)init_verbose_loggers)decrypt_value_helperencrypt_value_helper)_read_request_bodycheck_file_size_under_limit)!get_config_file_contents_from_gcsget_file_contents_from_s3)%remove_sensitive_info_from_deployment)
ProxyState)ERROR_RESPONSES)set_fine_tuning_config)init_guardrails_v2initialize_guardrails)perform_health_check)&_PROXY_VirtualKeyModelMaxBudgetLimiter)"_OPTIONAL_PromptInjectionDetection)_PROXY_failure_handler)_PROXY_track_cost_callback)add_litellm_data_to_request)user_update)delete_verification_tokenduration_in_secondsgenerate_key_helper_fn)update_team),get_disabled_non_admin_personal_key_creation)create_audit_log_for_update)set_files_config)!initialize_pass_through_endpoints)route_request)get_logging_payload)PrismaClientProxyLogging_cache_user_row_get_docs_url_get_projected_spend_over_limit_get_redoc_url_is_projected_spend_over_limit_is_valid_team_configsget_error_message_strget_instance_fn
hash_tokenreset_budgetupdate_spend)set_default_vertex_config)AssistantsTypedDict
DeploymentLiteLLM_ParamsModelGroupInfo)DefaultPrioritiesFlowItem	Scheduler)load_aws_kms)load_google_kms)
get_secretget_secret_boolget_secret_strstr_to_bool)SlackAlertingArgs)AnthropicMessagesRequestAnthropicResponse!AnthropicResponseContentBlockTextAnthropicResponseUsageBlock)HttpxBinaryResponseContent)	ModelInfo)RouterGeneralSettingsupdateDeployment)CustomHuggingfaceTokenizer)StandardLoggingPayload)!get_end_user_id_for_cost_tracking)versionz0.0.0T)Union)DependsFastAPIFileFormHeaderHTTPExceptionPathQueryRequestResponse
UploadFilestatus)jsonable_encoder)CORSMiddleware)get_openapi)FileResponseJSONResponseORJSONResponseRedirectResponseStreamingResponse)	APIRouter)OAuth2PasswordBearer)APIKeyHeader)StaticFilesSERVER_ROOT_PATH premium_user+LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES'global_max_parallel_request_retries_env   #global_max_parallel_request_retries1LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRY_TIMEOUT-global_max_parallel_request_retry_timeout_envg      N@)global_max_parallel_request_retry_timeout/ui/u(   👉 [```LiteLLM Admin Panel on /ui```](z). Create, Edit Keys with SSOuB   

💸 [```LiteLLM Model Cost Map```](https://models.litellm.ai/).zr[**Customize Swagger Docs**](https://docs.litellm.ai/docs/proxy/enterprise#swagger-docs---custom-routes--branding)
DOCS_TITLEzLiteLLM APIDOCS_DESCRIPTIONzJEnterprise Edition 

Proxy Server to call 100+ LLMs in the OpenAI format. 

z5Proxy Server to call 100+ LLMs in the OpenAI format. c                  ,    d a d ad ad ad ad ad ad ad ad a	y N)

master_keyuser_config_file_pathotel_logginguser_custom_authuser_custom_auth_pathuser_custom_key_generateuser_custom_ssouse_background_health_checkshealth_check_intervalprisma_client r   r   cleanup_router_config_variablesr   v  s<     J L #O#'  Mr   c                  b  K   t        j                  d       t        r1t        j                  d       t        j	                          d {    t
        j                  &t
        j                  j	                          d {    t        j                          d {    t        t        j                          d {    dt
        j                  v r.	 ddlm}  | | j                  j                          t!                y t!                y 7 7 7 u7 U# t        $ r Y t!                y w xY ww)Nz"Shutting down LiteLLM Proxy ServerzDisconnecting from Prismalangfuser   langFuseLogger)r1   infor   debug
disconnectlitellmcachejwt_handlerclosedb_writer_clientsuccess_callbacklitellm.utilsr   Langfuseflush	Exceptionr   r   s    r   proxy_shutdown_eventr     s     BC""#>?&&(((}} mm&&(((



#$$&&& W---	4)''--/ $%#%/ 	) 	) 	'  	 $%	sl   AD/
D5D/ DD/D!D/>D?D/"D 8D/D/D/D/	D,D/+D,,D/appc                  K   dd l }t                t        dd       at        4t        dd       }t
        j                  |t        t               d {   at        j                  dj                  t                     t        du rt        j                         at        d      }t        d      }t        j                  d	|       |`t         j"                  j%                  |      r`t&        j)                  |
      rIt&        j+                  t,        |       d {   \  aaan |t3        |t4              r]t         j"                  j%                  |      r>t&        j)                  |
      r(t&        j+                  t,        |       d {   \  aaant         j6                  j9                  d      8t3        |t4              r(t&        j+                  t,        |       d {   \  aaanYt3        |t:              rt=        di | d {    n5 |j>                  |      }t3        |t:              rt=        di | d {    t
        jA                  t,        t        tB               t
        jE                  t0        t        t               tF        rtI        jJ                  tM                      tN        tN        jQ                  t,               t        j                  dt               t        /t        )t
        jS                  t        t        tT        t0               t        -tV        jX                  dkD  rt
        j[                  tT               t        ;t
        j]                  t0        t        t^        t`        tb        t               d {    t
        je                          d  tg                d {    y 7 q7 7 :7 7 7 7 B7 w)Nr   LITELLM_MASTER_KEYDATABASE_URL)database_urlproxy_logging_objuser_api_key_cachezElitellm.proxy.proxy_server.py::startup() - CHECKING PREMIUM USER - {}FWORKER_CONFIGCONFIG_FILE_PATHzworker_config: %sconfig_file_pathr=   r   LITELLM_CONFIG_BUCKET_NAME)
llm_routerr   redis_usage_cache)general_settingsr   r   r<   zprisma_client: %s)r   r   litellm_proxy_admin_namer   )litellm_proxy_budget_name)r   r   !proxy_budget_rescheduler_min_time!proxy_budget_rescheduler_max_timeproxy_batch_write_atr   r   )4jsonrK   r   r   r   ProxyStartupEvent_setup_prisma_clientr   r   r1   r   r*   r   _license_check
is_premiumr   ospathisfileproxy_configis_yamlload_configr   llm_model_listr   
isinstancestrenvirongetdict
initializeloads_initialize_startup_loggingr   _initialize_jwt_authr   asynciocreate_task_run_background_health_checkprompt_injection_detection_objupdate_environment_add_master_key_hash_to_dbr   r   
max_budget_add_proxy_budget_to_db$initialize_scheduled_background_jobsr   r   r   _init_dd_tracerr   )r   r   _db_urlworker_configenv_config_yamls        r   proxy_startup_eventr    sf      0$7J!+ND!A/DD /1 E 
 
 OVV	

 u%002 1;?0KM%34F%GO2MB"77>>/*|/C/C, 0D 0
 #..!O /  	  
	"}c*}-$$m$D #..!M /  	  ZZ^^89E*3K
 #..!M /  	  t,-}--- 'DJJ}5M-. 1=11111++ 2  **)#- +  $(*	
 &1&999L2MB Z%;44!'%=-	 	5 	
  W%7%7!%;11&> 	2 	

  DD-'.O.O!5/ E 
 	
 	
 %%' 
 
   e
8 .
 2N	
 !s   AOOCOOA0OO	AO&O''OO5OOEOO*O=O>OO	OOOOOO)docs_url	redoc_urltitledescriptionr   	root_pathlifespanc                  l   t         j                  rt         j                  S t        t         j                  t         j                  t         j
                  t         j                        } t        j                  j                  }i }|D ]  }| d   |   ||<    || d<   | t         _        t         j                  S )N)r  r   r  routespaths)
r   openapi_schemar   r  r   r  r  LiteLLMRoutesopenai_routesvalue)r  r  paths_to_includeroutes       r   custom_openapir  4  s    
!!! iiOOzz	N "//55M"0"9%"@ .N7'Cr   DOCS_FILTEREDFalseTruec                       e Zd ZdZy)UserAPIKeyCacheTTLEnumr#   N)r   
__module____qualname__in_memory_cache_ttlr   r   r   r$  r$  K  s    r   r$  requestexcc           	         K   |j                   }t        |j                  rt        |j                        nt        j
                  d|j                  |j                  |j                  |j                  di|      S w)Nerrorr   typeparamcode)status_codecontentheaders)	r2  r   r/  intr   HTTP_500_INTERNAL_SERVER_ERRORr   r-  r.  )r(  r)  r2  s      r   openai_exception_handlerr5  O  sg      kkG XXCM6+P+P ;;	
  s   A9A;r;   _experimentalout/ui)	directoryhtmlui)namez.htmlz
index.html)exist_okz8server_root_path is set, forwarding any /ui requests to PROXY_BASE_URLhttpc                    K   | j                   j                  j                  d      r9| j                   j                  j                  dt         dd      }t        |      S  ||        d {   S 7 w)Nr8     )urlr   
startswithreplaceserver_root_pathr   )r(  	call_nextnew_paths      r   redirect_ui_middlewarerH    sb     {{**51";;++33E>N=Os;SUVW'11"7++++s   A*A3,A1-A3)allow_originsallow_credentialsallow_methodsallow_headers)DictFr   r   r   r   callback_settingszapi_log.jsonr   r   )default_in_memory_ttl)
dual_cacher   queuezlitellm-proxy-budgetdefault_user_idall)adminrS  ui_access_modeiU  i]  
   r  store_model_in_dbopen_telemetry_logger)r   r   r   )	call_idmodel_id	cache_keyapi_baser   model_regionresponse_cost!fastest_response_batch_completionrequest_datauser_api_key_dictrY  rZ  r[  r\  r   r]  r^  r_  r`  returnc        
         J   dd h}||||||t        |      t        | j                        t        | j                        |t        |      nd d
|
j                         D ci c]  \  }}|t        |       c}}}|	r:t	        |	      }|j                  |       t        |	      }|r|j                  |       	 |j                         D ci c]  \  }}||vr|t        |       c}}S c c}}w c c}}w # t        $ r$}t        j                  d|        i cY d }~S d }~ww xY w)Nr   )
x-litellm-call-idzx-litellm-model-idzx-litellm-cache-keyzx-litellm-model-api-basezx-litellm-versionzx-litellm-model-regionzx-litellm-response-costzx-litellm-key-tpm-limitzx-litellm-key-rpm-limitz+x-litellm-fastest_response_batch_completionzError setting custom headers: )
r   	tpm_limit	rpm_limititemsrI   updaterH   r   r1   r+  )ra  rY  rZ  r[  r\  r   r]  r^  r_  r`  kwargsexclude_valueskvr2  remaining_tokens_headerlogging_caching_headerskeyr  es                       r   get_custom_headersrq    sB    $ZN$&($,$".#&}#5#&'8'B'B#C#&'8'B'B#C 1< 12 "(
0A1c!f9
0G" "U#
 	./"=l"K"NN23 &mmo
-
UN* UO-
 	
 1

  ""%CA3#GH	s6    C)8C5 C/%C5 /C5 5	D">DD"D"c                 L  K   t        j                          }t        j                          |z
  dk  rnt        j                  d       d{    | j                          d{   r|j	                          t        dd      t        j                          |z
  dk  rmyy7 V7 @w)am  
    Asynchronously checks if the request is disconnected at regular intervals.
    If the request is disconnected
    - cancel the litellm.router task
    - raises an HTTPException with status code 499 and detail "Client disconnected the request".

    Parameters:
    - request: Request: The request object to check for disconnection.
    Returns:
    - None
    X  rA  Ni  zClient disconnected the requestr0  detail)timer  sleepis_disconnectedcancelr   )r(  llm_api_call_task
start_times      r   check_request_disconnectionr|  
  s      J
))+

"S
(mmA((*** $$&8  ))+

"S
(*s*   AB$	B 
B$!B"";B$B$"B$c                     ddl m} t        |       }|t        u r!t	        |       D ]  }t        ||      s|c S  yt        | t              rt        | t              r| S y)CResolve the actual TypedDict class from a potentially wrapped type.r   )_TypedDictMetaN)typing_extensionsr  r   r   r   r   r-  r   )typr  originargs       r   _resolve_typed_dict_typer  '  sT    0_FC=C#~.
 !
  
C	:c4#8
r   c                 
   t        |       }g }|t        u rJt        |       D ]:  }|t        |t	        d            rdt        |      vs*|j                  |       < |S t        | t              rt        | t              r| gS |S )r~  NNoneType)r   r   r   r   r-  r   append	BaseModel)r  r  typsr  s       r   _resolve_pydantic_typer  5  sx    _FDC=C"3T
3c#h.C  ! K 
C	:c9#=uKr   use_azure_key_vaultc                 F   | du ry 	 ddl m} ddlm} t	        j
                  dd       }|t        d       |       } |||      }|t        _        t        j                  t        _        y # t        $ r+}t        |      }t        j                  d|       Y d }~y d }~ww xY w)	NFr   )DefaultAzureCredential)SecretClientAZURE_KEY_VAULT_URIzMError when loading keys from Azure Key Vault: AZURE_KEY_VAULT_URI is not set.)	vault_url
credentialztError when loading keys from Azure Key Vault: %s .Ensure you run `pip install azure-identity azure-keyvault-secrets`)azure.identityr  azure.keyvault.secretsr  r   getenvr   r   secret_manager_clientKeyManagementSystemAZURE_KEY_VAULT_key_management_systemr   r1   	exception)r  r  r  KVUrir  clientrp  
_error_strs           r   load_from_azure_key_vaultr  F  s    e#
97 		/6=_  ,-
 *E(.%)<)L)L& 
V
&& C	
 	

s   A$A, ,	B 5!BB c                      t         ot        t        j                  t              rPt        j                  d       t        t        j                  vr$t        j                  j                  t               y y y y )Nz.setting litellm success callback to track cost)	r   r   r   _async_success_callbacklistr1   r   r\   r  r   r   r   cost_trackingr  e  sZ     g55t< &&'WX*73R3RR//667QR S = !r   c                      t         ot        t        j                  t              rPt        j                  d       t        t        j                  vr$t        j                  j                  t               y y y y )Nz.setting litellm failure callback to track cost)	r   r   r   failure_callbackr  r1   r   r[   r  r   r   r   error_trackingr  n  sZ     g..5 &&'WX&w/G/GG((//0FG H 6 !r   payloadspend_logs_urlc                 &   |q|ot        | d   t              r| d   j                         | d<   t        | d   t              r| d   j                         | d<   |j                  j	                  |        |S ||j                  j	                  |        |S )N	startTimeendTime)r   r   	isoformatspend_log_transactionsr  )r  r   r  s      r   _set_spend_logs_payloadr  w  s    
  ^%?gk*H5#*;#7#A#A#CGK gi((3!(!3!=!=!?GI,,33G<  
	",,33G<r   c
           
        	K   	 t        j                  d d|  d d        | .t        | t              r| j	                  d      rt        |       n| fd}
fd}fd	}fd
}	fd}t        j                   |
              t        j                   |              t        j                   |              t        j                   |              t        du r |        d {    nt        j                  d       t        j                  d       y 7 0# t        $ r- t        j                  dt        j                                 Y y w xY ww)Nz&Enters prisma db call, response_cost: z	, token: z; user_id: z; team_id: sk-tokenc            	      v  K   t         j                         d{   } | t        | t              rt	        di | } 	 t
        g}t        j                  dkD  r|j                  t               |D ]9  }|t
        j                  j                  |d      z   t
        j                  |<   ; 5t
        j                  j                  d      z   t
        j                  <   yyy7 # t        $ rC}t        j                  ddt!        |       dt#        j$                          z          Y d}~yd}~ww xY ww)zu
            - Update that user's row
            - Update litellm-proxy-budget row (global proxy spend)
            ro  Nr   z[91mz&Update User DB call failed to execute r   r   )r   async_get_cacher   r   LiteLLM_UserTabler   r   r  r  r   user_list_transactonsr   end_user_list_transactonsr   r1   r   r   	traceback
format_exc)existing_user_objuser_ids_idrp  end_user_idr^  user_ids       r   _update_user_dbz(update_database.<locals>._update_user_db  s>     '9&H&HW&H&U U ,<Mt1T$5$J8I$J! , 'yH**Q. (AB'? -"/"E"E"I"I#q"Q!R *??D  ( #.)+EEII +Q &??L / -	 !V0  $))>s1vhbI]I]I_H`ab s@   D9C(!D9 8C* 9A,C* %D9*	D639D1,D91D66D9c                  ,  K   	 t        j                  d d d       y t        5t        j                  j	                  d      z   t        j                  <   y y # t
        $ r(} t        j                  dt        |               | d } ~ ww xY ww)Nz'adding spend to key db. Response cost: z	. Token: .r   z'Update Key DB Call failed to execute - )r1   r   r   key_list_transactonsr   r   r  r   )rp  hashed_tokenr^  s    r   _update_key_dbz'update_database.<locals>._update_key_db  s     $**=m_IVbUccde  ' ,%'<<@@qQR "66|D -
  $..=c!fXF 	s2   BA  B:A  B 	B)#BBBc                  $  K   	 t         ;t              } | d<   t        | t        j                  d      t               a y y # t
        $ r=}t        j                  dt        |       dt        j                                 |d }~ww xY ww)N)ri  response_objr{  end_timer  spendSPEND_LOGS_URL)r  r  r   z)Update Spend Logs DB failed to execute - r   )r   rh   r  r   r  r   r1   r   r   r  r  )r  rp  completion_responser  r  ri  r^  r{  s     r   _insert_spend_log_to_dbz0update_database.<locals>._insert_spend_log_to_db  s      ,1%%8#-!)$/G (5GG$$; '')yy1A'B&3%M -   $**?Axr)J^J^J`Iab 	s)   BA A B	B8BBBc                    K   	 t        j                  d d d       t        j                  d       y t        rt        j                  j	                  d      z   t        j                  <   	 d d } t        j
                  j	                  | d      z   t        j
                  | <   y y # t        $ r Y y w xY w# t        $ r=}t        j                  dt        |       d	t        j                                 |d }~ww xY ww)
Nz(adding spend to team db. Response cost: z. team_id: r  zAtrack_cost_callback: team_id is None. Not tracking spend for teamr   z	team_id::z::user_id::z#Update Team DB failed to execute - r   )r1   r   r   team_list_transactonsr   team_member_list_transactonsr   r   r   r  r  )team_member_keyrp  r^  team_idr  s     r   _update_team_dbz(update_database.<locals>._update_team_db  s%    $**>}o[Y`Xaabc ?(..[  ,%'==AA'1MN "77@
 -6gYk'*S)+HHLL / &BB?S -  %  $))9#a&IDXDXDZC[\ 	sV   D
3C D
:C 4<B2 0D
2	B>;C <D
=B>>C 	D
8DDD
c                    K   	 t        j                  dj                               t        j                  d       y t        5t        j                  j                  d      z   t        j                  <   y y # t        $ r=} t        j                  dt        |        dt        j                                 | d } ~ ww xY ww)Nz6adding spend to org db. Response cost: {}. org_id: {}.z?track_cost_callback: org_id is None. Not tracking spend for orgr   z"Update Org DB failed to execute - r   )r1   r   r*   r   org_list_transactonsr   r   r   r   r  r  )rp  org_idr^  s    r   _update_org_dbz'update_database.<locals>._update_org_db  s     $**LSS%v
 >(..Y  ,%'<<@@KL "66v> -
  $))8Q9CWCWCYBZ[ 	s4   C<A> C:A> <C>	C8B??CCFzwdisable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur.zRuns spend update on all tablesz Error updating Prisma database: )r1   r   r   r   rC  rs   r  r  disable_spend_logsr   r   r  r  )r  r^  r  r  r  ri  r  r{  r  r  r  r  r  r  r  r  s    `````````     @r   update_databaser    sA    g
""4]O9UGS^_f^ggrszr{|	
 E3!7E<L<LU<S%E2L L"	J	&	 	4 	F	. 	O-.N,-O-.N,-&)+++ %% J 	""#DE ,  
"".y/C/C/E.FG	

s;   E+C1D2 ?D0 /D2 /E+0D2 23E(%E+'E((E+r  r  r  r  parent_otel_spanc                 `  
K   g 
dt         dt        f
fd}
fd}
fd}
fd}	|  ||        d{     |        d{     |        d{     |	        d{    t        j                  t        j                  
d	|
             y7 c7 T7 E7 6w)z`
    Use this to update the cache with new user spend.

    Put any alerting logic in here.
    r  r^  c           	        K   t        | t              r| j                  d      rt        |       }n| }t	        j
                  d|       t        j                  |       d {   }t	        j
                  d|        t	        j
                  d|        |y |j                  }||z   }|j                  du r|j                  t        ||j                  d   	      d
u rt        ||j                  j                  dd       	      \  }}|j                  j                  dt        d            }t        |j                   xs d||j"                  ||j$                  ||      }	t'        j(                  t*        j-                  d|	             |'t/        |dd       |j0                  xs d}
|
|z   |_        |'t/        |dd       |j2                  xs d}||z   |_        ||_        j5                  ||f       y 7 w)Nr  r  z"_update_key_cache: hashed_token=%sr  z&_update_key_cache: existing_spend_obj=z#_update_key_cache: existing spend: Fsoft_budget)current_spendsoft_budget_limitTinfr   )r  r  	key_aliasr  r  projected_spendprojected_exceeded_dateprojected_limit_exceeded)r-  	user_info
team_spendr   team_member_spend)r   r   rC  rs   r1   r   r   r  r  soft_budget_cooldownlitellm_budget_tablero   rm   r   floatCallInfor  r  r  r  r  r   budget_alertsgetattrr  r  r  )r  r^  r  existing_spend_objexisting_spend	new_spendr  r  
soft_limit	call_infoexisting_team_spendexisting_team_member_spendvalues_to_update_in_caches               r   _update_key_cachez'update_cache.<locals>._update_key_cacheN  s9    eS!e&6&6u&=%E2L L""#GVBTBdBdiuBdBv<v""45G4HI	
 	""12D1EF	
 %/55N"]2	 33u<"77C."+&8&M&M%'  8W'"4"I"I"M"M!4#84O4 ,@@DDuU|J !(..4",66%*22 /(?I !//3' 0  **L$?K"4"?"?"D1,?-,O) **,?FR);)M)M)RQR& +]: 0
 $- !((,8J)KLW =ws   A G?#G<$FG?c                     K   	g} 	 | D ]  }|t         j                  |       d {   }| y t        j                  d| d        t	        |t
              r|d   }n|j                  }|z   }t	        |t
              r||d<   
j                  ||f       ||_        
j                  ||j                         f        t         j                  dj                  t                     d {   }|y /|,|z   }
j                  dj                  t              |f       y y y 7 7 <# t        $ r@}t        j                  dt        |       dt        j                                 Y d }~y d }~ww xY ww)Nr  z!_update_user_db: existing spend: ; response_cost: r  z{}:spendz'An error occurred updating user cache: r   )r   r  r1   r   r   r   r  r  r   r*   r   r   r   r  r  )r  r  r  r  r  global_proxy_spend	incrementrp  r^  r  r  s           r   _update_user_cachez(update_cache.<locals>._update_user_cache  s    9*	;+=+M+MRU+M+V%V"%-$**78J7KK\]j\kl 0$7%7%@N%7%=%=N*]:	 0$72;&w/-44c;M5NO/8&,-44c;M;R;R;T5UV3  6 (:'I'I%%&>? (J ( " ")*/A/M.>	)00&&'?@)L 0N*; &W."  	 &&9#a&iFZFZF\E]^ 	sg   E>!D2 D-D2 E>CD2 3D04D2 :E>;/D2 *E>-D2 0D2 2	E;;6E61E>6E;;E>c                     K   y dj                        } 	 t        j                  |        d {   }|y t        j                  d| d        |d}n"t        |t              r|d   }n|j                  }|z   }t        |t              r||d<   j                  | |f       y ||_        j                  | |j                         f       y 7 # t        $ r+}t        j                  dt        |              Y d }~y d }~ww xY ww)Nzend_user_id:{}r  z%_update_end_user_db: existing spend: r  r   r  +An error occurred updating end user cache: )r*   r   r  r1   r   r   r   r  r  r   r   r  r   )r  r  r  r  rp  r  r^  r  s        r   _update_end_user_cachez,update_cache.<locals>._update_end_user_cache  s0    -"7%%k2	'9'I'Ic'I'R!R!)  &&78J7KK\]j\kl ")!"0$7%7%@N%7%=%=N&6I ,d3.7"7+)00#7I1JK+4"()00#7I7N7N7P1QR1 "S2  	 **=c!fXF 	sU   DC CC DA/C +D,(C DC 	D !DDDDc                    K   y dj                        } 	 t        j                  |        d {   }|y t        j                  d| d        |d}n"t        |t              r|d   }n|j                  }|d}|z   }t        |t              r||d<   j                  | |f       y ||_        j                  | |f       y 7 # t        $ r+}t        j                  dt        |              Y d }~y d }~ww xY ww)Nz
team_id:{}r  z!_update_team_db: existing spend: r  g        r  r  )r*   r   r  r1   r   r   r   r  r  r   r  r   )r  r  r  r  rp  r^  r  r  s        r   _update_team_cachez(update_cache.<locals>._update_team_cache  s5    ?m3!!'*"	 )88S8AA  ") &&34F3GGXYfXgh ")250$7%7%@N%7%=%=N%!$&6I ,d3.7"7+)00#7I1JK+4"()00#7I1JK7 B8  	 **=c!fXF 	sU   DC CC DA3C /D0C 
DC 	D!C<7D<DDN)r  r^  r#   )
cache_listttllitellm_parent_otel_span)r   r  r  r  r   async_set_cache_pipeline)r  r  r  r  r^  r  r  r  r  r  r  s    ````     @r   update_cacher  =  s      8:RMs RM5 RMj-`#L'R ]6e=III """$&&& """330%5 	4 	
 	J 	# 	' 	#sG   ;B.B&B.B(B.$B*%B.5B,61B.(B.*B.,B.c                      	 ddg} t        t        j                  d      5 }t        j                  | ||       d d d        y # 1 sw Y   y xY w# t
        $ r#}t        j                  d| d       Y d }~y d }~ww xY w)Nollamaservew)stdoutstderrzd
            LiteLLM Warning: proxy started with `ollama` model
`ollama serve` failed with Exceptionz). 
Ensure you run `ollama serve`
        )openr   devnull
subprocessPopenr   r1   r   )commandr
  rp  s      r   run_ollama_server  7  sx    

W%"**c"gWWWE #"" 
""eefdg h		
 	

s3   A AA AA A 	A:A55A:c                  h  K   t        j                  t              } | y	 t        | t               d{   \  }}|t
        d<   |t
        d<   t        |      t
        d<   t        |      t
        d<   t        5t        t        t              r!t        j                  t               d{    7 y7 w)z}
    Periodically run health checks in the background on the endpoints.

    Update health_check_results, based on this.
    N)
model_listdetailshealthy_endpointsunhealthy_endpointshealthy_countunhealthy_count)copydeepcopyr   rX   health_check_detailshealth_check_resultslenr   r   r  r  rw  )_llm_model_listr  r  s      r   r  r  E  s      mmN3O
7K&0D8
 2
..
 5F016I23034E0F_-256I2J./ ,!52
 -- 5666 2
 7s"   2B2B.A3B2(B0)B20B2c            	          e Zd ZdZd3dZdedefdZdedefdZ		 d4de
e   defd	Zd
ededefdZdefdZ	 d5d
edededefdZdedee   defdZdefdZdefdZd4de
e   defdZd
efdZd Zde
ej2                     defdZdefdZde
e   fd Zd6defd!Zd"edefd#Z d"edefd$Z!d%ed&e"fd'Z#d(e
e$   fd)Z%d*ed+ed,e&defd-Z'd.e(d
ed/e
e   fd0Z)d.e(defd1Z*d.e(d&e"fd2Z+y)7ProxyConfigz}
    Abstraction class on top of config loading/updating logic. Gives us one place to control all config updating logic.
    rb  Nc                     i | _         y r   configselfs    r   __init__zProxyConfig.__init__i  s	    &(r   r   c                     t         j                  j                  |      syt         j                  j                  |      \  }}|j	                         dk(  xs |j	                         dk(  S )NFz.yamlz.yml)r   r   r   splitextlower)r"  r   _file_extensions       r   r   zProxyConfig.is_yamll  sW    ww~~./GG,,-=>>##%0TN4H4H4Jf4TTr   	file_pathc           	          	 t        |d      5 }t        j                  |      xs i cddd       S # 1 sw Y   yxY w# t        $ r}t        d| dt	        |             d}~ww xY w)z,
        Load and parse a YAML file
        rNzError loading yaml file r   )r	  yaml	safe_loadr   r   )r"  r)  r   rp  s       r   _load_yaml_filezProxyConfig._load_yaml_files  s_    	Ni%~~d+1r &%% 	N6ykCF8LMM	Ns)   = 1	= := = 	A%A  A%c           	        K   |xs t         }||a t        j                  j                  |       r,t	        | d      5 }t        j                  |      }ddd       n|t        d|       g i i i d}| j                  t        j                  j                  t        j                  j                  |xs d                  }t        j                  dt        j                  |d	              |S # 1 sw Y   xY ww)
z
        Given a config file path, load the config from the file.
        Args:
            config_file_path (str): path to the config file
        Returns:
            dict: config
        Nr+  zConfig file not found: )r  r   router_settingslitellm_settingsr   )r   base_dirzloaded config=   indent)r   r   r   existsr	  r,  r-  r   _process_includesdirnameabspathr1   r   r   dumps)r"  r   r)  config_filer   s        r   _get_config_from_filez!ProxyConfig._get_config_from_file}  s      %=(=	'$4! 77>>YK)c*k4 +*"5i[ABB !$&#%$&	F ''BGGOOBGGOOIOQS4T$U ( 
 	""^DJJva4P3Q#RS% +*s   <D C4B D 4C=9D r   r2  c                    d|vr|S t        |d   t              st        d      |d   D ]  }t        j                  j                  ||      }t        j                  j                  |      st        d|       | j                  |      }|j                         D ]3  \  }}t        |t              r||v r||   j                  |       /|||<   5  |d= |S )au  
        Process includes by appending their contents to the main config

        Handles nested config.yamls with `include` section

        Example config: This will get the contents from files in `include` and append it
        ```yaml
        include:
            - model_config.yaml

        litellm_settings:
            callbacks: ["prometheus"]
        ```
        includez&'include' must be a list of file pathszIncluded file not found: )r   r  
ValueErrorr   r   joinr6  FileNotFoundErrorr.  rg  extend)r"  r   r2  include_filer)  included_configro  r  s           r   r7  zProxyConfig._process_includes  s     F"M&+T2EFF #9-LX|<I77>>),'*CI;(OPP"229=O-335
UeT*sf}3K&&u-"'F3K	 6 . 9r   
new_configc                 :  K   	 t         Pt        j                  dd      du st        r2|j	                  dd        t         j                  |d       d {    y t        t         d      5 }t        j                  ||d       d d d        y 7 8# 1 sw Y   y xY ww)	NrW  FTr  r   )data
table_namer  )default_flow_style)
r   r   r   rW  popinsert_datar	  r   r,  dump)r"  rE  r;  s      r   save_configzProxyConfig.save_config  s     	 $  !4e<D  NN<.+++QQQ ./#6+		*keL 76	 R 76s*   ABBB+B
BBBdepth	max_depthc                    ||kD  rt        j                  d| d       |S |j                         D ]  \  }}t        |t              r| j                  ||dz   |      ||<   1t        |t              r0|D ]*  }t        |t              s| j                  ||dz   |      }, qt        |t              s|j                  d      st        |      ||<    |S )a  
        Check for os.environ/ variables in the config and replace them with the actual values.
        Includes a depth limit to prevent infinite recursion.

        Args:
            config (dict): The configuration dictionary to process.
            depth (int): Current recursion depth.
            max_depth (int): Maximum allowed recursion depth.

        Returns:
            dict: Processed configuration dictionary.
        zMaximum recursion depth (z") reached while processing config.rA  )r   rN  rO  os.environ/)
r1   warningrg  r   r   _check_for_os_environ_varsr  r   rC  r   )r"  r   rN  rO  ro  r  items          r   rS  z&ProxyConfig._check_for_os_environ_vars  s     9 ((+I;6XY M ,,.JC%&"== 	Y > s E4(!D!$-#>>#'uqyI  ?   " E3'E,<,<],K(/s ) r   r  all_teams_configc                     i }|D ]   }d|vrt        d|       ||d   k(  s|} n |j                         D ]6  \  }}t        |t              s|j	                  d      s)t        |      ||<   8 |S )Nr  zteam_id missing from team: rQ  )r   rg  r   r   rC  r   )r"  r  rU  team_configteamrk  rl  s          r   _get_team_configzProxyConfig._get_team_config  s    $D$"=dV DEE$y/)" %  %%'DAq!S!all=&A!+AA ( r   c                     | j                         }|j                  di       }|j                  dd      }|i S | j                  ||      }|S )z^
        - for a given team id
        - return the relevant completion() call params
        r1  default_team_settingsN)r  rU  )get_config_stater   rY  )r"  r  r   r1  rU  rW  s         r   load_team_configzProxyConfig.load_team_config  sh     &&( "::&8"=+//0GN#I++.> , 
 r   cache_paramsc                 "   ddl m} d|v r|d   t         _        d|v r|d   t         _         |di |t         _        t         j                  Dt        t         j                  j                  t              rt         j                  j                  ay y y )Nr   CacherO  default_redis_ttldefault_in_redis_ttlr   )r   ra  rO  rb  r   r   r4   r   )r"  r^  ra  s      r   _init_cachezProxyConfig._init_cache$  sx    
 	""l2,89P,QG),.(45K(LG%--==$GMM4G4G)T ' 3 3 *U$r   c                   K   t         j                  j                  d      t         j                  j                  d      }t         j                  j                  d      }t         j                  j                  d      }t        j                  d||       |dk(  rt        ||       d{   }nt        ||      }|%t        d      | j                  |	       d{   }t        $| j                  |t        t        
       d{   }t        j                  |      }|j                  dd       | j                  |      }| j!                  |       |S 7 7 }7 Uw)a  
        Load config file
        Supports reading from:
        - .yaml file paths
        - LiteLLM connected DB
        - GCS
        - S3

        Args:
            config_file_path (str): path to the config file
        Returns:
            dict: config

        r   N LITELLM_CONFIG_BUCKET_OBJECT_KEYLITELLM_CONFIG_BUCKET_TYPEzbucket_name: %s, object_key: %sgcs)bucket_name
object_keyz(Unable to load config from given source.r   )r   r   rW  environment_variablesr  )r   r   r   r1   r   rP   rQ   r   r<  r   _update_config_from_dbrW  r  r  rJ  rS  update_config_state)r"  r   ri  rj  bucket_typer   printed_yamls          r   
get_configzProxyConfig.get_config7  sJ    $ ::>>67C**..)EFK(JKJ**..)EFK &&1;
 e#@ +
   3 +
 ~ JKK  55GW5XXF$66+"3 7  F }}V,0$7000?   /9 Ys7   B)E"+E,4E" E!)E"
E AE"E" E"c                     || _         y r   r  )r"  r   s     r   rm  zProxyConfig.update_config_stateo  s	    r   c                 @    t        j                  | j                        S )z
        Returns a deep copy of the config,

        Do this, to avoid mutating the config state outside of allowed methods
        )r  r  r   r!  s    r   r\  zProxyConfig.get_config_stater  s     }}T[[))r   r=   c                   K   | j                  |       d{   }|j                  dd      }|rv|j                         D ],  \  }}t        t	        ||            t
        j                  |<   . d|v r3t        j                  dd      t        _	        t        j                         a|j                  dd      a|j                  dd      }|i }|rd}d	}	|j                         D ]  \  }}|d
k(  r|du rt        | d       ddlm}
 i }d|v r|d   }|j!                  |       |j                  dd      }t#        j$                  d|       |dk(  s|dk(  rt'        |j)                               dk(  rt	        dd      }t	        dd      }d}|j!                  |||d       t	        dd      t	        dd      }|j!                  d|i       t#        j$                  d||	|       t#        j$                  d||	|       t#        j$                  d||	|       t#        j$                  d||	|       |dk(  r|j!                  ddi       |j                         D ]7  \  }}t+        |      t        u s|j-                  d      s*t	        |      ||<   9 | j/                  |       t0        j2                  t#        j$                  | d |	        |d
k(  r|d!u r|d"k(  r t5        |t        ||#      }|t0        _        |d$k(  rt9        |t        ||%       6|d&k(  r?t;        ||'      gt0        _        t#        j$                  d(t0        j<                          z|d)k(  rt?        |      t0        _         |d*k(  rAt?        |      t0        _!        t0        j@                  t0        jB                  t0        _         |d+k(  r<dd,l"m#} |D cg c]  }|d-   t;        |d.   |'      d/ c}t0        _$         |        |d0k(  rg t0        _%        |D ]  }d1|v r*t0        jJ                  jM                  t;        |2             1t0        jJ                  jM                  |       d3|v sUt        s#tO        tP        jR                  jT                        t#        j$                  d4       dd5l+m,}  |       }tZ        j]                  d6|        t        | d7t0        jJ                   d8|	        	|d9k(  rg t0        _/        |D ]O  }d1|v r*t0        j^                  jM                  t;        |2             1t0        j^                  jM                  |       Q t        | d:t0        j^                   d8|	        |dk(  r|d;k(  rQta        |      D ]  \  }}	 tc        d|i |  t#        j$                  | d>| d?| |	        ti        t0        ||       |d@k(  r|ote        |tf              r_|j                         D ]6  \  }}te        |t              s|j-                  d      s)t	        |      ||<   8 tk        d|i |t0        _6        ctO        dA|       t#        j$                  | d>| d?| |	        ti        t0        ||        |j                  dBi       }|i }|r|j                  dCd      }| jo                  |D       |j                  dEd      }|tq        d|i |t0        _9        |j                  dFd!      }tu        |G       |j                  dHd!      }tw        |I       | jy                  |J       |j                  dKd      } | rG| j-                  d      r6t#        j$                  dL       t	        |       } t#        j$                  dM|        |j                  dNt	        dOd            a=tz        r_tz        j-                  d      rJt	        tz              a=te        tz        t              s'tO        dPj}                  t+        tz                          tz        #te        tz        t              rt        tz              a@|j                  dQd      }!|! t        j                  t?        |!      dR       |j                  dSd!      aCt        d!aC|j                  dTd      }"|"t;        |"|'      aD|j                  dUd      }#|#t;        |#|'      aE|j                  dVd      }$|$t;        |$|'      aF|j                  dWd      t        |dW   X       d{    |j                  dYdZ      aH|j                  d[d      }%|%t        d!u rt        d\      |j                  d]t              aJ|j                  d^t              aK|j                  d_t              aL|j                  d`t              aM|j                  dad!      aN|j                  dbdc      aO|j                  ddd      aP|j                  de      .t        dur&t        dftP        jR                  jT                  z         dg|v r"|dg   t        _	        t        j                         adht0        j2                  dui}&|j                  did      }'|'r|'|&di<   t        dj       |'D ]  }(|(dk   j                         D ]9  \  })}*te        |*t              s|*j-                  d      s)t	        |*      |(dk   |)<   ; t        dl|(j                  dmdn       d	       |(dk   do   }+|(dk   j                  dpd      },dq|+v s|,t                 d}-|j                  drd      }.|.rx|.dk   j                         D ]W  \  })}*te        |*t              s|*j-                  d      s)|*j                  ddn      }t        j                  |      }*|*|.dk   |)<   Y t        d|i |.}-|j                  dsd      }/t        |/t       |j                  dud      }0t        |0t       |j                  dvd      }1t        |1t       |j                  dwd      }2|2rwte        |2tf              rgt        j                  t0        j                        }3dxdih}4|3j                  D 5cg c]	  }5|5|4vs|5 }6}5|2j                         D ]  \  })}*|)|6v s|*|&|)<    t1        j                  d|i |&|-t        dy      dz}d}7||j                  d"d      }7|7rt        |7|{       ||j                         |fS 7 c c}w # tN        $ rJ te        |tf              rtO        d<| d=|j)                                tO        d<| d=t+        |             w xY w7 :c c}5w w)}z<
        Load config values into proxy global state
        r   Nrk  )secret_namedefault_valueLITELLM_LICENSErN  r1  z[94mz[0mr   Tz
Setting Cache on Proxyr   r`  r^  r-  rediszpassed cache type=%szredis-semantic
REDIS_HOST
REDIS_PORT)r-  hostportREDIS_PASSWORDpasswordz%sCache Type:%s %sz%sCache Host:%s %sz%sCache Port:%s %sz%sCache Password:%s %sredis_semantic_cache_use_asyncrQ  )r^  zSet Cache on LiteLLM ProxyF
guardrails)guardrails_configr   r   r1  	callbacks)r  r   r   r1  post_call_rules)r  r   zlitellm.post_call_rules: max_internal_user_budget default_max_internal_user_budgetcustom_provider_map)custom_llm_setupprovidercustom_handler)r  r  r   r  r  
prometheusz'Starting Prometheus Metrics on /metrics)make_asgi_appz/metricsz! Initialized Success Callbacks - r%   r  z! Initialized Failure Callbacks - r[  z4team_id missing from default_team_settings at index=z
passed in value=z setting litellm.=upperbound_key_generate_paramsz=Invalid value set for upperbound_key_generate_params - value=r   key_management_system)r  key_management_settingsuse_google_kmsr  r  r  )r   r   zGOING INTO LITELLM.GET_SECRET!zRETRIEVED DB URL: %sr   r   z.Master key must be a string. Current type - {}user_api_key_cache_ttl)rO  rb  rW  custom_authcustom_key_generate
custom_ssopass_through_endpointsr  rU  rS  allowed_ipsz\allowed_ips is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your envionment.r   r   r   r  background_health_checksr   ,  r  enforced_paramszTrying to use `enforced_params`litellm_licensecache_responsesr  z<[32mLiteLLM: Proxy initialized with Config, Set models:[0mlitellm_paramsz	[32m    
model_namer   modelr\  r  assistant_settingsfinetune_settingsr  files_settingsdefault_vertex_configr0  r"  async_only_mode)assistants_configrouter_general_settings)all_guardrailsr   r   )^rp  r   rg  r   r   r   r   r  r   license_strr   r   rN  r)   litellm.caching.cachingra  rh  r1   r   r  keysr-  rC  rd  r   r   rW   guardrail_name_config_maprJ   rr   r  r  r  r  r   r  r  r   r  r   CommonProxyErrorsnot_premium_userr  prometheus_clientr  r   mountr  	enumerateTeamDefaultSettingsr   r   setattr#LiteLLM_UpperboundKeyGenerateParamsr  initialize_secret_managerKeyManagementSettings_key_management_settingsr   r  _load_alerting_settingsr   r*   rs   litellm_master_key_hashr   update_cache_ttlrW  r   r   r   rf   rU  r?  r   r   r   r  r   r   r  r  rD  rw   rU   re   rv   inspectgetfullargspecr/   argsr   rV   get_model_list)8r"  r=   r   r   rk  ro  r  r1  blue_color_codereset_color_codera  r^  cache_params_in_config
cache_type
cache_host
cache_portcache_passwordr  r  rT  callbackr  metrics_appidxteam_setting_k_vr   r  r  r  r  r   r  r  r  r  r  router_paramsr  r  rk  rl  litellm_model_namelitellm_model_api_baser  r  finetuning_configfiles_configr  r0  arg_specexclude_argsxavailable_argsguardrails_v2s8                                                           r   r   zProxyConfig.load_configz  sE     "__>N_OO !'

+BD I 399;
U"%jSPU&V"W

3 < !$99-/YY7H$-O*-88: #JJ':DA "::&8$?#!(O(.446
U'>etm_--EFG=#%L%)991A.1Q.$++,BC!-!1!1&'!BJ(../EzR #g-?O1Ol//12a7%/d%C
%/d%C
)-$++(2(2(2 &&6=I-78H$-ON(//$.!" -220+,&	 -220+,&	 -220+,&	 -224+,*	 "%55$++-Mt,TU '3&8&8&:
U;#-%2B2B=2Q0:50AL- ';
 $$,$?}}0,22.//IJZI[\ G^L(0E*/%1)9)9	1- 9RG5K'1#%1)9)9	 --'eFVW/G+ )..3G4K4K3LM 667<U|G4>>?DU|G<77?#DD  8 11> %*	3 %*D )-Z(8.=&*+;&<1A/ %*	3G/ %&../1G, %*(?#44;; /h ?
 $44;;HE+x7'3*3(9(J(J(P(P+& %& !5 : :$M!" !L /<o #		*k B+ %*, *++LWMeMeLffghxgyz ../1G, %*(?#44;; /h ?
 $44;;HE %* *++LWMeMeLffghxgyz N* 33-6.)\	/?,?	. )..*++<SE5'JZI[\ GS%0<<(Zt-D&+kkmFB)"c2r}}]7S,6rNb	 '4 @H%H  > ([\a[bc  )..*++<SE5'JZI[\ GS%0e 7j "::&8"=#!$4$8$89PRV$W!**AV*W&6&:&:)4'# '23H 4-40 .112BEJN>:"2"6"67Le"T%:MN((:J(K+//EL 7 7 F$**+KL),7$**+A<P)--j)=tDJ j33MB'
3
!*c2#HOO ,  %*Z*E*4Z*@'%5%9%9($&" &1"33*/0F*G&* 4 
 !1 4 45H% P ($)! +..}dCK&#2%8H$  #3"6"67Ld"S".+:-@P,( *--lDAJ%"1$7G#
  ##$<dCO7+;<T+U   .11 %N +..}dCK&<5+@ r  1A0D0D35V1- 1A0D0D35V1- $4#7#7&(<$  "2!5!5$&8"
 ,<+?+?*E,( %5$8$89PRU$V!#3#7#78NPT#U  !$$%67C , 5'88>>?  !$44-=>O-P*-88: w}} 

 ZZd3
*4M,'T $!"2399;DAq!!S)all=.I5?]./2 < UYY|R%@$AIJ%*+;%<W%E")./?)@)D)DZQU)V&116L6T$& $ <@#ZZ(<dC*+;<BBD1a%!,,}*E="5B		"A>?&'78;	 E
 !4 I6H I #JJ':DA&78 zz"2D9- !'

+BD I!)>? !**%6=z/4@--gnn=H L
 *2PA!<:OaNP'--/1&'(M!$ 0  

/$9 $%
 /3"JJ|T:M,?O v,,.0@@@y PV	3F  ) ),=&/&Z[^Z__qr~  sD  sD  sF  rG  %H'" !" #,"VWZV[[mnrs  oA  nB  !C# \T Qs   us3Iu0u2u6C.u&*us6-A;u)D*us;A1uu#K4uuFu/uAuuAu$u6C/u%	u/u3uA$u6u;Auuur   c                    ddl m} |j                  dd      }t        j                  d|        |y|D ]  }|dk(  rpt
        j                  |j                  dd      |j                  dd      |j                  d	d      |j                  d
d      |j                  dd      t               x|t        j                  v s ||ddd|j                  dd      i      }|t        j                  j                  |        y)z.
        Initialize alerting settings
        r   )$_init_custom_logger_compatible_classalertingNz_alerting_callbacks: slackalerting_thresholdrs  alert_typesalert_to_webhook_urlalerting_args)r  r  r  r  r  redis_cache)logging_integrationinternal_usage_cacher   custom_logger_init_args)*litellm.litellm_core_utils.litellm_loggingr  r   r1   r   r   update_valuesr   r   )_known_custom_logger_compatible_callbacksr  r  )r"  r   r  _alerting_callbacks_alert_loggers         r   r  z#ProxyConfig._load_alerting_settings@  s   	
 /22:tD""%:;K:L#MN&)F !//-11*dC'7';';<PRU'V 0 4 4]D I)9)=)=.* #3"6"6"M 1 0 	 WNNNB,2-1#'+-=-A-A/SW-X1	G *))0093 *4 	r   r  c                 4   ||t         j                  j                  k(  rt        d       y|t         j                  j                  k(  rt        d       y|t         j                  j                  k(  rddlm} |j                  d       y|t         j                  j                  k(  rt        d       y|t         j                  j                  k(  rdd	lm}  |        y|t         j                  j                  k(  rdd
lm}  |        yt%        d      y)z_
        Initialize the relevant secret manager if `key_management_system` is provided
        NTr  r  r   )AWSSecretsManagerV2)use_aws_secret_manager)use_aws_kms)GoogleSecretManager)HashicorpSecretManagerz&Invalid Key Management System selected)r  r  r  r  
GOOGLE_KMSr   AWS_SECRET_MANAGER-litellm.secret_managers.aws_secret_manager_v2r  load_aws_secret_managerAWS_KMSr~   GOOGLE_SECRET_MANAGER-litellm.secret_managers.google_secret_managerr  HASHICORP_VAULT0litellm.secret_managers.hashicorp_secret_managerr  r?  )r"  r  r  r  r  s        r   r  z%ProxyConfig.initialize_secret_managerh  s     !,$(;(K(K(Q(QQ)dC&*=*H*H*N*NNt4%&99??@ $;;SW;X&*=*E*E*K*KK.%)<)R)R)X)XX $%&*=*M*M*S*SS '( !IJJC -r   c                    t        |dd      }|||j                  d<   d|j                  d<   t        du rht        |dd      |j                  d<   t        |dd      |j                  d<   t        |dd      |j                  d<   t        |d	d      |j                  d	<   |j                  t        |j                  t              rld|j                  vr|j
                  |j                  d<   d|j                  v r |j                  d   d
u r||j                  d<   t        di |j                  }|S t        |j
                  |      }|S )z
        Common logic across add + delete router models
        Parameters:
        - deployment
        - db_model -> flag for differentiating model stored in db vs. config -> used on UI

        Return model info w/ id
        rZ  NidTdb_model
created_at
updated_at
created_by
updated_byF)r  r  r   )r  
model_infor   r   r   rZ  RouterModelInfo)r"  r  r  r  _model_infos        r   get_model_info_with_idz"ProxyConfig.get_model_info_with_id  sD    %UJ=?%(ET"+/EZ(4-4UL$-OE\*-4UL$-OE\*-4UL$-OE\*-4UL$-OE\*'Ju7G7G,N5+++).  &U---%2B2B:2NRW2W/7  ,)=E,<,<=K  *U^^hOKr   	db_modelsc                   K   g }t         t        |      dk(  ry|D ]<  }| j                  |      }|j                  "|j	                  |j                         > | j                  t               d{   }|j                  dd      }|r|D ]  }|d   j                         D ]9  \  }}	t        |	t              s|	j                  d      s)t        |	      |d   |<   ; |j                  di       j                  d	d      }
|
t         j                  |d
   |d         }
|j	                  |
        t         j                         }d}|D ]%  }
|
|vst         j                  |
      }|!|dz  }' |S 7 w)a5  
        (Helper function of add deployment) -> combined to reduce prisma db calls

        - Create all up list of model id's (db + config)
        - Compare all up list to router model id's
        - Remove any that are missing

        Return:
        - int - returns number of deleted deployments
        Nr   r  r   r  r  rQ  r   r  r  )model_groupr  r  rA  )r   r  r  r  r  rp  r   r   rg  r   r   rC  r   _generate_model_idget_model_idsdelete_deployment)r"  r  combined_id_listmr   r   r  r  rk  rl  rZ  router_model_idsdeleted_deployments
is_deleteds                 r   _delete_deploymentzProxyConfig._delete_deployment  s      Y1!4 A4414=J}}( ''
6  8MNNZZd3
#!"2399;DAq!!S)all=.I5?]./2 <
 !99\26::4F#)<<$),$7',-='>  =  H !''1 $  &335  (H//'99X9F
)'1,'	 )
 #"9 Os0   ;E:6E:4E75AE:<E:BE:E:.
E:c                 `   ddl }t        t        t        t              st	        dt               t
        yd}|D ]  }|j                  }t        |t              rq|j                         D ]R  \  }}t        |t              st        |      }|t	        dj                  |            t        |      dkD  sN|||<   T t        di |}nt        j                  d|        | j                  |d      }	t
        j!                  t#        |j$                  ||		      
      }
|
|dz  } |S )z
        Iterate through db models

        for any not in router - add them.

        Return - number of deployments added
        r   N7Master key is not initialized or formatted. master_key=r  zUnable to decrypt value={}HInvalid model added to proxy db. Invalid litellm params. litellm_params=T)r  r  r  r  r   )
deploymentrA  r   )base64r   r   r   r   r   r  r   rg  rL   r*   r  ry   r1   r+  r  upsert_deploymentrx   r  )r"  r  r  added_modelsr  _litellm_paramsrk  rl  _valuer  addeds              r   _add_deploymentzProxyConfig._add_deployment  sK    	Z
C%@I*V  A..O/40+113DAq!!S)!5A!>!>"+,H,O,OPQ,R"SSv;?17OA. 4 #1"C?"C %**^_n^op 55$ 6 K 00% ||#2* 1 E  !C D r   
new_modelsr   c           	      .  K   dd l }	 t        Ht        At        j                  dt        |              g }|D ]  }|j                  }t        |t              r5|j                         D ]  \  }}t        |      }	|	||<    t        di |}nt        j                  d|        m| j                  |      }
|j                  t        |j                   ||
      j#                  d              t        |      dkD  rt        j                  d	|        t%        j&                  |t)        d
            at        j                  dt                nMt        j                  dt        |              | j+                  |       d {    | j-                  |       t        t        j5                         at8        j;                          d {   }|j=                  di       xs i }|j=                  dd       }|j=                  dd       }|It        |t>              r9|D ]4  }|t$        j@                  vst$        j@                  j                  |       6 |It        |t>              r9|D ]4  }|t$        jB                  vst$        jB                  j                  |       6 |j=                  di       }|j                         D ]'  \  }}	 t        |      }	|	|	tD        jF                  |<   ) t        itH        ctH        jJ                  jL                  jO                  ddi       d {   }|-|jP                  !|jP                  }t        jR                  di | |j=                  di       }d|v r!tT        tT        j=                  dd       t        tT        d   t>              rm|j=                  dd       [t        |d   t>              rHt        j                  d       |d   tT        d<   tT        d   |_+        tT        d   |jX                  _+        ntT        5i a*|d   tT        d<   tT        d   |_+        tT        d   |jX                  _+        nFt        tT        t              r2|d   tT        d<   tT        d   |_+        tT        d   |jX                  _+        d|v rB|d   tT        d<   tT        d   |_-        |jX                  j]                  tT        d   t               d|v r5|d   tT        d<   |jX                  j]                  tT        d   t               y y 7 # t.        $ r,}t        j0                  dt3        |              Y d }~d }~ww xY w7 # t.        $ r+}t        j                  d|t3        |             Y d }~d }~ww xY w7 aw)Nr   zlen new_models: r  r  r  r  Texclude_nonez_model_list: r  )r  r  zupdated llm_router: )r  z+Error adding/deleting model to llm_router: r1  r   r  rk  z#Error setting env variable: %s - %s
param_namer0  wherer   r  z?Overriding Default 'alerting' values with db 'alerting' values.r  )r  r   r  )r  r   r   )/r  r   r   r1   r   r  r  r   r   rg  rL   ry   r+  r  r  rx   r  to_jsonr   r/   r   r  r  r   r  r   r  r   r   rp  r   r  r   r  r   r   r   dblitellm_config
find_firstparam_valueupdate_settingsr   r  slack_alerting_instancer  r  )r"  r  r   r  _model_listr  r  rk  rl  decrypted_valuer  rp  config_datar1  success_callbacksfailure_callbacksr   r  rk  db_router_settings_router_settings_general_settingss                         r   _update_llm_routerzProxyConfig._update_llm_router	  s     	/	!j&<$**-=c*o=N+OP$&#A&'&6&6O!/48$3$9$9$;DAq.B.KO1@OA. %< +9*K?*K,22fgvfwx !"&"="=A"="FK&&"'(||+:'2 "'t'4 $, {#a'(..{m/LM!(#.0E,01"J )..1Ej\/RS$**-=c*o=N+OP--
-CCC $$z$: !'668N )3355&??+=rBHb,001CTJ,001CTJ(Z8I4-P$5 #7+C+CC,,334DE %6
 (Z8I4-P$5 #7+C+CC,,334DE %6 !,0G L)//1DAq"6Q"?".$3BJJqM	 2 !m&?'4'7'7'F'F'Q'Q#%67 (R ( " #.&22>#5#A#A **>-=> (OO,>C** ,$((T:F/
;TB%))*d;G0<dC$**U 0A0 , .>j-I!*EUF!99B ")#% /@/L ,-=j-I!*EUF!99B ,d3/@/L ,-=j-I!*EUF!99B --.?.N]+,<],K)55CC,];
 D  "%667H&834 55CC%56L%M% D 	 7} D
  	 **=c!fXF 	 6,  $**91c!f "s   VFT  TT  $1VUA'V>A
V	A	V!U4=V1V2G+VT   	U)!U
VUV	V$ V
V
VVdb_general_settingsc                 2  K   |yt        |      }d|v r|d   t        d<   d|v r|d   t        d<   d|v r3|d   t        d<   t        j                  j	                  t        d          d|v r(|d   t        d<   t        t        d          d{    yy7 w)z;
        Pull from DB, read general settings value
        Nmax_parallel_requestsglobal_max_parallel_requestsr  )r  r  r  )r   r   r   r+  r  rf   )r"  r5  r3  s      r   _update_general_settingsz$ProxyConfig._update_general_settings	  s     
 & !45"&778I'945 *->>?P.@;<
 //0A/0R_-55CC.? D 
 $'889J(:56 4'78P'Q  	 9s   BBBBcurrent_configr"  db_param_valuec                     t        ||   t              r%|j                         D ]  \  }}|s	|||   |<    |S |||<   |S r   )r   r   rg  )r"  r:  r"  r;  ro  r  s         r   _update_config_fieldsz!ProxyConfig._update_config_fields	  sX     nZ0$7 -224
U6;N:.s3 5  *8N:&r   r   rW  c                   K   |durt        j                  d       |S g }g d}|D ]'  }|j                  d|d      }|j                  |       ) t	        j
                  |  d {   }|D ]j  }|t        |dd       }	|	dk(  r"t        j                  d|j                          t        |d	d       }
|	J|
M|	|v r| j                  ||	|

      }f|
||	<   l |S 7 uw)NTz4'store_model_in_db' is not True, skipping db updates)r   r0  r1  rk  r"  r   )ro  r  rH  r1  zlitellm_settings: r)  )r:  r"  r;  )	r1   r   get_generic_datar  r  gatherr  r)  r=  )r"  r   r   rW  _tasksr  rk  response	responsesr"  r)  s              r   rl  z"ProxyConfig._update_config_from_db	  s     D( %%F M
 A$55 h 6 H MM(#	  "..&11	!H#$X|TB
!33(--,X-A-A,BC &htD)k.E!V+!%!;!;+1'1+6 "< " .9z*% "( + 2s%   A#C%C&C2AC6C9#Cc                    K   	 |j                   j                  j                          d {   }|S 7 # t        $ r:}t	        j
                  dj                  t        |                   g }Y d }~|S d }~ww xY ww)NzQlitellm.proxy_server.py::add_deployment() - Error getting new models from DB - {})r&  litellm_proxymodeltable	find_manyr   r1   r  r*   r   )r"  r   r  rp  s       r   _get_models_from_dbzProxyConfig._get_models_from_db
  su     	,//GGQQSSJ  T 	 **cjjF
 J	s6   A:'4 24 A:4 	A7/A2,A:2A77A:c                   K   	 t         t        t         t              st        dt                | j	                  |       d{   }| j                  ||       d{    |j                  j                  j                  ddi       d{   }|%| j                  |j                         d{    yy7 w7 ^7 07 # t        $ r7}t        j                  d	j                  t        |                   Y d}~yd}~ww xY ww)
z{
        - Check db for new models
        - Check if model id's in router already
        - If not, add to router
        Nr  )r   )r  r   r"  r   r#  )r5  z>litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - {})r   r   r   r?  rG  r4  r&  r'  r(  r9  r)  r   r1   r  r*   )r"  r   r   r  r5  rp  s         r   add_deploymentzProxyConfig.add_deployment(
  s%    	!J)D Mj\Z   $77m7TTJ ))%9J *    )6(8(8(G(G(R(R#%78 )S ) #
 #.33(;(G(G 4    / U#  	 **PWWF 	sw   DAC B=C  B?!/C C%C 6C7C ;D=C ?C C C 	D-D ;D DD)rb  Nr   )r   rV  F),r   r%  r&  __doc__r#  r   boolr   r   r.  r   r<  r7  rM  r3  rS  r   rM  rY  r]  rd  rp  rm  r\  r   r/   r   r  r  r  r  r  r  r  rj   r4  Jsonr9  r   r=  ri   rl  rG  rI  r   r   r   r  r  d  s-   )U U UN N N 15$ ($	$L% % % %NMD M, >@###&#7:#	#J tDz d  &44&6# 6$ 6p$ *DAw~~.DABEDAL& &P%Kx} %KN @6#$ 6#3 6#p6 6# 6pTT (Tl!(4. !F  	
 
$/#/ / $D>	/b|  (#( ((r   r  c                  R    dd l } |j                  |       t        j                  d<   y )Nr   r   )r   r:  r   r   )rG  r   s     r   save_worker_configrO  V
  s    ",$**T"2BJJr   c                   K   t        j                  dd      j                         dk7  r
t                | a|a|du rddd l}ddlm}m	}m
} |j                  |j                          |j                  |j                          |j                  |j                         |du redd l}ddlm}m	}m
} |j                  |j                          |j                  |j                          |j                  |j                         n|du r|du rt         j                  j                  d	d      }||j!                         d
k(  rGdd l}ddlm	}m
}  |j                  |j                          |j                  |j                         nY|j!                         dk(  rFdd l}ddlm	}m
}  |j                  |j                          |j                  |j                         di t        i i}|r't"        j%                  t&        |       d {   \  aaa|r|a||t           d<   |r|a||t           d<   |r|t         j                  d<   |r||t           d<   |r|a||t           d<   |r|a||t           d<   |r||t           d<   |du rdt4        _        d|d   d<   |du rdt4        _        d|d   d<   |	r|	t4        _        |	|d   d<   t<        r	 |
ay 7 ƭw)NLITELLM_DONT_SHOW_FEEDBACK_BOXr   trueTr   )verbose_loggerr1   r2   )levelFLITELLM_LOGINFOr0   DEBUGgeneralr   r2  r\  AZURE_API_VERSION
max_tokenstemperaturerequest_timeoutaliasdrop_paramsadd_function_to_promptr  ) r   r  r&  r,   
user_model
user_debuglogginglitellm._loggingrS  r1   r2   setLevelrV  rW  r   r   upperr   r   r   r   r   user_headersuser_api_baseuser_temperatureuser_request_timeoutr   r^  r_  r  experimentaluser_telemetry)r  r]  r\  api_versionr   detailed_debugr[  rZ  r\  r  	telemetryr^  r_  r2  save	use_queuer   rb  rS  r1   r2   litellm_log_settingdynamic_configs                          r   r   r   \
  s    ( 
yy126<<>&HJJ}	
 	
 	gll3&&&W\\:%%%GLL9	
 	
 	gmm4&&&W]];%%%GMM:	%Ne3 jjnn]B?*"((*f4X /%..!,, .$--!,, %**,7X.%..!-- .$--!--  Z4N
 ***v*VV		
07z"9- 19z":. 	

&' 3=z"<0&4?z"=1.8Gz"#45.3z"7+d"37y!-0%)-&>By!":;'2<y!,/NA Ws   HKKCKc              #     K   t        j                  d       | D ]B  }t        j                  d|       	 dt        j                  |j	                                d D y # t
        $ r dt        j                  |       d Y lw xY ww)Ninside generatorzreturned chunk: %sdata: r   )r1   r   r   r:  r   r   )rB  chunks     r   data_generatorrw  
  s}     12""#7?	34::ejjl34D99   	34::e,-T22	3s(   1B
)A B
 $BB
BB
c                   K   t        j                  d       	 t        j                          | 4 d {   }t        j	                  ||       d {   }|2 3 d {   }|j                  d      }	 d| d &7 K7 .7 %# t        $ r}dt        |       d Y d }~Ld }~ww xY w6 d d d       d {  7   n# 1 d {  7  sw Y   nxY wd}d| d y # t        $ r}t        j                  dj                  t        |                   t        j                  |||	       d {  7   t        j                  d
| d       t        |t              r|t        j                         }t        |       d| }t        t!        |d|      t!        |dd      t!        |dd      t!        |dd            }	t#        j$                  d|	j'                         i      }
d|
 d Y d }~y d }~ww xY ww)Nrt  ra  rB  Tr   ru  r   [DONE]zTlitellm.proxy.proxy_server.async_assistants_data_generator(): Exception occured - {}ra  original_exceptionr`  [1;31mAn error occurred: P

 Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`r   r-  Noner.  r0    r,  r+  )r1   r   rv  r   async_post_call_streaming_hookmodel_dump_jsonr   r   r  r*   post_call_failure_hookr   r   r  r  ProxyExceptionr  r   r:  to_dict)rB  ra  r`  rv  crp  done_messageerror_traceback	error_msgproxy_exceptionerror_returneds              r   async_assistants_data_generatorr  
  s     12/,		8u ,JJ"3e K  E
 ! 0a%%4%80"1#T** 
0 ! 0"3q6($///0	 ! 8888   |nD)) ,&&biiA	

  66/ % 7 
 	
 	

 	""+A3  /A  B	
 a'G'224Oq6($&78I(Ay)4FF+!Wf-M3/	
 Wo.E.E.G$HI~&d+++7,s   G>C! A?C! C BC B-BB-!C 5	B>C ?C! C B-	B*B% C %B**C .C! 9B<:C!  CC	CC!  G>!
G;+AG63D64B=G61G>6G;;G>c                  K   t        j                  d       	 t        j                          | 2 3 d {   }t        j                  dj                  |             t        j                  ||       d {   }t        |t              r|j                  dd      }	 d| d z7 u7 4# t        $ r}dt        |       d Y d }~d }~ww xY w6 d}d| d y # t        $ r}t        j                  d	j                  t        |                   t        j                  |||
       d {  7   t        j                  d| d       t        |t              r|t        j                         }t        |       d| }t!        t#        |d|      t#        |dd      t#        |dd      t#        |dd            }t%        j&                  d|j)                         i      }	d|	 d Y d }~y d }~ww xY ww)Nrt  3async_data_generator: received streaming chunk - {}ry  T)r!  exclude_unsetru  r   rz  Ilitellm.proxy.proxy_server.async_data_generator(): Exception occured - {}r{  r}  r~  r   r-  r  r.  r0  r  r,  r+  )r1   r   rv  r*   r   r  r   r  r  r   r   r  r  r   r  r  r  r  r   r:  r  )
rB  ra  r`  rv  rp  r  r  r  r  r  s
             r   async_data_generatorr    s     122,		# 	,% &&ELLUS ,JJ"3e K  E %+--4t-T,ugT**	,
  ,s1vhd+++, $$  |nD)) ,&&W^^A	

  66/ % 7 
 	
 	

 	""+A3  /A  B	
 a'G'224Oq6($&78I(Ay)4FF+!Wf-M3/	
 Wo.E.E.G$HI~&d+++9,s   G?C" CB)C?C" 6B+7'C" 	B-(C" )C+C" -	C6CC" CC" !G?"
G<,AG74D75B=G72G?7G<<G?c                  K   t        j                  d       	 t        j                          | 2 3 d {   }t        j                  dj                  |             t        j                  ||       d {   }|j                  d      }	 d| dt        j                  |       d ~7 y7 8# t        $ r}d| dt        |       d Y d }~d }~ww xY w6 y # t        $ r}t        j                  dj                  t        |                   t        j                  |||	       d {  7   t        j                  d
| d       t        |t              r|t        j                          }t        |       d| }t#        t%        |d|      t%        |dd      t%        |dd      t%        |dd            }t        j                  d|j'                         i      }	d|	 d Y d }~y d }~ww xY ww)Nrt  r  ry  r-  zevent: z
data:r   r  r{  r}  r~  r   r  r.  r0  r  r,  r+  ru  )r1   r   rv  r*   r   r  r   r   r:  r   r   r  r  r   r   r  r  r  r  r  )
rB  ra  r`  rv  
event_typerp  r  r  r  r  s
             r   async_data_generator_anthropicr  L  s     12-,		# 	@% &&ELLUS ,JJ"3e K  E 6*J@
|74::e3D2ETJJ	@
  @
|73q6($???@ $  ,&&W^^A	

  66/ % 7 
 	
 	

 	""+A3  /A  B	
 a'G'224Oq6($&78I(Ay)4FF+!Wf-M3/	
 Wo.E.E.G$HI~&d+++9,s   G;C CB-C?C 6B/7C B1,C -C/C 1	C:CC CC G;
G8(AG30D31B=G3.G;3G88G;c                     t        | ||      S )NrB  ra  r`  )r  r  s      r   select_data_generatorr    s      +! r   r  c                     | j                  di       }| j                  di       j                  dd       }	 d|v r|j                  dd       }t        j                  |      }|S # t        $ r i cY S w xY w)Nr   r  r  azure
base_model)r   r   get_model_infor   )r  r   model_to_lookuplitellm_model_infos       r   get_litellm_model_infor    sz    <,Jii 0"599'4HOo%(nn\4@O$33OD!!  	s   ,A# #A10A1c                 6    t        j                  d| d          y )Nz$Backing off... this was attempt # %stries)r1   r   )r  s    r   
on_backoffr    s    EwwGWXr   c                 >   t        | t              xr; t        | dd       d uxr* t        | j                  t              xr d| j                  v  }t
        j                  d      du ry|r4t        j                  t        j                  dt	        |       d             |S )Nr   z"Max parallel request limit reached1disable_retry_on_max_parallel_request_limit_errorTgiveup)eventr  )r   r  r  r   r   r   r   r1   r   r   r:  )rp  results     r   r  r    s    1n% 	>Ay$'t3	>qyy#&	> 1AII=	F 	PQ	 !!$**xcRSf-U"VWMr   c                       e Zd Zedee   dedee   fd       Zede	dee
   defd       Zed	ede
d
ede	fd       Zedefd       Zede	de
dedededefd       Zedee   dededee
   fd       Zed        Zy)r   r   r   r   c                 R    t                t                |j                  ||       y)z*Initialize logging and alerting on startup)r   r   N)r  r  startup_event)clsr   r   r   s       r   r   z-ProxyStartupEvent._initialize_startup_logging  s*     	 	''!5F 	( 	
r   r   r   r   c                 (   |j                  dd      ^|d   j                         D ]9  \  }}t        |t              s|j	                  d      s)t        |      |d   |<   ; t        di |d   }n
t               }t        j                  |||       y)zInitialize JWT auth on startuplitellm_jwtauthNrQ  )r   r   r  r   )	r   rg  r   r   rC  r   LiteLLM_JWTAuthr   r  )r  r   r   r   rk  rl  r  s          r   r  z&ProxyStartupEvent._initialize_jwt_auth  s      148D():;AAC1a%!,,}*E=G]$%67: D .T0@AR0STO-/O&&'1+ 	' 	
r   r   r   c                 @   t        j                  dd      t        j                  d|      }|j                  d      du rt        j                  d       yt        ddg i i d||t        j                  dd	t        j                  i
      }t        j                  |       y)z,Adds master key hash to db for cost trackingPROXY_ADMIN_IDN$disable_adding_master_key_hash_to_dbTz&Skipping writing master key hash to dbuserr   update_data	user_role)request_typedurationmodelsaliasesr   r  r  r  r  
query_typeupdate_key_values)
r   r  r   r1   r   ra   LitellmUserRolesPROXY_ADMINr  r  )r  r   r   r   r   task_1s         r   r  z,ProxyStartupEvent._add_master_key_hash_to_db  s     99%t,8')yy ":($  FG4O %%&NO ,# 0*66(#.0@0L0L"MF 'r   r   c                    t         j                  t        d      t        j                  t        d|dg i i dt         j                  t         j                  dt         j                  t         j                  d             y)z Adds a global proxy budget to dbNzPbudget_duration not set on Proxy. budget_duration is required to use max_budget.r  r   r  )r  budget_duration)r  r  r  r  r  r   r  r  r  r  r  )r   r  r   r  r  ra   r  )r  r   s     r   r	  z)ProxyStartupEvent._add_proxy_budget_to_db  sy     ""*b 
 	"#1"-- ' 7 7(")"4"4'.'>'>#	
r   r   r   r   c           	        K   t               }t        j                  ||      }t        j                  |dz
  |dz         }	|j                  dd      du r|j	                  t
        d||g       |j	                  t        d|	|t        |g       t        dt              xs t        a	t        du rD|j	                  t        j                  dd||g       t        j                  ||	       d
{    |I|j                  j                  2|/t        d       |j                  dd      xs d}
t        |
d
d       }|
d   j!                         dk7  rt#        d      |j	                  |j                  j$                  d|t'        j(                         t+        d      z   |
g       |j	                  |j                  j,                  dd       t/        j0                  d      rXddlm} |j	                  |j                  j6                  ddd |d             |j                  j7                          d
{    |j9                          y
7 b7 w)z%Initializes scheduled background jobsr   disable_reset_budgetFinterval)secondsr  STORE_MODEL_IN_DBTrV  r   r   Nz3Alerting: Initializing Weekly/Monthly Spend Reportsspend_report_frequency7ddzBspend_report_frequency must be specified in days, e.g., '1d', '7d')r  )daysnext_run_timer  cronrA  )dayPROMETHEUS_URLr   )ZoneInfo	   zAmerica/Los_Angeles)hourminutetimezone)r!   r&   randintr   add_jobrt   ru   r   r   rW  r   rI  r+  r  r)   r3  r&  r?  send_weekly_spend_reportr   nowr   send_monthly_spend_reportr   r  zoneinfor  #send_fallback_stats_from_prometheusstart)r  r   r   r   r   r   r   	schedulerr  batch_writing_intervalr  r  r  s                r   r
  z6ProxyStartupEvent.initialize_scheduled_background_jobs  s     %&	>>-/P
 "( 1$&:Q&>"

  6>%Gj(-  
 	*!13DE	 	 	
 /1BCXGX 	 $++#%67	   --+?P .   
 )!99BBN)GH !$$%=tDL #
 -cr23D%b)//1S8 X  !99RR&llnB'(,-   !99SS   yy)*-!!%==aa%&;< "  (??cceeegb fs%   C*I,I-EI8I9IIr   rb  c                 L  K   d}|	 t        ||      }|j                          d{    t        j                  |j                                t        j                  |j                                |j                          d{    |S # t        $ r}|d}~ww xY w7 w7 w)zQ
        - Sets up prisma client
        - Adds necessary views to proxy
        N)r   r   )ri   r   connectr  r  check_view_exists(_set_spend_logs_row_count_in_proxy_statehealth_check)r  r   r   r   r   rp  s         r   r   z&ProxyStartupEvent._setup_prisma_clientz  s      15# ,!-AR!  ''))) //1 FFH
  ,,...!   * /sD   B$B B$B AB$B"B$	BBBB$"B$c                 N    t        dd      du rddl}|j                  d       yy)z
        Initialize dd tracer - if `USE_DDTRACE=true` in .env

        DD tracer is used to trace Python applications.
        Doc: https://docs.datadoghq.com/tracing/trace_collection/automatic_instrumentation/dd_libraries/python/
        USE_DDTRACEFTr   N)rb  )r   ddtrace	patch_all)r  r  s     r   r  z!ProxyStartupEvent._init_dd_tracer  s,     =%0D8d+ 9r   N)r   r%  r&  classmethodr   r/   rj   r4   r   r   ri   r3   r  r   r  r	  r3  r
  r   r  r   r   r   r   r     st   
V$
 (
 $J/	
 
" 

  -
 &	
 
( (( $( #&	(
 ( (@ 
 
 
6 bb $b ,/	b
 ,/b "b (b bH  sm  (  &	 
 
,	   D 
, 
,r   r   z
/v1/modelszmodel management)dependenciestagsz/modelsc           
      `  K   g }t        t              }t        g }n(t        j                         }t        j	                         }t        | ||      }t        | ||      }t        |||t        t        j                  dd            }t        |D cg c]	  }|dddd c}d	
      S c c}w w)z
    Use `/model/info` - to get detailed model information, example - pricing, mode, etc.

    This is just for compatibility with openai projects like aider.
    ra  proxy_model_listmodel_access_groupsinfer_model_from_keysF
key_modelsteam_modelsr  r`  r  r  ijNcopenai)r  objectcreatedowned_byr  )rG  r  )r-   r  r   get_model_namesget_model_access_groupsrC   rD   rB   r`  r   r   r   )ra  
all_modelsr  r  r  r  r  s          r   r  r    s      J0;D0A%557(@@B+)/J "+)/K
 )).223JERJ  $
 $ !%$	 $
  
s   BB.B) B.z/v1/chat/completionszchat/completionsz/chat/completionsz&/engines/{model:path}/chat/completionsz1/openai/deployments/{model:path}/chat/completions   r  zSuccessful response)r  r  rC  )	max_triesmax_timer  r  loggerfastapi_responsec                 `  K   i }	 t        |        d{   }t        j                  dj                  t	        j
                  |d                   t        || t        |t        t               d{   }t        j                  dd      xs t        xs	 |xs |d   |d<   t        r	t        |d	<   t        r	t        |d
<   t        r	t        |d<   t        r	t        |d<   t!        |d   t"              r.|d   t$        j&                  v rt$        j&                  |d      |d<   t(        j+                  ||d       d{   }| j,                  j                  dt#        t/        j0                                     |d<   t%        j2                  j4                  d>dt$        j2                  j7                         t9        j:                         d|\  }}||d<   g }|j=                  t(        j?                  ||d             tA        |dtB        t               d{   }|j=                  |       tE        jF                  | }| d{   }	|	d   }
tI        |
di       xs i }|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j                  dd      }|j                  di       xs i }tB        tB        jK                  |      |d<   tE        jL                  t(        jO                  |j                  dd      d !             d"|v rR|d"   d#u rKtQ        d>||jR                  |||t        |tI        |d$d      ||d%
|}tU        |
||&      }tW        |d'|(      S t(        jY                  |||
)       d{   }
tI        |
di       xs i }|j                  di       xs i }|j,                  j[                  tQ        d>||jR                  |||t        |tI        |d$d      ||d%
|       t]        |
*       d{    |
S 7 7 7 7 %7 7 7 # t^        $ r'}|j`                  }t(        jc                  |||+       d{  7   t%        jd                         }|jf                  |jh                  d,   jf                  _5        |j                  d"d      |d"   d#u rzt$        j2                  jm                  |d#-      }t%        jn                  ||j                  dd      d.|j                  dd      /      }tU        |||&      }tW        |d'0      cY d}~S t%        jp                  d,d,d,1      }||_9        |cY d}~S d}~wtt        $ r,}t        jv                  d2t#        |              t(        jc                  |||+       d{  7   tI        |d3d      }t        j                  d4||       t!        |tx              r^t{        tI        |d5t#        |            tI        |d6d7      tI        |d8d7      tI        |d9t|        j~                        tI        |d:i       ;      t#        |       }t{        tI        |d<|      tI        |d6d7      tI        |d8d7      tI        |d9d=      tI        |d:i       ;      d}~ww xY ww)?a  

    Follows the exact same API spec as `OpenAI's Chat API https://platform.openai.com/docs/api-reference/chat`

    ```bash
    curl -X POST http://localhost:4000/v1/chat/completions 
    -H "Content-Type: application/json" 
    -H "Authorization: Bearer sk-1234" 
    -d '{
        "model": "gpt-4o",
        "messages": [
            {
                "role": "user",
                "content": "Hello!"
            }
        ]
    }'
    ```

    r(  NzRequest received by LiteLLM:
{}r3  r4  rG  r(  r   ra  r   r   completion_modelr  r[  r\  rZ  r\  
completionra  rG  	call_typerd  litellm_call_idacompletion)original_function	rules_objr{  litellm_logging_objrG  ra  r  rG  
route_typer   r`  rA  _hidden_paramsrZ  r   r[  r^  r_  additional_headersrZ  r  successr  r   streamTallowed_model_region)
ra  rY  rZ  r[  r\  r   r^  r]  r_  r`  r  text/event-stream
media_typer2  rG  ra  rB  rB  r{  r   model_responseconvert_to_deltacached_response)completion_streamr  custom_llm_providerlogging_objr  prompt_tokenscompletion_tokenstotal_tokenszBlitellm.proxy.proxy_server.chat_completion(): Exception occured - litellm_debug_infoo[1;31mAn error occurred: %s %s

 Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`ru  r-  r  r.  r0  r2  )r   r-  r.  r/  r2  r   r  r   )@rN   r1   r   r*   r   r:  r]   r   r   r   r   r`  rh  ri  user_max_tokensrg  r   r   r   model_alias_mapr   pre_call_hookr2  uuiduuid4utilsfunction_setupRulesr   r  r  during_call_hookrg   r   r  r@  r  get_deploymentr  update_request_statusrq  r  r  r   post_call_success_hookrh  r?   r5   r`  r  ModelResponser   choicesr1  ModelResponseIteratorCustomStreamWrapperUsageusager   r  r   r  r   HTTP_400_BAD_REQUEST)r(  r  r  ra  rG  r  tasksllm_callllm_responsesrC  rB  hidden_paramsrZ  r[  r\  r^  r_  r  custom_headersselected_data_generatorrp  _data_chat_response	_iterator_streaming_response_usager$  r  s                               r   chat_completionrD    s    z D^
'88"".55djja6PQ	
 1-/%
 
   !3T:  G}	 	W "2D&:D"#!0D,D
 d7mS)d7mw?V?V.V#33DMBDM '44/dl 5 
 
 #*//"5"5TZZ\!2#
 $MM88 
+mm))+||~
 	
T '2"#.."3& / 	
 '$!!	
 
 	X  
 ('	Q<*:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB,9,=,=/-
) $1#4#45I2#N#TRT !!+!:!:H!:!MD33 $):B ?	 4 	
 h4!7/ "3#33!#!+$%68NPRS2S! %N '<!"3!'#
 %'.&  +AA):X B 
 

 H.39r 	 +../CRHNB  '' "3#33!#!+$%68NPRS2S! %	
 *8<<<{ 9

D
:
 (d
0 	=    66/  7 
 	
 	

 !..045IIq!))188Hd#/DNd4J;;- < I #*"="="+hhw+$5 HH%:DA	# '<,"3"'# %'.  Q!RST% 
&&PQTUVQWPXY	
  66/ATX 7 
 	
 	
 %Q(<bA"" C	
 a' 8SV4Q/a&1Qv/J/JK9b1  1vh	Ay)4FF+!Wf-M3/Ay"-
 	
1
s   Z.Q P3AQ 3P64CQ <P9=CQ P<,Q P?EQ Z.Q )Q*BQ ,Q-Q 2Z.3Q 6Q 9Q <Q ?Q Q Q 
Z+'U27Q:8CU2Z+Z. U2,Z+-Z.2Z+?<Z&;V><C*Z&&Z++Z.z/v1/completionscompletionsz/completionsz!/engines/{model:path}/completionsz,/openai/deployments/{model:path}/completionsc                 V	  K   i }	 t        |        d{   }t        j                  dd      xs t        xs	 |xs |d   |d<   t        r	t        |d<   t	        || t        |t
        t               d{   }t        r	t        |d<   t        r	t        |d<   t        r	t        |d<   t        r	t        |d	<   |d   t        j                  v rt        j                  |d      |d<   t        j                  ||d
       d{   }t        |dt         t               d{   }| d{   }t#        |di       xs i }|j                  dd      xs d}|j                  dd      xs d}	|j                  d	d      xs d}
|j                  dd      xs d}|j                  dd      xs d}t%        j&                  t        j)                  |j                  dd      d             t+        j,                  d|       d|v r:|d   du r3t/        ||||	|
t
        ||      }t1        |||      }t3        |d|      S t        j5                  |||       d{   }|j6                  j9                  t/        ||||	|
t
        ||             t;        |       d{    |S 7 7 .7 7 7 7 Y7 # t<        $ rH}|j>                  }t        jA                  |||       d{  7   |j                  dd      |d   du rt        jB                         }t        jD                  d d d !      }||_#        |jH                  |jJ                  d    jH                  _&        t        jN                  jQ                  |d"      }t        jR                  ||j                  dd      #      }t1        |||      }t3        |di       cY d}~S t        jT                         }|jH                  |jJ                  d    _+        |cY d}~S d}~wtX        $ r}t        jA                  |||       d{  7   t+        jZ                  d$j]                  t_        |                   t_        |       }ta        t#        |d%|      t#        |d&d'      t#        |d(d'      t#        |d)d*      +      d}~ww xY ww),a  
    Follows the exact same API spec as `OpenAI's Completions API https://platform.openai.com/docs/api-reference/completions`

    ```bash
    curl -X POST http://localhost:4000/v1/completions 
    -H "Content-Type: application/json" 
    -H "Authorization: Bearer sk-1234" 
    -d '{
        "model": "gpt-3.5-turbo-instruct",
        "prompt": "Once upon a time",
        "max_tokens": 50,
        "temperature": 0.7
    }'
    ```
    r  Nr   r  r  r[  r\  rZ  r\  text_completionr  atext_completionr
  r  rZ  r   r[  r^  r  r  r  final response: %sr  T)ra  rY  rZ  r[  r\  r   r^  r`  r  r  r  r  r  r{  r   r   r  r  r  z?litellm.proxy.proxy_server.completion(): Exception occured - {}r   r-  r  r.  r0  r  r,  )1rN   r   r   r`  r]   r   r   rh  ri  r&  rg  r   r'  r   r(  rg   r   r  r  r  r0  r1   r   rq  r  r   r1  r2  rh  r?   r5   r`  r  r2  r6  r7  r   r3  r1  r+  r4  TextCompletionStreamWrapperTextCompletionResponsetextr   r  r*   r   r  )r(  r  r  ra  rG  r:  rB  r<  rZ  r[  r\  r^  r  r=  r>  rp  r?  r@  rC  rA  rB  	_responser  s                          r   r  r    s    R De
'88   !3T:  G}	 	W &DM0-/%
 
 "2D&:D"#!0D,D
 =G333#33DMBDM '44/dFW 5 
 

 ')!!	
 
 ">*:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB'++,=tDJ 	33 $):B ?	 4 	
 	""#7Bh4!7/"3'!#!+!	N '<!"3!'# %'.&  +AA):X B 
 
 	  ''"3'!#!+!		
 *8<<<[ 9
4


 "T
  	= &66/  7 
 	
 	

 99Xt$0U8_5L$224N]]"#F
 $*N 89		N""1%--5;;- < I #*"E"E"+ii,#
 '<,"3!'# %'.   668I()		Ia % 
66/ATX 7 
 	
 	
 	&&MTTA	

 1vh	Ay)4FF+!Wf-M3/	
 	

s   R)J3 J AJ3 5J#6B	J3 ?J& J3 J)	J3 (J,)DJ3 9R):J3 J/AJ3 J1J3 R) J3 #J3 &J3 )J3 ,J3 /J3 1J3 3
R&='P $K'%CP R&R)3P :R&;R) R&R!'P*(A9R!!R&&R)z/v1/embeddings
embeddings)r  response_classr  z/embeddingsz /engines/{model:path}/embeddingsz+/openai/deployments/{model:path}/embeddingsc                 	  K   i }	 | j                          d{   }t        j                  |      }t        j                  dt        j                  |d             t        || t        |t        t               d{   }t        j                  dd      xs t        xs	 |xs |d   |d<   t        r	t        |d<   |d   t        j                  v rt        j                  |d      |d<   t        t        j                   ng }d|v rt#        |d   t$              rt'        |d         d	kD  rt#        |d   d	   t$              rt#        |d   d	   d	   t(              rt*        |d   |v r~t*        D ]u  }|d
   |d   k(  r0|d   d   t        j,                  v s|d   d   j/                  d      r>g }|d   D ](  }	|j1                  t        j2                  d|	             * ||d<    n t4        j7                  ||d       d{   }g }
|
j1                  t4        j9                  ||d             t;        |dt        t               d{   }|
j1                  |       t=        j>                  |
 }| d{   }|d   }t=        j@                  t4        jC                  |j                  dd      d             tE        |di       xs i }|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j                  di       xs i }|jF                  jI                  tK        d-||||t        |tE        |dd      ||d 	|       tM        |!       d{    |S 7 7 '7 7 b7 87 # tN        $ r}t4        jQ                  |||"       d{  7   tE        |d#d      }t        j                  d$||       t        jR                  d%jU                  tW        |                   t#        |tX              rIt[        |      }t]        |tE        |d&d'      tE        |d(d'      tE        |d)t^        j`                        *      tW        |       }t]        tE        |d+|      tE        |d&d'      tE        |d(d'      tE        |d)d,      *      d}~ww xY ww).a  
    Follows the exact same API spec as `OpenAI's Embeddings API https://platform.openai.com/docs/api-reference/embeddings`

    ```bash
    curl -X POST http://localhost:4000/v1/embeddings 
    -H "Content-Type: application/json" 
    -H "Authorization: Bearer sk-1234" 
    -d '{
        "model": "text-embedding-ada-002",
        "input": "The quick brown fox jumps over the lazy dog"
    }'
    ```

NzRequest received by LiteLLM:
%sr3  r4  r  embedding_modelr  inputr   r  r  zazure/zgpt-3.5-turbo)r  tokensrO  r  r	  
aembeddingr
  rA  r  r   r  r  r  rZ  r[  r\  r^  r  r  	ra  rZ  r[  r\  r   r^  r]  rY  r`  r  r{  r$  r%  z?litellm.proxy.proxy_server.embeddings(): Exception occured - {}r-  r  r.  r0  r,  r   r  r   )1bodyorjsonr   r1   r   r   r:  r]   r   r   r   r   r`  r   r'  r   model_namesr   r  r  r3  r   open_ai_embedding_modelsrC  r  decoder   r(  r.  rg   r  r@  r  r0  r  r2  rh  rq  r?   r   r  r  r*   r   r   rq   r  r   r8  )r(  r  r  ra  rG  rW  router_model_namesr  
input_listir9  r:  r;  rC  rB  r<  rZ  r[  r\  r^  r  r  rp  r$  r   r  s                             r   rO  rO    s    ` D\\\^#||D!"".JJtA&	
 1-/%
 
   !2D9  G}	 	W &DM
 =G333#33DMBDM7A7MZ33SUtO4=$/DM"Q&4=+T24=+A.4 )d7m?Q.Q'A$w-7*+G48X8XX-.w7BB8L &(
!%gA&-- '_Q O "/ )3W (" '44/dl 5 
 
 .."3& / 	
 '#!!	
 
 	X  
 ('	Q< 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB'++,=tDJ#0#4#45I2#N#TRT  '' "3!#!+$%68NPRS'! %	
 *8<<<w $
f

 (D 	=  66/ATX 7 
 	
 	
 %Q(<bA"" C	

 	&&MTTA	

 a'+A.G Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 3s   SN3 N"A$N3 >N%?FN3 N(AN3 	N+
,N3 6N.7D$N3 N1N3 !S"N3 %N3 (N3 +N3 .N3 1N3 3
S=SOC6SSSz/v1/images/generationsimagesz/images/generationsc                 |  K   i }	 | j                          d {   }t        j                  |      }t        || t        |t
        t               d {   }t        j                  dd       xs t        xs |d   |d<   t        r	t        |d<   |d   t        j                  v rt        j                  |d      |d<   t        j                  ||d       d {   }t        |dt        t               d {   }| d {   }t        j                   t        j#                  |j                  dd	      d
             t%        |di       xs i }|j                  dd       xs d	}|j                  dd       xs d	}	|j                  dd       xs d	}
|j                  dd       xs d	}|j                  dd       xs d	}|j&                  j)                  t+        |||	|
t
        |t%        |dd	      ||	             |S 7 7 7 +7 7 # t,        $ r#}t        j/                  |||       d {  7   t1        j2                  dj5                  t7        |                   t1        j8                  t;        j<                                t?        |t@              rRtC        t%        |dt7        |            t%        |dd      t%        |dd      t%        |dtD        jF                              t7        |       }tC        t%        |d|      t%        |dd      t%        |dd      t%        |dd            d }~ww xY ww)Nr  image_generation_modelr  image_generationr  aimage_generationr
  r  r   r  r  r  rZ  r[  r\  r^  r  rV  r{  zElitellm.proxy.proxy_server.image_generation(): Exception occured - {}r   r-  r  r.  r0  r,  r  )$rW  rX  r   r]   r   r   r   r   r`  r   r'  r   r(  rg   r   r  r  r0  r  r2  rh  rq  r   r  r1   r+  r*   r   r   r  r  r   r   r  r   r8  )r(  r  ra  rG  rW  r:  rB  r<  rZ  r[  r\  r^  r  rp  r  s                  r   rb  rb    s;    $ D`\\^#||D! 1-/%
 
   !94@ G} 	W
 &DM
 =G333#33DMBDM '44/dFX 5 
 

 '*!!	
 
 "> 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB'++,=tDJ  ''"3!#!+$%68NPRS'!
	
 K $
0


 ">  66/ATX 7 
 	
 	
 	""SZZA	

 	""9#7#7#9:a' 9c!f5Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 's   L<H G=9H H BH HH 7H8	H H	C:H <L<=H  H H H 	H 
L9L41H42DL44L99L<z/v1/audio/speechaudioz/audio/speechc                   K   i }	 | j                          d{   }t        j                  |      }t        || t        |t
        t               d{   }|j                  dd      |j                  |j                  |d<   t        r	t        |d<   t        j                  ||d       d{   }t        |dt        t               d{   }| d{   }t        j                  t        j!                  |j                  d	d
      d             t#        |di       xs i }|j                  dd      xs d
}|j                  dd      xs d
}	|j                  dd      xs d
}
|j                  dd      xs d
}|j                  d	d      xs d
}dt$        fd}t'        |||	|
t
        |t#        |dd
      d||
      }t)        |||       t+         ||      d|      S 7 7 7 =7 !7 # t,        $ r[}t/        j0                  dj3                  t5        |                   t/        j6                  t9        j:                                |d}~ww xY ww)z`
    Same params as:

    https://platform.openai.com/docs/api-reference/audio/createSpeech
    Nr  r  r  rb  r  aspeechr
  r  r   r  r  r  rZ  r[  r\  r^  rN  c                l   K   | j                  d       d {   }|2 3 d {   }| 7 7 6 y w)Ni   )
chunk_size)aiter_bytes)rN  
_generatorrv  s      r   generatezaudio_speech.<locals>.generatec  s:     (444EEJ) e Fzs$   4.4202424r  )
ra  rZ  r[  r\  r   r^  r]  r_  rY  r`  r  z
audio/mpegr  zAlitellm.proxy.proxy_server.audio_speech(): Exception occured - {})rW  rX  r   r]   r   r   r   r   r  r`  r   r(  rg   r   r  r  r0  r  r   rq  r  r   r   r1   r+  r*   r   r   r  r  )r(  r  ra  rG  rW  r:  rB  r<  rZ  r[  r\  r^  r  rk  r=  rp  s                   r   audio_speechrl    su    * DU\\^#||D! 1-/%
 
 88FD!).?.G.G.S,44DL&DM '44/dFX 5 
 

 ' !!	
 
 "> 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB'++,=tDJ	&@ 	
 ,/' !24JBO.2#
 	/	

 !X<
 	
Q $
 


 "Z  ""OVVA	

 	""9#7#7#9:s   IG5 G&9G5 G)AG5 /G,0G5 G/	G5 G2DG5 %I&G5 )G5 ,G5 /G5 2G5 5	I>AIIIz/v1/audio/transcriptionsz/audio/transcriptions.r   c                 h  K   i }	 | j                          d{   }|j                         D ci c]  \  }}|dk7  s|| }}}t        || t        |t        t
               d{   }|j                  dd      |j                  |j                  |d<   t        j                  dd      xs t        xs |d   |d<   t        r	t        |d<   t        t        j                  ng }|j                  t        dt        j                  dd	      t        |||
       |j!                          d{   }	t#        j$                  |	      }
|j                  |
_        |
|d<   	 t(        j+                  ||d       d{   }t-        |dt        t               d{   }| d{   }	 |
j5                          t7        j8                  t(        j;                  |j                  dd      d             t=        |di       xs i }|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j                  di       xs i }|j>                  jA                  tC        d%||||t        |t=        |dd      ||d	|       |S 7 c c}}w 7 m7 7 \7 @7 9# t.        $ r}t1        dt3        |            d}~ww xY w# |
j5                          w xY w# t.        $ r}t(        jE                  |||       d{  7   tG        jH                  djK                  t3        |                   tM        |t0              r\t        t=        |dt3        |jN                              t=        |d d!      t=        |d"d!      t=        |d#t        j                        $      t3        |       }t        t=        |d|      t=        |d d!      t=        |d"d!      t=        |d#d      $      d}~ww xY ww)&zq
    Same params as:

    https://platform.openai.com/docs/api-reference/audio/createTranscription?lang=curl
    Nr   r  r  moderation_modelr  z.File name is None. Please check your file namebad_request)r   r/  r-  r.  )r`  r   r\  audio_transcriptionr  atranscriptionr
  r  rt  r  r   r  r  r  rZ  r[  r\  r^  r  r  rV  r{  zHlitellm.proxy.proxy_server.audio_transcription(): Exception occured - {}r   r-  r  r.  r0  r,  r   )(formrg  r]   r   r   r   r   r  r`  r   rY  r   r  r   r8  rO   readioBytesIOr<  r   r(  rg   r   r   r   r   r  r  r0  r  r2  rh  rq  r  r1   r  r*   r   ru  )r(  r  r   ra  rG  	form_dataro  r  r\  file_contentfile_objectr:  rB  rp  r<  rZ  r[  r\  r^  r  r  r  s                         r   audio_transcriptionsry    s@    , D{!,,.(	-6__->P->zsE#-U
->P 1-/%
 
 88FD!).?.G.G.S,44DL   !3T: G} 	W
 &DM7A7MZ33SU==  H00"	  	$1	
 "YY[(jj.=="V	 *88"3/ 9  D ++%%	 H &~H  	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB'++,=tDJ#0#4#45I2#N#TRT  '' "3!#!+$%68NPRS'! %	
 C )P
J ) & 	@CA??	@ D  66/ATX 7 
 	
 	
 	&&V]]A	

 a' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 %s   P2L KL KK#L 'K(C
L 2K3/L #K# >K?K# K	K# 'K (K# -DL 
P2L L L K# K#  K# #	L,LLL
 
LL 
P/)P*MC%P**P//P2)r   	WebSocketWebSocketDisconnect)
_arealtimez/v1/realtimez	/realtime	websocketc                   K   dd l }| j                          d {    || d}	 t        |dt        t               d {   }| d {    y 7 47 7 	# |j
                  j                  $ rE}t        j                  d       | j                  |j                  d       d {  7   Y d }~y d }~wt        $ r4 t        j                  d       | j                  dd       d {  7   Y y w xY ww)	Nr   )r  r}  r|  r
  zInvalid status code)r/  reasonzInternal server errori  )
websocketsacceptrg   r   r`  
exceptionsInvalidStatusCoder1   r  r   r0  r   )r}  r  ra  r  rG  r:  rp  s          r   websocket_endpointr  '  s      



 DI&#!!	
 
 ! 
 	  22 P&&'<=oo1==9NoOOO I&&'>?oo40GoHHHIs   C1A	C1A A 	A 	A
A C1A A C..5B.#B&$B.)C1.7C.%C(&C.+C1-C..C1z/v1/assistants
assistantsz/assistantsc                 4  K   i }	 | j                          d{    t        || t        |t        t               d{   }t
        't        ddt        j                  j                  i      t        j                  di | d{   }t        j                  t        j                  |j                  dd      d	             t!        |d
i       xs i }|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j"                  j%                  t'        ||||t        t!        |dd      |             |S 7 =7 7 # t(        $ r-}	t        j+                  ||	|       d{  7   t-        j.                  dj1                  t3        |	                   t-        j4                  t7        j8                                t;        |	t              r\t=        t!        |	dt3        |	j>                              t!        |	dd      t!        |	dd      t!        |	dt@        jB                              t3        |	       }
t=        t!        |	d|
      t!        |	dd      t!        |	dd      t!        |	dd            d}	~	ww xY ww)z
    Returns a list of assistants.

    API Reference docs - https://platform.openai.com/docs/api-reference/assistants/listAssistants
    Nr  r  r+  rt  r  r   r  r  r  rZ  r[  r\  r  ra  rZ  r[  r\  r   r]  r`  r{  zClitellm.proxy.proxy_server.get_assistants(): Exception occured - {}r   r-  r  r.  r0  r,  r   )"rW  r]   r   r   r   r   r   r  no_llm_routerr  aget_assistantsr  r  r   r0  r   r  r2  rh  rq  r   r  r1   r+  r*   r   r   r  r  r   r  ru  r   r8  r(  r  ra  rG  rB  r<  rZ  r[  r\  rp  r  s              r   get_assistantsr  Q  s    * DGlln 1-/%
 
 2C2Q2Q2W2W(X  $33;d;; 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 Y 	
 <8  66/ATX 7 
 	
 	
 	""QXXA	

 	""9#7#7#9:a' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'i   JE E$E EA	E E	CE JE E E 
J(JFDJJJc                 ^  K   i }	 | j                          d{   }t        j                  |      }t        || t        |t
        t               d{   }t        't        ddt        j                  j                  i      t        j                  di | d{   }t        j                  t        j!                  |j#                  dd      d	             t%        |d
i       xs i }|j#                  dd      xs d}|j#                  dd      xs d}|j#                  dd      xs d}	|j&                  j)                  t+        ||||	t
        t%        |dd      |             |S 7 R7 7 # t,        $ r-}
t        j/                  ||
|       d{  7   t1        j2                  dj5                  t7        |
                   t1        j8                  t;        j<                                t?        |
t              r\tA        t%        |
dt7        |
jB                              t%        |
dd      t%        |
dd      t%        |
dtD        jF                              t7        |
       }tA        t%        |
d|      t%        |
dd      t%        |
dd      t%        |
dd            d}
~
ww xY ww)z~
    Create assistant

    API Reference docs - https://platform.openai.com/docs/api-reference/assistants/createAssistant
    Nr  r  r+  rt  r  r   r  r  r  rZ  r[  r\  r  r  r{  zElitellm.proxy.proxy_server.create_assistant(): Exception occured - {}r   r-  r  r.  r0  r,  r   )$rW  rX  r   r]   r   r   r   r   r   r  r  r  acreate_assistantsr  r  r   r0  r   r  r2  rh  rq  r   r  r1   r+  r*   r   r   r  r  r   r  ru  r   r8  )r(  r  ra  rG  rW  rB  r<  rZ  r[  r\  rp  r  s               r   create_assistantr    s    * DH\\^#||D! 1-/%
 
 2C2Q2Q2W2W(X  $66>>> 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 [ $
 ?8  66/ATX 7 
 	
 	
 	""SZZA	

 	""9#7#7#9:a' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'sk   J-E3 E+9E3 E.A	E3 E1CE3 *J-+E3 .E3 1E3 3
J*=J%FDJ%%J**J-z"/v1/assistants/{assistant_id:path}z/assistants/{assistant_id:path}assistant_idc                   K   i }	 t        || t        |t        t               d{   }t        't        ddt        j                  j                  i      t	        j                  dd|i| d{   }t        j                  t        j                  |j                  dd      d	
             t        |di       xs i }|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}	|j                   j#                  t%        ||||	t        t        |dd      |             |S 7 7 # t&        $ r-}
t        j)                  ||
|       d{  7   t+        j,                  dj/                  t1        |
                   t+        j2                  t5        j6                                t9        |
t
              r\t;        t        |
dt1        |
j<                              t        |
dd      t        |
dd      t        |
dt>        j@                              t1        |
       }t;        t        |
d|      t        |
dd      t        |
dd      t        |
dd            d}
~
ww xY ww)z~
    Delete assistant

    API Reference docs - https://platform.openai.com/docs/api-reference/assistants/createAssistant
    r  Nr  r+  rt  r  r  r   r  r  r  rZ  r[  r\  r  r  r{  zElitellm.proxy.proxy_server.delete_assistant(): Exception occured - {}r   r-  r  r.  r0  r,  r   )!r]   r   r   r   r   r   r  r  r  adelete_assistantr  r  r   r0  r   r  r2  rh  rq  r   r  r1   r+  r*   r   r   r  r  r   r  ru  r   r8  )r(  r  r  ra  rG  rB  r<  rZ  r[  r\  rp  r  s               r   delete_assistantr    ss    , DF 1-/%
 
 2C2Q2Q2W2W(X  $55X<XSWXX 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 S
 Y8  66/ATX 7 
 	
 	
 	""SZZA	

 	""9#7#7#9:a' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'Y   I? E E AE 2E3CE ?I? E E 
I<I7*E-+DI77I<<I?z/v1/threadsz/threadsc                 4  K   i }	 | j                          d{    t        || t        |t        t               d{   }t
        't        ddt        j                  j                  i      t        j                  di | d{   }t        j                  t        j                  |j                  dd      d	             t!        |d
i       xs i }|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j"                  j%                  t'        ||||t        t!        |dd      |             |S 7 =7 7 # t(        $ r-}	t        j+                  ||	|       d{  7   t-        j.                  dj1                  t3        |	                   t-        j4                  t7        j8                                t;        |	t              r\t=        t!        |	dt3        |	j>                              t!        |	dd      t!        |	dd      t!        |	dt@        jB                              t3        |	       }
t=        t!        |	d|
      t!        |	dd      t!        |	dd      t!        |	dd            d}	~	ww xY ww)zs
    Create a thread.

    API Reference - https://platform.openai.com/docs/api-reference/threads/createThread
    Nr  r  r+  rt  r  r   r  r  r  rZ  r[  r\  r  r  r{  zClitellm.proxy.proxy_server.create_threads(): Exception occured - {}r   r-  r  r.  r0  r,  r   )"rW  r]   r   r   r   r   r   r  r  r  acreate_threadr  r  r   r0  r   r  r2  rh  rq  r   r  r1   r+  r*   r   r   r  r  r   r  ru  r   r8  r  s              r   create_threadsr  r  s    * DGlln 1-/%
 
 2C2Q2Q2W2W(X  $22:T:: 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 Y 	
 ;8  66/ATX 7 
 	
 	
 	""QXXA	

 	""9#7#7#9:a' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'r  z/v1/threads/{thread_id}z/threads/{thread_id}	thread_idc                   K   i }	 t        || t        |t        t               d{   }t        't        ddt        j                  j                  i      t	        j                  dd|i| d{   }t        j                  t        j                  |j                  dd      d	
             t        |di       xs i }|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}	|j                   j#                  t%        ||||	t        t        |dd      |             |S 7 7 # t&        $ r-}
t        j)                  ||
|       d{  7   t+        j,                  dj/                  t1        |
                   t+        j2                  t5        j6                                t9        |
t
              r\t;        t        |
dt1        |
j<                              t        |
dd      t        |
dd      t        |
dt>        j@                              t1        |
       }t;        t        |
d|      t        |
dd      t        |
dd      t        |
dd            d}
~
ww xY ww)zs
    Retrieves a thread.

    API Reference - https://platform.openai.com/docs/api-reference/threads/getThread
    r  Nr  r+  rt  r  r  r   r  r  r  rZ  r[  r\  r  r  r{  z?litellm.proxy.proxy_server.get_thread(): Exception occured - {}r   r-  r  r.  r0  r,  r   )!r]   r   r   r   r   r   r  r  r  aget_threadr  r  r   r0  r   r  r2  rh  rq  r   r  r1   r+  r*   r   r   r  r  r   r  ru  r   r8  r(  r  r  ra  rG  rB  r<  rZ  r[  r\  rp  r  s               r   
get_threadr    sr    , DE 1-/%
 
 2C2Q2Q2W2W(X  $//L)LtLL 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 S
 M8  66/ATX 7 
 	
 	
 	""MTTA	

 	""9#7#7#9:a' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'r  z /v1/threads/{thread_id}/messagesz/threads/{thread_id}/messagesc                 b  K   i }	 | j                          d{   }t        j                  |      }t        || t        |t
        t               d{   }t        't        ddt        j                  j                  i      t        j                  dd|i| d{   }t        j                  t        j!                  |j#                  dd      d	
             t%        |di       xs i }|j#                  dd      xs d}|j#                  dd      xs d}	|j#                  dd      xs d}
|j&                  j)                  t+        |||	|
t
        t%        |dd      |             |S 7 T7 7 # t,        $ r-}t        j/                  |||       d{  7   t1        j2                  dj5                  t7        |                   t1        j8                  t;        j<                                t?        |t              r\tA        t%        |dt7        |jB                              t%        |dd      t%        |dd      t%        |dtD        jF                              t7        |       }tA        t%        |d|      t%        |dd      t%        |dd      t%        |dd            d}~ww xY ww)zv
    Create a message.

    API Reference - https://platform.openai.com/docs/api-reference/messages/createMessage
    Nr  r  r+  rt  r  r  r   r  r  r  rZ  r[  r\  r  r  r{  zAlitellm.proxy.proxy_server.add_messages(): Exception occured - {}r   r-  r  r.  r0  r,  r   )$rW  rX  r   r]   r   r   r   r   r   r  r  r  a_add_messager  r  r   r0  r   r  r2  rh  rq  r   r  r1   r+  r*   r   r   r  r  r   r  ru  r   r8  r(  r  r  ra  rG  rW  rB  r<  rZ  r[  r\  rp  r  s                r   add_messagesr  1  s    , DH\\^#||D! 1-/%
 
 2C2Q2Q2W2W(X  $11NINNN 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 [ $
 O8  66/ATX 7 
 	
 	
 	""OVVA	

 	""9#7#7#9:a' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'sk   J/E5 E-9E5 E0AE5 E3 CE5 ,J/-E5 0E5 3E5 5
J,?J'FDJ''J,,J/c                   K   i }	 t        || t        |t        t               d{   }t        't        ddt        j                  j                  i      t	        j                  dd|i| d{   }t        j                  t        j                  |j                  dd      d	
             t        |di       xs i }|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}	|j                   j#                  t%        ||||	t        t        |dd      |             |S 7 7 # t&        $ r-}
t        j)                  ||
|       d{  7   t+        j,                  dj/                  t1        |
                   t+        j2                  t5        j6                                t9        |
t
              r\t;        t        |
dt1        |
j<                              t        |
dd      t        |
dd      t        |
dt>        j@                              t1        |
       }t;        t        |
d|      t        |
dd      t        |
dd      t        |
dd            d}
~
ww xY ww)z
    Returns a list of messages for a given thread.

    API Reference - https://platform.openai.com/docs/api-reference/messages/listMessages
    r  Nr  r+  rt  r  r  r   r  r  r  rZ  r[  r\  r  r  r{  zAlitellm.proxy.proxy_server.get_messages(): Exception occured - {}r   r-  r  r.  r0  r,  r   )!r]   r   r   r   r   r   r  r  r  aget_messagesr  r  r   r0  r   r  r2  rh  rq  r   r  r1   r+  r*   r   r   r  r  r   r  ru  r   r8  r  s               r   get_messagesr    sp    , DD0-/%
 
 2C2Q2Q2W2W(X  $11NINNN 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 S
 O8  66/ATX 7 
 	
 	
 	""OVVA	

 	""9#7#7#9:a' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'r  z/v1/threads/{thread_id}/runsz/threads/{thread_id}/runsc                   K   i }	 | j                          d{   }t        j                  |      }t        || t        |t
        t               d{   }t        't        ddt        j                  j                  i      t        j                  dd|i| d{   }d|v r |d   du rt        t        |||	      d
      S t        j                   t"        j%                  |j'                  dd      d             t)        |di       xs i }|j'                  dd      xs d}|j'                  dd      xs d}	|j'                  dd      xs d}
|j*                  j-                  t/        |||	|
t
        t)        |dd      |             |S 7 x7 A7 # t0        $ r-}t"        j3                  |||       d{  7   t5        j6                  dj9                  t;        |                   t5        j<                  t?        j@                                tC        |t              r\tE        t)        |dt;        |jF                              t)        |dd      t)        |dd      t)        |dtH        jJ                              t;        |       }tE        t)        |d|      t)        |dd      t)        |dd      t)        |dd            d}~ww xY ww)zi
    Create a run.

    API Reference: https://platform.openai.com/docs/api-reference/runs/createRun
    Nr  r  r+  rt  r  r  Tra  rB  r`  r  r  r  r   r  r  r  rZ  r[  r\  r  r  r{  z?litellm.proxy.proxy_server.run_thread(): Exception occured - {}r   r-  r  r.  r0  r,  r   )&rW  rX  r   r]   r   r   r   r   r   r  r  r  arun_threadr   r  r  r  r   r0  r   r  r2  rh  rq  r   r  r1   r+  r*   r   r   r  r  r   r  ru  r   r8  r  s                r   
run_threadr    s    , DR\\^#||D!0-/%
 
 2C2Q2Q2W2W(X  $//L)LtLL h4!7$/&7%!%
 /  	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 q $
 MP  66/ATX 7 
 	
 	
 	""MTTA	

 	""9#7#7#9:a' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'sw   KF F9F FAF F 'F KCF KF F F 
K#K>G?DKKKz/v1/moderationsmoderationsz/moderationsc                   K   i }	 | j                          d{   }t        j                  |      }t        || t        |t
        t               d{   }t        j                  dd      xs t        xs |j                  d      |d<   t        r	t        |d<   t        j                  ||d       d{   }t        j                          t        |dt        t               d{   }| d{   }t        j                  t        j!                  |j                  d	d
      d             t#        |di       xs i }|j                  dd      xs d
}|j                  dd      xs d
}	|j                  dd      xs d
}
|j$                  j'                  t)        |||	|
t
        t#        |dd
      |             |S 7 7 q7 7 7 # t*        $ r}t        j-                  |||       d{  7   t/        j0                  dj3                  t5        |                   t7        |t8              rRt;        t#        |dt5        |            t#        |dd      t#        |dd      t#        |dt<        j>                              t5        |       }t;        t#        |d|      t#        |dd      t#        |dd      t#        |dd            d}~ww xY ww)a  
    The moderations endpoint is a tool you can use to check whether content complies with an LLM Providers policies.

    Quick Start
    ```
    curl --location 'http://0.0.0.0:4000/moderations'     --header 'Content-Type: application/json'     --header 'Authorization: Bearer sk-1234'     --data '{"input": "Sample text goes here", "model": "text-moderation-stable"}'
    ```
    Nr  rn  r  
moderationr  amoderationr
  r  r   r  r  r  rZ  r[  r\  r  r  r{  z@litellm.proxy.proxy_server.moderations(): Exception occured - {}r   r-  r  r.  r0  r,  r  ) rW  rX  r   r]   r   r   r   r   r`  r   r(  rv  rg   r   r  r  r0  r  r2  rh  rq  r   r  r1   r  r*   r   r   r   r  r   r8  )r(  r  ra  rG  rW  r:  rB  r<  rZ  r[  r\  rp  r  s                r   r  r  ]  s    : DX\\^#||D! 1-/%
 
   !3T: !!xx  	W
 &DM '44/dl 5 
 
 			 '$!!	
 
 "> 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 } $
$

 "8  66/ATX 7 
 	
 	
 	&&NUUA	

 a' 9c!f5Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 %s   KG G9G GA"G 6G72G )G
*	G 3G4CG  KG G G 
G G 	KK2G53CKKKz/v1/messagesz[beta] Anthropic `/v1/messages`)r  r  response_modelinclude_in_schemaanthropic_datac                 "  K   ddl m} ddlm} d|dgt         _        t        |       d{   }i |ddi}	 t        j                  d	d      xs t        xs |d
   |d
<   t        r	t        |d
<   t        ||t        |t        t               d{   }t        r	t        |d<   t        r	t        |d<   t        r	t        |d<   t        r	t        |d<   |d
   t         j                   v rt         j                   |d
      |d
<   t"        j%                  ||d       d{   }t&        t&        j(                  ng }d|v r*t+        j,                  t        j.                  d4i |      }	nt&        1|d
   |v r*t+        j,                  t'        j.                  d4i |      }	nt&        Ot&        j0                  ?|d
   t&        j0                  v r*t+        j,                  t'        j.                  d4i |      }	n\t&        C|d
   t&        j2                  v r.t+        j,                  t'        j.                  d4i |ddi      }	nt&        B|d
   t&        j5                         v r)t+        j,                  t'        j.                  d4i |      }	nt&        f|d
   |vr_t&        j6                  &t9        t&        j:                  j<                        dkD  r)t+        j,                  t'        j.                  d4i |      }	n_t        )t+        j,                  t        j.                  d4i |      }	n0t?        t@        jB                  dd|j                  d
d      z   i      |	 d{   }
tE        |
di       xs i }|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}t+        j,                  t"        jG                  |j                  dd      d             tI        jJ                  d |
       |jL                  jO                  tQ        ||||t        ||!             d"|v r"|d"   du rtS        |
||#      }tU        |d$%      S tI        jV                  d&jY                  |
             |
S 7 `7 7 w7 B# tZ        $ rH}|j\                  }t"        j_                  |||'       d{  7   |j                  d"d      |d"   du rt        j`                         }t        jb                  ddd(      }||_2        |jf                  |jh                  d   jf                  _5        t         jl                  jo                  |d)      }t        jp                  ||j                  d
d      *      }ts        |||#      }tU        |d$i +      cY d}~S t        jt                         }|jf                  |jh                  d   _;        |cY d}~S d}~wtx        $ r}t"        j_                  |||'       d{  7   tI        jz                  d,jY                  t}        |                   t}        |       }t        tE        |d-|      tE        |d.d/      tE        |d0d/      tE        |d1d2      3      d}~ww xY ww)5u   
    🚨 DEPRECATED ENDPOINT🚨

    Use `{PROXY_BASE_URL}/anthropic/v1/messages` instead - [Docs](https://docs.litellm.ai/docs/anthropic_completion).

    This was a BETA endpoint that calls 100+ LLMs in the anthropic format.
    r   )adapter_completion)anthropic_adapter	anthropic)r  adapterr  N
adapter_idr   r  r  r[  r\  rZ  r\  rG  r  api_keyspecific_deploymentTr+  z/completion: Invalid model name passed in model=r   rt  r  rZ  r[  r^  r  r  r  rI  )ra  rZ  r[  r\  r   r^  r`  r  r  r  r  z
Response from Litellm:
{}r{  r   r  rJ  r  zGlitellm.proxy.proxy_server.anthropic_response(): Exception occured - {}r   r-  r  r.  r0  r  r,  r   )@r   r  "litellm.adapters.anthropic_adapterr  adaptersrN   r   r   r`  r]   r   r   rh  ri  r&  rg  r'  r   r(  r   rY  r  r  aadapter_completionmodel_group_aliasdeployment_namesr
  default_deploymentr  pattern_routerpatternsr   r   r8  r  r0  r1   r   r2  rh  rq  r  r   r   r*   r5   r`  r  r2  r6  r7  r   r3  r1  r+  r4  rK  r  rL  rM  r   r  r   r  )r  r  r(  ra  r  r  r`  rG  r\  llm_responserB  r<  rZ  r[  r\  r^  r>  rp  r?  r@  rC  rA  rB  rN  r  s                            r   anthropic_responser    sR    ( +D*7HIJG ,G<<L<L<,<Dw
  !3T: G} 	W
 &DM0-/%
 
 "2D&:D"#!0D,D
 =G333#33DMBDM '44/dFW 5 
 

 8B7MZ33SU"..w/J/J/RT/RSL"tG}8J'J"..z/M/M/UPT/UVL",,8W!=!=="..z/M/M/UPT/UVL"tG}
8S8S'S"....PP4PL "tG}
8P8P8R'R"..z/M/M/UPT/UVL"W%77--9z0099:Q> #..z/M/M/UPT/UVL#"..w/J/J/RT/RSL"77Nhhw+,  &%*:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB 	33 $):B ?	 4 	
 	""#7B  ''"3!#!+!
	
 h4!7&D!"3!'#
 %'. 
 	!!"@"G"G"QRE =
4
d &X   &66/  7 
 	
 	

 99Xt$0U8_5L$224N]]"#F
 $*N 89		N""1%--5;;- < I #*"E"E"+ii,#
 '<,"3!'# %'.   668I()		Ia % 
66/ATX 7 
 	
 	
 	&&U\\A	

 1vh	Ay)4FF+!Wf-M3/	
 	

s   +ZRZAR RB	R RH7R RDR &Z'%R ZR R R 
Z#'W&
SCW&'Z(Z-3W& Z!Z&Z2ZXA9ZZZz/utils/token_counterz	llm utils)r  r  r  c                   K   ddl m} | j                  }| j                  }||t	        dd      d}d}d}t
        Lt
        j                  D ]9  }|d   | j                  k(  s|}t
        j                  || j                        } n |:|j                  d	i       j                  d
      }d|v r|j                  dd      d   }|xs | j                  }d}	|'t        t        t           |j                  dd            }	t         j                  j                  ||	      }
t!        |
d         } |||||
      }t#        || j                  ||      S w)r%   r   )token_counterN  z#prompt or messages must be providedrt  r  )r  received_model_namer  r  /rA  custom_tokenizer)r  r  r-  )r  rM  r    r  )r#  request_model
model_usedtokenizer_type)r   r  promptr    r   r   r  r  get_router_model_infor   splitr
   r   r   r+  _select_tokenizerr   TokenCountResponse)r(  r  r  r    r  r  r   _modelmodel_to_user  _tokenizer_usedtokenizer_usedr#  s                r   r  r    s     & ^^FH~(*$I
 	
 J)-J ++Fl#w}}4#
'==)(/ > 
  , '^^,<bAEEgN$$!3!9!9#q!A!!D 	+gmm  >B/0NN-t4
 mm55-= 6 O 01N (	L !mm%	 s   A!E$C1Ez/utils/supported_openai_params)r  r  c                    K   	 t        j                  |       \  } }}}dt        j                  | |      iS # t        $ r t	        dddj                  |       i      w xY ww)a-  
    Returns supported openai params for a given litellm model name 

    e.g. `gpt-4` vs `gpt-3.5-turbo` 

    Example curl: 
    ```
    curl -X GET --location 'http://localhost:4000/utils/supported_openai_params?model=gpt-3.5-turbo-16k'         --header 'Authorization: Bearer sk-1234'
    ```
    r  supported_openai_paramsr  r  r  r+  zCould not map model={}rt  )r   get_llm_providerget_supported_openai_paramsr   r   r*   )r  r  r'  s      r   r  r    sx     "

+2+C+C%+P("Aq%w'J'J1D(
 	

  
W.F.M.Me.T$U
 	

s   A#38 A#(A  A#model_paramsc                   K   | j                   j                  d      }| j                   j                  }|j                         D ]   \  }}t	        |      }|| j                   |<   " | j
                  j                  | j                  | j                   j                  d      | j
                  j                  d      |j                  xs t        |j                  xs t        d}| j
                  j                  | j
                  j                  |d<   |j                  j                  j                  |       d {   }	|	S 7 w)NTr   r  )rZ  r  r  r   r  r  rZ  rG  )r  r   r  rg  rM   r   r  r  r  r  r   r&  rE  create)
r  ra  r   _litellm_params_dict_orignal_litellm_model_namerk  rl  encrypted_valuer?  r  s
             r   _add_model_to_dbr    s;     (66;;;N"."="="C"C$**,1.Q7)8##A& - !++.."--&55EESWEX"--== > 
 (//K3K'//K3K	E !!-(3366j(++CCJJ K  N s   D=E?E Ec                 ,  K   | j                   j                  }| j                  }|yd| dt        j                          }|| _        t        | ||       d{   }t        t        |||i      |t        ddi      	       d{    |S 7 57 w)
a  
    If 'team_id' is provided,

    - generate a unique 'model_name' for the model (e.g. 'model_name_{team_id}_{uuid})
    - store the model in the db with the unique 'model_name'
    - store a team model alias mapping {"model_name": "model_name_{team_id}_{uuid}"}
    Nmodel_name_r'  r  ra  r   )r  model_aliasesr-  r?  )scope)rG  ra  http_request)	r   r  r  r)  r*  r  rb   UpdateTeamRequestr   )r  ra  r   _team_idoriginal_model_nameunique_model_namer  s          r   _add_team_model_to_dbr  :  s      &&..H&11%hZq?/L ,!+# N .0AB
 ,FF#34   !s$   ABB.B	B
BBc                     d}|j                   r|j                   t        j                  k(  ry| !|j                   t        j                  k7  rd}|S |j                  | k7  rd}|S )NTF)r  r  r  r  )r  ra  can_make_calls      r   check_if_team_id_matches_keyr  c  sl     M##''+;+G+GG&&*:*F*FF!M  $$/!Mr   z
/model/newz=Allows adding new models to the model list in the config.yaml)r  r  r  c                   K   	 dd l }t        t        dddi      | j                  j                  /t
        dur't        ddt        j                  j                  i      t        | j                  j                  |      st        ddd	i      d }t        du r	 	 | j                  }| j                  j                  t        | |t        
       d {   }nt        | |t        
       d {   }t        j                  t        t                d {    t"        j%                  dg       xs g }d|v r>t         j&                  j)                  | j                  || j                         d {    |S t        dddi      7 7 7 u7 # t*        $ r#}t-        j.                  d|        Y d }~|S d }~ww xY w# t*        $ r}t-        j0                  dj3                  t5        |                   t-        j6                  t9        j:                                t=        |t              rYt?        tA        |ddt5        |       d      tB        jD                  tA        |dd      tA        |dtF        jH                              t=        |t>              r|t?        dt5        |      z   tB        jD                  tA        |dd      tF        jH                        d }~ww xY ww)Nr   r  r+  VNo DB Connected. Here's how to do it - https://docs.litellm.ai/docs/proxy/virtual_keysrt  Ti  )r  ra  z,Team ID does not match the API key's team IDr  r  r  r  )r  r  passed_model_infozException in add_new_model: CSet `'STORE_MODEL_IN_DB='True'` in your env to enable this feature.zBlitellm.proxy.proxy_server.add_new_model(): Exception occured - {}ru  Authentication Error()r.  r  r0  r,  Authentication Error, )%r  r   r   r   r  r   r  r  r  r  rW  r  r  r  r   rI  r   r   r   r+  model_added_alertr   r1   r  r+  r*   r   r   r  r  r   r  r  ProxyErrorTypes
auth_errorr   r8  )r  ra  r  r  _original_litellm_model_name	_alertingrp  s          r   add_new_modelr  v  s    ]
 u  ""**6<t;S!2!C!C!I!IJ 
 , ++33GX
  !OP 
 $
S/;/F/F,**22:+;%1*;&3, &N ,A%1*;&3, &N
 #11"/CT 2    -00R@FB	i'+CCUU#/#:#:+G*6*A*A V       b 7&&

  S$..1MaS/QRR S  
""PWWA	

 	""9#7#7#9:a' 8/DSVHA-NO$//a&1Qv/J/JK	  >*G,s1v5 ++!Wf-,,	
 	
!
s   K#BG  7F F	F 0F1&F FAF 3F4F 8G  9K#:G  	F F F F 	F=F82G  7K#8F==G   
K 
DKK  K#z/model/updatezEdit existing model paramsc                 8  K   	 dd l }t        t        dddi      t        du rd }t	        | dd       }|t        d      |j                  }|t        d	      t        j                  j                  j                  d
|i       d {   }|6t        %t        j                  |      t        dddi      t        d      t        |j                        }| j                  t        d      | j                  j                  d      }|j                         D ]   \  }}	t        |	      }
|
| j                  |<   " i }| j                  j                         }|j                         D ]  \  }}||||<   ||v r||   	||   ||<    ! t!        j"                  |      |j$                  xs t&        d}t        j                  j                  j)                  d
|i|       d {   }|S y 7 h7 
# t
        $ r}t+        j,                  dj/                  t1        |                   t+        j2                  t5        j6                                t9        |t              rYt;        t	        |ddt1        |       d      t<        j>                  t	        |dd      t	        |dt@        jB                              t9        |t:              r|t;        dt1        |      z   t<        j>                  t	        |dd      t@        jB                        d }~ww xY ww)Nr   r  r+  r  rt  Tr   zmodel_info not providedzmodel_info.id not providedrZ  r#  r  r  zOCan't edit model. Model in config. Store model in db via `/model/new`. to edit.zmodel not foundzlitellm_params not providedr   r  )r  r  r$  rG  zAlitellm.proxy.proxy_server.update_model(): Exception occured - {}ru  r  r  r.  r  r0  r,  r  )"r  r   r   rW  r  r   r  r&  rE  find_uniquer   r/  r   r  rg  rM   r   r:  r  r   rh  r1   r+  r*   r   r   r  r  r   r  r  r  r   r8  )r  ra  r  	_model_idr  _existing_litellm_params_existing_litellm_params_dict_new_litellm_params_dictrk  rl  r  merged_dictionary_mpro  r  r?  r  rp  s                     r   update_modelr    sB    e
  u  $I!,dCK" 9::#I  <==#&&>>JJ%y1 K   %
 (/*"1191EQ'$'#%v     122,0(77-) **2 =>>'3'B'B'G'G! (H ($
 16681"6Q"?1@++A. 9
 !#--224C!iik
U$-2%c*885c:F-J3-O%c* * #'**->"?/77S;SE $1#3#3#K#K#R#R!9- $S $ N
 "!E %f  
""OVVA	

 	""9#7#7#9:a' 8/DSVHA-NO$//a&1Qv/J/JK	  >*G,s1v5 ++!Wf-,,	
 	
!
sO   LBG7 
G2EG7 *G5+G7 0L2G7 5G7 7
LDLLLz/v2/model/infozv2 - returns all the models set on the config.yaml, shows 'user_access' = True if the user has access to the model. Provides more info about each model in /models, including config.yaml descriptions (except api key and api base))r  r  r  r  z!Specify the model name (optional))r  r   c                   K   t         t        t         t              st        dddt          di      t        j                          d{    t        j                  t               }t        
|t        gz  }t        | j                        dkD  r	 ||D cg c]  }|d   |k(  s| }}|D ]  }|j                  d	i       }|d
u r7d}t        t        j                  |i d      xs d}nd}t        |      }||d<   t        |      }	|	i k(  r;|j                  di       }
|
j                  dd      }	 t!        j"                  |      }	|	i k(  rc|j                  di       }
|
j                  dd      }|j'                  d      }t        |      dkD  r|d   }	 t!        j"                  ||d         }	|	j)                         D ]  \  }}||vs|||<    ||d	<   |d   j+                  dd       |d   j+                  dd       |d   j+                  dd       |d   j+                  dd        t-        j.                  d|       d|iS 7  c c}w # t$        $ r i }	Y w xY w# t$        $ r i }	Y w xY ww)zQ
    BETA ENDPOINT. Might change unexpectedly. Use `/v1/model/info` for now.
    Nr  r+  zNo model list passed, models=zI. You can add a model through the config.yaml or on the LiteLLM Admin UI.rt  r   r  r   Tr  async)r  ri  client_typellm_router_is_Noneopenai_clientr  r  r  r  r  r  r  vertex_credentialsaws_access_key_idaws_secret_access_keyall_models: %srG  )r   r   r  r   r   rp  r  r  r`  r  r  r   r   _get_clientr   r  r   r  r   r  rg  rJ  r1   r   )ra  r  r   r  r  r  r   _openai_clientr	  r  r  litellm_modelsplit_modelrk  rl  s                  r   model_info_v2r  U  s    & Z%E88H  IR  S
 	
 
!
!
###~.Jzl"
 ##$q(!+HAq%/Ga
H ZZb1
D=#N%**#)"' +   	  "6/M&3F?# 4&A ##ZZ(8"=N*..w=M(%,%;%;-%P" ##ZZ(8"=N*..w=M'--c2K;!# +B(%,%;%;'[^&"
 ',,.DAq
" !
1 /  *| 	 $$Y5 $$%94@ $$%8$? $$%<dCk n /<JO $ I>  (%'"(  (%'"(s|   A	I6IAI6I&I*B
I65IAI6I%3I6A>I6I6I"I6!I""I6%I30I62I33I6z/model/streaming_metricsz1View time to first token for models in spend logs)r  r  r  r  _selected_model_groupr  r  c                   K   t         5t        t        j                  j                  ddt
        j                        |xs! t        j                         t        d      z
  }|xs t        j                         }|j                         |j                         k(  }|rd}nd}t               }t         j                  j                  ||||       d {   }i }||D ]  }	|	d   }
|	d	   }|	d
   }d}|r|	d   }|}||vri ||<   n|	d   }|}|	d
   }||vri ||<   t        |      }d|
v rt        |
      }d|v r|j                  d      d   }|j!                  |       |||   |<    	 g }t#        t%        |j'                         d             }|D ]@  }dt        |      i}||   j'                         D ]
  \  }}|||<    |j)                  |       B |t+        |      dS y 7 w)Ninternal_errorr  r,     r  a  
            SELECT
                api_base,
                model_group,
                model,
                "startTime",
                request_id,
                EXTRACT(epoch FROM ("completionStartTime" - "startTime")) AS time_to_first_token
            FROM
                "LiteLLM_SpendLogs"
            WHERE
                "model_group" = $1 AND "cache_hit" != 'True'
                AND "completionStartTime" IS NOT NULL
                AND "completionStartTime" != "endTime"
                AND DATE("startTime") = DATE($2::timestamp)
            GROUP BY
                api_base,
                model_group,
                model,
                request_id
            ORDER BY
                time_to_first_token DESC;
        a  
            SELECT
                api_base,
                model_group,
                model,
                DATE_TRUNC('day', "startTime")::DATE AS day,
                AVG(EXTRACT(epoch FROM ("completionStartTime" - "startTime"))) AS time_to_first_token
            FROM
                "LiteLLM_SpendLogs"
            WHERE
                "startTime" BETWEEN $2::timestamp AND $3::timestamp
                AND "model_group" = $1 AND "cache_hit" != 'True'
                AND "completionStartTime" IS NOT NULL
                AND "completionStartTime" != "endTime"
            GROUP BY
                api_base,
                model_group,
                model,
                day
            ORDER BY
                time_to_first_token DESC;
        r\  r  time_to_first_tokenr   
request_idr  https:///openai/r   c                     | d   S Nr   r   rT  s    r   <lambda>z)model_streaming_metrics.<locals>.<lambda>2  	    dSTgr   r  daterG  all_api_bases)r   r  r  db_not_connected_errorr  r   r4  r   r  r   r!  setr&  	query_rawr   r  addr   sortedrg  r  r  )ra  r  r  r  is_same_day	sql_query_all_api_basesdb_response_daily_entries
model_data	_api_baser  r  
unique_key_request_id_day_combined_model_namerB  r  entry	model_keylatencys                         r   model_streaming_metricsr7    s9     %<<BB!66	
 	
 ?X\\^iQ.??I'G.."glln4K	0	. UN%((22()W K N%J":.I(F",-B"CJ(6(
n424N;/!%(!
&01F&G#~-+-N4(#&v; Y&'*9~$11';'A'A*'Ma'P$34?RN:&';<1 &4	  " f^%9%9%;AUVW!CSX&E&4S&9&?&?&A"	7#*i  'BOOE"	 " !.1
 	
_ 	s   CG&G#DG&z/model/metricsz>View number of requests & avg latency per model on config.yamlz	gpt-4-32kr  customerc                 @  K   t         t        dddt        j                        |xs! t	        j
                         t        d      z
  }|xs t	        j
                         }||dk(  rd}||dk(  rd}d	}t               }t         j                  j                  ||||||       d {   }i }	||D ]j  }
|
d
   }|
d   }|
d   }|
d   }||	vri |	|<   t        |      }d|v rt        |      }d|v r|j                  d      d   }|j                  |       ||	|   |<   l 	 g }t        t        |	j                         d             }	|	D ]@  }dt        |      i}|	|   j                         D ]
  \  }}|||<    |j!                  |       B |t#        |      dS y 7 w)N Prisma Client is not initializedr  r  r,     r  	undefinednulla  
        SELECT
            api_base,
            model_group,
            model,
            DATE_TRUNC('day', "startTime")::DATE AS day,
            AVG(EXTRACT(epoch FROM ("endTime" - "startTime")) / NULLIF("completion_tokens", 0)) AS avg_latency_per_token
        FROM
            "LiteLLM_SpendLogs"
        WHERE
            "startTime" >= $2::timestamp AND "startTime" <= $3::timestamp
            AND "model_group" = $1 AND "cache_hit" != 'True'
            AND (
                CASE
                    WHEN $4 != 'null' THEN "api_key" = $4
                    ELSE TRUE
                END
            )
            AND (
                CASE
                    WHEN $5 != 'null' THEN "end_user" = $5
                    ELSE TRUE
                END
            )
        GROUP BY
            api_base,
            model_group,
            model,
            day
        HAVING
            SUM(completion_tokens) > 0
        ORDER BY
            avg_latency_per_token DESC;
    r\  r  r  avg_latency_per_tokenr  r  r   c                     | d   S r  r   r  s    r   r  zmodel_metrics.<locals>.<lambda>  r   r   r  r!  r"  )r   r  r   r4  r   r  r   r%  r&  r&  r   r  r'  r   r(  rg  r  r  )ra  r  r  r  r  r8  r*  r+  r,  r-  r.  r/  r  r2  _avg_latency_per_tokenr3  rB  r  r4  r5  r6  s                        r   model_metricsrA  ?  s      6!66	
 	
 @X\\^iR.@@I'G'[08{2!ID UN%((22()Wgx K N%J":.I(Fe$D%/0G%H">)')t$#&v; Y&'*9~$11';'A'A*'Ma'P$349ON4 !56 & 	  " f^%9%9%;AUVW!CSX&E&4S&9&?&?&A"	7#*i  'BOOE"	 " !.1
 	
K s   B&F(F)C4Fz/model/metrics/slow_responsesz/View number of hanging requests per model_groupc           	        K   t         t        dddt        j                        ||dk(  rd}||dk(  rd}|xs! t	        j
                         t        d      z
  }|xs t	        j
                         }t        j                  j                  xs d	}t        |      }d
}t         j                  j                  |||||||       d {   }|9|D ]4  }	|	j                  d      xs d}
d|
v r|
j                  d      d   }
|
|	d<   6 |S 7 Aw)Nr:  r  r  r,  r<  r=  r;  r  r  a  
SELECT
    api_base,
    COUNT(*) AS total_count,
    SUM(CASE
        WHEN ("endTime" - "startTime") >= (INTERVAL '1 SECOND' * CAST($1 AS INTEGER)) THEN 1
        ELSE 0
    END) AS slow_count
FROM
    "LiteLLM_SpendLogs"
WHERE
    "model_group" = $2
    AND "cache_hit" != 'True'
    AND "startTime" >= $3::timestamp
    AND "startTime" <= $4::timestamp
    AND (
        CASE
            WHEN $5 != 'null' THEN "api_key" = $5
            ELSE TRUE
        END
    )
    AND (
        CASE
            WHEN $6 != 'null' THEN "end_user" = $6
            ELSE TRUE
        END
    )
GROUP BY
    api_base
ORDER BY
    slow_count DESC;
    r\  r   r  r   )r   r  r   r4  r   r  r   r   r+  r  r3  r&  r&  r   r  )ra  r  r  r  r  r8  r  r*  r,  rowr/  s              r   model_metrics_slow_responsesrD    s5      6!66	
 	
 '[08{2@X\\^iR.@@I'G 	11DDK  /0IB &((22 K C
+1rIY&%OOJ7:	'C
O	 
 !s   CDD
	ADz/model/metrics/exceptionsz7View number of failed requests per model on config.yamlc                   K   t         t        dddt        j                        |xs! t	        j
                         t        d      z
  }|xs t	        j
                         }||dk(  rd}	 d	}t         j                  j                  |||||       d {   }g }t               }		 ||D ]  }
|
j                  d
d      }|
j                  dd      }|
j                  di       }||d}|j                  |       |j                  |       |j                         D ]  \  }}|	j                  |         |t        |	      dS 7 w)Nr:  r  r  r,  r;  r  r<  r=  aD  
        WITH cte AS (
            SELECT 
                CASE WHEN api_base = '' THEN litellm_model_name ELSE CONCAT(litellm_model_name, '-', api_base) END AS combined_model_api_base,
                exception_type,
                COUNT(*) AS num_rate_limit_exceptions
            FROM "LiteLLM_ErrorLogs"
            WHERE 
                "startTime" >= $1::timestamp 
                AND "endTime" <= $2::timestamp 
                AND model_group = $3
            GROUP BY combined_model_api_base, exception_type
        )
        SELECT 
            combined_model_api_base,
            COUNT(*) AS total_exceptions,
            json_object_agg(exception_type, num_rate_limit_exceptions) AS exception_counts
        FROM cte
        GROUP BY combined_model_api_base
        ORDER BY total_exceptions DESC
        LIMIT 200;
    combined_model_api_baser   total_exceptionsr   exception_counts)r  rG  )rG  exception_types)r   r  r   r4  r   r  r   r&  r&  r%  r   rh  r  rg  r'  r  )ra  r  r  r  r  r8  r*  r,  rB  rI  r.  r  rG  rH  curr_rowrk  rl  s                    r   model_metrics_exceptionsrK    s]      6!66	
 	
 @X\\^iR.@@I'G'[0I, &((229g'<g K HeO %JNN#<bAE)~~.@!D)~~.@"E$4H OO,-OOH%(..01##A& 1 & o1FGG?s   BEEB/Ez/model/infoz/v1/model/infolitellm_model_idc           
      :  K   t         e	 t        t        t        j                  t                     }t        dt        t               |      }|j                         }t        |      }d|iS t        t        ddd	i
      t        t        dddi
      |Kt        j                  |      }|t        ddd| di
      |j                         }t        |      }d|iS g }t        t              }t        g }n(t        j!                         }t        j#                         }t%        | ||      }	t'        | ||      }
t)        |	|
|t         t*        j-                  dd            }t/        |      dkD  rO|}t        j1                         a
t        1t        D cg c]  }|d   |v s| }}t3        j4                  |      }ng }|D ]  }|j-                  di       }t7        |      }|i k(  r;|j-                  di       }|j-                  dd      }	 t        j                  |      }|i k(  rc|j-                  di       }|j-                  dd      }|j9                  d      }t/        |      dkD  r|d   }	 t        j                  ||d         }|j;                         D ]  \  }}||vs|||<    ||d<   t        |      } t=        j>                  d|       d|iS # t
        $ r i }Y w xY wc c}w # t
        $ r i }Y w xY w# t
        $ r i }Y w xY ww)a  
    Provides more info about each model in /models, including config.yaml descriptions (except api key and api base)

    Parameters:
        litellm_model_id: Optional[str] = None (this is the value of `x-litellm-model-id` returned in response headers)

        - When litellm_model_id is passed, it will return the info for that specific model
        - When litellm_model_id is not passed, it will return the info for all models

    Returns:
        Returns a dictionary containing information about each model.

    Example Response:
    ```json
    {
        "data": [
                    {
                        "model_name": "fake-openai-endpoint",
                        "litellm_params": {
                            "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
                            "model": "openai/fake"
                        },
                        "model_info": {
                            "id": "112f74fab24a7a5245d2ced3536dd8f5f9192c57ee6e332af0f0512e08bed5af",
                            "db_model": false
                        }
                    }
                ]
    }

    ```
    Nr  r;   r  )deployment_dictrG  r  r+  zLLM Model List not loaded in. Make sure you passed models in your config.yaml or on the LiteLLM Admin UI. - https://docs.litellm.ai/docs/proxy/configsrt  zLLM Router is not loaded in. Make sure you passed models in your config.yaml or on the LiteLLM Admin UI. - https://docs.litellm.ai/docs/proxy/configsr  i  zModel id = z not found on litellm proxyr  r  Fr  r   r  r   r  r  r  r  r  r  ) r`  r
   rM  r   r  r   rx   ry   
model_dumprR   r   r   r   r/  r-   r  r  r  rC   rD   rB   r   r   r  r  r  r  r  r  rg  r1   r   )ra  rL  r   _deployment_info_deployment_info_dictdeployment_infor  r  r  r  r  all_models_strrY  r  _relevant_modelsr  r  r  r  r  rk  rl  s                         r   model_info_v1rU  c  s    ` 	#D'*@*@z*RSJ &)  "
 !1 ; ; = E1!
 -..  r
 	
   q
 	
 #$33=M3N"{+;*<<WX  !0 : : < E1!
 -..J0;D0A%557(@@B+)/J
 "+)/K
 -).223JERN >Q$#224%) )aQ|_-K>    '78JJYY|R0
 4%@ #"YY'7<N*..w=M(%,%;%;-%P" #"YY'7<N*..w=M'--c2K;!# +B(%,%;%;'[^&"
 ',,.DAq
" !
1 / )l5eLI L /<Jw  	J	Z ,  (%'"(  (%'"(s   L(K" E+LK4*K4.A&LK9+AL9L
L.4L"K1-L0K11L9LLLL
LLLLrS  r  c                 t    g }|D ]0  }|||k7  r| j                  |      }| |j                  |       2 |S )N)r  )get_model_group_infor  )r   rS  r  model_groupsr  _model_group_infos         r   _get_model_group_inforZ    sT     *,L"{e';&;;;N( 12   r   z/model_group/infoc           
        K   t         t        dddi      t        t        dddi      t        t              }t        g }n(t        j                         }t        j                         }t        | ||      }t        | ||      }t        |||t        t        j                  dd      	      }t        t        ||
      }d|iS w)a#  
    Get information about all the deployments on litellm proxy, including config.yaml descriptions (except api key and api base)

    - /model_group/info returns all model groups. End users of proxy should use /model_group/info since those models will be used for /chat/completions, /embeddings, etc.
    - /model_group/info?model_group=rerank-english-v3.0 returns all model groups for a specific model group (`model_name` in config.yaml)

    

    Example Request (All Models):
    ```shell
    curl -X 'GET'     'http://localhost:4000/model_group/info'     -H 'accept: application/json'     -H 'x-api-key: sk-1234'
    ```

    Example Request (Specific Model Group):
    ```shell
    curl -X 'GET'     'http://localhost:4000/model_group/info?model_group=rerank-english-v3.0'     -H 'accept: application/json'     -H 'Authorization: Bearer sk-1234'
    ```

    Example Request (Specific Wildcard Model Group): (e.g. `model_name: openai/*` on config.yaml) 
    ```shell
    curl -X 'GET'     'http://localhost:4000/model_group/info?model_group=openai/tts-1' 
    -H 'accept: application/json'     -H 'Authorization: Bearersk-1234'
    ```

    Learn how to use and set wildcard models [here](https://docs.litellm.ai/docs/wildcard_routing)

    Example Response:
    ```json
        {
            "data": [
                {
                "model_group": "rerank-english-v3.0",
                "providers": [
                    "cohere"
                ],
                "max_input_tokens": null,
                "max_output_tokens": null,
                "input_cost_per_token": 0.0,
                "output_cost_per_token": 0.0,
                "mode": null,
                "tpm": null,
                "rpm": null,
                "supports_parallel_function_calling": false,
                "supports_vision": false,
                "supports_function_calling": false,
                "supported_openai_params": [
                    "stream",
                    "temperature",
                    "max_tokens",
                    "logit_bias",
                    "top_p",
                    "frequency_penalty",
                    "presence_penalty",
                    "stop",
                    "n",
                    "extra_headers"
                ]
                },
                {
                "model_group": "gpt-3.5-turbo",
                "providers": [
                    "openai"
                ],
                "max_input_tokens": 16385.0,
                "max_output_tokens": 4096.0,
                "input_cost_per_token": 1.5e-06,
                "output_cost_per_token": 2e-06,
                "mode": "chat",
                "tpm": null,
                "rpm": null,
                "supports_parallel_function_calling": false,
                "supports_vision": false,
                "supports_function_calling": true,
                "supported_openai_params": [
                    "frequency_penalty",
                    "logit_bias",
                    "logprobs",
                    "top_logprobs",
                    "max_tokens",
                    "max_completion_tokens",
                    "n",
                    "presence_penalty",
                    "seed",
                    "stop",
                    "stream",
                    "stream_options",
                    "temperature",
                    "top_p",
                    "tools",
                    "tool_choice",
                    "function_call",
                    "functions",
                    "max_retries",
                    "extra_headers",
                    "parallel_tool_calls",
                    "response_format"
                ]
                },
                {
                "model_group": "llava-hf",
                "providers": [
                    "openai"
                ],
                "max_input_tokens": null,
                "max_output_tokens": null,
                "input_cost_per_token": 0.0,
                "output_cost_per_token": 0.0,
                "mode": null,
                "tpm": null,
                "rpm": null,
                "supports_parallel_function_calling": false,
                "supports_vision": true,
                "supports_function_calling": false,
                "supported_openai_params": [
                    "frequency_penalty",
                    "logit_bias",
                    "logprobs",
                    "top_logprobs",
                    "max_tokens",
                    "max_completion_tokens",
                    "n",
                    "presence_penalty",
                    "seed",
                    "stop",
                    "stream",
                    "stream_options",
                    "temperature",
                    "top_p",
                    "tools",
                    "tool_choice",
                    "function_call",
                    "functions",
                    "max_retries",
                    "extra_headers",
                    "parallel_tool_calls",
                    "response_format"
                ]
                }
            ]
            }
    ```
    r  r+  zLLM Model List not loaded inrt  zLLM Router is not loaded inr  r  Fr  )r   rS  r  rG  )r   r   r   r-   r  r  r  rC   rD   rB   r`  r   r   rZ  )ra  r  r  r  r  r  rS  rX  s           r   model_group_infor\  #  s     B W.L$M
 	
 W.K$L
 	
 1<D0A%557(@@B+)/J
 "+)/K
 -).223JERN *?n+*L L!!s   CCz/model/deletez;Allows deleting models in the model list in the config.yamlr   c                 2  K   	 	 t         t        dddi      t        du r	 t         j                  j                  j                  d| j                  i       d {   }|t        ddd	| j                   d
i      t         t        j                  | j                         dd|j                   diS t        dddi      7 j# t        $ r}t        |t              rYt        t        |ddt        |       d      t        j                   t        |dd      t        |dt"        j$                              t        |t              r|t        dt        |      z   t        j                   t        |dd      t"        j$                        d }~ww xY ww)Nr  r+  r  rt  TrZ  r#  r  zModel with id=z not found in dbr  r   zModel: z deleted successfullyr  ru  r  r  r.  r  r0  r,  r  )r   r   rW  r&  rE  deleter  r   r  rZ  r   r   r  r  r   r  r  r   r8  )r   r  rp  s      r   delete_modelr_    s    ?
	  u  $
 )++CCJJ!:==1 K  F ~# ##~jmm_DT%UV  %,,
,>(99NOPPb !.  
a' 8/DSVHA-NO$//a&1Qv/J/JK	  >*G,s1v5 ++!Wf-,,	
 	

s=   FAC	 CAC	 7F8C	 		FB=FFFz/model/settingszMReturns provider name, description, and required parameters for each providerc            	         K   g } t         j                  D ]2  }| j                  t        |t        j                  |                   4 | S w)a?  
    Used by UI to generate 'model add' page
    {
        field_name=field_name,
        field_type=allowed_args[field_name]["type"], # string/int
        field_description=field_info.description or "", # human-friendly description
        field_value=general_settings.get(field_name, None), # example value
    }
    )r  )r<  fields)r   provider_listr  ProviderInfoget_provider_fields)returned_listr  s     r   model_settingsrf  8  sL     $ M))22xP	
 * s   A
Az/alerting/settingszFReturn the configurable alerting param, description, and current valuer  c                   K   	 t         't        ddt        j                  j                  i      | j
                  t        j                  k7  rAt        dddj                  t        j                  j                  | j
                        i      t         j                  j                  j                  ddi       d {   }|E|j                  9t        |j                        }|j                  di       }|j                  d	      }ni }d }d
did
did
did
did
did
did
did
did
did	}t         j"                  }|j$                  j'                         }g }d}	t(        j                  d	      r$t+        t(        d	   t,              rdt(        d	   v rd}	t/        d|d   d
   d|	|dndd d      }
|j1                  |
       t2        j4                  j7                         D ]m  \  }}||v sd }||v rd}nd}t/        |||   d
   |j8                  xs d|j                  |d       ||j:                  |dk(  rdnd      }
|j1                  |
       o |S 7 w)Nr  r+  rt  {}, your role={}r"  r   r#  r  r  r-  BooleanInteger)	slack_alertingdaily_report_frequencyreport_check_intervalbudget_alert_ttloutage_alert_ttlregion_outage_alert_ttlminor_outage_alert_thresholdmajor_outage_alert_thresholdmax_outage_alert_list_sizeFr  Trk  zhEnable slack alerting for monitoring proxy in production: llm outages, budgets, spend tracking failures.)
field_name
field_typefield_descriptionfield_valuestored_in_dbfield_default_valuepremium_fieldr   rp  )r   r   r  r$  r  r  r  r  r*   not_allowed_accessr&  r'  r(  r)  r   r   r   r+  r  rO  r   r   r  
ConfigListr  r   model_fieldsrg  r  r   )ra  r5  db_general_settings_dictalerting_args_dictalerting_valuesallowed_args_slack_alerting_slack_alerting_args_dict
return_valis_slack_enabled_response_objrt  
field_info_stored_in_dbs                 r   alerting_settingsr  Y  s     .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 !. 0 0 ? ? J J/0 !K !  &+>+J+J+V#'(;(G(G#H #;#?#?QS#T*B*F*Fz*R "9-#)9"5"()!4#Y/#Y/$*I#6)/(;)/(;'-y&9
L &7%N%NO / = = H H JJJ'J$d- &z22## 01&9 E$,8Te M m$"3"@"@"F"F"H
J%,0M// $ %&%'
3F;","8"8">B599*dK*$.$6$6&*CCD
M m,) #I* Ks   B<I>I?D)I)A(Iz/queue/chat/completionsrj  )r  r  r  c                   K   	 i }	 | j                          d {   }t        | j                        | j                  t	        | j
                        t        j                  |      d|d<   t        j                  d|       t        j                  dd       xs t        xs	 |xs |d   |d<   |j                  dd       |j                  |j                  |d<   d|vri |d<   |j                  |d   d<   |j                  |d   d	<   t	        | j
                        }|j                  d
d        ||d   d<   t!        |dd       |d   d<   |j                  |d   d<   t!        |dd       |d   d<   t        | j                        |d   d<   t"        r	t"        |d<   t$        r	t$        |d<   t&        r	t&        |d<   t(        r	t(        |d<   t*        't-        ddt.        j0                  j2                  i      t+        j4                  d)i | d {   }d|v r |d   du rt7        t9        |||      d      S |j
                  j;                  dt        |d         i       |S 7 E7 V# t<        $ r}t>        jA                  |||        d {  7   tC        |t,              rYtE        t!        |d!d"t        |       d#      tF        jH                  t!        |d$d%      t!        |d&tJ        jL                        '      tC        |tD              r|tE        d(t        |      z   tF        jH                  t!        |d$d%      tJ        jL                  '      d }~ww xY ww)*N)rB  methodr2  rW  proxy_server_requestzreceiving data: %sr   r  r  metadatauser_api_keyuser_api_key_metadataauthorizationr2  r  user_api_key_aliasuser_api_key_user_idr  user_api_key_team_idendpointr[  r\  rZ  r\  r  r+  rt  r  Tr  r  r  zx-litellm-prioritypriorityr{  ru  r  r  r.  r  r0  r,  r  r   )'r   r   rB  r  r   r2  r  r1   r   r   r   r`  r  r  r  rJ  r  rh  ri  r&  rg  r   r   r  r  r  schedule_acompletionr   r  rh  r   r   r  r   r  r  r  r   r8  )r(  r  r  ra  rG  _headersrB  rp  s           r   async_queue_requestr    sr     D\
\\^# w{{#nnGOO,IIdO	(
#$ 	""#7>  !3T:  G}	 	W 88FD!).?.G.G.S,44DLT!!D+<+D+DZ(4E4N4NZ01(T	
 '/Z#18{D2
Z-. 4E3L3LZ/03:y$4
Z/0 (+7;;'7Z$ "2D&:D"#!0D,D2C2Q2Q2W2W(X  $88@4@@ h4!7$$&7%!%
 /  	  '')=s4
CS?T(UVQ $r A   
66/ATX 7 
 	
 	
 a' 8/DSVHA-NO$//a&1Qv/J/JK	  >*G,s1v5 ++!Wf-,,	
 	

sd   MI$ IG0I$ I"'I$ 3M4*I$ MI$ "I$ $	M-MJ	CMMMz/fallback/login)r  r  c                   K   t        j                  dt        | j                              }t        j                  d      }|j	                  d      r|dz  }n|dz  }|ddlm}  |t        d	      S ddlm}  |t        d	      S w)
z
    Create Proxy API Keys using Google Workspace SSO. Requires setting PROXY_BASE_URL in .env
    PROXY_BASE_URL should be the your deployed proxy endpoint, e.g. PROXY_BASE_URL="https://litellm-production-7002.up.railway.app/"
    Example:
    r>  UI_USERNAMEr  zsso/callbackz/sso/callbackr   )HTMLResponser  )r1  r0  )r   r  r   base_urlendswithfastapi.responsesr  rG   )r(  redirect_urlui_usernamer  s       r   fallback_loginr  <  s|      99-s73C3C/DEL))M*KS!&' 	3I3??2I3??s   BBz/login)r  c                 `
  K   	 dd l }t        +t        dt        j                  dt        j                        | j                          d {   }t        |j                  d            }t        |j                  d            }t        j                  dd	      }t        j                  d
d       }|t        t        t              nd }|+t        dt        j                  d
t        j                        d }d }t        6t        j                   j"                  j%                  dd|ii       d {   }t'               }		 t)        j*                  ||      rt)        j*                  ||      rt,        j.                  }t0        }
t0        }t        j                  dd       t        j2                  d   |
k(  s	|
t0        k(  rt        j                  dt0              }t5        t7        ||             d {    t        j                  d      <t9        d0ddit,        j.                  dt:        j<                  g i i d|dd	 d {   }n+t        dt        j                  dt        j                        |d   }t        j                  dd      }|j?                  d      r|dz  }n|dz  }dd l }|jC                  |
|d |dtD        tF        j                  d d!      |	d"t        d#$      }|d%|
z   z  }tI        |d&'      }|jK                  d|(       |S |	 tM        |d)d*      }
tM        |d+t,        jN                        }tM        |dd*      }tM        |dd*      }tQ        |,      }t)        j*                  ||      st)        j*                  ||      rt        j                  d      .t9        d0ddi|dt:        j<                  g i i d|
dd	 d {   }n+t        dt        j                  dt        j                        |d   }t        j                  dd      }|j?                  d      r|dz  }n|dz  }dd l }|jC                  |
|||dtD        tF        j                  d d!      |	d"t        d#$      }|d%|
z   z  }tI        |d&'      }|jK                  d|(       |S t        d-| t        j                  d.t        jR                        t        d/t        j                  d.t        jR                        # t        $ r t        j                  g d       Y w xY w7 7 7 07 7 [w)1Nr   )pipinstallzpython-multipartMaster Key not set for Proxy. Please set Master Key to use Admin UI. Set `LITELLM_MASTER_KEY` in .env or set general_settings:master_key in config.yaml.  https://docs.litellm.ai/docs/proxy/virtual_keys. If set, use `--detailed_debug` to debug issue.r   r,  usernamer}  r  rT  UI_PASSWORDzset Proxy master key to use UI. https://docs.litellm.ai/docs/proxy/virtual_keys. If set, use `--detailed_debug` to debug issue.
user_emailequalsr#  r  )r  r  r  r   r  ro  24hrlitellm-dashboard	r  r  key_max_budgetr  r  r   r  r  r  z_No Database connected. Set DATABASE_URL in .env. If set, use `--detailed_debug` to debug issue.r  r>  r   r  zui/r   username_passwordlitellm_key_header_nameAuthorizationr  ro  r  r  login_methodr   auth_header_name(disabled_non_admin_personal_key_creationHS256	algorithmz?userID=i/  )rB  r0  )ro  r  r  unknownr  r  zAInvalid credentials used to access UI.
Not valid credentials for invalid_credentialszVInvalid credentials used to access UI.
Check 'UI_USERNAME', 'UI_PASSWORD' in .env filer   )*	multipartImportErrorr  runr   r  r  r  r   r4  rr  r   r   r   r  r   r&  litellm_usertabler(  rc   secretscompare_digestr  r  r   r   r^   UpdateUserRequestra   r   max_ui_session_budgetr  jwtencoder   r   r   
set_cookier  INTERNAL_USER_VIEW_ONLYrs   HTTP_401_UNAUTHORIZED)r(  r  rr  r  r}  r  ui_password	_user_rowr  r  r  key_user_idrB  ro  litellm_dashboard_uir  	jwt_tokenredirect_responser  	_passwordhash_passwords                        r   loginr  W  s|    
?  P ++66	
 	
 D488J'(H488J'(H))M73K))M40K)3)?c*oT V ++66	
 	
 I 	   '**<<GG(H!56 H 
 
	 	56 -
 h49O9O+: %00	* /II&-9

+,700))$46NOK
 "##
 	
 	
 99^$03 " "2!=!= &&-&C&C ! *2
 H !y$//$::	  w!yy)92>((- E)  F* JJ""& 3 ,$4$8$8-% =e   
	  	
W 44,1ESVW$$	$B  			
 )Y	:{$4$L$L
	 Yi@
Iz9=	 #2!!(I6':P:P9;
 yy(4!7 "!&" &/$**1*G*G"$#%"$!"#*#6
"  %}(33(>>	  7#C#%99-=r#B #,,S1$-$$.$

&",!*$7$0(8(<(<1?) Ai ! # I  !J$88  0(c! ((WI(F$$ \]e\fg$//+11	  m ++'--	
 	
Q  ?=>?  4
<	
Hs   T.S8 AT.TC%T.2T"3CT.5T%6AT.T(F
T.T+D'T.8 TT.TT."T.%T.(T.+T.z/onboarding/get_tokeninvite_linkc                   K   t         +t        dt        j                  dt        j
                        t        't        ddt        j                  j                  i      t        j                  j                  j                  d| i	       d{   }|t        d
ddi      t        j                  j!                         j#                         }|j$                  j#                         }||k  rt        d
ddi      t        j                  j!                         }t        j                  j                  j'                  d| i||d|j(                  d       d{   }t        j                  j*                  j                  d|j(                  i	       d{   }|t        d
ddi      |j,                  }t/        d&ddi|j0                  dt        j2                  g i i d|j(                  dd	 d{   }|d   }	t5        j6                  dd      }
|
j9                  d      r|
dz  }
n|
dz  }
ddl}t=               }|j?                  |j(                  |	|j,                  |j0                  dt@        tB        jE                  dd       |d!t         d"#      }|
d$jG                  ||      z  }
|
||d%S 7 %7 [7  7 ĭw)'z
    - Get the invite link
    - Validate it's still 'valid'
    - Invalidate the link (prevents abuse)
    - Get user from db
    - Pass in user_email if set
    Nr  r   r,  r  r+  rt  r  r#    %Invitation link does not exist in db.Invitation link has expired.T)accepted_atr  is_acceptedr  r  r  User does not exist in db.r  ro  r  r   r  r  r  r>  r   r  zui/onboardingz/ui/onboardingr  r  r  r  r  r  z?token={}&user_email={})	login_urlr  r  r   )$r   r  r  r  r   r4  r   r   r  r$  r  r&  litellm_invitationlinkr  r   r+  get_utc_datetimer!  
expires_atrh  r  r  r  ra   r  r  r   r  r  r  rc   r  r   r   r   r*   )r  
invite_objutc_now_dateexpires_at_datecurrent_timer'  user_objr  rB  ro  r  r  r  r  s                 r   
onboardingr  .  s
      P ++66	
 	
 .EEKKL
 	

 %''>>JJ[! K  J W.U$V
 	

 ==11388:L ++002O%W.L$M
 	

 ==113L55<<[!'&$,,	
 =  	A #%%77CC*,,- D  H W.J$K
 	
 $$J+  "++%;;''*

 H 7
C99%5r:$$S)/ 00 	56 - 

''"--!++/( 0 4 4)?! 9a	
 	  I" 5<<Y
SS)  m&	sK   BK J5CK J8=K J;AK 9J>:B<K 8K ;K >K z/onboarding/claim_tokenrG  c                 h  K   t         't        ddt        j                  j                  i      t         j
                  j                  j                  d| j                  i       d{   }|t        dddi      t        j                  j                         j                         }|j                  j                         }||k  rt        ddd	i      |j                  d
u rnt        dddi      |j                  | j                  k7  r3t        dddj!                  | j                  |j                        i      t#        | j$                        }t         j
                  j&                  j)                  d|j                  id|i       d{   }|t        dddi      |S 7 D7 w)a&  
    Special route. Allows UI link share user to update their password.

    - Get the invite link
    - Validate it's still 'valid'
    - Check if user within initial session (prevents abuse)
    - Get user from db
    - Update user password

    This route can only update user password.
    Nr  r+  rt  r  r#  r  r  r  TzThe invitation link was never validated. Please file an issue, if this is not intended - https://github.com/BerriAI/litellm/issues.zwInvalid invitation link. The user id submitted does not match the user id this link is attached to. Got={}, Expected={}r  r  r}  r  r  )r   r   r  r$  r  r&  r  r  invitation_linkr   r+  r  r!  r  r  r  r*   rs   r}  r  rh  )rG  r  r  r  r  r  s         r   claim_onboarding_linkr    s     .EEKKL
 	

 %''>>JJT))* K  J W.U$V
 	

 ==11388:L ++002O%W.L$M
 	
 %  _
 	
 T\\)  S  Z  ZLL*"4"4
 	
 T]]3M"%%77>>*,,-Z4O ?  H W.J$K
 	
 OiVs%   A'F2)F-*D+F2F0F20F2z
/get_imagec                     t         j                  j                  t         j                  j                  t                    } t         j                  j                  | d      }t        j                  d|      }t        j                  d|       |j                  d      rt               }|j                  |      }|j                  dk(  r]t         j                  j                  | d      }t        |d      5 }|j                  |j                         ddd       t!        |d	
      S t!        |d	
      S t!        |d	
      S # 1 sw Y   0xY w)zGet logo to show on admin UIzlogo.jpgUI_LOGO_PATHzReading logo from path: %s)zhttp://r  r  zcached_logo.jpgwbNz
image/jpegr  )r   r   r8  r9  __file__r@  r  r1   r   rC  r:   r   r0  r	  r   r1  r   )current_dirdefault_logo	logo_pathr  rB  
cache_pathfs          r   	get_imager    s    
 ''//"''//(";<K77<<Z8L		.,7I;YG 34::i(3&k3DEJj$'1(() (  
|DD  FF I,?? ('s   4D??Ez/invitation/newzInvite Linksc           	        K   t         't        ddt        j                  j                  i      |j
                  t        j                  k7  rAt        dddj                  t        j                  j                  |j
                        i      t        j                  j                         }|t        d      z   }	 t         j                  j                  j!                  | j"                  |||j"                  xs t$        ||j"                  xs t$        d	       d{   }|S 7 # t&        $ r9}d
t)        |      v rt        dddi      t        ddt)        |      i      d}~ww xY ww)%  
    Allow admin to create invite links, to onboard new users to Admin UI.

    ```
    curl -X POST 'http://localhost:4000/invitation/new'         -H 'Content-Type: application/json'         -d '{
            "user_id": "1234" // 👈 id of user in 'LiteLLM_UserTable'
        }'
    ```
    Nr  r+  rt  rh  r  r  )r  r  r  r  r  r  r  z*Foreign key constraint failed on the fieldzYUser id does not exist in 'LiteLLM_UserTable'. Fix this by creating user via `/user/new`.r  )r   r   r  r$  r  r  r  r  r*   r{  r   r+  r  r   r&  r  r  r  r   r   r   )rG  ra  r  r  rB  rp  s         r   new_invitationr    si    . .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 ==113L	q 11JG&))@@GG<<*(/77S;S*/77S;S H 	
 	
 	
  G73q6Ax  Wc!f4EFFGs=   B9E,<A"D' D%D' $E,%D' '	E)04E$$E))E,z/invitation/infoinvitation_idc                   K   t         't        ddt        j                  j                  i      |j
                  t        j                  k7  rAt        dddj                  t        j                  j                  |j
                        i      t         j                  j                  j                  d| i       d{   }|t        dddi      |S 7 w)	r  Nr  r+  rt  rh  r  r#  -Invitation id does not exist in the database.)r   r   r  r$  r  r  r  r  r*   r{  r&  r  r  )r  ra  rB  s      r   invitation_infor  T  s     . .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 #%%<<HH]# I  H LM
 	
 Os   B;C=C>Cz/invitation/updatec           	        K   t         't        ddt        j                  j                  i      |j
                  (t        dddj                  |j
                        i      t        j                  j                         }t         j                  j                  j                  d| j                  i| j                  | j                  |||j
                  d	       d{   }|t        ddd
i      |S 7 w)uZ  
    Update when invitation is accepted
    
    ```
    curl -X POST 'http://localhost:4000/invitation/update'         -H 'Content-Type: application/json'         -d '{
            "invitation_id": "1234" // 👈 id of invitation in 'LiteLLM_InvitationTable'
            "is_accepted": True // when invitation is accepted
        }'
    ```
    Nr  r+  rt  r  z'Unable to identify user id. Received={}r  )r  r  r  r  r  r  r  )r   r   r  r$  r  r  r*   r   r+  r  r&  r  rh  r  r  )rG  ra  r  rB  s       r   invitation_updater    s    2 .EEKKL
 	

   (BII%--
 	
 ==113L"%%<<CCT''($$++'&+33
 D 	 	H LM
 	
 O!	s   CC: C8!C:z/invitation/deletec                   K   t         't        ddt        j                  j                  i      |j
                  t        j                  k7  rAt        dddj                  t        j                  j                  |j
                        i      t         j                  j                  j                  d| j                  i       d{   }|t        dddi      |S 7 w)	u  
    Delete invitation link
    
    ```
    curl -X POST 'http://localhost:4000/invitation/delete'         -H 'Content-Type: application/json'         -d '{
            "invitation_id": "1234" // 👈 id of invitation in 'LiteLLM_InvitationTable'
        }'
    ```
    Nr  r+  rt  rh  r  r#  r  )r   r   r  r$  r  r  r  r  r*   r{  r&  r  r^  r  r  s      r   invitation_deleter    s     0 .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 #%%<<CCT''( D  H LM
 	
 Os   CC!CC!z/config/updatezconfig.yamlconfig_infoc                   K   	 ddl }	 t        t        d      t        durt	        dddi      | j                  d	      }t        j                  |      }|j                         D ]H  \  }}|d
k(  st        j                  j                  j                  d|i||dd|id       d{    J t        j                          d{   }t        j                  d|       | j                  |j!                  di        | j                  j#                  d	      }|d   }|j                         D ]G  \  }}|dk(  r8d|vrddgi}n.t%        |d   t&              rd|d   vr|d   j)                  d       |||<   I ||d<   | j*                  l|j!                  di        | j*                  }|j                         D ]  \  }}t-        |      }	|	||<    |d   }
|j                         D ]  \  }}||   |
|<    | j.                  |j!                  di        | j.                  }i ||d   |d<   d|v rZd|d   v rSt%        |d   d   t&              r=t%        |d   t&              r*|d   d   |d   z   }t'        t1        |            }||d   d<   t        j3                  |       d{    t        j5                  t        t6               d{    ddiS 7 ,7 7 57 # t        $ r}t        j8                  dj;                  t=        |                   t        j                  t?        j@                                t%        |t              rYtC        tE        |ddt=        |       d       tF        jH                  tE        |d!d"      tE        |d#tJ        jL                        $      t%        |tB              r|tC        d%t=        |      z   tF        jH                  tE        |d!d"      tJ        jL                  $      d}~ww xY ww)&z
    For Admin UI - allows admin to update config via UI

    Currently supports modifying General Settings + LiteLLM settings
    r   NzNo DB ConnectedTr  r+  r  rt  r   r0  r"  r"  r)  r)  r  rh  r  zLoaded config: %sr   r  r  r  rk  r  r1  r   )rE  r  r   zConfig updated successfullyzBlitellm.proxy.proxy_server.update_config(): Exception occured - {}ru  r  r  r.  r  r0  r,  r  )'r  r   r   rW  r   r   jsonify_objectrg  r&  r'  upsertr   rp  r1   r   r   
setdefaultr   r   r  r  rk  rM   r1  r%  rM  rI  r   r+  r*   r   r  r  r  r  r  r  r   r8  )r  r  updated_settingsrk  rl  r   updated_general_settings_existing_settings_updated_environment_variablesr  _existing_env_variablesupdated_litellm_settingscombined_success_callbackrp  s                 r   update_configr    s    @
	  -..D(b  '+++>(778HI$**,DAq%%#&&55<<'+121"E#0!"4 =    - $..00""#6? ''30"5'2'C'C'H'H! (I ($ "((:!;06681..!);;.87)-D*#$6z$BDI"*<Z*HH.z:AA'J()"1% 9 *<F%&,,85r:-8-N-N* 7<<>1"6Q"?4C.q1 ? '--D&E#6<<>1-KA-N'* ?
 ''30"5'2'C'C$***+,*F%& #&>>&&1C*DD -./ABD !9:L!MtT123EF23EFG . 15S9R5S0T- 2 -.*
 &&&&999))';L * 
 	
 	
 899i 1F 	:	

  
""PWWA	

 	""9#7#7#9:a' 8/DSVHA-NO$//a&1Qv/J/JK	  >*G,s1v5 ++!Wf-,,	
 	
!
sy   PA/K" 48K" ,K-K" 
KGK" (K)&K" K K" PK" K" K"  K" "
P,DO==PPz/config/field/updatec           
        K   	 t         't        ddt        j                  j                  i      |j
                  t        j                  k7  r't        ddt        j                  j                  i      | j                  t        j                  vr(t        dddj                  | j                        i      	 t        di | j                  | j                  i t         j                   j"                  j%                  ddi	       d{   }||j&                  i }nt)        |j&                        }| j                  || j                  <   t         j                   j"                  j+                  ddidt-        j.                  |      d
dt-        j.                  |      id       d{   }|S # t        $ r2 t        dddj                  t        | j                              i      w xY w7 7 Fw)z=
    Update a specific field in litellm general settings
    Nr  r+  rt  Invalid field={} passed in.z)Invalid type of field value={} passed in.r"  r   r#  r  r)  r  r  r   )r   r   r  r$  r  r  r  r  r{  rt  ConfigGeneralSettingsr}  r*   rw  r   r-  r&  r'  r(  r)  r   r  r   r:  rG  ra  r5  r   rB  s        r   update_config_general_settingsr    s     .EEKKL
 	

 ""&6&B&BB.AAGGH
 	

 3@@@:AA$//RS
 	



D$2B2B CD !. 0 0 ? ? J J/0 !K ! 
 "&9&E&E&M 3 ? ?@ )-(8(8T__%"%%44;;/0%7

ScHde$djj1A&BC
 <  H OC  
DKK))*
 	

s=   B7G5:!F3 /G5
G1B!G5,G3-G53;G..G53G5z/config/field/infort  c                   K   	 t         't        ddt        j                  j                  i      |j
                  t        j                  k7  r't        ddt        j                  j                  i      | t        j                  vrt        dddj                  |       i      t         j                  j                  j                  ddi       d {   }||j                  t        dddj                  |       i      t!        |j                        }| |v rt#        | ||    	      S t        dddj                  |       i      7 ww)
Nr  r+  rt  r
  r"  r   r#  zField name={} not in DB)rt  rw  )r   r   r  r$  r  r  r  r  r{  r  r}  r*   r&  r'  r(  r)  r   ConfigFieldInfo)rt  ra  r5  r   s       r   get_config_general_settingsr    st    
 .EEKKL
 	

 ""&6&B&BB.AAGGH
 	

 .;;;:AA*MN
 	
 !. 0 0 ? ? J J/0 !K ! 
 "&9&E&E&M6==jIJ
 	

   3 ? ?@))"%3CJ3O   !:!A!A*!MN %s   CEEA8Ez/config/listconfig_typec                   K   	 t         't        ddt        j                  j                  i      |j
                  t        j                  k7  rAt        dddj                  t        j                  j                  |j
                        i      t         j                  j                  j                  ddi       d{   }|"|j                  t        |j                        }ni }d	d
id	d
id	d
id	d
id	did}g }t        j                   j#                         D ]  \  }}||v s||   d	   }|dk(  r/|dk(  rt$        g}	ng }	|	D ]  }
|
j&                  j#                         D cg c]3  \  }}t)        ||j*                  dt,        j/                  |d      d      5 }}}d}|
j                   j#                         D ]6  \  }}t1        |d      r |j2                  |j2                  ||   _        |dz  }8 d}||v rd}n
|t,        v rd}t7        |||   d	   |j2                  xs dt,        j/                  |d      ||j8                  |      }|j;                  |        Id}d}||v rd}n
|t,        v rd}t7        |||   d	   |j2                  xs dt,        j/                  |d      ||j8                  |      }|j;                  |        |S 7 c c}}w w)z
    List the available fields + current values for a given type of setting (currently just 'general_settings'user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),)
    Nr  r+  rt  rh  r"  r   r#  r-  rj  PydanticModel)r7  r8  max_request_size_mbmax_response_size_mbr  r  r   )rt  ru  rv  ry  rx  r   r  rA  TF)rt  ru  rv  rw  rx  ry  nested_fields)r   r   r  r$  r  r  r  r  r*   r{  r&  r'  r(  r)  r   r  r}  rg  PassThroughGenericEndpoint__annotations__FieldDetailr   r   r   hasattrr  rv  r|  r   r  )r  ra  r5  r~  r  r  rt  r  typed_dict_typepydantic_class_listpydantic_class	sub_fieldsub_field_typer  r  sub_field_infor  r  s                     r   get_config_listr!      sb     
 .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 !. 0 0 ? ? J J/0 !K !  &+>+J+J+V#'(;(G(G#H #%  #))!4)/(; &	2!' 3#)?";L J"7"D"D"J"J"L
J% +:6v>O/1!99+E*F'*,'&9N :H9W9W9]9]9_	% :`5I~ $'0'5'>'>.00@0D0DYPT0U)- :` " 	% C (44::<!& $NMB . : : F !/ : : *#.@ q = %)M!%==(,#'77(-$.#-#/
#;F#C*4*@*@*FB$4$8$8T$J%2,6,>,>&3%M %%m4U ':Z !% $!99$(M#33$)M *)+J7?&0&<&<&B 0 4 4Z F!.(2(:(:"/! !!-0W #MZ AD	%s-   B<K>K?A'K'A K'8K
D5KKz/config/field/deletec                   K   	 t         't        ddt        j                  j                  i      |j
                  t        j                  k7  rAt        dddj                  t        j                  j                  |j
                        i      | j                  t        j                  vr(t        dddj                  | j                        i      t         j                  j                  j                  ddi	       d{   }||j                   (t        ddd
j                  | j                        i      t#        |j                         }|j%                  | j                  d       t         j                  j                  j'                  ddidt)        j*                  |      ddt)        j*                  |      id       d{   }|S 7 7 w)z|
    Delete the db value of this field in litellm general settings. Resets it to it's initial default value on litellm.
    Nr  r+  rt  rh  r
  r"  r   r#  zField name={} not in configr  r)  r  r  )r   r   r  r$  r  r  r  r  r*   r{  rt  r  r}  r&  r'  r(  r)  r   rJ  r  r   r:  r  s        r   delete_config_general_settingsr#     s    
 .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 3@@@:AA$//RS
 	
 !. 0 0 ? ? J J/0 !K ! 
 "&9&E&E&M:AA$//RS
 	

   3 ? ?@ $/"%%44;;/0%7

ScHde$djj1A&BC
 <  H O3"s%   D GGC	GGGGz/get/config/callbacks)r  r  r  c                  0  K   	 ddl } t               }t        j                          d{   }|j	                  di       }|j	                  di       }|j	                  di       }|j	                  dg       }g }	 |D ]  }|dk7  r|dk(  rd	g}	n1|d
k(  rdg}	n(|dk(  rdg}	n|dk(  rdg}	n|dk(  rg d}	n|dk(  rg d}	ng }	i }
|	D ]-  }|j	                  |d      }|d|
|<   t        |      }||
|<   / |j                  ||
d       |dk(  sg d}i }|D ]-  }|j	                  |d      }|d||<   t        |      }|||<   / |j                  ||d        |j	                  dg       }g }d|v rdg}i }|D ]C  }|j	                  |d      }|t        j                  dd      }|||<   3t        |      }|||<   E t        j                  j                  }t        j                  j                         }t        j                  j                  }|j                  d|||d       g d}i }|D ]-  }|j	                  |d      }|d||<   t        |      }|||<   / |j                  d|d       t        i }nt        j                         }d||||dS 7 o# t         $ r}t#        j$                  dj'                  t)        |                   t+        |t,              rYt/        t1        |d d!t)        |       d"      t2        j4                  t1        |d#d$      t1        |d%t6        j8                        &      t+        |t.              r|t/        d't)        |      z   t2        j4                  t1        |d#d$      t6        j8                  &      d}~ww xY ww)(z~
    For Admin UI - allows admin to view config via UI
    # return the callbacks and the env variables for the callback

    r   Nr1  r   rk  r   r   	openmeterOPENMETER_API_KEY
braintrustBRAINTRUST_API_KEY	traceloopTRACELOOP_API_KEYcustom_callback_apiGENERIC_LOGGER_ENDPOINTotel)OTEL_EXPORTEROTEL_ENDPOINTOTEL_HEADERS	langsmith)LANGSMITH_API_KEYLANGSMITH_PROJECTLANGSMITH_DEFAULT_RUN_NAMEr  )r<  	variables)LANGFUSE_PUBLIC_KEYLANGFUSE_SECRET_KEYLANGFUSE_HOSTr  r  SLACK_WEBHOOK_URL)r<  r5  active_alertsalerts_to_webhook)	SMTP_HOST	SMTP_PORTSMTP_USERNAMESMTP_PASSWORDSMTP_SENDER_EMAILTEST_EMAIL_ADDRESSEMAIL_LOGO_URLEMAIL_SUPPORT_CONTACTemailr  )r   r  alertsr0  available_callbacksz?litellm.proxy.proxy_server.get_config(): Exception occured - {}ru  r  r  r.  r  r0  r,  r  )r  AllCallbacksr   rp  r   rL   r  r   r  r   r+  r  _all_possible_alert_typesr  r   get_settingsr   r1   r  r*   r   r   r   r  r  r  r  r   r8  )r  all_available_callbacksr.  _litellm_settingsr3  rk  _success_callbacks_data_to_return	_callbackenv_varsenv_vars_dict_varenv_variabler-  _langfuse_vars_langfuse_env_varsr  alerting_data_slack_vars_slack_env_varsr  _decrypted_value_alerting_types_all_alert_types_alerts_to_webhook_email_vars_email_env_varsr2  rp  s                                r   rp  rp     s8    o
"..(3355'OO,>C'OO,>C +0G L /223ErJ	 ,IJ&++ H ,., H +- 34H"77 9:H&(QH+- H  "H "$D#8#<#<T4#HL#+.2d+ +?\*R.=d+ %  &&	'VWj("
 &("*D#8#<#<T4#HL#+37*40 +?\*R3B*40 +  &&&5GHi ,r &))*b9	i#K !O#488tD'YY':DAF,2OD) (<,'O$,<OD) $ 0GGSSO!99SSU  "99NN    #!0%4);		
 D044T4@L#(,% $8l#K (8%   	,	
 !)668  (#/#:
 	
_ 6l  
&&MTTA	

 a' 8/DSVHA-NO$//a&1Qv/J/JK	  >*G,s1v5 ++!Wf-,,	
 	

sA   N%J JC!J FJ NJ 	N$C*NNNz/config/yamlc                    K   ddiS w)a  
    This is a mock endpoint, to show what you can set in config.yaml details in the Swagger UI.

    Parameters:

    The config.yaml object has the following attributes:
    - **model_list**: *Optional[List[ModelParams]]* - A list of supported models on the server, along with model-specific configurations. ModelParams includes "model_name" (name of the model), "litellm_params" (litellm-specific parameters for the model), and "model_info" (additional info about the model such as id, mode, cost per token, etc).

    - **litellm_settings**: *Optional[dict]*: Settings for the litellm module. You can specify multiple properties like "drop_params", "set_verbose", "api_base", "cache".

    - **general_settings**: *Optional[ConfigGeneralSettings]*: General settings for the server like "completion_model" (default model for chat completion calls), "use_azure_key_vault" (option to load keys from azure key vault), "master_key" (key required for all calls to proxy), and others.

    Please, refer to each class's description for a better understanding of the specific attributes within them.

    Note: This is a mock endpoint primarily meant for demonstration purposes, and does not actually provide or change any configurations.
    helloworldr   )r  s    r   config_yaml_endpointra  !  s     . Ws   z/get/litellm_model_cost_map)r  r  c                     K   	 t         j                  } | S # t        $ r}t        ddt	        |       d      d }~ww xY ww)Nr  zInternal Server Error (r  rt  )r   
model_costr   r   r   )_model_cost_maprp  s     r   get_litellm_model_cost_mapre  !  sK     
!,, 
,SVHA6
 	

s   A A	>9>Ar  )r  c                    K   yw)NzLiteLLM: RUNNINGr   r  s    r   homerg  !  s	     s   z/routesc            	        K   g } t         j                  D ]d  }t        |dd      }|t        |dd      t        |dd      t        |dd      t        |dd      r|j                  ndd}| j	                  |       f d| iS w)zD
    Get a list of available routes in the FastAPI application.
    r  Nr   methodsr<  )r   ri  r<  r  r  )r   r  r  r   r  )r  r  endpoint_route
route_infos       r   
get_routesrl  !  s     
 F 
D9%vt4"5)T:vt4 uj$7 #++	J MM*%  fs   %B AB )NNrJ  r   )NNNNNNNN)NNNNFFNNrs  NFTTNFFN(  r  r  r  rt  r   r&   r  r  sysrv  r  r)  warningsr   r   typingr   r   r   r   r	   r
   r   r   r   opentelemetry.tracer   _Spanr   filterwarningsUserWarningr    r  r  r   insertr9  rb  backofffastapirX  r,  apscheduler.schedulers.asyncior!   r  rp  r(   r,   collectionsr-   
contextlibr.   r   r/   rc  r1   r2   r  r3   r4   litellm.exceptionsr5   1litellm.integrations.SlackAlerting.slack_alertingr6   'litellm.litellm_core_utils.core_helpersr7   r8   &litellm.llms.custom_httpx.http_handlerr9   r:   litellm.proxy._types5litellm.proxy.analytics_endpoints.analytics_endpointsr=   analytics_routerlitellm.proxy.auth.auth_checksr>   litellm.proxy.auth.auth_utilsr?   litellm.proxy.auth.handle_jwtr@   "litellm.proxy.auth.litellm_licenserA   litellm.proxy.auth.model_checksrB   rC   rD   $litellm.proxy.auth.user_api_key_authrE   rF   )litellm.proxy.batches_endpoints.endpointsbatches_routerlitellm.proxy.caching_routescaching_router)litellm.proxy.common_utils.admin_ui_utilsrG   )litellm.proxy.common_utils.callback_utilsrH   rI   rJ   &litellm.proxy.common_utils.debug_utilsrK   debugging_endpoints_router0litellm.proxy.common_utils.encrypt_decrypt_utilsrL   rM   -litellm.proxy.common_utils.http_parsing_utilsrN   rO   ,litellm.proxy.common_utils.load_config_utilsrP   rQ   0litellm.proxy.common_utils.openai_endpoint_utilsrR   &litellm.proxy.common_utils.proxy_staterS   (litellm.proxy.common_utils.swagger_utilsrT   -litellm.proxy.fine_tuning_endpoints.endpointsfine_tuning_routerrU   ,litellm.proxy.guardrails.guardrail_endpointsguardrails_router(litellm.proxy.guardrails.init_guardrailsrV   rW   litellm.proxy.health_checkrX   0litellm.proxy.health_endpoints._health_endpointshealth_router,litellm.proxy.hooks.model_max_budget_limiterrY   .litellm.proxy.hooks.prompt_injection_detectionrZ   )litellm.proxy.hooks.proxy_failure_handlerr[   -litellm.proxy.hooks.proxy_track_cost_callbackr\   $litellm.proxy.litellm_pre_call_utilsr]   >litellm.proxy.management_endpoints.budget_management_endpointsbudget_management_router5litellm.proxy.management_endpoints.customer_endpointscustomer_router:litellm.proxy.management_endpoints.internal_user_endpointsinternal_user_routerr^   ;litellm.proxy.management_endpoints.key_management_endpointsr_   r`   ra   key_management_router9litellm.proxy.management_endpoints.organization_endpointsorganization_router:litellm.proxy.management_endpoints.team_callback_endpointsteam_callback_router1litellm.proxy.management_endpoints.team_endpointsteam_routerrb   )litellm.proxy.management_endpoints.ui_ssorc   ui_sso_router+litellm.proxy.management_helpers.audit_logsrd   4litellm.proxy.openai_files_endpoints.files_endpointsopenai_files_routerre   >litellm.proxy.pass_through_endpoints.llm_passthrough_endpointsllm_passthrough_router;litellm.proxy.pass_through_endpoints.pass_through_endpointsrf   pass_through_router(litellm.proxy.rerank_endpoints.endpointsrerank_routerlitellm.proxy.route_llm_requestrg   7litellm.proxy.spend_tracking.spend_management_endpointsspend_management_router1litellm.proxy.spend_tracking.spend_tracking_utilsrh   7litellm.proxy.ui_crud_endpoints.proxy_setting_endpointsui_crud_endpoints_routerlitellm.proxy.utilsri   rj   rk   rl   rm   rn   ro   rp   rq   rr   rs   rt   ru   4litellm.proxy.vertex_ai_endpoints.langfuse_endpointslangfuse_router2litellm.proxy.vertex_ai_endpoints.vertex_endpointsvertex_routerrv   litellm.routerrw   rx   ry   rz   litellm.schedulerr{   r|   r}   *litellm.secret_managers.aws_secret_managerr~   "litellm.secret_managers.google_kmsr   litellm.secret_managers.mainr   r   r   r   )litellm.types.integrations.slack_alertingr   litellm.types.llms.anthropicr   r   r   r   litellm.types.llms.openair   litellm.types.routerr   r  r   r   litellm.types.utilsr   ModelMapInfor   r   r   litellm._versionr   r   suppress_debug_infor   r   r   r   r   r   r   r   r   r   r   r   r   r   fastapi.encodersr   fastapi.middleware.corsr   fastapi.openapi.utilsr   r  r   r   r   r   r   fastapi.routingr   fastapi.securityr   fastapi.security.api_keyr   fastapi.staticfilesr   litellm.proxy.enterpriseproxy
enterpriser  rE  r   r   r   rL  r   r   proxy_stater   r3  r   r   r  ui_link
ui_messagecustom_swagger_message_title_descriptionr   r   r  r   r  openapienumEnumr$  exception_handlerr  r5  originsr8  r  r  r@  ui_pathr  listdirr   r  r%  folder_namefolder_pathmakedirssrcdstrenamer)   r   
middlewarerH  add_middlewarerM  rg  r`  ra  r&  ri  rh  rk  user_configrf  r   local_loggingrj  r   r   r   r   rN  log_filer  r   r   r   r'  r  r   model_max_budget_limiterr  r  r   r   r   r   r   rp  r   r  r  rQ  r   r   rU  Literalr   r   r   r  r  r   r  rW  rX  r   async_resultcelery_app_conn	celery_fnr   UserAPIKeyAuthrq  r|  r  r  r  r  r  SpendLogsPayloadr  r  r  r  r  r  r   rO  r   rw  r  r  r  r  r  r  r  r   r   r  poston_exceptionexporD  r  rO  rb  rl  ry  rz  r{  r|  r}  r  r  r  r^  r  r  r  r  r  r  r  r  r  TokenCountRequestr  r  r  r  r  r  r  r  r7  rA  rD  rK  rU  rZ  r\  ModelInfoDeleter_  rf  r  r  r  r  r  InvitationClaimr  r  InvitationModelInvitationNewr  r  InvitationUpdater  InvitationDeleter  
ConfigYAMLr  ConfigFieldUpdater  r  r  r|  r!  ConfigFieldDeleter#  rp  ra  re  rg  rl  include_routerr   r   r   <module>r
     sL$      	 	    
     (
 
 
 1DD# #    	K 8
 $  rwww	U?	 8 $ *   H 9 3 K Q " : E 4 ; 
 O B ? 
 H W > D V P T < T M T L S 
 T I N S R M 9 R    W X  E D C >  H  A = H : 9 6 ;( #       . - 2 2 - -    & % 1 1 1 1 + +11 299/4 #..0d 09B1: '#  l*2/0'0*-.U*V'?Hryy7@ -x}  187;-u;0551- d
#.wi7TU  T T
 N  4@<	/] 	 BII
VWmVnnrs}r~
 AAW@XX\]g\h	i  &> ~!7 ~! ~!B _
 & 299_g&&0\ CKTYY  ~&G .  '& 
% 	''//"''//(";<Kggll;?GIIe[7>TIJBJJw'W%(l*B''**84Q7K'',,w<KBKKd3 '',,w1C'',,{L9CBIIc3 ( 2FGWFXX[\	
 299%&.+;BJJ'(			,' 	, 
 	,   %%   

  '+ x} +#
HV #!% % $  4  
HSM  (,x% ,0DDJJ  B!      1 2 8J'    # 	   t 2 , */' /$' !$' !   lOS )K L S 4 '+ x} + )  	/3 (+, 3 ""#"!"&158<#%2%2 c]2 sm	2
 }2 sm2 c]2 3-2 E%*-.2 (0~2 4.2 
2jw :4 "
4 
>SH %)4))* SM& s
lwC=wc]w #w c]	w
 E?w tnwt
7>l l^' }3 

	#sn33,!/3,?C3,l6,!/6,?C6,r1,!/1,?C1,h!/?C *, $ Y
&x, x,x (9 :;CUBV   W%678@R?S   )00A(B.%..b +,-
	  
 +,-
	  
 ,+,-
	  
 7+,-
	]$9:NoN	   LL16  (/0A(B	~
~
~
 C=~
 &	~
<~
B W->%?$@   '*;"<!=]O   '+,-
  
 2+,-
    (/0A(B	


 C=
 &	
 
D +,-!
	   +,-!
	   &+,-!
	   1+,-!
	    (/0A(B	uuu C=u &	u0up +,-!
	   +,-!
	   )00A(Bggg &ggT +,-
  
 +,-
   )00A(Baaa &aaH +,-
  
 +,-
   Cy(/0A(B	HHH H &	HH` < ;  ~{ 9:III  IP +,-
  
 +,-
   )00A(BSSS &SSl +,-
  
 +,-
   )00A(BTTT &TTn (+,-
  
 %+,-
   )00A(B	SSS S &	SSl +,-
  
 +,-
   )00A(BSSS &SSl +,-
  
 +,-
   )00A(B	RRR R &	RRj &+,-
  
 #+,-
   )00A(B	UUU U &	UUp &+,-
  
 #+,-
   )00A(B	QQQ Q &	QQh "+,-
  
 +,-
   )00A(B	___ _ &	__D +,-!
	   +,-!
	   )00A(Bjjj &jj` 
+	,+,-$   )00A(B	L
,L
L
 L
 &	L
L
r 
+,-%	  :!2 ::z $
+,-  

 


8%  :&&%&  &Rc]/=	& O
	+,-	   )00A(Bb
b
%b
b
L ,
	+,-	   )00A(Bj
"j
%j
j
Z  w
	+,-   )00A(B(7=== "\ %\ C=\ 
 D>\ \ ~ C
	+,-   )00A(B+/$("&	z
%z
#C=z
 !z
 h	z
z
z P
	+,-   )00A(B+6$("&!"i
%i
#C=i
 !i
 h	i

 c]i
 smi
i
X #A
	+,-   )00A(B+6$("&!"O%O#C=O !O h	O
 c]O smOOd I
	+,-   )00A(B+/$("&!"NH%NH#C=NH !NH h	NH
 c]NH smNHNHb 
	+,-  
 
	+,-   )00A(B&*e %e sme e P(,S	@H	. 
	+,-   )00A(B!%B"%B"#B"
B"L M
	+,-	  A
? A
A
H _
	+,-  4 X
+,-   )00A(Bd%ddP 
	+,-	    (/0A(B	k
k
k
 C=k
 &	k
k
\ 	.!1UK@' @ L@4   Q
 Q
Q
h 	 E:q# q ;qh 
#u=Ho H >HV /@ 0@D 
	+,-"   >EEV=W8G
8G,:8G8Gv 
	+,-"   =DDU<V**+9**Z 
	+,-"   )00A(B3
3%33l 
	+,-"   )00A(B+
+%++^ 
+,-	  G
Z G
G
V 
+,-	   )00A(BE
E%EEP 
+,-"   )00A(B44%44n 
+,-	   )00A(BA+,A%A 
*AAH 
+,-	   )00A(B?
?%??D 
+,-	  v
v
r 
+,-	  J ( !+,-  



 Cw'89:;  < IW->%?$@A B`   6    > "   = !   % &   = !   ) *   ? #   & '   = !   ( )   ' (   ;    = !   & '   ? #   * +   > "   # $   $ %   - .   + ,   & '   ' (   + ,YO  U
+A3.RS
TTUD  GN   	B
  		s   >Aj
 Aj# #Aj1 %B"Ak Ak DAk j
Aj jAjjAj j#Aj.j-Aj.j1Akj7Aj>j=Akj>AkkAkkAkkAkkAkkAkkAk