
    g                    T   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZmZm	Z	m
Z
mZmZmZ d dlmZ d dlZd dlZd dlZd dlZd dlmZmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lm Z m!Z! d dl"m#Z#m$Z$m%Z%m&Z& d dl' ddl( ddl)m*Z*m+Z+ ddl,m-Z-m.Z. er	d dl/m0Z1 e1Z0neZ0 G d de      Z2y)    N)	timedelta)TYPE_CHECKINGAnyDictListLiteralOptionalUnion)APIError)verbose_loggerverbose_proxy_logger)	DualCache)CustomBatchLogger)duration_in_seconds)_add_key_name_and_team_to_alert)get_async_httpx_clienthttpxSpecialProvider)	AlertTypeCallInfoVirtualKeyEventWebhookEvent)*      )send_to_webhooksquash_payloads)_add_langfuse_trace_id_to_alert process_slack_alerting_variables)Routerc                       e Zd ZdZddg edi dfdee   dee   dee   dee	   dee
e	eee   ef   f      dee   f fd	Z	 	 	 	 	 	 dXdee   dee   deee	      dee
e	eee   ef   f      d
ee
   dee   fdZd Zd Zd Zd ZdYdZd ZdedefdZdefdZ	 	 	 	 dZdeej6                     deej6                     ded   dee   fdZdedefdZded    d!e fd"Z!d!e defd#Z"d$ee   d%ee   d&ee   d'ee   d(ee   f
d)Z#d*ee   defd+Z$d,ed-   d.ed/   d0ed1ed2ee   d3e%defd4Z&d5e'd6eddfd7Z(d5e'd6eddfd8Z)d9ed:ed;e*fd<Z+d9efd=Z,d>e-defd?Z.	 	 d[d@edAee   dBee   fdCZ/d>e-defdDZ0d>e-d,edefdEZ1	 dYdFedGedH   d,e	dIed!ee-   f
dJZ2dK Z3dL Z4dM Z5defdNZ6dYdee*   fdOZ7	 d\dPefdQZ8dR Z9dS Z:dTe;d,e	dUefdVZ<dee   defdWZ= xZ>S )]SlackAlertingz(
    Class for sending Slack Alerts
    Ninternal_usage_cachealerting_thresholdalertingalert_typesalert_to_webhook_urldefault_webhook_urlc                 f   |d}|| _         || _        || _        |xs
 t               | _        t        t        j                        | _        t        |      | _
        d| _        t        di || _        || _        t        j                          | _        t%        	| L  di |d| j"                  i y )Ni,  )llm_providerr&   F
flush_lock )r#   r$   r%   r   r"   r   r   LoggingCallbackasync_http_handlerr   r&   
is_runningSlackAlertingArgsalerting_argsr'   asyncioLockr+   super__init__)
selfr"   r#   r$   r%   r&   r1   r'   kwargs	__class__s
            f/var/www/openai/venv/lib/python3.12/site-packages/litellm/integrations/SlackAlerting/slack_alerting.pyr5   zSlackAlerting.__init__1   s     %!$"4 &$8$GIK!"8-==#
 %E!5%
!  .??#6 !,,.>6>doo>    r1   
llm_routerc                 N   |*|| _         t        j                  | j                                ||| _        ||| _        |t        di || _        |I| j                  t        |      | _        n+t        |      xs i }| j                  j                  |       ||| _        y y )Nr*   r,   )r$   r2   create_taskperiodic_flushr#   r%   r0   r1   r&   r   updater;   )r6   r$   r#   r%   r&   r1   r;   _new_valuess           r9   update_valueszSlackAlerting.update_valuesR   s     $DM 3 3 56)&8D#"*D$!2!C]!CD+((0,L)=-)
 5-A  	  ))00=!(DO "r:   c                    K   y wNr,   r6   s    r9   deployment_in_cooldownz$SlackAlerting.deployment_in_cooldownu   	        c                    K   y wrC   r,   rD   s    r9    deployment_removed_from_cooldownz.SlackAlerting.deployment_removed_from_cooldownx   rF   rG   c                      t        t              S rC   )listr   rD   s    r9   _all_possible_alert_typesz'SlackAlerting._all_possible_alert_types{   s     Ir:   c                 >   	 ||z
  }|j                         }|j                  di       }|j                  dd      }t        j                  ||      }|j                  dd       }	|	|j                  dd       }	t	        |	      d d }
||||
fS # t
        $ r}|d }~ww xY w)Nlitellm_paramsmodel rO   optional_paramsmessagesinputd   )total_secondsgetlitellmget_api_basestr	Exception)r6   r7   
start_timeend_timetime_differencetime_difference_floatrN   rO   api_baserS   	_messageses               r9   )_response_taking_too_long_callback_helperz7SlackAlerting._response_taking_too_long_callback_helper   s    	&3O$3$A$A$C!#ZZ(8"=NJJw+E++%XHzz*d3H!::gt4 Hds+I(%9DD 	G	s   B	B 	BBBc           	      (   |y d|v r|	 d}|d   }t        |      dk(  ry d }	 t        |j                         d       }t        |      }|y |j                         D ]  \  }}|d| dt        |d       d	z  } d
|z   d
z   }|S y # t        $ r Y Gw xY w)N_latency_per_deploymentrP   r   c                     | d   S )Nr   r,   )xs    r9   <lambda>zBSlackAlerting._get_deployment_latencies_to_alert.<locals>.<lambda>   s    1r:   key
: r   sz```)lensorteditemsdictr[   round)r6   metadata_message_to_send_deployment_latencies_deployment_latency_mapr`   latencys          r9   "_get_deployment_latencies_to_alertz0SlackAlerting._get_deployment_latencies_to_alert   s    $0
  "$,-F$G!()Q.6:#(.)//1~)% +//D*E' '.%<%B%B%D!' b
"U715E4Fa$HH  &E$'77%?##9 1&  s   'B 	BBc                   K   | j                   | j                  y | j                  |||      \  }}}}t        j                  st        j
                  rd}d| d| d| d}	dt        |d       d	| j                   d
}
i }|| j                  kD  rv|Ed|v rAd|d   v r:|d   d   }t        |	|      }	| j                  |      }||	d| z  }	d|v r|d   }| j                  |
|	z   dt        j                  |       d {    y y 7 w)N)r7   r\   r]   >Message not logged. litellm.redact_messages_in_exceptions=True
Request Model: `z`
API Base: ``
Messages: ``z`Responses are slow - r   z&s response time > Alerting threshold: zs`rN   rs   request_infors   rs   z 
Available Deployment Latencies
alerting_metadataLowmessagelevel
alert_typer   )r$   r%   rc   rX   turn_off_message_loggingredact_messages_in_exceptionsrr   r#   r   rx   
send_alertr   llm_too_slow)r6   r7   completion_responser\   r]   r_   rO   r`   rS   r   slow_messager   	_metadatarv   s                 r9   !response_taking_too_long_callbackz/SlackAlerting.response_taking_too_long_callback   s     == D$4$4$< ::%! ;  	9uh ++w/T/TWH+E7.
.YaXbbcd/6KA0N/OOuvz  wN  wN  vO  OQ  R"$ 4#:#:: "$.&)9"::"()9"::"F	>!-	  +/*Q*Q& +R +' +6 <=T<UVL ')3(12E(F%//$|3$11"3	 "   - ;,s   C?D
DD
deployment_metricsreturnc                   K   d}	 |j                   r]| j                  j                  dj                  |j                  t
        j                  j                        dd       d{    |dz  }|j                  g| j                  j                  dj                  |j                  t
        j                  j                        |j                  d       d{    |dz  }|S 7 ~7 # t        $ r Y yw xY ww)ax  
        Store the perf by deployment in cache
        - Number of failed requests per deployment
        - Latency / output tokens per deployment

        'deployment_id:daily_metrics:failed_requests'
        'deployment_id:daily_metrics:latency_per_output_token'

        Returns
            int - count of metrics set (1 - if just latency, 2 - if failed + latency)
        r   {}:{}r   N)rj   valueparent_otel_span)failed_requestr"   async_increment_cacheformatidSlackAlertingCacheKeysfailed_requests_keyr   latency_per_output_tokenlatency_keyr[   )r6   r   
return_vals      r9   async_update_daily_reportsz(SlackAlerting.async_update_daily_reports   s	     
	!00//EE*--.BBHH %) F    a
 "::F//EE*--/E/Q/Q/W/W -EE%) F    a
/  		sN   C7AC( %C$&A2C( C&
C( #C7$C( &C( (	C41C73C44C7c                   K   |j                         }|D cg c],  }dj                  |t        j                  j                        . }}|D cg c],  }dj                  |t        j
                  j                        . }}||z   }| j                  j                  |       d{   }|yd}|D ]  }	|	|	dkD  sd} n |ry|dt        |       }
|t        |      d }d}|
D cg c]  }||n|
 c}t        t        t                    fdd      dd	 }|D cg c]  }|   dkD  s| }}d}|D cg c]  }||n|
 c}t        t        t                    fd
d      dd	 }|D cg c]  }|   dkD  s| }}dt        j                          d}|dz  }|s|dz  }t        t        |            D ]  }|||      j                  d      d   }|j                  |      }t        |t              r|d   j!                  dd      }n yt#        j$                  |||d   ni       }|d}||      }|d|dz    d| d| d| d	z  } |dz  }|s|dz  }t        t        |            D ]  }|||      j                  d      d   }|j                  |      }||d   j!                  dd      }nd}t#        j$                  |||d   ni       }t'        ||      d      }|d|dz    d| d| d| d	z  } |D cg c]  }|df }}|D cg c]  }|df }}||z   }| j                  j)                  |       d{    |d t        j                         | j*                  j,                  z    d!z  }| j/                  |d"t0        j2                  i #       d{    yc c}w c c}w 7 c c}w c c}w c c}w c c}w c c}w c c}w 7 7 2w)$a  
        Send a daily report on:
        - Top 5 deployments with most failed requests
        - Top 5 slowest deployments (normalized by latency/output tokens)

        Get the value from redis cache (if available) or in-memory and send it

        Cleanup:
        - reset values in cache -> prevent memory leak

        Returns:
            True -> if successfuly sent
            False -> if not sent
        r   )keysNFTr   c                     |    S rC   r,   )ireplaced_failed_valuess    r9   rh   z2SlackAlerting.send_daily_reports.<locals>.<lambda>Z  s    03r:   )rj   reverse   c                     |    S rC   r,   )r   replaced_slowest_valuess    r9   rh   z2SlackAlerting.send_daily_reports.<locals>.<lambda>l  s    1!4r:   z	

Time: `u(   `s
Here are today's key metrics 📈: 

u7   

*❗️ Top Deployments with Most Failed Requests:*

z	None
:rN   rO   rP   rQ   	r   z. Deployment: `z`, Failed Requests: `z`,  API Base: ``
u#   

*😅 Top Slowest Deployments:*

   z`, Latency per output token: `zs/token`,  API Base: ``

)
cache_listz

Next Run is at: `z`sr   r   )get_model_idsr   r   r   r   r   r"   async_batch_get_cachern   ro   rangetimesplitget_model_info
isinstancerq   rW   rX   rY   rr   async_set_cache_pipeliner1   daily_report_frequencyr   r   daily_reports)r6   routeridsr   failed_request_keyslatency_keyscombined_metrics_keyscombined_metrics_valuesall_nonevalfailed_request_valueslatency_valuesplaceholder_valuer   top_5_failedindextop_5_slowestr   r   rj   _deploymentdeployment_namer`   latency_cache_keysfailed_request_cache_keyscombined_metrics_cache_keysr   r   s                             @@r9   send_daily_reportsz SlackAlerting.send_daily_reports  s<      ""$
 
 NN25IIOOP 	 

 TW
SVRGNN25AAGGHSV 	 
 !4l B(,(A(A(W(W& )X )
 #
 #**C37  +
  7&c%&!
 15H1I1KL  /"
. &E,==."
 #,-.3
 1	  ,
+e/Ee/Lq/PE| 	 
  (#
' &E,=='#
 #-./4
 1	  -
,e0G0NQR0RE} 	 

  		},WXPPz!Gs<()A%l1o6<<SA!DC //4K+t,"-.>"?"C"CGR"P++%5@5LK 01RTH *<?;EAaC5/@@UV[U\\kltkuuxyyG# *& 	<<z!Gs=)*A}Q/066s;A>C //4K&"-.>"?"C"CGR"P"$++%5@5LK 01RTH 1-2BCQGEAaC5/@@^_d^ee{  }E  |F  FK  L  LG +" 3??,3sAh,?9L$M9L#c1X9L!$M&8;T&T#''@@2 A 
 	
 	
 	*499;9K9K9b9b+b*cceff oo .. 	  
 	
 	
 C

#
0"

#

b @$M	
	
s   O!1N2
O!1N7&O!'N<(O!9O!?*O!)N?6-O!#O1O5O!=O	
-O!7OO	FO!OO!O)&O!OAO!,O-O!?O!O!r\   r]   type)hanging_requestslow_responserequest_datac                   K   | j                   | j                  y d}|u|j                  dd      }|j                  dd       }||j                  dd       }	 t        |      }|d d }t
        j                  st
        j                  rd}d| d| d	}nd}|d
k(  rt        j                  | j                         d {    i }| j                  |       d {   du ry |b|j                  dd       Jt        |d   t              r7t        j                  ||d   j                  di             }	|	d}	|d|	 z  }n_|j                  dd       Mt        |d   t              r:|d   }
|
j                  dd      }	t        ||
      }|	d}	d|
v r|
d   }|d|	 d	z  }d| j                   d}dt
        j                   v r*t#        |       d {   }||dj%                  |      z  }| j'                  |j                  di             }||d| z  }| j)                  ||z   dt*        j,                  |       d {    y y y # t        $ r d}Y w xY w7 7 7 7 !w)NrP   rO   rS   rT   rU   rz   r{   r|   r}   r   )r   T
deploymentrN   rQ   z
API Base: rs   r`   r~   r   z
API Base: `z`Requests are hanging - zs+ request time`langfuseu   
🪢 Langfuse Trace: {}r   z
Deployment Latencies
Mediumr   )r$   r%   rW   rZ   r[   rX   r   r   r2   sleepr#   _request_is_completedr   rq   rY   r   success_callbackr   r   rx   r   r   llm_requests_hanging)r6   r\   r]   r   r   rO   rS   r   r   	_api_baser   alerting_messagelangfuse_urlrv   s                 r9   response_taking_too_longz&SlackAlerting.response_taking_too_long  s     == D$4$4$<# $$Wb1E#''
D9H'++GT:x=#DS>
 0088 U  0wnXJaPLL$$--''   ')//\/JJdR'##L$7C
 .I !( 4 4#(4\(B(F(F,b)!I !($&	 l9+$>>L!%%j$7C
 ,dI
 '3:&>I )j" =I#B%1I$L !($&	*i7,56I,J) mI;a$@@L /t/F/F.GGWX ! !9!99)H%1* $L $/$(C(J(J<(XX +/*Q*Q)--j"= +R +' +6 &>?V>W$XXLoo,|;"(==&7	 &   k ( %   KT$so   AI&I
 )AI&?I I&IC6I&I"A/I&I$I&
II&II&I&"I&$I&error_messagefailing_modelc                   K   | j                   | j                  yd| j                  vry| j                  }d|z   }dj                  |      }|j	                  |       d{   }|\| j                  |dt        j                  i        d{    |j                  |d| j                  j                  	       d{    yy7 c7 87 
w)
z
        Raise alert when tracking failed for specific model

        Args:
            error_message (str): Error message
            failing_model (str): Model that failed tracking
        Nfailed_tracking_spendzFailed Tracking Cost for z budget_alerts:failed_tracking:{}ri   Highr   SENTrj   r   ttl)r$   r%   r"   r   async_get_cacher   r   r   async_set_cacher1   budget_alert_ttl)r6   r   r   _cacher   
_cache_keyresults          r9   failed_tracking_alertz#SlackAlerting.failed_tracking_alert  s      == D$4$4$<"$*:*:: 55-=7>>}M
--*-==>//$::"$	 "    ((&&77 )     >s6   A C"C#,CC/C?C
 CC
C)token_budgetsoft_budgetuser_budgetteam_budgetproxy_budgetprojected_limit_exceeded	user_infoc                   K   | j                   }| j                  | j                  y d| j                  vry d}|j                  d      }| j	                  |      }d }d }d}	d }
|dk(  rd}|	dz  }	nz|d	k(  rd}|	d
z  }	nm|dk(  rd}|	dz  }	|j
                  xs |}nP|dk(  rd}|	dz  }	|j                  xs |}n3|dk(  rd}|	dz  }	|j                  }n|dk(  rd}|	dz  }	d}|j                  }|j                  |j                  y d}|j                  5|j                  dkD  r&|j                  |j                  z
  |j                  z  }|j                  I|j                  |j                  k\  rd}|	d|j                   dz  }	nA|dk  rd}|	dz  }	n4|dk  r/d}|	dz  }	n'|j                  |j                  |j                  k\  rd}||d j                  ||      }|j                  |!       d {   }|qt        d(|||	d"|}
| j                  |	d#z   |z   d$t        j                   |
i %       d {    |j#                  |d&| j$                  j&                  '       d {    y y 7 y7 87 
w))Nbudget_alerts
default_idTexclude_nonerP   r   proxyzProxy Budget: r   zSoft Budget Crossed: r   internal_userzUser Budget: r   teamzTeam Budget: r   rj   zKey Budget: r   z$Key Budget: Projected Limit Exceededr   budget_crossedzBudget Crossed
 Total Budget:`r}   g?threshold_crossedz5% Threshold Crossed g333333?z15% Threshold Crossedsoft_budget_crossedzbudget_alerts:{}:{}ri   )eventevent_groupevent_message

r   )r   r   r   r   r   r   r   r,   )r"   r$   r%   
model_dump_get_user_info_struser_idteam_idtoken
max_budgetr   spendr   r   r   r   r   r   r   r1   r   )r6   r   r   r   _iduser_info_jsonuser_info_strr   r   r   webhook_eventpercent_leftr   r   s                 r9   r   zSlackAlerting.budget_alerts9  s!      !55== D$4$4$<$"2"22)"--4-@//	:  	  	  04>!!K--M]"!K44M]")K_,M##*sC]" K_,M##*sC^#K^+M//C//KCCM.E//C 'I,A,A,I+##a'((9??:(( )
 +)"6"66(5i6J6J5K1M %+!88%+!88"".)"7"77-!8.55eSAJ!11j1AAF~ , ! +"/! %	! oo)F2]B (66+&( &    ,," **;; -    - Bs7   G,I..I(/AI.1I*2/I.!I,"I.*I.,I.c                     |j                  d      }|j                  d       d}|j                         D ]  \  }}|d| d| dz  } |S )z>
        Create a standard message for a budget alert
        Tr   r  rP   r   :* `r   )r  poprp   )r6   r   _all_fields_as_dictmsgkvs         r9   r  z SlackAlerting._get_user_info_str  sb     (222E('--/DAqQqcaS$$C 0 
r:   r  	key_aliasend_user_idresponse_costr  c                    K   | j                   [d| j                   v rL|I|F|Ct        ||||d d d |d d dddj                  ||            }| j                  |       d {    y y y y y y 7 
w)Nwebhookspend_trackedcustomerz-Customer spend tracked. Customer={}, spend={})r  r  r  customer_idr  r  
user_emailr  projected_exceeded_dateprojected_spendr   r   r   r  )r$   r   r   send_webhook_alert)r6   r  r  r  r  r  r   s          r9   customer_spend_alertz"SlackAlerting.customer_spend_alert  s      MM%T]]*'!) !#%'#(, $%&MTTE$ )))>>>+ * " ( + &2 ?s   AA- A+!A-alertsc                     dddd}|D ]5  }|dk(  r|dxx   dz  cc<   |dk\  r|dxx   dz  cc<   )|dxx   dz  cc<   7 d	}|j                         D ]   \  }}|dkD  s|d
j                  ||      z  }" |S )z
        Parameters:
        - alerts: List[int] -> list of error codes (either 408 or 500+)

        Returns:
        - str -> formatted string. This is an alert message, giving a human-friendly description of the errors.
        r   )Timeout Errors
API ErrorsUnknown Errors  r$  r     r%  r&  rP   z
{}: {}
)rp   r   )r6   r"  error_breakdownalert	error_msgrj   r   s          r9   _count_outage_alertsz"SlackAlerting._count_outage_alerts  s     ./aSTUE| 01Q61#-2- 01Q61  	)//1JCqy\00e<<	 2 r:   r   )MajorMinorrj   )ModelRegionkey_valproviderr`   outage_valuec                     | d|d|i}|||d<   d}|j                         D ]  \  }	}
|d|	 d|
 dz  } d| d	| d
| j                  |d          dt        t        j                         |d   z
  d       d	S )z!Format an alert message for slackz NameProviderzAPI Baserk   r   r  r   u   


*⚠️ z Service Outage*

z

*Errors:*
r"  )r"  z

*Last Check:* `last_updated_at   z	s ago`


)rp   r,  rr   r   )r6   r   rj   r1  r2  r`   r3  headersheaders_strr  r  s              r9   _outage_alert_msg_factoryz'SlackAlerting._outage_alert_msg_factory  s     U%=':x@"*GJMMODAqQqcaS,,K $		   ,x"89 : ;diikL1B$CCQGH I	 		r:   	exceptiondeployment_idc                   K   | j                   y| j                   j                  |      }|y|j                  j                  }|j                  j                  }|t        j                  |      \  }}}}|j                  j                  }|+t
        j                  j                  ||j                        }|y||z   }| j                  j                  |       d{   }	t        |dd      *|j                  dk7  r|j                  dk  s| j                   y|	t               }
|
j                  |       t!        ||j                  gd	d	t#        j"                         |

      }	| j                  j%                  ||	| j&                  j(                         d{    yt+        |	d         | j&                  j,                  k  r|	d   j/                  |j                         n	 |	d   }
|
j                  |       |
|	d<   t#        j"                         |	d<   |	d   d	u r{t+        |	d         | j&                  j0                  k\  rVt+        |
      dkD  rH| j3                  dd|d|	|      }| j5                  |dt6        j8                  i        d{    d|	d<   n|	d   d	u rzt+        |	d         | j&                  j:                  k\  rUt+        |
      dkD  rG| j3                  dd|d|	|      }| j5                  |dt6        j8                  i        d{    d|	d<   | j                  j%                  ||	       d{    y7 c7 7 7 77 w)z
        Send slack alert if specific provider region is having an outage.

        Track for 408 (Timeout) and >=500 Error codes
        Nmodel_idrO   )custom_llm_providerrN   ri   status_coder'  r(  F)provider_region_idr"  minor_alert_sentmajor_alert_sentr6  deployment_idsr   r"  rF  r6  rD  r   r.  r0  r   rj   r1  r`   r3  r2  r   r   TrE  r-  r   rj   r   )r;   get_deploymentrN   rO   rA  rX   get_llm_providerregion_nameutils_get_model_regionr"   r   getattrrB  setaddProviderRegionOutageModelr   r   r1   region_outage_alert_ttlrn   max_outage_alert_list_sizeappendminor_outage_alert_thresholdr:  r   r   outage_alertsmajor_outage_alert_threshold)r6   r;  r<  r   rO   r2  _rK  	cache_keyr3  _deployment_setr  s               r9   region_outage_alertsz"SlackAlerting.region_outage_alerts  s     ??"__33]3K
))//,,@@$+$<$<5$I!E8Q !//;;!--99$,Z=V=V : K  {*	 ++;;	;JJ 	
 I}d3;%%,))C/&!eO.4#,!--.!&!& $		.L ++;;"&&>> <   
 |H%&););)V)VV")))*?*?@&'78M*)8%&*.))+&' +,5L*+!!>>?O$q(00"#)! 1 C //$22"$	 "    04L+, +,5L*+!!>>?O$q(00"#)! 1 C //$22"$	 "    04L+, ''77 8 
 	
 	
} K4@4	
s]   C M"M#B<MM C3MMBMM)M?M MMMMMc                   K   	 | j                   j                  |       d{   }t        |dd      *|j                  dk7  r|j                  dk  s| j                  y| j                  j                  |      }|y|j                  j                  }|j                  j                  }|	 t        j                  |      \  }}}}t        j                  ||j                  	      }|ht        ||j                  gd
d
t        j                               }| j                   j                  ||| j                   j"                         d{    yt%        |d         | j                   j&                  k  r|d   j)                  |j                         n	 t        j                         |d<   |d   d
u rmt%        |d         | j                   j*                  k\  rH| j-                  dd||||      }	| j/                  |	dt0        j2                  i        d{    d|d<   ns|d   d
u rlt%        |d         | j                   j4                  k\  rG| j-                  dd||||      }	| j/                  |	dt0        j2                  i        d{    d|d<   | j                   j                  ||       d{    y7 # t        $ r d}Y w xY w7 7 7 I7 !# t        $ r Y yw xY ww)a
  
        Send slack alert if model is badly configured / having an outage (408, 401, 429, >=500).

        key = model_id

        value = {
        - model_id
        - threshold
        - alerts []
        }

        ttl = 1hr
        max_alerts_size = 10
        ri   NrB  r'  r(  r>  r@  rP   rQ   F)r?  r"  rD  rE  r6  r   r"  r6  rD  r.  r/  rG  r   r   TrE  r-  r   rH  )r"   r   rN  rB  r;   rI  rN   rO   rA  rX   rJ  r[   rY   OutageModelr   r   r1   outage_alert_ttlrn   rS  rT  rU  r:  r   r   rV  rW  )
r6   r;  r<  r3  r   rO   r2  rX  r`   r  s
             r9   rV  zSlackAlerting.outage_alerts  sG    &j	8<8Q8Q8a8afs8a8t2tL	=$7?))S0!--3??* 777OJ!--33E!00DDH",3,D,D5,Q)E8Q ++Z-F-FH #**%112%*%*$(IIK  //??%&**;; @   
  L*+$$??@ X&--i.C.CD.2iikL*+ /0E9X./%%BBC 44&!%!-% 5  oo"(66&(	 &    48/0/0E9X./%%BBC 44&!%!-% 5  oo (66&(	 &    48/0 ++;;! <   K 3u* ! "!H" @,  		s   K*K J=;K K* K >K*?.K .K  	BK KK K*CK KA3K K)K 7K8K <K*=K  KK KK K K K 	K'$K*&K''K*
model_namelitellm_model_namepassed_model_infoc                   K   t        |dd       }i }d}|'t        j                  j                  |i       }d| d}n t        j                  j                  |i       }d}|j	                         D ]+  \  }}	|dk(  s|dk(  rdj                  |	      }	|| d|	 d	z  }- d
| d| dt        j                  dd       d| d| d}
| j                  |
dt        j                  i       }|!t        j                  |      r| d {    y y y 7 w)N
base_modelrP   zBase Model: `r   input_cost_per_tokenoutput_cost_per_tokenz{:.8f}rl   rk   u%   
*🚅 New Model Added*
Model Name: `zn

Usage OpenAI Python SDK:
```
import openai
client = openai.OpenAI(
    api_key="your_api_key",
    base_url=PROXY_BASE_URLhttp://0.0.0.0:4000z:
)

response = client.chat.completions.create(
    model="z", # model to send to the proxy
    messages = [
        {
            "role": "user",
            "content": "this is a test request, write a short poem"
        }
    ]
)
```

Model Info: 
```
z
```
r   r   )rN  rX   
model_costrW   rp   r   osgetenvr   r   new_model_addedr2   iscoroutine)r6   r_  r`  ra  base_model_from_user
model_inforc  model_info_strr  r  r   	alert_vals               r9   model_added_alertzSlackAlerting.model_added_alert  s^      ''8,M

+ ++//0DbIJ()=(>cBJ ++//0BBGJ$$&DAq**a3J.JOOA&2aSm+N '\   ii(*?@A B <    5< OO 00 	 $ 
	  W%8%8%COO &D s   D DD
Dc                    K   y wrC   r,   )r6   r_  s     r9   model_removed_alertz!SlackAlerting.model_removed_alertS  rF   rG   r  c                   K   t        j                  dd      }|t        d      |j                         }ddi}| j                  j                  |||       d{   }|j                  dk(  ryt        d	|j                         y
7 +w)z
        Sends structured alert to webhook, if set.

        Currently only implemented for budget alerts

        Returns -> True if sent, False if not.

        Raises Exception
            - if WEBHOOK_URL is not set
        WEBHOOK_URLNz$Missing webhook_url from environmentContent-typeapplication/json)urlr8  data   Tz#Error sending webhook alert. Error=F)	ri  rj  r[   model_dump_jsonr.   postrB  printtext)r6   r  webhook_urlpayloadr8  responses         r9   r   z SlackAlerting.send_webhook_alertV  s      iit4BCC//1!#560055 6 
 

 3&7G
s   AB	B,B	premium_useremail_logo_urlemail_support_contactc                 p   K   ddl m}m} |dur&||"t        d|j                  j
                         y w)Nr   )CommonProxyErrorsr  Tz$Trying to Customize Email Alerting
 )litellm.proxy.proxy_serverr  r  
ValueErrornot_premium_userr   )r6   r  r  r  r  s        r9   %_check_if_using_premium_email_featurez3SlackAlerting._check_if_using_premium_email_featureu  sJ      	Ot#)-B-N ;<M<^<^<d<d;ef  	s   46c                 x  K   	 ddl m} | j                  d| j                  vr!t        j                  d| j                         yddlm}m} t        j                  dt        j                  dd             }t        j                  d	d       }| j                  |||       d {    |t        }|t        }|j                  }|j                  }|j                  }	|B|	@|>|j                   j"                  j%                  d
|	i       d {   }
|
|
j                  }|j&                  }|j(                  }t        j                  dd      }d}|%t        j                  d|j+                                |j,                  dk(  rt.        j1                  ||||||      }n|j,                  dk(  ro|j2                  }d}|D|B|j                   j4                  j%                  d|i       d {   }||j6                  xs d}t8        j1                  |||||      }n%t        j                  d|j;                                |j=                          |d| |d} ||d   |d   |d          d {    y7 7 q7 7 # t>        $ r)}t        j                  d tA        |             Y d }~yd }~ww xY ww)!Nr   
send_emailemailz;Error sending email alert - 'email' not in self.alerting %sF)r  prisma_clientSMTP_SENDER_LOGOEMAIL_LOGO_URLEMAIL_SUPPORT_CONTACTr  )whererf  rg  Alert from LiteLLM Server*Trying to send email alert to no recipientextrakey_created)r  recipient_email
key_budget	key_tokenbase_urlr  internal_user_createdzDefault Teamr  -)r  r  	team_namer  r  z3Trying to send email alert on unknown webhook event	LiteLLM: tosubjecthtmlr  r  r  receiver_emailr  r  TzError sending email alert %s)!litellm.proxy.utilsr  r$   r   errorr  r  r  ri  rj  r  LITELLM_LOGO_URLLITELLM_SUPPORT_CONTACTr   r  r  dblitellm_usertablefind_uniquer  r  rq   r   KEY_CREATED_EMAIL_TEMPLATEr   r  litellm_teamtable
team_aliasUSER_INVITED_EMAIL_TEMPLATEr  r{  r[   rZ   )r6   r  r  r  r  r  r  
event_namer  recipient_user_iduser_rowr  r  r  email_html_contentr  r  team_rowemail_eventrb   s                       r9   &send_key_created_or_user_invited_emailz4SlackAlerting.send_key_created_or_user_invited_email  s    c	6}}$t}}(D$**QMM NYY"BII.>$EN %'II.Et$L!<<n.C   %!1$,(?%&44J+66O - 5 5'%1!-!.!1!1!C!C!O!O$&78 "P "  '&.&9&9O%++I&11Jyy!13HIH!<&$**@',,.
 ""m3%?%F%F#1$3)'%*? &G &" $$(??'//*	&=+D%2%5%5%G%G%S%S('2 &T &  H  +$,$7$7$>3	%@%G%G#1$3'%*? &H &" %**I'224
 ))+%&zl3*K *40#I. (   _ @ 2  	 &&'Es1vN	s   J:A J J:AJ #I;$A)J I>C J .J/BJ 5J6J :J:;J >J J J 	J7J2-J:2J77J:c           	        K   ddl m} ddlm} t	        j
                  dt	        j
                  dd            }t	        j
                  dd      }| j                  |||       d{    |t        }|t        }|j                  }|j                  }|j                  }	|j                  }
d}|%t        j                  d	|j                         
       |j                   dk(  rd| d|	 d|
 d| d	}|j#                          |d| |d} ||d   |d   |d          d{    |j$                  dk(  rddlm}  ||       d{    y7 7 -7 	w)z
        Sends structured Email alert to an SMTP server

        Currently only implemented for budget alerts

        Returns -> True if sent, False if not.
        r   )r  r  r  r  Nr  r  r  r  r   z
            <img src="zD" alt="LiteLLM Logo" width="150" height="50" />

            <p> Hi ze, <br/>

            Your LLM API usage this month has reached your account's <b> monthly budget of $a   </b> <br /> <br />

            API requests will be rejected until either (a) you increase your monthly budget or (b) your monthly usage resets at the beginning of the next calendar month. <br /> <br />

            If you have any questions, please send an email to zY <br /> <br />

            Best, <br />
            The LiteLLM team <br />
            r  r  r  r  r  r  r   )send_team_budget_alertr  F)r  r  r  r  ri  rj  r  r  r  r   r  r  r  r   r  rq   r   r{  r   #litellm.integrations.email_alertingr  )r6   r  r   r  r  r  r  r  r  	user_namer  r  r  r  s                 r9   send_email_alert_using_smtpz)SlackAlerting.send_email_alert_using_smtp  s     	<2		*:D A
 !#		*A4 H88.*?
 	
 	
 !-N ($;!"00
'22!))	"--
8" &&<MDVDVDX "22&%& '; ]]g\h i@ AV?V W" 	%%'!":,/&
 &t,	*V$
 	
 	

 $$.R(}EEEi	
T	
 Fs7   A#E%E&B;E!E"%EEEEEr   r   )r   r   r   r   c                   K   | j                   yd| j                   v r!|dk(  r|| j                  |       d{    d| j                   v r"|dk(  r|| j                  ||       d{    d| j                   vry|| j                  vrydd	lm}  |j
                         j                  d
      }t        j                  dd      }	|dk(  s|dk(  r|}
nd| d| d| }
|r$|j                         D ]  \  }}|
d| d| dz  }
 |r$|j                         D ]  \  }}|
d| d| dz  }
 |		|
d|	 dz  }
| j                  || j                  v r| j                  |   }n/| j                  | j                  }nt        j                  dd      }|t        d      d|
i}ddi}t        |t              r(|D ]"  }| j                  j!                  ||||d       $ n | j                  j!                  ||||d       t#        | j                        | j$                  k\  r| j'                          d{    yy7  7 7 w)a  
        Alerting based on thresholds: - https://github.com/BerriAI/litellm/issues/1298

        - Responses taking too long
        - Requests are hanging
        - Calls are failing
        - DB Read/Writes are failing
        - Proxy Close to max budget
        - Key Close to max budget

        Parameters:
            level: str - Low|Medium|High - if calls might fail (Medium) or are failing (High); Currently, no alerts would be 'Low'.
            message: str - what is the alert about
        Nr  r   r  r  )r  r   slackr   )datetimez%H:%M:%Srf  r   rk  zLevel: `z`
Timestamp: `z`

Message: r   : `r   z

*Alerting Metadata*: 
z

Proxy URL: `r}   SLACK_WEBHOOK_URLz*Missing SLACK_WEBHOOK_URL from environmentr~  rv  rw  )rx  r8  r  r   )r$   r   r  r%   r  nowstrftimeri  rj  rp   r&   r'   r  r   rK   	log_queuerT  rn   
batch_sizeflush_queue)r6   r   r   r   r   r   r7   r  current_time_proxy_base_urlformatted_messagerj   r   slack_webhook_urlr  r8  rx  s                    r9   r   zSlackAlerting.send_alert3  s    . ==  &o-%))	)BBB t}}$o-% 22'J 3    $--'T---% $x||~..z:))$4d;(J:K,K ' 5'nWIV  $lln
U!tC5E7%%@@! -/557
U!'B3%s5'QV%WW! 8&#3O3DA!FF %%1d777 ))*5  %%1 $ 8 8 "		*=t D$IJJ,-!#56'.(%%"#*#*&0	 ) NN!!,&&",	 t~~$//1""$$$ 2Y CJ %s4   8IH:/I*H=+GI3I 4I=I Ic           	      $  K   | j                   sy t        | j                         }|j                         D cg c]  }t        | |d   |d          }}t	        j
                  |  d {    | j                   j                          y c c}w 7 $w)Nitemcount)slackAlertingInstancer  r  )r  r   valuesr   r2   gatherclear)r6   squashed_queuer  taskss       r9   async_send_batchzSlackAlerting.async_send_batch  s     ~~(8
 '--/	
 0 &*fT'] 0	 	 
 nne$$$
 	%s   5BB	B)B*%Bc           	        K   	 d| j                   v r-|j                  di       xs i }|j                  di       xs i }|j                  dd      xs d}||z
  }|}	t        |t        j                        rgt        |d      r[|j                  Ot        |j                  d      r9|j                  j                  }
|
!|
d	kD  rt        |j                         |
z        }	t        |	t              r|	j                         }	| j                  t        |d
|	t        j                  j                                      d{    yy7 # t        $ r+}t!        j"                  dt%        |              Y d}~yd}~ww xY ww)zLog deployment latencyr   rN   rn  r   rP   usageNcompletion_tokensr   Fr   r   r   
updated_atzR[Non-Blocking Error] Slack Alerting: Got error in logging LLM deployment latency: )r%   rW   r   rX   ModelResponsehasattrr  r  floatrV   r   r   DeploymentMetricsrL  get_utc_datetimer[   r   r  rZ   )r6   r7   response_objr\   r]   rN   rn  r?  
response_sfinal_valuer  rb   s               r9   async_log_success_eventz%SlackAlerting.async_log_success_event  sp    "	$"2"22!',<b!A!GR+//bAGR
%>>$39r(0:(=
(lG,A,ABL'2$**6 2 24GH(4(:(:(L(L%(49JQ9N&+&4469JJ' k95"-";";"=K55%#',1<#*==#A#A#C	  + 3*  	 &&dehijekdlm 		sA   E:D6E :E;E ?E:E 	E7!E2-E:2E77E:c           	        K   |j                  di       }|j                  di       xs i }|j                  dd      }	 d| j                  v rC	 | j                  t        |ddt        j
                  j                                      d{    t        |j                  d
d      t              rZd| j                  v r| j                  |d
   |       d{    d| j                  v r| j                  |d
   |       d{    yyy7 # t        $ r+}t        j                  d	t        |              Y d}~d}~ww xY w7 l7 B# t        $ r Y yw xY ww)z Log failure + deployment latencyrN   rn  r   rP   r   TNr  zException raises -r;  rV  )r;  r<  r[  )rW   r%   r   r  rX   rL  r  r[   r   debugrZ   r   r   rV  r[  )	r6   r7   r  r\   r]   _litellm_params_model_infor?  rb   s	            r9   async_log_failure_eventz%SlackAlerting.async_log_failure_event  ss     **%5r:%)),;Ar??4,	$"2"22
H99)'+/59'.}}'E'E'G	   &**["5x@"d&6&66,,"("5&. -   
 *T-=-==33"("5X 4    > A ! H"((+=c!fX)FGGH
  		s   ;EE =D 
D
D AE E+E EE E
D 	E !D;6E ;E  E E 	EEEEc                 T  K   d}| j                   j                  t        j                  j                  d       d{   }t        j
                         }|?| j                   j                  t        j                  j                  |       d{    |S t        |t              rw| j                  j                  }||z
  |k\  rY| j                  |       d{    | j                   j                  t        j                  j                  |       d{    d}|S 7 7 7 I7 w)z[
        Returns:
        - True -> report sent
        - False -> report not sent
        FN)rj   r   rH  )r   T)r"   r   r   report_sent_keyr   r   r   r   r  r1   r   r   )r6   r;   report_sent_boolreport_sentr  interval_secondss         r9   _run_scheduler_helperz#SlackAlerting._run_scheduler_helper  s0     ! 55EE&66<<! F 
 

 yy{++;;*::@@" <   $   U+#11HHk)-==--Z-@@@//??.>>DD& @    $( 5
 AsG   ;D(D AD(D"A	D(D$<D(D&	D("D($D(&D(c                 H  K   || j                   yd| j                   v r}	 | j                  |       d{    t        j                  | j                  j
                  dz
  | j                  j
                  dz         }t        j                  |       d{    |y7 g7 w)z
        If 'daily_reports' enabled

        Ping redis cache every 5 minutes to check if we should send the report

        If yes -> call send_daily_report()
        Nr   )r;   r   )r%   r  randomrandintr1   report_check_intervalr2   r   )r6   r;   intervals      r9   _run_scheduled_daily_reportz)SlackAlerting._run_scheduled_daily_report  s      !1!1!9d...00J0GGG!>>&&<<q@&&<<q@ mmH---  	 H
 .s"   4B"BA B"B B" B"
time_rangec                   K   | j                   d| j                  vry	 ddlm} t	        |dd       }|d   j                         dk7  rt        d      t        j                  j                         j                         }|t        j                  |      z
  }d	|j                  d
       d|j                  d
       }| j                  j                  |       d{   ry ||j                  d
      |j                  d
             d{   }||g g fk(  ry|\  }}	d|j                  d       d|j                  d       d| d}
|3|
dz  }
|D ])  }t        t        |d         d      }|
d|d    d| dz  }
+ |	3|
dz  }
|	D ])  }t        t        |d         d      }|
d|d    d| dz  }
+ | j!                  |
dt"        j$                  i        d{    | j                  j'                  |dt)        |              d{    y7 47 7 :7 # t        $ r"}t+        j,                  d!|        Y d}~yd}~wt.        $ r"}t+        j,                  d"|        Y d}~yd}~ww xY ww)#z
        Send a spend report for a configurable time range.

        Args:
            time_range: A string specifying the time range for the report, e.g., "1d", "7d", "30d"
        Nspend_reportsr    _get_spend_report_for_time_rangedz0Time range must be specified in days, e.g., '7d'daysweekly_spend_report_sent_%Y-%m-%drX  ri   
start_dateend_dateu   *💸 Spend Report for `%m-%d-%Y - z` (z days)*

*Team Spend Report:*
total_spendr7  Team: `r  ` | Spend: `$r   
*Tag Spend Report:*
Tag: `individual_request_tagr   r   r   r   zInvalid time range format: zError sending spend report: )r$   r%   7litellm.proxy.spend_tracking.spend_management_endpointsr  intlowerr  r  r  dater   r  r"   r   rr   r  r   r   r  r   r   r   r  r[   )r6   r  r  r  todays_dater  _event_cache_key_respspend_per_teamspend_per_tag_spend_messager  _team_spend
_tag_spendverb   s                   r9   send_weekly_spend_reportz&SlackAlerting.send_weekly_spend_report/  s     == O4;K;K$K:	K
 z#2'D"~##%, !STT"++//1668K$x'9'9t'DDJ!::;N;Nz;Z:[[\]h]q]qr|]}\~..>>CS>TTT:%..z:$--j9 E }"b 1,1)NM7
8K8KJ8W7XX[\g\p\pq{\|[}  ~A  BF  AG  GP  QN)"<<+E"'eM.B(CQ"GK"!%"5!6mK=PSTN , (";;*E!&uU=-A'BA!FJ"u5M/N.O}]g\hhk&llN + //&$22"$	 "    ++;;$'
3 <   G U2  	K &&)DRD'IJJ 	K &&)EaS'IJJ	Ks   I3B?H HH #I3$+H HH I3B?H H.H HH I3H H H H 	I0%I=I3I0I+&I3+I00I3c                 n  K   	 ddl m} ddlm} t        j                  j                         j                         }|j                  d      } ||j                  |j                        \  }}|t	        j                  |dz
        z   }d|j                  d       d	|j                  d       }| j                  j                  |
       d{   ry ||j                  d      |j                  d             d{   }||g g fk(  ry|\  }	}
d|j                  d       d|j                  d       d}|	7|dz  }|	D ]-  }|d   }t        |      }t        |d      }|d|d    d| dz  }/ |
7|dz  }|
D ]-  }|d   }t        |      }t        |d      }|d|d    d| dz  }/ | j!                  |dt"        j$                  i        d{    | j                  j'                  |dd       d{    y7 07 7 17 # t(        $ r }t+        j,                  d |       Y d}~yd}~ww xY ww)! r   )
monthranger  r   )dayr  monthly_spend_report_sent_r  rX  ri   Nr   u    *💸 Monthly Spend Report for `r  r  z` *
r  r  r7  r  r  r  r   r	  r
  r  r   r   r   i ' r   $Error sending weekly spend report %s)calendarr  r  r  r  r  r  replaceyearmonthr   r  r"   r   r  rr   r   r   r  r   r[   r   r;  )r6   r  r  r  first_day_of_monthrX  last_day_of_monthr  r  monthly_spend_per_teammonthly_spend_per_tagr  r  r  r  rb   s                   r9   send_monthly_spend_reportz'SlackAlerting.send_monthly_spend_reportx  s    @	V+ #++//1668K!,!4!4!4!;#-k.>.>@Q@Q#R A  2X5G5G&*6 ! "<<N<W<WXb<c;ddefw  gA  gA  BL  gM  fN   O..>>CS>TTT:-66zB*33J? E
 }"b 1<A9"$9?@R@[@[\f@g?hhkl}  mG  mG  HR  mS  lT  TY  ZN%1"<<3E"'"6K"'"4K"'Q"7K"!%"5!6mK=PSTN 4 %0";;2E!&}!5J!&z!2J!&z1!5J"u5M/N.O}]g\hhk&llN 3 //&$22"$	 "    ++;;$& <   U U@  	V **+QSTUU	Vs   H5CH	 G?H	 H5+H	  HH	 H5CH	 H%H	 9H:H	 >H5?H	 H	 H	 H	 		H2H-(H5-H22H5c                    K   	 ddl m}  |        d{   }d| }| j                  |dt        j                  i        d{    y7 57 # t
        $ r }t        j                  d|       Y d}~yd}~ww xY ww)z
        Helper to send fallback statistics from prometheus server -> to slack

        This runs once per day and sends an overview of all the fallback statistics
        r   )#get_fallback_metric_from_prometheusNz*Fallback Statistics:*
r   r   r  )6litellm.integrations.prometheus_helpers.prometheus_apir*  r   r   fallback_reportsr[   r   r  )r6   r*  !falllback_success_info_prometheusfallback_messagerb   s        r9   #send_fallback_stats_from_prometheusz1SlackAlerting.send_fallback_stats_from_prometheus  s     	R :;; .
 ++L*MN  //($55"$	 "    	# <  	R &&'MqQQ	RsP   A:A A
/A AA 	A:
A A 	A7A2-A:2A77A:	key_eventr  c                   K   	 d| d}|j                         }|dz  }|j                         D ]  \  }}d|v s|| d| dz  } |dz  }|j                  }|j                         D ]  \  }}|dk(  r|| d| dz  } | j                  |d|i 	       d
{    y
7 # t        $ r }	t        j                  d|	       Y d
}	~	y
d
}	~	ww xY ww)z
        Handles sending Virtual Key related alerts

        Example:
        - New Virtual Key Created
        - Internal User Updated
        - Team Created, Updated, Deleted
        r}   r   z*Action Done by:*

created_byr  z
*Arguments passed:*
user_api_key_dictr   r   Nz-Error sending send_virtual_key_event_slack %s)r  rp   request_kwargsr   r[   r   r  )
r6   r0  r   r  r   key_event_dictrj   r   r4  rb   s
             r9   send_virtual_key_event_slackz*SlackAlerting.send_virtual_key_event_slack  s    	*S)G&113N ,,G,224
U3&#c%44G 5
 00G&55N,224
U--cU#eWC00 5
 //%"$	 "    	  	 &&?  		sF   C5B! AB! BB! CB! !	C
*C CC

Cc                   K   |y|j                  dd      dk7  rh|j                  dd      dk7  rS|j                  dd      }| j                  j                  dj                  |      d	
       d{   }||dk(  s|dk(  ry	y7 w)z[
        Returns True if the request is completed - either as a success or failure
        NFlitellm_statusrP   successfaillitellm_call_idzrequest_status:{}T)rj   
local_only)rW   r"   r   r   )r6   r   r;  statuss       r9   r   z#SlackAlerting._request_is_completed  s       -r2i?  !126&@ +../@"EO*.*C*C*S*S'..?D +T + %F !v':f>N%s   A/B1B2B)NNNNNNrC   )NNr   N)NN)7d)?__name__
__module____qualname____doc__DEFAULT_ALERT_TYPESr	   r   r  r   r   r   r
   rZ   r5   r   rA   rE   rI   rL   rc   rx   r   r  r  r   boolr   r  r   rq   r   r   r   r   r  r!  r,  BaseOutageModelr:  r   r[  rV  r   rq  rs  r   r   r  r  r  r   r  r  r  r  r  r  r(  r/  r   r6  r   __classcell__)r8   s   @r9   r!   r!   +   s    59 #%': -1?&y1? %
? 4.? )_? 'E$s)S.112
? &c]?F $(.215QU(,'+!)4.!) %UO!) d9o.	!)
 'tIuT#Y^7L,L'MN!)  ~!) V$!)F2 $D1f,"3,	,\T$ Tp 3704<M'+eX../e 8,,-e 89	e
 tneN S >q(
q qf
H 
 
"?}"? C="? c]	"?
  "? UO"?H49  0,- &' 	
  3- & 
8G
G
 G
 
	G
R}} } 
	}~99369KN9vC l t D )-/3	 !  (}	f)f	fPE)E7:E	EZ -1l%l% ./l% 	l%
  l% L)l%\$LB"  " HHSM . GKGKRBVH@.". . 	.` 4 r:   r!   )3r2   r  ri  r  r   r   typingr   r   r   r   r   r	   r
   openair   rX   litellm.litellm_core_utils*litellm.litellm_core_utils.litellm_logginglitellm.typeslitellm._loggingr   r   litellm.caching.cachingr   (litellm.integrations.custom_batch_loggerr   *litellm.litellm_core_utils.duration_parserr   2litellm.litellm_core_utils.exception_mapping_utilsr   &litellm.llms.custom_httpx.http_handlerr   r   litellm.proxy._typesr   r   r   r   )litellm.types.integrations.slack_alertingemail_templates.templatesbatching_handlerr   r   rL  r   r   litellm.routerr   _Routerr!   r,   r:   r9   <module>rX     s{      	    K K K   ! 1  A - F J T S 7 ) > T0FFs% sr:   