
    g                     &   d dl Z d dlZd dlmZmZ d dlmZmZmZmZm	Z	m
Z
mZmZ d dlmZ d dlmZ d dlZd dlmZmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZ d dlm Z m!Z! erd dl"m#Z$ d dl%m&Z' e$Z#e'Z&neZ#eZ& G d de      Z( G d de      Z)y)    N)datetime	timedelta)TYPE_CHECKINGAnyListLiteralOptionalTuple	TypedDictUnion)HTTPException)	BaseModel)	DualCacheModelResponseverbose_proxy_logger)(RATE_LIMIT_ERROR_MESSAGE_FOR_VIRTUAL_KEY)CustomLogger)!_get_parent_otel_span_from_kwargs)CurrentItemRateLimitUserAPIKeyAuth)get_key_model_rpm_limitget_key_model_tpm_limit)Span)InternalUsageCachec                   ^    e Zd ZU ee   ed<   ee   ed<   ee   ed<   ee   ed<   ee   ed<   y)CacheObjectcurrent_global_requestsrequest_count_api_keyrequest_count_user_idrequest_count_team_idrequest_count_end_user_idN)__name__
__module____qualname__r	   dict__annotations__     a/var/www/openai/venv/lib/python3.12/site-packages/litellm/proxy/hooks/parallel_request_limiter.pyr   r   !   s4    %d^+#D>)#D>)#D>)'~-r)   r   c                   :    e Zd ZdefdZd Zdededede	de
d	e
d
e
dee   de	ded   deeeef      fdZdefdZ	 d!dee	   defdZ	 d!dee	   dee	   dee	   dee	   dee	   dee   defdZdededede	fdZd Zd Zde	dedee   fdZdedef fd Z xZS )"!_PROXY_MaxParallelRequestsHandlerinternal_usage_cachec                     || _         y N)r-   )selfr-   s     r*   __init__z*_PROXY_MaxParallelRequestsHandler.__init__+   s
    $8!r)   c                     	 t        j                  |       t        j                  rt	        |       y y # t
        $ r Y y w xY wr/   )r   debuglitellmset_verboseprint	Exception)r0   print_statements     r*   print_verbosez/_PROXY_MaxParallelRequestsHandler.print_verbose.   s=    	 &&7""o& # 		s   04 	A A user_api_key_dictcachedata	call_typemax_parallel_requests	tpm_limit	rpm_limitcurrentr   rate_limit_type)usercustomerteamvalues_to_update_in_cachec                   K   |G|dk(  s
|dk(  s|dk(  r| j                  d|
 d| d| d|       S dddd}|j                  |	|f       y t        |d	         |k  r6|d
   |k  r.|d   |k  r&|d	   dz   |d
   |d   d}|j                  |	|f       y t        dd|
 d|d    d| d|d
    d| 
dt	        | j                               i      w)Nr   Hit limit for z). Current limits: max_parallel_requests: , tpm_limit: , rpm_limit: additional_details   current_requestscurrent_tpmcurrent_rpmrO   rP   rQ     z1LiteLLM Rate Limit Handler for rate limit type = z'. Crossed TPM, RPM Limit. current rpm: z, rpm limit: z, current tpm: z, tpm limit: retry-afterstatus_codedetailheaders)raise_rate_limit_errorappendintr   strtime_to_next_minute)r0   r:   r;   r<   r=   r>   r?   r@   rA   r   rB   rF   new_vals                r*   check_key_in_limitsz5_PROXY_MaxParallelRequestsHandler.check_key_in_limits6   s    $ ?$)Y!^yA~22)77HHq  sH  rI  IV  W`  Va  an  ox  ny  (z 3   %&  G
 &,,.CW-MN*+,/DD&2&2 %,,>$?!$C&}5&}5G
 &,,.CW-MNJ?J[  \C  DK  LY  DZ  C[  [h  ir  hs  sB  CJ  KX  CY  BZ  Zg  hq  gr  s&D,D,D,F(GH s   CCreturnc                     t        j                         }|t        d      z   j                  dd      }||z
  j	                         }|S )NrM   )minutesr   )secondmicrosecond)r   nowr   replacetotal_seconds)r0   rd   next_minuteseconds_to_next_minutes       r*   r\   z5_PROXY_MaxParallelRequestsHandler.time_to_next_minuteg   sK    lln Yq11::!QR:S #."3!B!B!D%%r)   rL   c           	      n    d}||dz   |z   }t        dd| dt        | j                               i      )zX
        Raise an HTTPException with a 429 status code and a retry-after header
        "Max parallel request limit reached rR   z#Max parallel request limit reached rS   rT   )r   r[   r\   )r0   rL   error_messages      r*   rX   z8_PROXY_MaxParallelRequestsHandler.raise_rate_limit_errors   sQ     =))C/2DDM89K8LM"C(@(@(B$CD
 	
r)   r   r    r!   r"   parent_otel_spanc                    K   |||||g}| j                   j                  ||       d {   }|t        d d d d d       S t        |d   |d   |d   |d   |d         S 7 5w)N)keysrm   )r   r   r    r!   r"   r   rM            )r-   async_batch_get_cacher   )	r0   r   r   r    r!   r"   rm   ro   resultss	            r*   get_all_cache_objectsz7_PROXY_MaxParallelRequestsHandler.get_all_cache_objects   s      $!!!%
 11GG- H 
 

 ?(,&*&*&**.  $+AJ")!*")!*")!*&-aj
 	

s   (A"A 6A"c                   K   | j                  d       |j                  }|j                  }|t        j                  }|i }|j                  di       j                  dd       }t        |dt        j                        }|t        j                  }t        |dt        j                        }	|	t        j                  }	g }
d }|d}| j                  j                  |d|j                         d {   }|d}||k\  r| j                  d	| d
|       S | j                  j                  |dd|j                         d {    t        j                         j                  d      }t        j                         j                  d      }t        j                         j                  d      }| d| d| }| j                  |dnd || d| dnd |j                   |j                    d| dnd |j"                  |j"                   d| dnd |j$                  |j$                   d| dnd |j                         d {   }|!| d| d}|d   }| j                  d|        |t        j                  k(  r'|t        j                  k(  r|	t        j                  k(  rn|dk(  s
|dk(  s|	dk(  r%| j                  dt&         d| d| d| d|	 
      S |dddd}|
j)                  ||f       nvt+        |d         |k  r6|d   |k  r.|d   |	k  r&|d   dz   |d   |d   d}|
j)                  ||f       n/| j                  dt&         d| d | d!|d    d"|	 d#|d    d$      S t-        |      t/        |      ||j                  d%d       }| d| d| d}| j                  j                  ||j                  &       d {   }d }d }t-        |      }t/        |      }|&|r|j                  |      }|r|j                  |      }|dddd}|
j)                  ||f       n|||d   dz   |d   |d   d}|1|d   |k\  r)| j                  d'| d(t&         d| d | d!|d    d$      S |1|d   |k\  r)| j                  d)| d(t&         d| d*| d+|d    d$      S |
j)                  ||f       d }d }|r|||d   z
  }|||d   z
  }d,| |d-| |i}d|vri |d<   |d   j1                  |       |j                   }|y|j2                  }|j4                  }|t        j                  }|t        j                  }| d| d}| j7                  ||||t        j                  |d.   |||d/|
0       d {    |j"                  } | y|j8                  }!|j:                  }"|!t        j                  }!|"t        j                  }"|  d| d}| j7                  ||||t        j                  |d1   ||!|"d2|
0       d {    |j$                  rt        |d3t        j                        }#t        |d4t        j                        }$|#t        j                  }#|$t        j                  }$|j$                   d| d}| j7                  ||||t        j                  ||d5   |#|$d6|
7       d {    t=        j>                  | j                  jA                  |
d8|j                  9             y 7 	7 7 7 57 7 7 Qw):Nz)Inside Max Parallel Request Pre-Call Hookmetadataglobal_max_parallel_requestsr?   r@   Tkey
local_onlylitellm_parent_otel_spanrM   zHit Global Limit: Limit=z, current: rK   rz   valuer{   r|   %Y-%m-%d%H%M-::::request_count)r   r   r    r!   r"   rm   r   z	current: r   rH   z: z. max_parallel_requests: rI   rJ   rN   rO   rP   rQ   z. tpm_limit: z, current_tpm z , rpm_limit: z current rpm rk   modelrz   r|   zHit TPM limit for model: z on zHit RPM limit for model: z. rpm_limit: z, current_rpm zlitellm-key-remaining-tokens-zlitellm-key-remaining-requests-r    rC   )r:   r;   r<   r=   r>   rA   r   r?   r@   rB   rF   r!   rE   end_user_tpm_limitend_user_rpm_limitr"   rD   )r:   r;   r<   r=   r>   r   rA   r?   r@   rB   rF   <   
cache_listttlr|   )!r9   api_keyr>   sysmaxsizegetgetattrr-   async_get_cacherm   rX   async_increment_cacher   rd   strftimeru   user_idteam_idend_user_idr   rY   rZ   r   r   updateuser_tpm_limituser_rpm_limitr^   team_tpm_limitteam_rpm_limitasynciocreate_taskasync_batch_set_cache)%r0   r:   r;   r<   r=   r   r>   rx   r?   r@   rF   r]   _keyr   current_datecurrent_hourcurrent_minuteprecise_minutecache_objectsr   rA   _modeltpm_limit_for_modelrpm_limit_for_model_tpm_limit_for_key_model_rpm_limit_for_key_model_remaining_tokens_remaining_requests_remaining_limits_datar   r   r   r   r   r   r   r   s%                                        r*   async_pre_call_hookz5_PROXY_MaxParallelRequestsHandler.async_pre_call_hook   s4
     	FG#++ 1 G G ($'KK!<D'+xx
B'?'C'C*D(
$ -{CKKH	I-{CKKH	I  	" #''31D,0,E,E,U,U):)K)K -V - '# '.*+'&*FF22)AB^A__j  lC  kD  (E 3  
 //EE#->-O-O	 F     ||~..z:||~..t4!006(><..9IJ+/+E+E 0; / & )2n-_= %,,8 %,,-R/?O %,,8 %,,-R/?O %00< %001N3C?S.??5 ,F ,
 &
8 '.ir.1A$Q! $$;<G7)45%4,,&!+yA~a22)78`7aacdkcl  mF  G\  F]  ]j  kt  ju  uB  CL  BM  (N 3   ()#$#$
 *002G1QRG./03HHM*Y6M*Y6 )00B(Ca(G#*=#9#*=#9
 *002G1QR22)78`7aacdkclly  {D  zE  ES  T[  \i  Tj  Sk  ky  zC  yD  DQ  RY  Zg  Rh  Qi  ij  (k 3   $$56B&'89EXXgt,F)2fXR'7G " !55EE)):)K)K F  G
 #'"&'>?P'Q$'>?P'Q$!+*B*F*Fv*N'+*B*F*Fv*N'()#$#$
 *002G1QR$04G4S )00B(Ca(G#*=#9#*=#9 (3.2EE66-FvhdS{R||~  @G  H  HU  Vi  Uj  jx  y@  AN  yO  xP  PQ  ,R 7   (3.2EE66-FvhdS{R||~  @G  H  HU  Vi  Uj  jx  y@  AN  yO  xP  PQ  ,R 7   .446KW5UV $"&&2(;gm>T(T%&2*=@V*V' 0x8:K1&:<O&"
 %#%Z ##$:; $++.==N.==N%!$%!$'.ir.1A$Q!**"3#&)kk%&=>&;(( &*C +     $++.==N.==N%!$%!$'.ir.1A$Q!**"3#&)kk%&=>&;(( &*C +     ((!(!#7" ")!#7" ")%([["!)%([[" %001N3C?S "
 **"3#&)kk&;%&AB,, **C +    	%%;;4):)K)K < 	
 	]'&
^^8Fs   C/\1[:2A\[=D\\ F\\F6\\B\\	B,\9\:A\=\ \\\	\\c                 
  K   ddl m} t        |      }	 | j                  d       |d   d   j	                  dd       }|d   d   d   }|d   d   j	                  d	d       }	|d   d   j	                  d
d       }
|d   d   j	                  dd       }|j	                  d      }|d   d   j	                  di       xs i }|)d}| j
                  j                  |dd|       d {    t        j                         j                  d      }t        j                         j                  d      }t        j                         j                  d      }| d| d| }d}t        |t              r|j                  j                  }g }|| d| d}| j
                  j                  ||       d {   xs dddd}t        |d   dz
  d      |d   |z   |d   dz   d}| j                  d| d|        |j!                  ||f        ||      }||d|v sd |v s|| d| d| d}| j
                  j                  ||       d {   xs dddd}t        |d   dz
  d      |d   |z   |d   dz   d}| j                  d| d|        |j!                  ||f       |	d}t        |t              r|j                  j                  }|	 d| d}| j
                  j                  ||       d {   xs d|dd}t        |d   dz
  d      |d   |z   |d   dz   d}| j                  d| d|        |j!                  ||f       |
d}t        |t              r|j                  j                  }|
 d| d}| j
                  j                  ||       d {   xs d|dd}t        |d   dz
  d      |d   |z   |d   dz   d}| j                  d| d|        |j!                  ||f       |d}t        |t              r|j                  j                  }| d| d}| j
                  j                  ||       d {   xs d|dd}t        |d   dz
  d      |d   |z   |d   dz   d}| j                  d| d|        |j!                  ||f       | j
                  j#                  |d!|"       d {    y 7 7 7 7 7 77 7 # t$        $ r}| j                  |       Y d }~y d }~ww xY ww)#Nr   )#get_model_group_from_litellm_kwargskwargsz5INSIDE parallel request limiter ASYNC SUCCESS LOGGINGlitellm_paramsrw   rx   user_api_keyuser_api_key_user_iduser_api_key_team_iduser_api_key_model_max_budgetrC   user_api_key_metadataTr}   r   r   r   r   r   r   r   rM   rN   rO   rP   rQ   zupdated_value in success call: z, precise_minute: model_rpm_limitmodel_tpm_limitr   r   ))litellm.proxy.common_utils.callback_utilsr   r   r9   r   r-   r   r   rd   r   
isinstancer   usagetotal_tokensr   maxrY   r   r7   )r0   r   response_obj
start_timeend_timer   r|   rx   r   r   r   r   user_api_key_end_user_idr   r   r   r   r   r   r   rF   r   rA   r]   model_groupes                             r*   async_log_success_eventz9_PROXY_MaxParallelRequestsHandler.async_log_success_event  s    	
 7X7
 ]	"VW+12B+CJ+O+S+S.,( ""23J?OL#)*:#;J#G#K#K&$  $**:#;J#G#K#K&$  -33C,DZ,P,T,T/-) (.zz&'9$ '(4889PRTU  " ,75//EE#-E	 F    $<<>22:>L#<<>2248L%\\^44T:N ,~Q|nAn=MNNL,6+11>> )+%'#nB~&6oF & !% 9 9 I I--E !J !   )*#$#$  ),G4F,G!,KQ(O#*=#9L#H#*=#9A#= ""5gY>PQ_P`a *002G1QR
 >fEK(+%)>>(,AA4@ $nB{m2n5E_U & !% 9 9 I I--E !J !   )*#$#$  ),G4F,G!,KQ(O#*=#9L#H#*=#9A#= ""5gY>PQ_P`a *002G1QR
 $/ lM:#/#5#5#B#BL ,,B~.>oN & !% 9 9 I I--E !J !   )*#/#$  ),G4F,G!,KQ(O#*=#9L#H#*=#9A#= ""5gY>PQ_P`a *002G1QR
 $/ lM:#/#5#5#B#BL ,,B~.>oN & !% 9 9 I I--E !J !   )*#/#$  ),G4F,G!,KQ(O#*=#9L#H#*=#9A#= ""5gY>PQ_P`a *002G1QR
 (3 lM:#/#5#5#B#BL 00>2B/R & !% 9 9 I I--E !J !   )*#/#$  ),G4F,G!,KQ(O#*=#9L#H#*=#9A#= ""5gY>PQ_P`a *002G1QR++AA4)A B   q8JBBB(
  	"q!!	"s   TCS %S&CS 1S2BS SB*S 6S7B*S !S"B*S SA9S SS TS S S S S S S 	T(S>9T>TTc                 L  K   	 | j                  d       t        |      }|d   j                  di       xs i }|j                  dd       }|j                  dd       }| j                  d|        |y dt        |d	         v ry |Od}	| j                  j                  |	d
|       d {    | j                  j                  |	dd
|       d {    t        j                         j                  d      }
t        j                         j                  d      }t        j                         j                  d      }|
 d| d| }| d| d}| j                  j                  ||       d {   xs dddd}t        |d   dz
  d      |d   |d   d}| j                  d|        | j                  j                  ||d|       d {    y 7 ,7 7 j7 # t        $ r7}t        j                  dj                  t        |                   Y d }~y d }~ww xY ww)Nz(Inside Max Parallel Request Failure Hookr   r   rw   rx   r   zuser_api_key: rj   	exceptionTry   r   r}   r   r   r   r   r   r   r   rM   r   rN   rO   rP   rQ   zupdated_value in failure call: r   )r   r|   z;Inside Parallel Request Limiter: An exception occurred - {})r9   r   r   r[   r-   r   r   r   rd   r   r   async_set_cacher7   r   r   format)r0   r   r   r   r   r|   	_metadatarx   r   r   r   r   r   r   r   rA   r]   r   s                     r*   async_log_failure_eventz9_PROXY_MaxParallelRequestsHandler.async_log_failure_event  s    O	IJ1@ % /044ZDJI+4==.,( %==>L~>?# 4s6+;N7OO 0;9D"77GG $'+5M H    33II  #'1I	 J     (||~66zB'||~66t<!)!8!8!>$0><..AQ!R $nB~&6oF & !% 9 9 I I--E !J !   )*#$#$  ),G4F,G!,KQ(O#*=#9#*=#9 ""%DWI#NO//??)-E	 @   U&   	 **MTTF 	s   H$A0G! 4H$5G! H$%G! +G,&G! GBG! 2G3AG! GG! H$G! G! G! G! !	H!*-HH$H!!H$r   c                 &  K   ddl m} ddlm} ddlm} 	  |||| j                  j                  d|j                  d       d{   }|y|j                         S 7 # t        $ r&} |j                  dt        |             Y d}~yd}~ww xY ww)	a?  
        Helper to get the 'Internal User Object'

        It uses the `get_user_object` function from `litellm.proxy.auth.auth_checks`

        We need this because the UserApiKeyAuth object does not contain the rpm/tpm limits for a User AND there could be a perf impact by additionally reading the UserTable.
        r   r   )get_user_object)prisma_clientFN)r   r   user_api_key_cacheuser_id_upsertrm   proxy_logging_objz3Parallel Request Limiter: Error getting user object)litellm._loggingr   litellm.proxy.auth.auth_checksr   litellm.proxy.proxy_serverr   r-   
dual_cacherm   
model_dumpr7   r3   r[   )r0   r   r:   r   r   r   _user_id_rate_limitsr   s           r*   get_internal_user_objectz:_PROXY_MaxParallelRequestsHandler.get_internal_user_object  s      	:B<	)8+#'#<#<#G#G$!2!C!C"&* $  $+'2244$  	& &&Es1v 		sL   B/A AA BA BA 	B(B	B	BBc                   K   |j                   }t        j                         j                  d      }t        j                         j                  d      }t        j                         j                  d      }| d| d| }| d| d}	| j                  j                  |	|j                         d{   }
d}d}d}d}|
T|j                  |j                  |
d	   z
  }|j                  }|j                  |j                  |
d
   z
  }|j                  }t        |d      rt        |d      }nd}|t        |t              st        |t              r~t        |t              r|j                         }|j                  di       xs i }|||d<   |||d<   |||d<   |||d<   t!        |di |d|i       t"        | I  |||       d{   S yy7  7 	w)z;
        Retrieve the key's remaining rate limits.
        r   r   r   r   r   r   r   NrQ   rP   _hidden_paramsadditional_headerszx-ratelimit-remaining-requestszx-ratelimit-limit-requestszx-ratelimit-remaining-tokenszx-ratelimit-limit-tokens)r   r   rd   r   r-   r   rm   r@   r?   hasattrr   r   r   r&   r   r   setattrsuperasync_post_call_success_hook)r0   r<   r:   responser   r   r   r   r   r   rA   key_remaining_rpm_limitkey_rpm_limitkey_remaining_tpm_limitkey_tpm_limitr   _additional_headers	__class__s                    r*   r   z>_PROXY_MaxParallelRequestsHandler.async_post_call_success_hook;  s@     $++||~..z:||~..t4!006(><..9IJ#*)2n-=_ M++;;)):)K)K <   	 26'+15'+ **6%//'-2HH ( !2 ; ; **6%//'-2HH ( !2 ; ;8-.$X/?@N!N%~y1ZPT5U.)4!/!:!:!<"0"4"45I2"N"TRT&2+ $$DE (DQ#$@A&2+ $$BC (BO#$>? M>M#79LM ='  5 6V &5js%   B2G5G6DGGGGr/   )r#   r$   r%   r   r1   r9   r   r   r&   r[   rZ   r	   r   r   r
   r   r^   floatr\   r   rX   r   r   ru   r   r   r   r   r   __classcell__)r   s   @r*   r,   r,   )   s   9-? 9/)/ / 	/
 /  #/ / / $/  #/ !!;</ $(c3h#8/b
&U 
& 37
"*3-
	
, ,0$
!)#$
  (}$
  (}	$

  (}$
 $,C=$
 #4.$
 
$
Lr)r r 	r
 rh	g"RPd"" *" 
$	"HDD-;D Dr)   r,   )*r   r   r   r   typingr   r   r   r   r	   r
   r   r   fastapir   pydanticr   r4   r   r   r   r   litellm.constantsr   "litellm.integrations.custom_loggerr   'litellm.litellm_core_utils.core_helpersr   litellm.proxy._typesr   r   litellm.proxy.auth.auth_utilsr   r   opentelemetry.tracer   _Spanlitellm.proxy.utilsr   _InternalUsageCacher   r,   r(   r)   r*   <module>r      sr     
 ( W W W !   , 1 F ; U E
 1MD,D.) .V Vr)   