
    gY                         d dl Z d dlmZmZ d dlmZmZmZmZmZm	Z	 d dl
Z
d dl
mZmZmZ d dlmZ d dlmZ d dlmZ d dlmZ er	d d	lmZ eZneZ G d
 de      Z G d de      Zy)    N)datetime	timedelta)TYPE_CHECKINGAnyDictListOptionalUnion)ModelResponsetoken_counterverbose_logger)	DualCache)CustomLogger)!_get_parent_otel_span_from_kwargs)LiteLLMPydanticObjectBase)Spanc                   8    e Zd ZU dZeed<   dZeed<   dZeed<   y)RoutingArgsi  ttlr   lowest_latency_buffer
   max_latency_list_sizeN)	__name__
__module____qualname__r   float__annotations__r   r   int     [/var/www/openai/venv/lib/python3.12/site-packages/litellm/router_strategy/lowest_latency.pyr   r      s"    C#$5$!#3#r    r   c                   ^   e Zd ZU dZeed<   dZeed<   dZeed<   i fde	de
defd	Zd
 Zd Zd Z	 	 	 	 ddede
deeeeef         deeeef      dee   dee   fdZ	 	 	 ddede
deeeeef         deeeef      dee   f
dZ	 	 	 ddede
deeeeef         deeeef      dee   f
dZy)LowestLatencyLoggingHandlerF	test_flagr   logged_successlogged_failurerouter_cache
model_listrouting_argsc                 @    || _         || _        t        di || _        y )Nr   )r'   r(   r   r)   )selfr'   r(   r)   s       r!   __init__z$LowestLatencyLoggingHandler.__init__!   s#     )$'7,7r    c                    	 	 |d   j                  d      y |d   d   j                  dd       }|d   j                  di       j                  dd       }||y t        |t              rt        |      }	 | d}t	        j
                         j                  d      }t	        j
                         j                  d      }	t	        j
                         j                  d	      }
| d
|	 d
|
 }||z
  }d }|j                  dd       |d   du r|j                  d|      |z
  }|}d }d}t        |t              rat        |dd       }|R|j                  }|j                  }t        |j                         |z        }|t        |j                         |z        }t        |      }| j                  j                  ||      xs i }||vri ||<   t!        ||   j                  dg             | j"                  j$                  k  r%||   j'                  dg       j)                  |       n,||   d   d | j"                  j$                  dz
   |gz   ||   d<   |t!        ||   j                  dg             | j"                  j$                  k  r%||   j'                  dg       j)                  |       n,||   d   d | j"                  j$                  dz
   |gz   ||   d<   |||   vri ||   |<   ||   |   j                  dd      |z   ||   |   d<   ||   |   j                  dd      dz   ||   |   d<   | j                  j+                  ||| j"                  j,                         | j.                  r| xj0                  dz  c_        y y # t2        $ r7}t5        j6                  dj9                  t        |                   Y d }~y d }~ww xY w)Nlitellm_paramsmetadatamodel_group
model_infoid_map%Y-%m-%d%H%M-streamTcompletion_start_timer   usagekeyparent_otel_spanlatency   time_to_first_tokentpmrpmr<   valuer   `litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {})get
isinstancer   strr   nowstrftimer   getattrcompletion_tokenstotal_tokensr   total_secondsr   r'   	get_cachelenr)   r   
setdefaultappend	set_cacher   r$   r%   	Exceptionr   	exceptionformat)r+   kwargsresponse_obj
start_timeend_timer0   r2   latency_keycurrent_datecurrent_hourcurrent_minuteprecise_minuteresponse_ms!time_to_first_token_response_timefinal_valuer@   rM   _usagerL   r=   request_count_dictes                         r!   log_success_eventz-LowestLatencyLoggingHandler.log_success_event(   ss   F	 &'++J7?$%56zBFF!4 ,-11,CGGdS&"*C(RB
	 "-T2'||~66zB'||~66t<!)!8!8!>$0><..AQ!R)1J)>IM1::h-9fX>NRV>V 

#:HE
R 6 8C7;# lM:$\7DAF),2,D,D)'-':':&+'557:KK' =H27 A O O Q"3!43/ $EV#L %%//':J 0   	 # //-/&r* *2.229bAB''==> 'r*55iDKKKX8J28N!9C))??!C9EHS}9U&r*95
 '2.r2667LbQR++AAB +2.9912 &!45 /r23HIId//EEI 0 +2.1 ");B)??=?&r*>: 'r*>:>>uaH<W #2&~6u= 'r*>:>>uaH1L #2&~6u= !!++#+=4CTCTCXCX , 
 >>''1,' " 	$$ryyF
 	s$   N	 AN	 L-N	 		O	-OO	c                   K   	 |j                  dd      }t        |t        j                        rj|d   j                  d      y|d   d   j                  dd      }|d   j                  di       j                  dd      }||yt        |t              rt        |      }	 | d}| j                  j                  |	       d{   xs i }	||	vri |	|<   t        |	|   j                  d
g             | j                  j                  k  r%|	|   j                  d
g       j                  d       n,|	|   d
   d| j                  j                  dz
   dgz   |	|   d
<   | j                  j                  ||	| j                  j                         d{    yy7 7 # t        $ r7}
t!        j"                  dj%                  t        |
                   Y d}
~
yd}
~
ww xY ww)zR
        Check if Timeout Error, if timeout set deployment latency -> 100
        rU   Nr.   r/   r0   r1   r2   r3   r<   r>   g     @@r?   rC   rE   )rF   rG   litellmTimeoutr   rH   r'   async_get_cacherP   r)   r   rQ   rR   async_set_cacher   rT   r   rU   rV   )r+   rW   rX   rY   rZ   
_exceptionr0   r2   r[   rd   re   s              r!   async_log_failure_eventz3LowestLatencyLoggingHandler.async_log_failure_event   s    >	K6J*goo6*+//
;C"()9"::"F"J"J%t#K   0155lBGKKDRVWB"*bj#B, W
	 &1M"6K"//??K?PPVTV ' !3313*2. .r266y"EF++AAB +2.99)RHOOPVW<Nr<R%=GD--CCaG=ILR8=T*2.y9 ++;;'0 --11 <    1 Q"  	$$ryyF
 	sn   G#AF  G#AF  G#A F  F	CF  FF  G#F  F   	G )-GG#G  G#c                   K   	 	 |d   j                  d      y |d   d   j                  dd       }|d   j                  di       j                  dd       }||y t        |t              rt        |      }	 | d}t	        j
                         j                  d      }t	        j
                         j                  d      }	t	        j
                         j                  d	      }
| d
|	 d
|
 }||z
  }d }|j                  dd       |d   du r|j                  d|      |z
  }|}d}d }t        |t              rat        |dd       }|R|j                  }|j                  }t        |j                         |z        }|t        |j                         |z        }t        |      }| j                  j                  ||d       d {   xs i }||vri ||<   t!        ||   j                  dg             | j"                  j$                  k  r%||   j'                  dg       j)                  |       n,||   d   d | j"                  j$                  dz
   |gz   ||   d<   |t!        ||   j                  dg             | j"                  j$                  k  r%||   j'                  dg       j)                  |       n,||   d   d | j"                  j$                  dz
   |gz   ||   d<   |||   vri ||   |<   ||   |   j                  dd      |z   ||   |   d<   ||   |   j                  dd      dz   ||   |   d<   | j                  j+                  ||| j"                  j,                         d {    | j.                  r| xj0                  dz  c_        y y 7 7 *# t2        $ r7}t5        j6                  dj9                  t        |                   Y d }~y d }~ww xY ww)Nr.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   Tr9   r   r:   )r<   r=   
local_onlyr>   r?   r@   rA   rB   rC   z\litellm.router_strategy.lowest_latency.py::async_log_success_event(): Exception occured - {})rF   rG   r   rH   r   rI   rJ   r   rK   rL   rM   r   rN   r   r'   rk   rP   r)   r   rQ   rR   rl   r   r$   r%   rT   r   rU   rV   )r+   rW   rX   rY   rZ   r0   r2   r[   r\   r]   r^   r_   r`   ra   rb   rM   r@   rc   rL   r=   rd   re   s                         r!   async_log_success_eventz3LowestLatencyLoggingHandler.async_log_success_event   s    G	 &'++J7?$%56zBFF!4 ,-11,CGGdS&"*C(RB

 "-T2'||~66zB'||~66t<!)!8!8!>$0><..AQ!R)1J)>IM1::h-9fX>NRV>V 

#:HE
R 6 8C 7;#lM:$\7DAF),2,D,D)'-':':&+'557:KK' =H27 A O O Q"3!43/ $EV#L ++;;')9#' <   
  # //-/&r* *2.229bAB''==> 'r*55iDKKKX8J28N!9C))??!C9EHS}9U&r*95
 '2.r2667LbQR++AAB +2.9912 &!45 /r23HIId//EEI 0 +2.1 ");B)??=?&r*>: 'r*>:>>uaH<W #2&~6u= 'r*>:>>uaH1L #2&~6u= ''77#+=4CTCTCXCX 8   
 >>''1,' "uj  	$$nuuF
 	sk   O$N! O$AN! O$E,N! N	F+N! 4N5%N! O$N! N! !	O!*-OO$O!!O$Nr0   healthy_deploymentsmessagesinputrequest_kwargsrd   c                    i }t        d      }t        j                         j                  d      }	t        j                         j                  d      }
t        j                         j                  d      }|	 d|
 d| }d}|y|}|D ]   }|d   d   |vsd	d
g|d
d
di||d   d   <   " 	 t	        ||      }|j                         }t        j                  t        |      t        |            }t        |      }g }|j                         D ]  \  }}d}|D ]  }||d   d   k(  s|} |!|j                  dd      xsS |j                  di       j                  dd      xs/ |j                  di       j                  dd      xs t        d      }|j                  dd      xsS |j                  di       j                  dd      xs/ |j                  di       j                  dd      xs t        d      }|j                  d	g       }|j                  dg       }|j                  |i       j                  dd
      }|j                  |i       j                  dd
      }d}|E|j                  dd      3|d   du r,t        |      d
kD  r|D ]  }t        |t               s||z  } n|D ]  }t        |t               s||z  } |t        |      z  }|j                  di       j                  dd      } | ||| <   ||z   |kD  s|dz   |kD  r|j                  ||f        t        |      d
k(  ryt        |d       }!|!d
   d   }| j                   j"                  |z  }"|!D #cg c]  }#|#d   ||"z   k  s|# }$}#t        j$                  |$      }%|%d
   }|d|v r||d   d<   |S # t
        $ r d
}Y w xY wc c}#w )z>Common logic for both sync and async get_available_deploymentsinfr4   r5   r6   r7   Nr1   r2   r>   r   )rA   rB   )rs   textrA   r.   rB   r@   g        r8   Tapi_base r?   c                     | d   S )Nr?   r   )xs    r!   <lambda>zHLowestLatencyLoggingHandler._get_available_deployments.<locals>.<lambda>  s    1r    rh   r/   _latency_per_deployment)r   r   rI   rJ   r   rT   itemsrandomsamplelistrP   dictrF   rG   rR   sortedr)   r   choice)&r+   r0   rr   rs   rt   ru   rd   r~   lowest_latencyr\   r]   r^   r_   
deploymentall_deploymentsdinput_tokens_items_all_deploymentspotential_deploymentsitemitem_map_deploymentm_deployment_tpm_deployment_rpmitem_latencyitem_ttft_latencyitem_rpmitem_tpmtotal_call_latency_deployment_api_basesorted_deploymentsbufferr|   valid_deploymentsrandom_valid_deployments&                                         r!   _get_available_deploymentsz6LowestLatencyLoggingHandler._get_available_deployments  sN    #%u||~..z:||~..t4!006(><..9IJ
%,$At$O;s"Aa$8:, 56 %	((GL
 !&&(!==fs6{C/0 !#-335ND(K(1\?400"#K ) " t,  ??#3R8<<UDI ??<488E  <	  t,  ??#3R8<<UDI ??<488E  <	  $<<	26L (-BB G||NB7;;E1EH||NB7;;E1EH E*"&&x6B"8,4)*Q.%6M!-7. &7 &2M!-7. &2 !3|#44L $/??3CR#H#L#LB$  $/@L'(<= <'/9a</1%,,k<-HIC 6F $%* $$9~N ,A.q1 ""88>I *
)!QqT^f5L-LA) 	 

 #)--0A"B,Q/
%**F ( :&) M  	L	t
s   .N/ .O?O/N>=N>c                    K   | d}t        |      }| j                  j                  ||       d {   xs i }| j                  ||||||      S 7 w)Nr3   r;   )r   r'   rk   r   	r+   r0   rr   rs   rt   ru   r[   r=   rd   s	            r!   async_get_available_deploymentsz;LowestLatencyLoggingHandler.async_get_available_deployments  s      %T*+L,
 ##332B 4    	 	 ..
 	
s   1AAAc                     | d}t        |      }| j                  j                  ||      xs i }| j                  ||||||      S )z>
        Returns a deployment with the lowest latency
        r3   r;   )r   r'   rO   r   r   s	            r!   get_available_deploymentsz5LowestLatencyLoggingHandler.get_available_deployments/  su     %T*+L,
 ''2B (   	 	 ..
 	
r    )NNNN)NNN)r   r   r   r$   boolr   r%   r   r&   r   r   r   r,   rf   rn   rq   rH   r	   r   r   r
   r   r   r   r   r    r!   r#   r#      s   ItNCNC OQ8%8378GK8IVBHJ` 48,0)--1LL "L 4S#X/0	L
 c4i()L !L %TNLd 48,0)-

 "
 4S#X/0	

 c4i()
 !
D 48,0)-

 "
 4S#X/0	

 c4i()
 !
r    r#   )r   r   r   typingr   r   r   r   r	   r
   ri   r   r   r   litellm.caching.cachingr   "litellm.integrations.custom_loggerr   'litellm.litellm_core_utils.core_helpersr   litellm.types.utilsr   opentelemetry.tracer   _Spanr   r#   r   r    r!   <module>r      sS     ( B B  @ @ - ; U 91DD$+ $r
, r
r    