gddlZddlZddlmZddlmZddlmZddlm Z m Z GddejZ Gdd ejZ Gd d eZGd d Zy)N)Optional) BaseModel) print_verbose) DualCache RedisCacheceZdZdZdZy)SchedulerCacheKeyszscheduler:queueN)__name__ __module__ __qualname__queuedefault_in_memory_ttlF/var/www/openai/venv/lib/python3.12/site-packages/litellm/scheduler.pyr r s Err ceZdZdZdZdZy)DefaultPrioritiesrN)r r r HighMediumLowrrrrrs D F Crrc,eZdZUeed<eed<eed<y)FlowItempriority request_id model_nameN)r r r int__annotations__strrrrrrsMOOrrceZdZUeed< ddeedeefdZde fdZ de d e d e d e fd Zde d e d e d e fd ZdZd e d e fdZde d e d dfdZy) SchedulercacheNpolling_interval redis_cachecg|_d}|tjj}t |||_|xsd|_y)z_ polling_interval: float or null - frequency of polling queue. Default is 3ms. N)r&rgQ?)rr rvaluerr$r%)selfr%r&rs r__init__zScheduler.__init__sJ 15  "$6$L$L$R$R !#;P  !1 8DrrequestcK|j|jd{}tj||j|j f|j ||jd{y7V7w)Nr)rr) get_queuerheapqheappushrr save_queue)r)r+rs r add_requestzScheduler.add_request1sknn0B0BnCC uw//1C1CDEooEg6H6HoIII D Js" A=A9AA=3A;4A=;A=idrhealth_deploymentsreturncHK|j|d{}|stdj|tdt |t |dk(rAtd|d||dd|k(r$t j |td |y y y 7w) a/ Return if request can be processed. Returns: - True: * If healthy deployments are available * OR If request at the top of queue - False: * If no healthy deployments available * AND request not at the top of queue r-N-Incorrectly setup. Queue is invalid. Queue={}zlen(health_deployments): rzqueue: z , seeking id=z Popped id: TF)r. Exceptionformatrlenr/heappopr)r3rr4rs rpollzScheduler.poll;snn n;;?FFuM  1#6H2I1JKL ! "a ' GE7-t< =Qx{b  e$ B401- Return a queue for that specific model group N{}:{})key)r$r:r rr(async_get_cache isinstancelist)r)r _cache_keyresponses rr.zScheduler.get_queueus| :: ! (:(@(@(F(F SJ!ZZ77J7GGHz(D'A Hd+zz HsABB7BrcK|jOdjtjj|}|jj ||d{y7w)z; Save the updated queue of the model group NrD)rEr()r$r:r rr(async_set_cache)r)rrrIs rr1zScheduler.save_queuesU :: ! (:(@(@(F(F SJ**,,5,I I I JsAA!AA!)NN)r r r rr rfloatrr*rr2r!rHboolr>r@rBr.r1rrrr#r#s -1,09"5/9j)9$JJ"S"c"t"PT"HSctPT$ # $ drr#)enumr/typingrpydanticrlitellmrlitellm.caching.cachingrrEnumr rrr#rrrrUsO !9   y mmr