g,l'ddlZddlZddlZddlZddlZddlZddlmZddl Z ddl m Z ejjejdZej ddZedk(re ddlmZdZGddeZd efd Zd Zd Ze j4e j6d ddde j6dddde j6dddde j6ddde j6ddde j6d d!dd"e j6d#dd$e j6d%dd"e j6d&dd'e j6d(d)ed*+e j6d,d-d)ed.d/0e j6d1d-d)ed2d30e j6d4d-d)ed56e j6d7ded89e j6d:ded;9e j6dd)d?@e j6dAd)dB@e j6dCdDddEe j6dFdedG9e j6dHd)edI9e j6dJdedK9e j6dLdMd-d)edN6e j6dOd)dPQe j6dRd)dSQe j6dTd-d)dUVe j6dWd-d)dXVe j6dYdZed[9e j6d\d-d)d]Ve j6d^d-d)d_Ve j6d`dedadbce j6dddededfce j6dgd)d-dhidjZe dkk(reyy)lN) load_dotenvzlitellm.secrets LITELLM_MODEDEV)EnumceZdZdZdZy)LiteLLMDatabaseConnectionPool <N)__name__ __module__ __qualname__database_connection_pool_limit database_connection_pool_timeoutL/var/www/openai/venv/lib/python3.12/site-packages/litellm/proxy/proxy_cli.pyrrs%'"')$rrreturncdddlm}|jd||jd|tj|}tj|j }|j |tj|d}tj|j|}|S)Nr)verbose_proxy_loggerzurl: zparams: T)doseq)query) litellm._loggingrdebugurlparseparse_qsrupdate urlencode urlunparse_replace)urlparamsr parsed_url parsed_query encoded_query modified_urls rappend_query_paramsr&s5se}-&23""3'J$$Z%5%56L&&|4@M&&z':':':'OPL rc ddg}ttjd5}tj|||dddy#1swYyxYw#t $r}t d|dYd}~yd}~wwxYw)Nollamaservew)stdoutstderrzd LiteLLM Warning: proxy started with `ollama` model `ollama serve` failed with Exceptionz). Ensure you run `ollama serve` )openosdevnull subprocessPopen Exceptionprint)commandr/es rrun_ollama_server6*sr  W% "**c "g   WWW E# " "   eefdgh     s3AAAA A A A0A++A0cddl}|j|j|j5}|jd|fdk(cdddS#1swYyxYw)Nr localhost)socketAF_INET SOCK_STREAM connect_ex)portr9ss ris_port_in_user?8sA v~~v'9'9 :a||[$/0A5 ; : :s A  Az--hostz0.0.0.0z!Host for the server to listen on.HOST)defaulthelpenvvarz--portzPort to bind the server to.PORTz --num_workerszONumber of uvicorn / gunicorn workers to spin up. By default, 1 uvicorn is used. NUM_WORKERSz --api_basez API base URL.)rArBz --api_versionz2024-07-01-previewz$For azure - pass in the api version.z--modelz-mz)The model name to pass to litellm expectsz--aliaszThe alias for the model - use this to give a litellm model name (e.g. "huggingface/codellama/CodeLlama-7b-Instruct-hf") a more user-friendly name ("codellama")z --add_keyz --headerszheaders for the API callz--saveTzSave the model-specific config)is_flagtyperBz--debugFzTo debug the inputDEBUG)rArHrIrBrCz--detailed_debugzTo view detailed debug logsDETAILED_DEBUGz --use_queuez)To use celery workers for async endpoints)rArHrIrBz --temperaturezSet temperature for the model)rArIrBz --max_tokenszSet max tokens for the modelz--request_timeoutz+Set timeout in seconds for completion callsz --drop_paramszDrop any unmapped params)rHrBz--add_function_to_promptz5If function passed but unsupported, pass it as promptz--configz-cz]Path to the proxy configuration file (e.g. config.yaml). Usage `litellm --config config.yaml`z --max_budgetz`Set max budget for API calls - works for hosted models like OpenAI, TogetherAI, Anthropic, etc.`z --telemetryzZHelps us know if people are using this feature. Turn this off by doing `--telemetry False`z --log_configz&Path to the logging configuration filez --versionz-vzPrint LiteLLM versionz--healthz:Make a chat/completions request to all llms in config.yaml) flag_valuerBz--testz4proxy chat completions url to make a test request toz --test_asyncz9Calls async endpoints /queue/requests and /queue/response)rArHrBz--iam_token_db_authz!Connects to RDS DB with IAM tokenz--num_requestsr z-Number of requests to hit async endpoint withz--run_gunicornzTStarts proxy via gunicorn, instead of uvicorn (better for managing multiple workers)z--run_hypercornz@Starts proxy via hypercorn, instead of uvicorn (supports HTTP/2)z--ssl_keyfile_pathz^Path to the SSL keyfile. Use this when you want to provide SSL certificate when starting proxySSL_KEYFILE_PATH)rArIrBrCz--ssl_certfile_pathz_Path to the SSL certfile. Use this when you want to provide SSL certificate when starting proxySSL_CERTFILE_PATHz--localzfor local debugging)rHrArBc! Rdet}!|r ddlm}"m}#m}$m}%n ddlm}"m}#m}$m}%|dur9tjjd}'tjd|'dy|rd |vr  tddld|dur*ddl}(ddled |d defd })|}*g}+ej!},|(j"j%|* 5}-t'|*D]"}.|+j)|-j+|)$ ddd|(j"j-|+d}/d}0|+D].}1|1j/s|1j1|/dz }/*|0dz }00ej!}2t3d|2|,z t3dt3d|*t3d|/t3d|0y|durSt3ddj5d |d d}3t3t7j8|3j7dy|dur|xsd}4tjd|4ddl}5|dur d |d n||5j=d}6|6j>j@jC|4dddgd !}3tjd"|3t3d#|4|6j>j@jC|4dddgd$}3|3D]}7tjd%|7t3d&|6j@jC|4d'}3t3|3y|rt7jD|}|%did(|d)|d*d+|d,| d-| d.| d/| d0| d1|d2|d3|d4|d5|d6|d7|d8| ddl#}8tHjJd9k(rnddl&}9d;}:d<};i}<|rdd=l(m)}=tIjTd>}>tIjTd?}?tIjTd@}@tIjTdA}AtIjTdB}B|=|>|?|@C}CdD|@d |CdE|>d |?dF|A }D|BrDdGBz }DDtHjVdH<dItHjVdJ<ddKl,m-}EtIjTdLdJtIjTdLdIk(r2E}F|Fj]D]\}G}H|HtHjV|G<|_ ddl/}I|#}JIja|Jjc|N}K|Kj5dOd}L|L&dPLvr"LdPdurddl2}Md|M_3|MjiKj5dQi}<|<i}<|}QtIjTdU}RtIjTdV}StIjTdA}T|Qr'Rr%Sr#Tr!dDRd SdEQdFT}P|PtHjVdH<|._make_openai_completion*s(#0STDzz"EDzQH}}H"5/C%-Jse"4K',yy'=$'7'<'<'>$8:JK-h7F+(2%k]*VHNScRdeJJsO!4a8sAB8%B B=' B88B=) max_workerszElapsed Time: zLoad test Summary:zTotal Requests: zSuccessful Calls: zFailed Calls: Fz) LiteLLM: Health Testing models in configz/health)r )indentrXzE LiteLLM: Making a test ChatCompletions request to your proxy. Model=z My API Key)api_keybase_urlrYz*this is a test request, write a short poemrZ)r]r^ max_tokensz LiteLLM: response from proxy zN LiteLLM: Making a test ChatCompletions + streaming r equest to proxy. Model=)r]r^streamz'LiteLLM: streaming response from proxy z$ making completion request to proxy)r]promptr]aliasrj api_versionrdetailed_debug temperaturertrequest_timeout max_budget telemetry drop_paramsadd_function_to_promptheaderssaveconfig use_queuentzLuvicorn, gunicorn needs to be imported. Run - `pip install 'litellm[proxy]'`dr )generate_iam_auth_token DATABASE_HOST DATABASE_PORT DATABASE_USER DATABASE_NAMEDATABASE_SCHEMA)db_hostdb_portdb_userz postgresql://@/z?schema= DATABASE_URLTrueIAM_TOKEN_DB_AUTH)decrypt_env_var USE_AWS_KMSz?yaml needs to be imported. Run - `pip install 'litellm[proxy]'`)config_file_pathlitellm_settings json_logsgeneral_settingskey_management_systemkey_management_settings database_urlDATABASE_USERNAMEDATABASE_PASSWORDrdatabase_connection_timeoutz os.environ/z../..)get_secret_str) default_value DIRECT_URL) get_secret)connection_limit pool_timeoutprisma)capture_output)check_prisma_schema_diff)should_update_schemadisable_prisma_schema_update)db_url)rdbpushz--accept-data-losszError: )startstopzYUnable to connect to DB. DATABASE_URL found in environment, but prisma package not found.rDii)rRzlitellm.proxy.proxy_server:app)rRhostr=zUsing log_config: log_configz,Using json logs. Setting log_config to None.z/LiteLLM Proxy: Using SSL with certfile: z and keyfile: z ssl_keyfile ssl_certfileuvloop)loopworkersc0eZdZdfd ZdZdZxZS))run_server..StandaloneApplicationc|xsi|_||_t| dd}|dz}t t dt d|dt dt ddy) Nz curl --location 'http://0.0.0.0:z/chat/completions' \a --header 'Content-Type: application/json' \ --data ' { "model": "gpt-3.5-turbo", "messages": [ { "role": "user", "content": "what llm are you" } ] }' zLiteLLM: Test your local proxy with: "litellm --test" This runs an openai.ChatCompletion request to your proxy [In a new terminal tab] z8LiteLLM: Curl Command Test for your local proxy z  z;Docs: https://docs.litellm.ai/docs/simple_proxy z5See all Router/Swagger docs on http://0.0.0.0:)options applicationsuper__init__r3)selfrRr _endpoint_str curl_command __class__r=s rrz2run_server..StandaloneApplication.__init__s#*=bDL'*D$G$&;4&@UV"&  ! GtVWcVddno\RSWRXXbcrcZ|jH|jjDcic]"\}}||jjvr|||$}}}ni}|jD]<\}}|j|jj |j |>ycc}}wN)cfgritemssettingssetlower)rkeyvaluers r load_configz5run_server..StandaloneApplication.load_configsxx+/3ll.@.@.B".B U"dhh&7&77E.StandaloneApplication.loads+++rr)r r r rrr __classcell__)rr=s@rStandaloneApplicationrs$L =,rrz)LiteLLM Proxy: Starting server on z with z workers zuvicorn.workers.UvicornWorker-iXz+%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s)bindr worker_classpreload accesslogtimeoutaccess_log_formatcertfilekeyfile)rRr)r))Configz using Hypercorn r)`locals proxy_serverrPrQrRrS ImportErrorstr importlibmetadataversionclickechor6rk concurrentrlfuturesThreadPoolExecutorrangeappendsubmitwaitdonerbr3rdr_dumpsopenaiOpenAIchat completionscreateloadsuvicornr.namegunicorn.app.baser2 litellm.proxy.auth.rds_iam_tokenrgetenvenviron*litellm.secret_managers.aws_secret_managerrrasynciorun get_configrTr _turn_on_jsoninitialize_secret_manager_key_management_settingsrrrr startswithgetcwdsyspathinsertabspathrchdir isinstancelitellm.secret_managers.mainrr&r0FileNotFoundError litellm.proxy.db.check_migrationrlitellm.proxy.db.prisma_clientr__file__dirnameCalledProcessErrorrerandom randranger?randint_turn_on_debuglitellm.proxy.proxy_serverbaseBaseApplicationhypercorn.asyncior)hypercorn.configrrrr)frr=rjrxr]rwadd_keyrrrryrzrtr{r~rrr|r}testlocal num_workers test_asynciam_token_db_auth num_requestsrhealthr run_gunicorn run_hypercornssl_keyfile_pathssl_certfile_pathrargsrPrQrRrSr5 pkg_versionrrmconcurrent_callsr start_timeexecutor_successful_calls failed_callsfutureend_timerg request_modelrclientchunkrgunicorndb_connection_pool_limitdb_connection_timeoutrrrrrdb_name db_schematoken_db_urlr new_env_varkvr proxy_config_configrrTrrr database_hostdatabase_usernamedatabase_password database_name original_dirrrr!r%is_prisma_runnablerrrdname uvicorn_argsrgunicorn_optionsr)rrkrlsf `` @@r run_serverrF?s B 8D      $$((00;  2;-rBC U"x'7TTF!D6* @(YY[    2 2(3 +,x/FGH-  ( F{{}==?.$)$ A%L  99; x 2345 "$  !1 234 "#3"456 |n-. U :;9974&$w!?9@ djj34 50  TUbTc d   4< av.HH|hG;;**11#K 2    4XJ?@ ]^k]l m ;;**11#K 2  E JJ@H I 56%%,,(T-  h jj)G    $    *  $ " , "   $ $:   ! " # &  ww$( $' "  Pii0Gii0Gii0Gii0G "34I+'E &gYaway'!G9UGXi[11)0BJJ~ &.4BJJ* + O IImT * 6 -(F2*+K#))+1 ! 1 ,     '=Lkk,"9"96"9"RSG '{{+=tD  ,#33$[1T9$(!%%'&{{+=rB '#% (8(<(<+T)%667LM&6&:&:)4' #'23H4-40,//EL# " / : $&II.A$B!$&II.A$B! " / : "))%&33D2EQGXFYYZ[hZiijkxjy#zL1=BJJ~.'7';';0-LLRR( $%5$8$8--NNTT% ! 7 7 F!yy{ rwww/2-l$O &'J|S,I-9 >* IInd + 7yyt,8 +C99^T2>-E(=F$.nD#QL#6|V#LL1=BJJ~.99\40<-E(=F$&99\#:L#6|V#LL/;BJJ|,z$?%)""UO)(,,-KL -D9"1X(*yy{ "$''//("; " 8 3&NN N"HH\2'&*o 4>$.D T !  " " $ 34  ! &zl3 4)3L &    @ A)-L & 5 ]e%; ,1A1MHIZH[[ijzi{|EF/? ]+/@ ^, GKK  #  T !7 , (9(9(I(I7 ,r >tfAdV6R]Q^^op  &$(& ? %R  !,1A1MHIZH[[ijzi{|EF0A ,.> + !!1 ce d "  / / >tfAdVKde XF"V1TF+,FK ,1A1MHIZH[[ijzi{|EF#4!1 KKc6* +)#S 3q6) @  Z ^  l !U T% +%*" +8 *<<J'!GA3-0 JJv'7'7aa'HII JJHH\2BHH\2sx j 1k/k9k5B3l "l k 'kk kk25l  llm428m/*n/m44nn&__main__)!rr_r.rr0r urllib.parseparserrdotenvrrrrconfig_filenamer litellm_modeenumrr}rrr&r6r?r4optionboolfloatintrFr rrrrRs     #ryy/ 5M  *D*    6  i&IRXh+HQWX  Z   lD?  /  tT(S  k $Ok4.HI h46VW         &     4 T4SDs1O  6  ot2LM   @   h     k    e    1      E   ?    D     ,    8     _     K    i    j  iu;PQp,R            N XJ  @ Y|p,f zLr