ge6 d dl Z d dlmZ d dlmZmZmZmZ d dlZd dl Z d dl
m Z d dl
mZmZ d dlmZmZ d dlmZ d dlmZmZ G d d
e Zd ed ed
ededede f dZ G d d Z y) N)partial)CallableListOptionalUnion)Logging)
custom_promptprompt_factory)AsyncHTTPHandlerget_async_httpx_client) TextChoices)CustomStreamWrapperTextCompletionResponsec b e Zd Z ddeej
deej f fd
Z xZS ) TextCompletionCodestralErrorrequestresponsec || _ || _ ||| _ n t j dd | _ ||| _ n&t j || j | _ t | ! | j y )NPOSTz8https://docs.codestral.com/user-guide/inference/rest_api)methodurl) status_coder ) r messager httpxRequestr Responsesuper__init__)selfr r r r __class__s ^/var/www/openai/venv/lib/python3.12/site-packages/litellm/llms/codestral/completion/handler.pyr z%TextCompletionCodestralError.__init__ su ' "DL == N DL $DM !NN ' DM LL
)NN) __name__
__module__ __qualname__r r r r r
__classcell__r s @r! r r s;
,0-1
%--(
5>>*
r" r clientapi_baseheadersdatamodelmessagesc K | j |||d d { }|j dk7 r!t |j |j |j }|j |d|d|i |S 7 ]w)NTr* r+ stream r r complete_input_dictinputapi_keyoriginal_responseadditional_args)postr r text aiter_lines post_call) r( r) r* r+ r, r- logging_objr completion_streams r! make_callr@ 4 s [[7d[SSHs"* ,,hmm
!,,.+.5 ! Ts A;A9A A;c e Zd Zd fd Zdee dedefdZdefdZd ed
e j d e d e d
e
dededeeef dede fdZdddi fd edededed e dedededeee j& f dedee ef fdZddi fd ededed e dedededeee j& f de fdZdddi fd ededed e dededeee j& f defdZd Z xZS ) CodestralTextCompletionreturnNc " t | y N)r r )r r s r! r z CodestralTextCompletion.__init__Q s
r" r7 user_headersc x | t d ddj | d}|t |t ri ||}|S )NzVMissing CODESTRAL_API_Key - Please add CODESTRAL_API_Key to your environment variableszapplication/jsonz Bearer {})z content-type
Authorization)
ValueErrorformat
isinstancedict)r r7 rF r* s r! _validate_environmentz-CodestralTextCompletion._validate_environmentT sX
? h
/ ( / / 8
#
<(F11L1Gr" generated_textc g d}|D ]k }|j j | r|j |dd }|j | sG|ddd j |ddd dd ddd }m |S )z
Parse the output text to remove any special characters. In our current approach we just check for ChatML tokens.
Initial issue that prompted this - https://github.com/BerriAI/litellm/issues/763
)z
<|assistant|>z
<|system|>z<|user|>zzr3 N)strip
startswithreplaceendswith)r rN chat_template_tokenstokens r!
output_parserz%CodestralTextCompletion.output_parsere s
*E ##%007!/!7!7r1!E &&u-!/"!5!=!=eDbDk2q!QRVTVRV!W *
r" r, r model_responser0 r> optional_paramsr+ r- c f |j | ||j d|i |
d|j |j dk7 r*t t |j |j |j } | j dg }
g }|
D ] }d }d}d }d }|j d i }t j j |j d
|j d |j d |j d
}|j | t j | j d || j d | j d | j d d| j d }|S # t $ r t |j d w xY w)Nr4 r5 z#codestral api: raw model_response: r1 )r r i choicesr r
finish_reasonindexcontentlogprobs)r] r^ r; r` idcreatedr, usageFobject)ra r\ rb r, rc r0 rd )
r= r; r r strjson Exceptiongetlitellmutilsr
appendr )r r, r rY r0 r> rZ r7 r+ r-
print_verboseencodingcompletion_response_original_choices_choiceschoice_finish_reason_index_text _logprobs_choice_message_choice _responses r! process_text_completion_responsez8CodestralTextCompletion.process_text_completion_responsey s &mm 2D 9
;HMM?KL 3 &.HMM* $ 0 0
W"*--/ 0 3 3Ir B&('F "NFE I $jjB 7O mm// $jj 9 jj)$((3 J/ 0 G
OOG $3 (6 22"&&t, '++I6%))'2%))'2&**84
S W.x}}RUV V Ws 1F !F0r) custom_prompt_dictrl timeoutr* c | j || }|
j dd du r|}n|xs d}||v r || }t |d |d |d | }n
t || }t j
j
}|j D ] \ }}||
vs ||
|< |
j d d
}||d |
}|}| j |||||| d
| du rC|du r | j ||||||||| |
|
||| S | j ||||||||| |
d
|
||| S |du rUt j j ||t j | | }t |j! |d| }|S t j j ||t j | }| j# ||||
j% d d
| |
||||| S )Ncustom_endpointTz/https://codestral.mistral.ai/v1/fim/completionsrolesinitial_prompt_valuefinal_prompt_value) role_dictr r r- )r, r- r0 F)r, prompt)r4 r* r) acompletion)r6 r7 r9 )r, r- r+ r) rY rl rm r7 r> rZ litellm_params logger_fnr* r{ )r, r- r+ r) rY rl rm r7 r> rZ r0 r r r* r{ r/ codestral)custom_llm_providerr> )r r* r+ ) r, r rY r0 r> rZ r7 r+ r- rl rm )rM popr r
ri CodestralTextCompletionConfig
get_configitemspre_callasync_streamingasync_completionmodule_level_clientr: rf dumpsr
iter_linesry rh )r r, r- r) rz rY rl rm r7 r> rZ r{ r r r r* completion_urlmodel_prompt_detailsr configkvr0 r+
input_textr rx s r!
completionz"CodestralTextCompletion.completion s $ ,,Wg> 0$ 74 ? %N M M
& e#< ".w7%9:P%Q#78L#M! F $%(CF 66AACLLNDAq (%&" # !$$Xu5
'+" * *
$ ~++ % +#1"/ % # +$3#1 ' # # , $ ,, % +#1"/ % # +$3 #1 ' # # - & T> 2277 ZZ% 8 H ,##%$/ ' I 2277" ZZ% 8 H
44 )"&&x7# + ' 5
r" c
K t t j j d| i } |j ||t j |
d { }| j! |||| |||
||| | S 7 # t j $ rI}t |j j dj |j j d }~wt $ r*}t ddj t | d }~ww xY ww) Nr{ ) llm_providerparams)r* r+ zHTTPStatusError - {}r2 i z{}) r, r rY r0 r> r7 r+ r- rl rZ rm )r ri LlmProvidersTEXT_COMPLETION_CODESTRALr: rf r r HTTPStatusErrorr r r rJ r; rg re ry )r r, r- r) rY rl rm r7 r> r0 r+ rZ r{ r r r*
async_handlerr es r! r z(CodestralTextCompletion.async_completionQ s & / --GG w'
* / /'
40@ 0 H 44 )# ' + 5
$$ . JJ 2 2.55ajjooF
. SV)<
sG (D*A8 A6A8 D6A8 8D AC D%D DDc K d| d< t d t t ||t j | ||| |d| }|S w)NTr0 )r) r* r+ r, r- r> ztext-completion-codestral)r? r@ r, r r> )r r r@ rf r )r r, r- r) rY rl rm r7 r> r+ r{ rZ r r r*
streamwrappers r! r z'CodestralTextCompletion.async_streaming sW " X+ " ! ZZ%! ' ;#
s ?Ac y rE )r argskwargss r! embeddingz!CodestralTextCompletion.embedding s r" )rC N)r# r$ r% r r re rL rM rX r r r boolLiteLLMLoggingr listry r floatTimeoutr r r r r r& r' s @r! rB rB P sK #
" C (GG ..G / G
G $
G G G D#I G G
Gl !M
M
M
M
! M
/
M
M
M
M
uemm+,M
!M
"
%':: ;#M
z !1
1
1
1
/ 1
1
1
1
uemm+,1
"
#1
~ " " " "
/ "
" " uemm+,"
!" H
r" rB ) rf functoolsr typingr r r r r ri *litellm.litellm_core_utils.litellm_loggingr r 3litellm.litellm_core_utils.prompt_templates.factoryr r
&litellm.llms.custom_httpx.http_handlerr r litellm.types.utilsr
litellm.utilsr r rg r re rL r r@ rB r r" r! r s 2 2 P , E
9
8
8Y
Y
r"