g"wLdZddlZddlZddlZddlmZddlmZmZm Z m Z ddl Z ddl Z ddl mZmZmZddlmZddlmZddlmZGd d eZGd d ZGd dZGddeeZGddZGddZGddeeZGddeeZGddeZGddeeZ GddeeZ!dZ" d0d e e#d!e e#d"e e#d#e e#d$e e#d%e e#d&e e#d'e e#d(e e$d)e e e%e jLffd*Z'Gd+d,Z(d-e#d.e#fd/Z)y)1zF Common utilities used across bedrock chat/embedding/image generation N)Enum)AnyListOptionalUnion) BaseConfigBaseLLMExceptionLiteLLMLoggingObj) get_secret)AllMessageValues) ModelResponsec eZdZy) BedrockErrorN)__name__ __module__ __qualname__V/var/www/openai/venv/lib/python3.12/site-packages/litellm/llms/bedrock/common_utils.pyrrsrrcReZdZdZdefdZdedefdZdeefdZ deefdZ y ) AmazonBedrockGlobalConfigcyNrselfs r__init__z"AmazonBedrockGlobalConfig.__init__s rreturnc ddiS)zS Mapping of common auth params across bedrock/vertex/azure/watsonx region_nameaws_region_namerrs rget_mapped_special_auth_paramsz8AmazonBedrockGlobalConfig.get_mapped_special_auth_params s011rnon_default_paramsoptional_paramscp|j}|jD]\}}||vs ||||<|Sr)r!items)rr"r# mapped_paramsparamvalues rmap_special_auth_paramsz1AmazonBedrockGlobalConfig.map_special_auth_params&sF;;= .446LE5 %8= e 457rc gdS)D Source: https://www.aws-services.info/bedrock.html )z eu-west-1z eu-west-3z eu-central-1rrs rget_eu_regionsz(AmazonBedrockGlobalConfig.get_eu_regions-s   rc gdS)r+)z us-east-2z us-east-1z us-west-2z us-gov-west-1rrs rget_us_regionsz(AmazonBedrockGlobalConfig.get_us_regions7s   rN) rrrrdictr!r)rstrr,r.rrrrrsG 22 $QU S    S  rrc eZdZdZdededeeejfde fdZ dede e d ed ededef d Z dded ejdededede e d ed ededeedeedefdZ ddedede e d edeedeedefdZy )AmazonInvokeMixinzG Base class for bedrock models going through invoke_handler.py error_message status_codeheadersrct|||S)N)messager4r5)r)rr3r4r5s rget_error_classz!AmazonInvokeMixin.get_error_classHs!#  rmodelmessagesr#litellm_paramsctd)NzGtransform_request not implemented for config. Done in invoke_handler.pyNotImplementedError)rr9r:r#r;r5s rtransform_requestz#AmazonInvokeMixin.transform_requestQs" U  rN raw_responsemodel_response logging_obj request_dataencodingapi_key json_modec td)NzHtransform_response not implemented for config. Done in invoke_handler.pyr=) rr9r@rArBrCr:r#r;rDrErFs rtransform_responsez$AmazonInvokeMixin.transform_response]s" V  rapi_basectd)NzJvalidate_environment not implemented for config. Done in invoke_handler.pyr=)rr5r9r:r#rErIs rvalidate_environmentz&AmazonInvokeMixin.validate_environmentos" X  rNN)rrr__doc__r0intrr/httpxHeadersr r8rr r?Responser r rrboolrHrKrrrr2r2Css   /2 =B4CV=W      '(           ."&$(  nn &  '    '(    # D>   0"&"&      '(    #  3-     rr2c eZdZUdZdZeeed<dZee ed<dZ ee ed<dZ eeed< ddeedee dee deeddf dZ ed Zd ed ed ed eeeeffdZd edeefdZdeded ededef dZy)AmazonTitanConfigaw Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=titan-text-express-v1 Supported Params for the Amazon Titan models: - `maxTokenCount` (integer) max tokens, - `stopSequences` (string[]) list of stop sequence strings - `temperature` (float) temperature for model, - `topP` (int) top p for model N maxTokenCount stopSequences temperaturetopPrct}|jD]%\}}|dk7s |t|j||'yNrlocalsr%setattr __class__)rrUrVrWrXlocals_keyr(s rrzAmazonTitanConfig.__init__s;(!--/JCf}!2U3*rc |jjDcic]`\}}|jdsJ|jds9t|tj tj ttfs|||bc}}Scc}}wN___abc __dict__r% startswith isinstancetypes FunctionTypeBuiltinFunctionType classmethod staticmethodclskvs r get_configzAmazonTitanConfig.get_config **, ,1<<%LL(&&--   qD,   A%Bsupported_paramsproviderr9stopcd}d|vrZtjrJ|dk(rEd|vrAg}t|tr/|D]*}t j d|s|j |,|||d<|S)z filter params to fit the required provider format, drop those that don't fit if user sets `litellm.drop_params = True`. Nrwbedrockamazonz ^(\|+|User:)$)litellm drop_paramsrhlistrematchappend)rrurvr9rw filtered_stopss r_map_and_modify_argz%AmazonTitanConfig._map_and_modify_argsx % %'*=*=9$U): " dD)!88$4a8)003"  $'4 V $rc gdS)N) max_tokensmax_completion_tokensrwrWtop_pstreamrrr9s rget_supported_openai_paramsz-AmazonTitanConfig.get_supported_openai_paramss   rr"r#r|c|jD]W\}}|dk(s|dk(r||d<|dk(r||d<|dk(r|jd|id||}|d|d<|d k(r||d <|d k(sS||d <Y|S) NrrrUrWrwry)rvr9rwrVrrXr)r%r)rr"r#r9r|rprqrs rmap_openai_paramsz#AmazonTitanConfig.map_openai_paramss',,.DAqL A)@$@340M!12 .F{ $ 8 8QK)5q!9! 4A3H0G|*+'H},-)/r)NNNN)rrrrMrUrrN__annotations__rVr}rWfloatrXrrlrrr/r0rrrrrRrrrrrTrT}s, $(M8C='$(M8D>(#'K%'D(3-(,(,'+" 4} 4 ~ 4e_ 4 sm 4  4  $     DIsN#  .  c       rrTceZdZUdZdZeeed<dZee ed<dZ ee ed<dZ ee ed<dZ ee ed <dZeeed <dZeee ed < ddeedee d dfd ZedZdZdedefdZy)AmazonAnthropicClaude3Configa~ Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=claude https://docs.anthropic.com/claude/docs/models-overview#model-comparison Supported Params for the Amazon / Anthropic Claude 3 models: - `max_tokens` Required (integer) max tokens. Default is 4096 - `anthropic_version` Required (string) version of anthropic for bedrock - e.g. "bedrock-2023-05-31" - `system` Optional (string) the system prompt, conversion from openai format to this is handled in factory.py - `temperature` Optional (float) The amount of randomness injected into the response - `top_p` Optional (float) Use nucleus sampling. - `top_k` Optional (int) Only sample from the top K options for each subsequent token - `stop_sequences` Optional (List[str]) Custom text sequences that cause the model to stop generating irzbedrock-2023-05-31anthropic_versionNsystemrWrtop_kstop_sequencesrct}|jD]%\}}|dk7s |t|j||'yrZr[)rrrr_r`r(s rrz%AmazonAnthropicClaude3Config.__init__s; (!--/JCf}!2U3*rc |jjDcic]O\}}|jds9t|tj tj ttfs|||Qc}}Scc}}wNrcrerns rrrz'AmazonAnthropicClaude3Config.get_config y **, ,1<<%&&--   qD,   AA6c gdS)N) rrtools tool_choicerrwrWr extra_headersrrs rrz8AmazonAnthropicClaude3Config.get_supported_openai_paramss   rr"r#c|jD]G\}}|dk(s|dk(r||d<|dk(r||d<|dk(r||d<|dk(r||d<|dk(r||d<|dk(sC||d<I|S) NrrrrrwrrWrr%rr"r#r'r(s rrz.AmazonAnthropicClaude3Config.map_openai_params(s.446LE5 $1H(H05 -+0( ,1)49 01 %16 .+0(7rrL)rrrrMrrrNrrr0rrWrrrrrrrlrrrr/rrrrrrs !%J $';x}; FHSM #'K%'!E8E?!E8C=*.NHT#Y'.%)+/4SM4$C=4  4  "  D4rrceZdZUdZej Zeee d<dZ ee e d<dZ ee e d<dZeee d<dZeee d<dZeee d< ddeedee dee deedeedeed dfd Zed Zd Zd edefdZy)AmazonAnthropicConfiga Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=claude Supported Params for the Amazon / Anthropic models: - `max_tokens_to_sample` (integer) max tokens, - `temperature` (float) model temperature, - `top_k` (integer) top k, - `top_p` (integer) top p, - `stop_sequences` (string[]) list of stop sequences - e.g. ["\n\nHuman:"], - `anthropic_version` (string) version of anthropic for bedrock - e.g. "bedrock-2023-05-31" max_tokens_to_sampleNrrWrrrrct}|jD]%\}} |dk7s | t|j|| 'yrZr[) rrrrWrrrr_r`r(s rrzAmazonAnthropicConfig.__init__Ns;(!--/JCf}!2U3*rc |jjDcic]O\}}|jds9t|tj tj ttfs|||Qc}}Scc}}wrrerns rrrz AmazonAnthropicConfig.get_config\rrc gdS)N)rrrWrwrrrrs rrz1AmazonAnthropicConfig.get_supported_openai_paramsns   rr"r#c|jD]B\}}|dk(s|dk(r||d<|dk(r||d<|dk(r||d<|dk(r||d<|dk(s9|d us>||d<D|S) NrrrrWrrwrrTrrs rrz'AmazonAnthropicConfig.map_openai_paramszs.446LE5 $1H(H:? 67 %16 .+0(49 01 Ud],1)7r)NNNNNN)rrrrMr{rrrrNrrr}rWrrrrr0rrlrrrr/rrrrrr9s +2*<*<(3-<%)NHTN)#'K%'E8C=E8C='+x}+/3)-'+##+/ 4&sm 4! 4e_ 4 } 4 } 4$C= 4  4  "   D 4 rrc eZdZUdZdZeeed<dZee ed<dZ ee ed< ddeedee dee ddfdZ e dZd e dee fd Zd ed ed e d edef dZy)AmazonCohereConfiga* Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=command Supported Params for the Amazon / Cohere models: - `max_tokens` (integer) max tokens, - `temperature` (float) model temperature, - `return_likelihood` (string) n/a NrrWreturn_likelihoodrct}|jD]%\}}|dk7s |t|j||'yrZr[)rrrWrr_r`r(s rrzAmazonCohereConfig.__init__; (!--/JCf}!2U3*rc |jjDcic]`\}}|jdsJ|jds9t|tj tj ttfs|||bc}}Scc}}wrbrerns rrrzAmazonCohereConfig.get_configrsrtr9c gdS)N)rrWrrrs rrz.AmazonCohereConfig.get_supported_openai_paramss   rr"r#r|ct|jD]$\}}|dk(r||d<|dk(r||d<|dk(s ||d<&|S)NrrWrrrr"r#r9r|rprqs rrz$AmazonCohereConfig.map_openai_paramssX',,.DAqH},-)M!12 .L 01 - /rNNN)rrrrMrrrNrrWrrr0rrlrrrrr/rRrrrrrrs!%J $#'K%''+x}+%)'++/ 4SM 4e_ 4$C= 4  4  $  c       rrc<eZdZUdZdZeeed<dZee ed<dZ ee ed<dZ ee ed<dZ eeed<dZeeed<dZeeed < ddeedee dee dee deedeed eed dfd Zed Zd ed efdZdeded eded ef dZy)AmazonAI21ConfigaK Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=j2-ultra Supported Params for the Amazon / AI21 models: - `maxTokens` (int32): The maximum number of tokens to generate per result. Optional, default is 16. If no `stopSequences` are given, generation stops after producing `maxTokens`. - `temperature` (float): Modifies the distribution from which tokens are sampled. Optional, default is 0.7. A value of 0 essentially disables sampling and results in greedy decoding. - `topP` (float): Used for sampling tokens from the corresponding top percentile of probability mass. Optional, default is 1. For instance, a value of 0.9 considers only tokens comprising the top 90% probability mass. - `stopSequences` (array of strings): Stops decoding if any of the input strings is generated. Optional. - `frequencyPenalty` (object): Placeholder for frequency penalty object. - `presencePenalty` (object): Placeholder for presence penalty object. - `countPenalty` (object): Placeholder for count penalty object. N maxTokensrWrXrVfrequencePenaltypresencePenalty countPenaltyrct}|jD]%\} } | dk7s | t|j| | 'yrZr[) rrrWrXrVrrrr_r`r(s rrzAmazonAI21Config.__init__s;(!--/JCf}!2U3*rc |jjDcic]`\}}|jdsJ|jds9t|tj tj ttfs|||bc}}Scc}}wrbrerns rrrzAmazonAI21Config.get_configrsrtr9c gdSN)rrWrrrrs rrz,AmazonAI21Config.get_supported_openai_params    rr"r#r|c|jD].\}}|dk(r||d<|dk(r||d<|dk(r||d<|dk(s*||d<0|S)NrrrWrrXrrrs rrz"AmazonAI21Config.map_openai_paramssi',,.DAqL /0 ,M!12 .G|*+'H},-)/r)NNNNNNN)rrrrMrrrNrrWrrXrVr}rr/rrrrlrrr0rrrRrrrrrrsN( $Ix}##'K%' D(5/ $(M8D>('+htn+&*OXd^*#'L(4.'$('+ $(,+/*.'+ 4C= 4e_ 4uo 4  ~ 4 #4. 4"$ 4tn 4  4  $         rrceZdZdZdZy)AnthropicConstantsz Human: z Assistant: N)rrr HUMAN_PROMPT AI_PROMPTrrrrr(s  L!Irrc eZdZUdZdZeeed<dZee ed<dZ ee ed< ddeedee deeddfdZ e d Z d edefd Zd ed ed ededef dZy)AmazonLlamaConfigaB Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=meta.llama2-13b-chat-v1 Supported Params for the Amazon / Meta Llama models: - `max_gen_len` (integer) max tokens, - `temperature` (float) temperature for model, - `top_p` (float) top p for model N max_gen_lenrWrXrUrct}|jD]%\}}|dk7s |t|j||'yrZr[)rrUrWrXr_r`r(s rrzAmazonLlamaConfig.__init__<rrc |jjDcic]`\}}|jdsJ|jds9t|tj tj ttfs|||bc}}Scc}}wrbrerns rrrzAmazonLlamaConfig.get_configGrsrtr9c gdSrrrs rrz-AmazonLlamaConfig.get_supported_openai_paramsZrrr"r#r|c|jD].\}}|dk(r||d<|dk(r||d<|dk(r||d<|dk(s*||d<0|S)NrrrWrrrrs rrz#AmazonLlamaConfig.map_openai_paramsbsi',,.DAqL 12 .M!12 .G|+,(H},-)/rr)rrrrMrrrNrrWrrXrrlrrr0rrr/rRrrrrrr-s"&K#%#'K%' D(5/ (,'+" 4} 4e_ 4sm 4  4  $         rrc eZdZUdZdZeeed<dZee ed<dZ ee ed<dZ ee ed<dZ ee eed< ddeedee deedee dee eddf d Zed Zd ede efd Zd eded ededef dZy)AmazonMistralConfiga Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-mistral.html Supported Params for the Amazon / Mistral models: - `max_tokens` (integer) max tokens, - `temperature` (float) temperature for model, - `top_p` (float) top p for model - `stop` [string] A list of stop sequences that if generated by the model, stops the model from generating further output. - `top_k` (float) top k for model NrrWrrrwrct}|jD]%\}}|dk7s |t|j||'yrZr[) rrrWrrrwr_r`r(s rrzAmazonMistralConfig.__init__s;(!--/JCf}!2U3*rc |jjDcic]`\}}|jdsJ|jds9t|tj tj ttfs|||bc}}Scc}}wrbrerns rrrzAmazonMistralConfig.get_configrsrtr9c gdS)N)rrWrrwrrrs rrz/AmazonMistralConfig.get_supported_openai_paramssGGrr"r#r|c|jD]8\}}|dk(r||d<|dk(r||d<|dk(r||d<|dk(r||d<|dk(s4||d<:|S)NrrWrrwrrrs rrz%AmazonMistralConfig.map_openai_paramssz',,.DAqL 01 -M!12 .G|+,(F{*+'H},-)/r)NNNNN)rrrrMrrrNrrWrrrrwrr0rrlrrrr/rRrrrrrrus  !%J $#'K%'!E8E?!!E8E?! $D(49 $%)'+#!%$( 4SM 4e_ 4} 4  4 tCy! 4  4  $HHcH      rrcfd}|S)z,Closure to capture the headers and add them.cnjD]!\}}|jj||#y)z.Actual callback function that Boto3 will call.N)r%r5 add_header)requestkwargs header_name header_valuer5s rcallbackz#add_custom_header..callbacks,)0 %K OO & &{L A*9rr)r5rs` radd_custom_headerrsB Oraws_access_key_idaws_secret_access_keyr aws_bedrock_runtime_endpointaws_session_nameaws_profile_name aws_role_nameaws_web_identity_tokenrtimeoutc `tdd} tdd} ||||||||g} t| D](\}}|s |jdst|| |<*| \}}}}}}}}tjdt j }|rn|r|}n| r| }n| r| }n tddtd}|r|}n |r|}nd |d }d dl}t| tr|jj| | }nft| tjr2|jj| j| j }n|jj}|p|n|lt|}| td d|j#d}|j%|||d}|j#d|dd|dd|dd||||}n|U|S|j#d||}|j'||}|j#d|dd|dd|dd||||}nZ||j#d||||||}n?|'|j)|j#d||||}n|j#d||||}| r/|j*j,j/dt1| |S)NAWS_REGION_NAME AWS_REGIONz os.environ/ SSL_VERIFYzRAWS region not set: set AWS_REGION_NAME or AWS_REGION env variable or in .env filei)r7r4AWS_BEDROCK_RUNTIME_ENDPOINTzhttps://bedrock-runtime.z.amazonaws.comr)connect_timeout read_timeoutz6OIDC token could not be retrieved from secret manager.stsi)RoleArnRoleSessionNameWebIdentityTokenDurationSecondszbedrock-runtime Credentials AccessKeyIdSecretAccessKey SessionToken) service_namerraws_session_tokenr endpoint_urlconfigverify)rr)rr)rrrrrrr) profile_name)rrrrrzbefore-sign.bedrock-runtime.*)r enumeratergosgetenvr{ ssl_verifyrboto3rhrsessionConfigrOTimeoutconnectreadclientassume_role_with_web_identity assume_roleSessionmetaeventsregisterr)rrrr rrrrrrrlitellm_aws_region_namestandard_aws_region_nameparams_to_checkir'r env_aws_bedrock_runtime_endpointrrr oidc_token sts_client sts_responsers rinit_bedrock_clientrs)):DA),= $ Oo.5 U%%m4!+E!2OA /  $<););O"P*=9.I#%    "'7'C\\ /"7" "--!3C. **=9-H".}"=>O"P*=9.I#%    &*/"7#%   %,<=DD*#% E *#%   ## +->}-M  Mrc*eZdZdZdZdZdZdZy)ModelResponseIteratorc ||_d|_y)NF)rAis_done)rrAs rrzModelResponseIterator.__init__s, rc|Srrrs r__iter__zModelResponseIterator.__iter__ rcL|jrtd|_|jSNT)r StopIterationrArs r__next__zModelResponseIterator.__next__s" <<  """rc|Srrrs r __aiter__zModelResponseIterator.__aiter__rrcTK|jrtd|_|jSwr)rStopAsyncIterationrArs r __anext__zModelResponseIterator.__anext__s& <<$ $ """s&(N)rrrrrrrrrrrrrs ##rrresponse_tool_namercx|tjjvrtjj|}|S)z If litellm formatted the input tool name, we need to convert it back to the original name. Args: response_tool_name (str): The name of the tool as received from the response. Returns: str: The original name of the tool. )r{bedrock_tool_name_mappings cache_dict)r s rget_bedrock_tool_namer$s:W??JJJ$??JJ   r) NNNNNNNNNNN)*rMrr~rienumrtypingrrrrrOr{)litellm.llms.base_llm.chat.transformationrr r litellm.secret_managers.mainr litellm.types.llms.openair litellm.types.utilsr rrr2rTrrrrrrrrr0r/rrrrr$rrrr+s --  46- # $ $ N7 7 tf):fRPPfMM`B*JBJW(*Wt"" E):EPG+ZGT'++/%)26&*&*#',0$(59s}s$C=sc] s #+3- s sm ssmsC=s%SMsD>seE5==01 2sl##4ccr