
    gfc                         d dl Z d dlmZ d dlmZmZmZmZmZ d dl	Z	d dl
Z
d dlmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZmZmZmZmZ d
dlmZmZ ddlm Z   e        Z!	  G d de      Z"y)    N)deepcopy)AnyCallableListOptionalUnion)verbose_logger)asyncify)
BaseAWSLLM)_get_httpx_clientget_async_httpx_client)AllMessageValues)CustomStreamWrapperEmbeddingResponseModelResponseUsage
get_secret   )AWSEventStreamDecoderSagemakerError   )SagemakerConfigc                   j   e Zd ZdefdZ	 ddededee   dededee   fd	Z	di ddd
i fdede
dededededeeeej                   f      dedefdZ	 ddededefdZdee   dededee   dededee   dededefdZdee   dededee   dededededee   dedefdZi ddfdede
dededef
dZy)SagemakerLLMoptional_paramsc                 p   	 ddl m} |j                  dd       }|j                  dd       }|j                  dd       }|j                  dd       }|j                  dd       }|j                  d	d       }|j                  d
d       }	|j                  dd        |j                  dd       }
|j                  dd       }|Dt	        dd       }|t        |t              r|}t	        dd       }|t        |t              r|}|d}| j                  ||||||	||
|	      }||fS # t        $ r t        d      w xY w)Nr   )Credentials7Missing boto3 to call bedrock. Run 'pip install boto3'.aws_secret_access_keyaws_access_key_idaws_session_tokenaws_region_nameaws_role_nameaws_session_nameaws_profile_nameaws_bedrock_runtime_endpointaws_web_identity_tokenaws_sts_endpointAWS_REGION_NAME
AWS_REGION	us-west-2)	r    r   r!   r"   r$   r%   r#   r'   r(   )botocore.credentialsr   ImportErrorpopr   
isinstancestrget_credentials)selfr   r   r   r    r!   r"   r#   r$   r%   r'   r(   litellm_aws_region_namestandard_aws_region_namecredentialss                  ^/var/www/openai/venv/lib/python3.12/site-packages/litellm/llms/sagemaker/completion/handler.py_load_credentialszSagemakerLLM._load_credentials'   s   	Y8
 !0 3 34KT R+//0CTJ+//0CTJ)--.?F'++OTB*../A4H*../A4H*D	
 "1!4!45Mt!T*../A4H "&01BD&I#&2z'8 #:'1,'E$'3
(#9 #;&"-#'#7#7/"7/+--'#9- $8 
$
 O++_  	YWXX	Ys   D   D5Nmodeldatamessagesr"   extra_headersc                    	 ddl m} ddlm}	  ||d|      }
|j                  d      du r
d| d	| d
}n	d| d	| d}|j                  dd       }||}t        j                  |      j                  d      }t        j                  ||||      } |	d|||      }|
j                  |       |d|v r|d   |j                  d<   |j                         }|S # t        $ r t	        d      w xY w)Nr   )	SigV4Auth)
AWSRequestr   	sagemakerstreamTzhttps://runtime.sagemaker.z.amazonaws.com/endpoints/z/invocations-response-streamz/invocationssagemaker_base_urlutf-8)headersr8   r:   r   POST)methodurlr9   rC   Authorization)botocore.authr=   botocore.awsrequestr>   r-   getjsondumpsencodesagemaker_configvalidate_environmentadd_authrC   prepare)r2   r5   r8   r9   r:   r   r"   r;   r=   r>   sigv4api_baserA   encoded_datarC   requestprepped_requests                    r6   _prepare_requestzSagemakerLLM._prepare_request^   s7   	Y/6 +{ODx(D03O3DD]^c]d  eA  BH3O3DD]^c]ddpqH,001EtL))Hzz$'..w7"77!+	 8 
 xlG
 	w%/]*J/<_/MGOOO,!//+?  	YWXX	Ys   C C-Fmodel_responseprint_verboselitellm_paramstimeoutacompletionrC   c                    | j                  |      \  }}t        |      }|j                  dd       }|j                  dd       }t        j
                  j                         }|j                         D ]  \  }}||vs|||<    |du r9|du r | j                  |||
||||||||||      }|S t        j                  |||||      }| j                  ||||||      }||j                  j                  d|i       t               }|j                  |j                   |j                  ||      }|j"                  d	k7  r.t%        |j"                  t'        |j)                               
      t+        d      }|j-                  |j-                  d            }t/        ||d|      }|j1                  |d|d|i       |S |du r| j3                  |||
||||||||||      S t        j                  |||||      }||||||d}  | j                  d#i | }	 ||j                  j                  d|i       d}	t               }|j5                  g d||j                   |j                  d       	 |j                  |j                   |j                  ||	      }|j"                  d	k7  r!t%        |j"                  |j6                  
      	 t        jA                  |||||||||"	      S # t8        $ r'}!|j1                  g dt'        |!      d|i       |!d }!~!ww xY w# t8        $ r}!t;        j<                  dt'        |!             t?        |!di       j                  di       j                  dd      }"t?        |!di       j                  di       j                  dt'        |!            }#d |#v r|#d!z  }#t%        |"|#
      d }!~!ww xY w)$Nr@   model_idT)r:   r8   custom_prompt_dicthf_model_namer   encodingrX   logging_objr^   r"   r5   rC   rZ   r8   r:   r   rZ   rC   )r8   r9   r:   r   r5   r"   z$X-Amzn-SageMaker-Inference-Component)rF   rC   rK   r@      status_codemessage r8      
chunk_sizer?   completion_streamr8   custom_llm_providerrb   complete_input_dictinputapi_keyoriginal_responseadditional_args)r:   r8   r_   r`   rX   ra   rb   r^   r   r5   r"   rC   rZ   r8   r9   r   r5   r"   r:        r@rp   rS   rC   rr   rs   ru   rF   rC   rK   r[   zSagemaker error %sresponseResponseMetadataHTTPStatusCode  ErrorMessage+Inference Component Name header is requiredJ
 pass in via `litellm.completion(..., model_id={InferenceComponentName})`	r8   raw_responserX   rb   request_datar:   r   ra   rZ    )!r7   r   r.   rJ   litellmr   
get_configitemsasync_streamingrN   transform_requestrW   rC   updater   postrF   rf   r   r0   readr   
iter_bytesr   	post_callasync_completionpre_calltext	Exceptionr	   errorgetattrtransform_response)$r2   r8   r:   rX   rY   ra   rb   r   rZ   r[   r_   r`   	logger_fnr\   rC   r5   r"   inference_paramsr@   r^   configkvr{   r9   prepared_requestsync_handlersync_responsedecoderrn   streaming_response_dataprepared_request_argserf   error_messages$                                       r6   
completionzSagemakerLLM.completion   s   & (,'='=o'N$_#O4!%%h5"&&z48 ((335LLNDAq))&' #	 # T>d"//%'9"/$3%#1 +%$3 +##1 0   '99%$3#1# :  $(#8#8%$3 +$3 $9 $  ' %,,33?J  12 , 1 1(,,,44!	 !2 ! !,,3($1$=$= #M$6$6$8 9 
 0b9$+$6$6!,,,=%! &9&7(3 +	&" !!"4!6 =	 "  &%
 $((!#5+-!'! /' /- )  " !22+) 3 
 .&. !
 1400I3HI9	Q# !((//;XF
 G,.L  +0 0 4 4/77! !  , 1 1(,,,44#	 !2 ! !,,3($1$=$= - 2 2  48  22&)#+) 3 

 
	
/  %%&)!f%:E$B	 &    	Q  !5s1v>:r*',%s+  :r*..w;??	3q6R  =M!nn [-PP	Qs9   AL 2AK* *	L3"LLL 	O&BN>>OrS   c                 z  K   	 |$t        t        j                  j                        }|j	                  |||d       d {   }|j
                  dk7  r!t        |j
                  |j                        t        d      }|j                  |j                  d	            }|S 7 c# t        j                  $ r<}	|	j                  j
                  }
t        |
|	j                  j                        d }	~	wt        j                  $ r t        d
d      t        $ r}t        dt!        |            d }~ww xY ww)Nllm_providerT)rC   rK   r@   rd   re   rh   ri   rj   rk   i  zTimeout error occurred.r~   )r   r   LlmProviders	SAGEMAKERr   rf   r   r   r   aiter_bytesr   httpxHTTPStatusErrorr{   TimeoutExceptionr   r0   )r2   rS   rC   r9   rb   clientr{   r   rn   err
error_coder   s               r6   make_async_callzSagemakerLLM.make_async_callc  s.    &	B~/!(!5!5!?!? $[[	 )  H ##s*$ ( 4 4hmm  ,"5G ' 3 3$$$5! %$#6 $$ 	T11J ZARARSS%% 	U S:STT 	B S#a&AA	BsG   D;>B( B&A"B( %D;&B( (D8;7C22+D8D33D88D;r_   r`   r^   rb   c                 t  K   t         j                  ||i |ddi||       d {   }t        | j                        }||||||d} |di | d {   }| j	                  |j
                  |j                  ||       d {   }t        ||d|      }|j                  g dd	d
|i       |S 7 7 _7 0w)Nr@   Trc   rv   )rS   rC   r9   rb   r?   rm   rh   zfirst stream response receivedrp   rq   r   )	rN   async_transform_requestr
   rW   r   rF   rC   r   r   )r2   r:   r8   r_   r`   r5   r"   r   ra   rX   r^   rb   rZ   rC   r9   asyncified_prepare_requestr   r   rn   r   s                       r6   r   zSagemakerLLM.async_streaming  s      &==??$?) > 
 
 &.d.C.C%D".&. !
 "<!T>S!TT"&"6"6%))$,,#	 #7 #
 
 1/ +#	
 	>2D9	 	 	
 "!M
  U
s3   $B8B2-B8B40B8B6-B84B86B8c                 n  K   d}t        t        j                  j                        }t        j                  |||	||       d {   }t        | j                        }|||	|||d} |di | d {   }|
j                  g d||j                  |j                  d       	 ||j                  j                  d|i       	 |j                  |j                  |j                  ||	       d {   }|j                  d
k7  r!t        |j                  |j                        	 t        j'                  ||||
|||	||	      S 7 7 7 W# t         $ r*}|
j#                  |d   dt%        |      d|i       |d }~ww xY w# t         $ r'}t%        |       }d|v r|dz  }t        d|      d }~ww xY ww)Nrw   r   rc   rv   rh   rx   ry   z#X-Amzn-SageMaker-Inference-Componenrz   rd   re   inputsrp   rq   r   r   r~   r   r   )r   r   r   r   rN   r   r
   rW   r   rF   rC   r   r   rf   r   r   r   r   r0   r   )r2   r:   r8   r_   r`   r5   r"   ra   rX   r   rb   r^   rC   rZ   r[   async_handlerr9   r   r   r   r{   r   r   s                          r6   r   zSagemakerLLM.async_completion  s!      . --77
 &==+) > 
 
 &.d.C.C%D".&. !
 "<!T>S!TT'+,00+33 	 	
!	I# !((//:HE!.!3!3(,,,44#	 "4 "  ''3.($,$8$8(--  /$  22!)#+) 3 

 
	

$ U(  %%x.&)!f%:D$A	 &    	I"1vhM<M!nn S-HH		Isx   AF5E-F54E50F5&F ,E 2E
34E ' F5F5
E 	E?%E::E??F 	F2"F--F22F5rr   c           	         ddl }|j                  dd      }|j                  dd      }|j                  dd      }||j                  d|||      }n&t        d      xs |xs d	}|j                  d|
      }t	        |      }|j                  dd       t
        j                  j                         }|j                         D ]  \  }}||vs|||<    t        j                  d|i      j                  d      }d| d| d}|j                  |d||d       	 |j                  |d|d      }t        j&                  |d   j)                         j+                  d            }|j-                  |d|d |i!        |d"|        d#|vrt%        dd$      |d#   }t/        |t0              st%        d%d&|       g }t3        |      D ]  \  }}|j5                  d#||d'        d(|_        ||_        ||_        d}|D ]  } |t=        |j                  |             z  }! t?        |d)tA        |d|*             |S # t        $ ru}t        |di       j!                  di       j!                  dd      }t        |di       j!                  di       j!                  dt#        |            }t%        ||      d}~ww xY w)+zG
        Supports Huggingface Jumpstart embeddings like GPT-6B
        r   Nr   r    r"   zsagemaker-runtime)service_namer    r   region_namer)   r+   )r   r   r@   text_inputsrB   zE
        response = client.invoke_endpoint(
            EndpointName=z?,
            ContentType="application/json",
            Body=zK, # type: ignore
            CustomAttributes="accept_eula=true",
        )rh   )rp   request_strry   zapplication/jsonzaccept_eula=true)EndpointNameContentTypeBodyCustomAttributesr{   r|   r}   r~   r   r   re   r   utf8rp   rq   zraw model_response: 	embeddingzembedding not found in responsei  z"Response not in expected format - )objectindexr   listusage)prompt_tokenscompletion_tokenstotal_tokens)!boto3r.   r   r   r   r   r   r   r   rK   rL   rM   r   invoke_endpointr   r   rJ   r0   r   loadsr   decoder   r/   r   	enumerateappendr   r9   r8   lensetattrr   )!r2   r8   rr   rX   rY   ra   rb   r   r_   rZ   r   r   r   r    r"   r   r   r   r   r   r   r9   r   r{   r   rf   r   
embeddingsoutput_dataidxr   input_tokensr   s!                                    r6   r   zSagemakerLLM.embedding+  sQ   " 	 !0 3 34KT R+//0CTJ)--.?F( \\0"3&;+	 " F ,- " 
 \\0' " F $O4Xt, ((335LLNDAq))&' #	 # zz=%0188A ! 
	 	48U 	 	
	Q--".!3	 . H" ::hv.335<<VDE&2D9	 	 	
 	,XJ78h& )J  k*
*d+ <ZLI 
 '
3NC&9M 4
 !')$DC 566L  	*"#)	
 s  		Q:r*',%s+  :r*..w;??	3q6R  ![-PP		Qs   H? ?	J=A0J88J=)N)__name__
__module____qualname__dictr7   r0   r   r   r   rW   r   r   r   r   floatr   Timeoutboolr   r   r   r   r   r   r   r       r6   r   r   %   sc   5,5,~ )-, , 	,
 '(, , ,  ~,p :>!U
U
 U
 &	U

  U
 U
 U
 %u}} 456U
 U
 U
z .B.B .B 	.B`6"'(6" 6" !	6"
  }6" 6" &6" 3-6" 6" 6" 6"p^
'(^
 ^
 !	^

  }^
 ^
 &^
 ^
 ^
 3-^
 ^
 ^
R NN N *	N
  N Nr   r   )#rK   copyr   typingr   r   r   r   r   r   r   litellm._loggingr	   #litellm.litellm_core_utils.asyncifyr
   !litellm.llms.bedrock.base_aws_llmr   &litellm.llms.custom_httpx.http_handlerr   r   litellm.types.llms.openair   litellm.utilsr   r   r   r   r   common_utilsr   r   transformationr   rN   r   r   r   r6   <module>r      sY      7 7   + 8 8 7  A +"$ T
: T
r   