
    gW                         d dl Z d dlmZ d dlZd dlmZmZ d dlmZm	Z	 da
 G d de      ZdefdZi ddfded	ed
ededef
dZdi fded	efdZd Zy)    N)Callable)custom_promptprompt_factory)ModelResponseUsagec                        e Zd Z fdZ xZS )	VLLMErrorc                     || _         || _        t        j                  dd      | _        t        j
                  || j                        | _        t        | !  | j                         y )NPOSTzhttp://0.0.0.0:8000)methodurl)status_coderequest)	r   messagehttpxRequestr   Responseresponsesuper__init__)selfr   r   	__class__s      Y/var/www/openai/venv/lib/python3.12/site-packages/litellm/llms/vllm/completion/handler.pyr   zVLLMError.__init__   sO    &}}F8MN;ULL	
    )__name__
__module____qualname__r   __classcell__)r   s   @r   r	   r	      s    
 
r   r	   modelc                     	 ddl m}m} t        	 ||       at        |fS # t        $ r}t        dt        |            d }~ww xY w)Nr   )LLMSamplingParamsr   r   r   )vllmr!   r"   llm	Exceptionr	   str)r   r!   r"   es       r   validate_environmentr*      sE    7,;E"CN"" 7As1v667s   ! 	AA  Amessagesmodel_responseprint_verboseoptional_paramsc
                 J   	 t        |       \  a}
 |
di |}| |v r||    }t        |d   |d   |d   |      }nt        | |      }|j                  |d	d
|i       t        rt        j                  ||      }nt        dd      d|v r|d   du rt        |      S |j                  |d	|d
|i        |d|        |d   j                  d   j                  |j                  d   j                  _        t!        |d   j"                        }t!        |d   j                  d   j$                        }t'        t)        j(                               |_        | |_        t/        ||||z         }t1        |d|       |S # t        $ r}t        dt	        |            d }~ww xY w)Nr#   r   r$   rolesinitial_prompt_valuefinal_prompt_value	role_dictr1   r2   r+   r   r+    complete_input_dict)inputapi_keyadditional_args/Need to pass in a model name to initialize vllmstreamT)r8   r9   original_responser:   zraw model_response: prompt_tokenscompletion_tokenstotal_tokensusage )r*   r&   r'   r	   r(   r   r   pre_callgenerateiter	post_calloutputstextchoicesr   contentlenprompt_token_ids	token_idsinttimecreatedr   r   setattr)r   r+   r,   r-   encodinglogging_objr.   custom_prompt_dictlitellm_params	logger_fnr"   r)   sampling_paramsmodel_prompt_detailspromptrH   r?   r@   rB   s                      r   
completionr[   '   s   72?^ %77O""1%8*73!56L!M34HI	
  eh? .@   ,,v7#T
 	

 ?"x'@D'HG} 	%2OD	 	 	
 	,WI674;AJ4F4Fq4I4N4Nq!))1 GAJ778
 2 21 5 ? ?@!$TYY[!1$'/&)::

 	/q  7As1v667s   E> >	F"FF"c                 <   	 t        |       \  }} |di |}g }	| |v r6||    }
|D ]+  }t	        |
d   |
d   |
d   |      }|	j                  |       - n%|D ]   }t        | |	      }|	j                  |       " |r|j                  |	|      }nt        dd
      g }|D ]  }t               }|j                  d   j                  |j                  d   j                  _        t        |j                        }t        |j                  d   j                         }t#        t%        j$                               |_        | |_        t+        ||||z         }t-        |d|       |j                  |        |S # t        $ r}t        |      }t        d|      d}~ww xY w)a  
    Example usage:
    import litellm
    import os
    from litellm import batch_completion


    responses = batch_completion(
        model="vllm/facebook/opt-125m",
        messages = [
            [
                {
                    "role": "user",
                    "content": "good morning? "
                }
            ],
            [
                {
                    "role": "user",
                    "content": "what's the time? "
                }
            ]
        ]
    )
    r#   r   r$   Nr0   r1   r2   r3   r5   r;   r>   rB   rC   )r*   r'   r(   r	   r   appendr   rE   r   rH   rI   rJ   r   rK   rL   rM   rN   rO   rP   rQ   r   r   rR   )r   r+   r.   rU   r&   r"   r)   	error_strrX   promptsrY   r   rZ   rH   final_outputsoutputr,   r?   r@   rB   s                       r   batch_completionsrb   q   s   8:2?^ %77OG""1%8G".w7%9:P%Q#78L#M 	F NN6"    G#%'BFNN6"   ,,w8#T
 	
 M&4:NN14E4J4Jq!))1 F334q 1 ; ;<!$TYY[!1$'/&)::

 	/^,# $ _  :F	Ay99:s   E5 5	F>FFc                       y )NrC   rC   r   r   	embeddingrd      s    r   )rP   typingr   r   3litellm.litellm_core_utils.prompt_templates.factoryr   r   litellm.utilsr   r   r&   r'   r	   r(   r*   listdictr[   rb   rd   rC   r   r   <module>rj      s       /

	 
	7 	7( GGG "G 	G GV 15MMM`	r   