
    gA(                         d dl Z d dlZd dlZd dlZddlmZmZ ddlm	Z	 ddl
mZ dgZ ej                         Zd adadad Z	 	 	 	 	 	 	 	 	 	 ddZ G d	 d
e      Zy)    N   )ProcessPoolExecutorEXTRA_QUEUED_CALLS)	cpu_count)get_contextget_reusable_executorc                  \    t         5  t        } t        dz  a| cddd       S # 1 sw Y   yxY w)zEnsure that each successive executor instance has a unique, monotonic id.

    The purpose of this monotonic id is to help debug and test automated
    instance creation.
    r   N)_executor_lock_next_executor_id)executor_ids    \/var/www/openai/venv/lib/python3.12/site-packages/joblib/externals/loky/reusable_executor.py_get_next_executor_idr      s$     
'Q 
s   "+c
                 J    t         j                  | |||||||||	
      \  }
}|
S )a  Return the current ReusableExectutor instance.

    Start a new instance if it has not been started already or if the previous
    instance was left in a broken state.

    If the previous instance does not have the requested number of workers, the
    executor is dynamically resized to adjust the number of workers prior to
    returning.

    Reusing a singleton instance spares the overhead of starting new worker
    processes and importing common python packages each time.

    ``max_workers`` controls the maximum number of tasks that can be running in
    parallel in worker processes. By default this is set to the number of
    CPUs on the host.

    Setting ``timeout`` (in seconds) makes idle workers automatically shutdown
    so as to release system resources. New workers are respawn upon submission
    of new tasks so that ``max_workers`` are available to accept the newly
    submitted tasks. Setting ``timeout`` to around 100 times the time required
    to spawn new processes and import packages in them (on the order of 100ms)
    ensures that the overhead of spawning workers is negligible.

    Setting ``kill_workers=True`` makes it possible to forcibly interrupt
    previously spawned jobs to get a new instance of the reusable executor
    with new constructor argument values.

    The ``job_reducers`` and ``result_reducers`` are used to customize the
    pickling of tasks and results send to the executor.

    When provided, the ``initializer`` is run first in newly spawned
    processes with argument ``initargs``.

    The environment variable in the child process are a copy of the values in
    the main process. One can provide a dict ``{ENV: VAL}`` where ``ENV`` and
    ``VAL`` are string literals to overwrite the environment variable ``ENV``
    in the child processes to value ``VAL``. The environment variables are set
    in the children before any module is loaded. This only works with the
    ``loky`` context.
    )
max_workerscontexttimeoutkill_workersreusejob_reducersresult_reducersinitializerinitargsenv)_ReusablePoolExecutorr   )r   r   r   r   r   r   r   r   r   r   	_executor_s               r   r   r   %   sD    h )>>!!' ? LIq     c                   x     e Zd Z	 	 	 	 	 	 	 	 	 d fd	Ze	 	 	 	 	 	 	 	 	 	 dd       Z fdZd Zd Z fdZ	 xZ
S )	r   c           
      P    t         |   |||||||	|
       || _        || _        y )N)r   r   r   r   r   r   r   r   )super__init__r   _submit_resize_lock)selfsubmit_resize_lockr   r   r   r   r   r   r   r   r   	__class__s              r   r!   z_ReusablePoolExecutor.__init__i   sA     	#%+# 	 		
 '#5 r   c           
         t         5  t        }||du r||j                  }nt               }n|dk  rt	        d| d      t        |t              rt        |      }||j                         dk(  rt	        d      t        ||||||	|
      }|Ed}t        j                  j                  d	| d       t               }|a | t         f||d
|xa}n|dk(  r	|t        k(  }|j                  j                   s|j                  j"                  s|s|j                  j                   rd}n|j                  j"                  rd}nd}t        j                  j                  d| d| d       |j#                  d|       d xax}a | j$                  dd|i|cd d d        S t        j                  j                  d|j                   d       d}|j'                  |       d d d        ||fS # 1 sw Y   fS xY w)NTr   z(max_workers must be greater than 0, got .forkz4Cannot use reusable executor with the 'fork' context)r   r   r   r   r   r   r   Fz#Create a executor with max_workers=)r   r   autobrokenshutdownzarguments have changedz)Creating a new executor with max_workers=z, as the previous instance cannot be reused (z).)waitr   r   z+Reusing existing executor with max_workers= )r
   r   _max_workersr   
ValueError
isinstancestrr   get_start_methoddictmputildebugr   _executor_kwargs_flagsr*   r+   r   _resize)clsr   r   r   r   r   r   r   r   r   r   executorkwargs	is_reusedr   reasons                   r   r   z+_ReusablePoolExecutor.get_reusable_executor   sJ     H"D=X%9"*"7"7K"++K! >{m1M  '3'%g."w'?'?'AV'K J  ) /'!F !	9+aH 45#) '*"( + +( 	( 	H F?"&66EOO**// --!)!11!+!9GGMMC&- (##)(".
 %%4l%K>BBIB+;4344 $/39C ^J GGMM''/'<'<&=Q@ !%I$$[1U X ""Y X ""s   FG."A G..G:c                 n    | j                   5  t        |   |g|i |cd d d        S # 1 sw Y   y xY wN)r"   r    submit)r#   fnargsr<   r%   s       r   rA   z_ReusablePoolExecutor.submit   s.    %%7>"6t6v6 &%%s   +4c                    | j                   5  |t        d      || j                  k(  r
	 d d d        y | j                  || _        	 d d d        y | j	                          | j
                  5  t        | j                  j                               }t        d |D              }|| _        t        ||      D ]  }| j                  j                  d         	 d d d        t        | j                        |kD  rZ| j                  j                  sDt!        j"                  d       t        | j                        |kD  r| j                  j                  sD| j%                          t        | j                  j                               }t'        d |D              s(t!        j"                  d       t'        d |D              s(d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nz&Trying to resize with max_workers=Nonec              3   <   K   | ]  }|j                           y wr@   is_alive.0ps     r   	<genexpr>z0_ReusablePoolExecutor._resize.<locals>.<genexpr>   s     'Hi

i   MbP?c              3   <   K   | ]  }|j                           y wr@   rF   rH   s     r   rK   z0_ReusablePoolExecutor._resize.<locals>.<genexpr>  s     :	1!**,	rL   )r"   r/   r.   _executor_manager_thread_wait_job_completion_processes_management_locklist
_processesvaluessumrange_call_queueputlenr8   r*   timesleep_adjust_process_countall)r#   r   	processesnb_children_aliver   s        r   r9   z_ReusablePoolExecutor._resize   s   %%" !IJJ 1 11	 &% ,,4 %0! &% %%'
 00 !7!7!9:	$''Hi'H$H!$/!{,=>A$$((. ?	 1 DOO${24;;;M;M

4  DOO${24;;;M;M &&(T__3356I:	::

4  :	::? &%$ 10% &%s7   GGG-A)GA9GA,GG	GGc                    | j                   rGt        j                  dt               t        j
                  j                  d| j                   d       | j                   r#t        j                  d       | j                   r"yy)z8Wait for the cache to be empty before resizing the pool.z\Trying to resize an executor with running jobs: waiting for jobs completion before resizing.z	Executor z, waiting for jobs completion before resizingrM   N)
_pending_work_itemswarningswarnUserWarningr4   r5   r6   r   rZ   r[   )r#   s    r   rP   z*_ReusablePoolExecutor._wait_job_completion  sm     ##MM?
 GGMMD,,- ." "
 &&JJt &&r   c                 R    dt               z  t        z   }t        |   |||       y )N   )
queue_size)r   r   r    _setup_queues)r#   r   r   rg   r%   s       r   rh   z#_ReusablePoolExecutor._setup_queues  s/     _'99
/j 	 	
r   )	NNNr   NNNr-   N
NN
   Fr)   NNNr-   N)__name__
__module____qualname__r!   classmethodr   rA   r9   rP   rh   __classcell__)r%   s   @r   r   r   h   sv     64  Y# Y#v7!!F"
 
r   r   ri   )rZ   rb   	threadingmultiprocessingr4   process_executorr   r   backend.contextr   backendr   __all__RLockr
   r   r   r7   r   r   r   r-   r   r   <module>rw      s        E &  "
# !" 	 
 
@Fu
/ u
r   