a
    zez                     @   s~  d Z ddlZddlmZ ddlZddlZddlZddlm	Z	 ddl
Z
ddlZddlmZ ddlZddlZddlZe ZdaG dd dZd	d
 Ze
e dZdZG dd deZG dd dZdd ZG dd deZG dd deZG dd deZ G dd de	Z!dd Z"dd Z#d-dd Z$d!d" Z%G d#d$ d$e
j&Z'da(da)d%d& Z*d'd( Z+G d)d* d*ej,Z-G d+d, d,ej.Z/dS ).z"Brian Quinlan (brian@sweetapp.com)    N)_base)Queue)partialFc                   @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
_ThreadWakeupc                 C   s   d| _ tjdd\| _| _d S )NF)duplex)_closedmpPipe_reader_writerself r   0/usr/lib/python3.9/concurrent/futures/process.py__init__C   s    z_ThreadWakeup.__init__c                 C   s$   | j s d| _ | j  | j  d S NT)r   r   closer
   r   r   r   r   r   G   s    
z_ThreadWakeup.closec                 C   s   | j s| jd d S )N    )r   r   
send_bytesr   r   r   r   wakeupM   s    z_ThreadWakeup.wakeupc                 C   s    | j s| j r| j  qd S N)r   r
   poll
recv_bytesr   r   r   r   clearQ   s    
z_ThreadWakeup.clearN)__name__
__module____qualname__r   r   r   r   r   r   r   r   r   B   s   r   c                  C   s@   da tt } | D ]\}}|  q| D ]\}}|  q*d S r   )_global_shutdownlist_threads_wakeupsitemsr   join)r    _thread_wakeuptr   r   r   _python_exitW   s    
r%      =   c                   @   s   e Zd Zdd Zdd ZdS )_RemoteTracebackc                 C   s
   || _ d S r   tb)r   r*   r   r   r   r   w   s    z_RemoteTraceback.__init__c                 C   s   | j S r   r)   r   r   r   r   __str__y   s    z_RemoteTraceback.__str__N)r   r   r   r   r+   r   r   r   r   r(   v   s   r(   c                   @   s   e Zd Zdd Zdd ZdS )_ExceptionWithTracebackc                 C   s8   t t|||}d|}|| _d | j_d| | _d S )N z

"""
%s""")	tracebackformat_exceptiontyper!   exc__traceback__r*   )r   r1   r*   r   r   r   r   }   s
    
z _ExceptionWithTraceback.__init__c                 C   s   t | j| jffS r   )_rebuild_excr1   r*   r   r   r   r   
__reduce__   s    z"_ExceptionWithTraceback.__reduce__N)r   r   r   r   r4   r   r   r   r   r,   |   s   r,   c                 C   s   t || _| S r   )r(   	__cause__)r1   r*   r   r   r   r3      s    
r3   c                   @   s   e Zd Zdd ZdS )	_WorkItemc                 C   s   || _ || _|| _|| _d S r   )futurefnargskwargs)r   r7   r8   r9   r:   r   r   r   r      s    z_WorkItem.__init__Nr   r   r   r   r   r   r   r   r6      s   r6   c                   @   s   e Zd ZdddZdS )_ResultItemNc                 C   s   || _ || _|| _d S r   )work_id	exceptionresult)r   r=   r>   r?   r   r   r   r      s    z_ResultItem.__init__)NNr;   r   r   r   r   r<      s   r<   c                   @   s   e Zd Zdd ZdS )	_CallItemc                 C   s   || _ || _|| _|| _d S r   )r=   r8   r9   r:   )r   r=   r8   r9   r:   r   r   r   r      s    z_CallItem.__init__Nr;   r   r   r   r   r@      s   r@   c                       s*   e Zd Zd fdd	Z fddZ  ZS )
_SafeQueuer   c                   s&   || _ || _|| _t j||d d S )N)ctx)pending_work_itemsshutdown_lockr#   superr   )r   max_sizerB   rC   rD   r#   	__class__r   r   r      s    z_SafeQueue.__init__c                    s   t |trtt|||j}tdd||_	| j
|jd }| j | j  W d    n1 sj0    Y  |d ur|j| nt || d S )Nz

"""
{}"""r-   )
isinstancer@   r.   r/   r0   r2   r(   formatr!   r5   rC   popr=   rD   r#   r   r7   set_exceptionrE   _on_queue_feeder_error)r   eobjr*   	work_itemrG   r   r   rM      s    
(z!_SafeQueue._on_queue_feeder_error)r   )r   r   r   r   rM   __classcell__r   r   rG   r   rA      s   rA   c                 g   s,   t | }tt|| }|s d S |V  qd S r   )ziptuple	itertoolsislice)	chunksize	iterablesitchunkr   r   r   _get_chunks   s
    rZ   c                    s    fdd|D S )Nc                    s   g | ]} | qS r   r   ).0r9   r8   r   r   
<listcomp>   r   z"_process_chunk.<locals>.<listcomp>r   )r8   rY   r   r\   r   _process_chunk   s    	r^   c              
   C   s`   z|  t|||d W nB tyZ } z*t||j}|  t||d W Y d }~n
d }~0 0 d S )N)r?   r>   r>   )putr<   BaseExceptionr,   r2   )result_queuer=   r?   r>   rN   r1   r   r   r   _sendback_result   s    
rc   c              
   C   s   |d ur:z||  W n$ t y8   tjjddd Y d S 0 | jdd}|d u r`|t  d S z|j|j	i |j
}W n@ t y } z(t||j}t||j|d W Y d }~nd }~0 0 t||j|d ~~q:d S )NzException in initializer:T)exc_infoblockr_   )r?   )ra   r   LOGGERcriticalgetr`   osgetpidr8   r9   r:   r,   r2   rc   r=   )
call_queuerb   initializerinitargs	call_itemrrN   r1   r   r   r   _process_worker   s$    &rq   c                       sl   e Zd Z fddZdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd Zdd Z  ZS )_ExecutorManagerThreadc                    sf   |j | _|j| _| j| jfdd}t||| _|j| _|j	| _
|j| _|j| _|j| _t   d S )Nc                 S   s<   t jd | |  W d    n1 s.0    Y  d S )Nz?Executor collected: triggering callback for QueueManager wakeup)r   utildebugr   )r"   r#   rD   r   r   r   
weakref_cb  s    z3_ExecutorManagerThread.__init__.<locals>.weakref_cb)_executor_manager_thread_wakeupr#   _shutdown_lockrD   weakrefrefexecutor_reference
_processes	processes_call_queuerl   _result_queuerb   	_work_idswork_ids_queue_pending_work_itemsrC   rE   r   )r   executorru   rG   r   r   r     s    	
z_ExecutorManagerThread.__init__c                 C   s   |    |  \}}}|r(| | d S |d urX| | ~|  }|d urV|j  ~|  r |   | j	s | 
  d S q d S r   )add_call_item_to_queuewait_result_broken_or_wakeupterminate_brokenprocess_result_itemrz   _idle_worker_semaphorereleaseis_shutting_downflag_executor_shutting_downrC   join_executor_internals)r   result_item	is_brokencauser   r   r   r   run:  s"    


z_ExecutorManagerThread.runc                 C   s~   | j  rd S z| jjdd}W n tjy6   Y d S 0 | j| }|j rn| j j	t
||j|j|jdd q | j|= q q d S )NFre   T)rl   fullr   ri   queueEmptyrC   r7   set_running_or_notify_cancelr`   r@   r8   r9   r:   )r   r=   rP   r   r   r   r   Z  s"    



z-_ExecutorManagerThread.add_call_item_to_queuec           
   
   C   s   | j j}| jj}||g}dd t| j D }tj|| }d }d}d }||v rz|	 }d}W q t
y }	 z tt|	|	|	j}W Y d }	~	qd }	~	0 0 n||v rd}| j | j  W d    n1 s0    Y  |||fS )Nc                 S   s   g | ]
}|j qS r   )sentinelr[   pr   r   r   r]   {  r   zG_ExecutorManagerThread.wait_result_broken_or_wakeup.<locals>.<listcomp>TF)rb   r
   r#   r   r|   valuesr   
connectionwaitrecvra   r.   r/   r0   r2   rD   r   )
r   result_readerwakeup_readerreadersworker_sentinelsreadyr   r   r   rN   r   r   r   r   q  s&    ,(z3_ExecutorManagerThread.wait_result_broken_or_wakeupc                 C   sr   t |tr2| j|}|  | jsn|   d S n<| j|jd }|d urn|jr`|j	
|j n|j	|j d S r   )rI   intr|   rK   r!   r   rC   r=   r>   r7   rL   
set_resultr?   )r   r   r   rP   r   r   r   r     s    
z*_ExecutorManagerThread.process_result_itemc                 C   s   |   }tp|d u p|jS r   )rz   r   _shutdown_thread)r   r   r   r   r   r     s    z'_ExecutorManagerThread.is_shutting_downc                 C   s   |   }|d ur d|_d|_d }td}|d urHtdd| d|_| j D ]\}}|j	
| ~qR| j  | j D ]}|  q~|   d S )NzKA child process terminated abruptly, the process pool is not usable anymoreTz^A process in the process pool was terminated abruptly while the future was running or pending.z
'''
r-   z''')rz   _brokenr   BrokenProcessPoolr(   r!   r5   rC   r    r7   rL   r   r|   r   	terminater   )r   r   r   bper=   rP   r   r   r   r   r     s"    

z'_ExecutorManagerThread.terminate_brokenc                 C   s   |   }|d ur|d|_|jr|i }| j D ]\}}|j s*|||< q*|| _z| j  W qL t	j
yr   Y qvY qL0 qLd|_d S )NTF)rz   r   _cancel_pending_futuresrC   r    r7   cancelr   
get_nowaitr   r   )r   r   new_pending_work_itemsr=   rP   r   r   r   r     s    

z2_ExecutorManagerThread.flag_executor_shutting_downc              	   C   sl   |   }d}||k rh|   dkrht|| D ]8}z| jd  |d7 }W q, tjyb   Y  qY q,0 q,qd S )Nr   r&   )get_n_children_aliverangerl   
put_nowaitr   Full)r   n_children_to_stopn_sentinels_sentir   r   r   shutdown_workers  s    
z'_ExecutorManagerThread.shutdown_workersc                 C   sh   |    | j  | j  | j | j  W d    n1 sB0    Y  | j D ]}|  qVd S r   )	r   rl   r   join_threadrD   r#   r|   r   r!   r   r   r   r   r   r     s    

(z._ExecutorManagerThread.join_executor_internalsc                 C   s   t dd | j D S )Nc                 s   s   | ]}|  V  qd S r   )is_aliver   r   r   r   	<genexpr>  r   z>_ExecutorManagerThread.get_n_children_alive.<locals>.<genexpr>)sumr|   r   r   r   r   r   r     s    z+_ExecutorManagerThread.get_n_children_alive)r   r   r   r   r   r   r   r   r   r   r   r   r   r   rQ   r   r   rG   r   rr     s   + 
&rr   c               	   C   sh   t rtrttda ztd} W n ttfy:   Y d S 0 | dkrHd S | dkrTd S d|  attd S )NTSC_SEM_NSEMS_MAX   z@system provides too few semaphores (%d available, 256 necessary))_system_limits_checked_system_limitedNotImplementedErrorrj   sysconfAttributeError
ValueError)	nsems_maxr   r   r   _check_system_limits  s     r   c                 c   s&   | D ]}|   |r| V  qqd S r   )reverserK   )iterableelementr   r   r   _chain_from_iterable_of_lists,  s    r   c                   @   s   e Zd ZdS )r   N)r   r   r   r   r   r   r   r   8  s   r   c                       s~   e Zd ZdddZdd Zdd Zd	d
 Zdd Zdd Ze	j
jje_ddd fdd
ZdddddZe	j
jje_  ZS )ProcessPoolExecutorNr   c                 C   sJ  t   |d u r6t pd| _tjdkrntt| j| _n8|dkrHtdn tjdkrh|tkrhtdt || _|d u r~t	
 }|| _| jjdddk| _|d urt|std	|| _|| _d | _i | _d| _t | _td| _d| _d| _i | _d| _t | _| jt  }t!|| j| j| j| jd
| _"d| j"_#|$ | _%t&' | _(d S )Nr&   win32r   z"max_workers must be greater than 0zmax_workers must be <= F)
allow_noneforkzinitializer must be a callable)rF   rB   rC   rD   r#   T))r   rj   	cpu_count_max_workerssysplatformmin_MAX_WINDOWS_WORKERSr   r   get_context_mp_contextget_start_method#_safe_to_dynamically_spawn_childrencallable	TypeError_initializer	_initargs_executor_manager_threadr{   r   	threadingLockrw   	Semaphorer   r   _queue_countr   r   r   rv   EXTRA_QUEUED_CALLSrA   r}   _ignore_epipeSimpleQueuer~   r   r   r   )r   max_workers
mp_contextrm   rn   
queue_sizer   r   r   r   @  sZ    






zProcessPoolExecutor.__init__c                 C   s<   | j d u r8| js|   t| | _ | j   | jt| j < d S r   )r   r   _launch_processesrr   startrv   r   r   r   r   r   _start_executor_manager_thread  s    


z2ProcessPoolExecutor._start_executor_manager_threadc                 C   s2   | j jddrd S t| j}|| jk r.|   d S )NF)blocking)r   acquirelenr{   r   _spawn_process)r   process_countr   r   r   _adjust_process_count  s
    

z)ProcessPoolExecutor._adjust_process_countc                 C   s$   t t| j| jD ]}|   qd S r   )r   r   r{   r   r   )r   r"   r   r   r   r     s    z%ProcessPoolExecutor._launch_processesc                 C   s8   | j jt| j| j| j| jfd}|  || j|j	< d S )N)targetr9   )
r   Processrq   r}   r~   r   r   r   r{   pidr   r   r   r   r     s    z"ProcessPoolExecutor._spawn_processc                O   s   | j  | jrt| j| jr&tdtr2tdt }t||||}|| j	| j
< | j| j
 |  j
d7  _
| j  | jr|   |   |W  d    S 1 s0    Y  d S )Nz*cannot schedule new futures after shutdownz6cannot schedule new futures after interpreter shutdownr&   )rw   r   r   r   RuntimeErrorr   r   Futurer6   r   r   r   r`   rv   r   r   r   r   )r   r8   r9   r:   fwr   r   r   submit  s"    

zProcessPoolExecutor.submitr&   )timeoutrV   c                   s:   |dk rt dt jtt|t|d|i|d}t|S )Nr&   zchunksize must be >= 1.rV   )r   )r   rE   mapr   r^   rZ   r   )r   r8   r   rV   rW   resultsrG   r   r   r     s    zProcessPoolExecutor.mapTF)cancel_futuresc                C   s   | j 0 || _d| _| jd ur(| j  W d    n1 s<0    Y  | jd ur^|r^| j  d | _d | _| jd ur|r| j	  d | _d | _
d | _d S r   )rw   r   r   rv   r   r   r!   r}   r~   r   r{   )r   r   r   r   r   r   shutdown  s    
(

zProcessPoolExecutor.shutdown)NNNr   )T)r   r   r   r   r   r   r   r   r   r   Executor__doc__r   r   rQ   r   r   rG   r   r   ?  s     
U

r   )NN)0
__author__rj   concurrent.futuresr   r   multiprocessingr   multiprocessing.connectionmultiprocessing.queuesr   r   rx   	functoolsr   rT   r   r.   WeakKeyDictionaryr   r   r   r%   _register_atexitr   r   	Exceptionr(   r,   r3   objectr6   r<   r@   rA   rZ   r^   rc   rq   Threadrr   r   r   r   r   BrokenExecutorr   r   r   r   r   r   r   <module>.   sN   



)  