a
    ze+                     @   s`  d dgZ ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
mZ ddl
mZmZ ddlmZ dZd	Zd
ZdZe Zdd Zdd ZG dd deZG dd dZdd ZG dd deZd*ddZdd ZG dd deZ G d d  d e!Z"G d!d" d"e!Z#e#Z$G d#d$ d$e#Z%G d%d& d&e!Z&G d'd( d(e&Z'G d)d de"Z(dS )+Pool
ThreadPool    N   )util)get_contextTimeoutError)waitINITRUNCLOSE	TERMINATEc                 C   s   t t|  S N)listmapargs r   */usr/lib/python3.9/multiprocessing/pool.pymapstar/   s    r   c                 C   s   t t| d | d S )Nr   r   )r   	itertoolsstarmapr   r   r   r   starmapstar2   s    r   c                   @   s   e Zd Zdd Zdd ZdS )RemoteTracebackc                 C   s
   || _ d S r   tb)selfr   r   r   r   __init__:   s    zRemoteTraceback.__init__c                 C   s   | j S r   r   r   r   r   r   __str__<   s    zRemoteTraceback.__str__N)__name__
__module____qualname__r   r   r   r   r   r   r   9   s   r   c                   @   s   e Zd Zdd Zdd ZdS )ExceptionWithTracebackc                 C   s0   t t|||}d|}|| _d| | _d S )N z

"""
%s""")	tracebackformat_exceptiontypejoinexcr   )r   r(   r   r   r   r   r   @   s    
zExceptionWithTraceback.__init__c                 C   s   t | j| jffS r   )rebuild_excr(   r   r   r   r   r   
__reduce__E   s    z!ExceptionWithTraceback.__reduce__N)r   r    r!   r   r*   r   r   r   r   r"   ?   s   r"   c                 C   s   t || _| S r   )r   	__cause__)r(   r   r   r   r   r)   H   s    
r)   c                       s,   e Zd Z fddZdd Zdd Z  ZS )MaybeEncodingErrorc                    s.   t || _t || _tt| | j| j d S r   )reprr(   valuesuperr,   r   )r   r(   r.   	__class__r   r   r   T   s    

zMaybeEncodingError.__init__c                 C   s   d| j | jf S )Nz(Error sending result: '%s'. Reason: '%s')r.   r(   r   r   r   r   r   Y   s    zMaybeEncodingError.__str__c                 C   s   d| j j| f S )Nz<%s: %s>)r1   r   r   r   r   r   __repr__]   s    zMaybeEncodingError.__repr__)r   r    r!   r   r   r2   __classcell__r   r   r0   r   r,   P   s   r,   r   Fc              
   C   s  |d ur(t |tr|dks(td||j}| j}t| drR| j  |j	  |d urb||  d}|d u s~|r||k rz
| }	W n& t
tfy   td Y qY n0 |	d u rtd q|	\}
}}}}zd||i |f}W nJ ty4 } z0|r|turt||j}d|f}W Y d }~n
d }~0 0 z||
||f W nT ty } z:t||d }td	|  ||
|d|ff W Y d }~n
d }~0 0 d  }	 }
 } } }}|d7 }qftd
|  d S )Nr   zMaxtasks {!r} is not valid_writerr   z)worker got EOFError or OSError -- exitingzworker got sentinel -- exitingTFz0Possible encoding error while sending result: %szworker exiting after %d tasks)
isinstanceintAssertionErrorformatputgethasattrr4   close_readerEOFErrorOSErrorr   debug	Exception_helper_reraises_exceptionr"   __traceback__r,   )inqueueoutqueueinitializerinitargsZmaxtaskswrap_exceptionr9   r:   Z	completedtaskjobifuncr   kwdsresultewrappedr   r   r   workera   sN    





(
rQ   c                 C   s   | d S r   r   )exr   r   r   rB      s    rB   c                       s.   e Zd Zdd fdd
Z fddZ  ZS )
_PoolCacheNnotifierc                  s   || _ t j|i | d S r   )rU   r/   r   )r   rU   r   rM   r0   r   r   r      s    z_PoolCache.__init__c                    s    t  | | s| jd  d S r   )r/   __delitem__rU   r9   )r   itemr0   r   r   rV      s    z_PoolCache.__delitem__)r   r    r!   r   rV   r3   r   r   r0   r   rS      s   rS   c                   @   s  e Zd ZdZedd ZdKddZeje	fdd	Z
d
d Zdd Zedd Zedd Zdd Zedd Zedd Zdd Zdd Zdi fddZdLddZdMd d!ZdNd"d#Zd$d% ZdOd'd(ZdPd)d*Zdi ddfd+d,ZdQd-d.ZdRd/d0ZedSd1d2Zed3d4 Z ed5d6 Z!ed7d8 Z"ed9d: Z#d;d< Z$d=d> Z%d?d@ Z&dAdB Z'edCdD Z(edEdF Z)dGdH Z*dIdJ Z+dS )Tr   Tc                 O   s   | j |i |S r   Process)ctxr   rM   r   r   r   rY      s    zPool.ProcessNr   c                 C   s  g | _ t| _|pt | _|   t | _| j | _	t
| j	d| _|| _|| _|| _|d u rjt phd}|dk rztd|d urt|std|| _z|   W nF ty   | j D ]}|jd u r|  q| j D ]}|  qւ Y n0 |  }tjtj| j| j| j| j| j| j | j | j!| j| j| j| j"|| j	fd| _#d| j#_$t%| j#_| j#&  tjtj'| j| j(| j!| j | jfd| _)d| j)_$t%| j)_| j)&  tjtj*| j!| j+| jfd| _,d| j,_$t%| j,_| j,&  t-j.| | j/| j| j | j!| j | j	| j#| j)| j,| jf	dd| _0t%| _d S )	NrT   r   z&Number of processes must be at least 1zinitializer must be a callabletargetr   T   )r   exitpriority)1_poolr	   _stater   _ctx_setup_queuesqueueSimpleQueue
_taskqueue_change_notifierrS   _cache_maxtasksperchild_initializer	_initargsos	cpu_count
ValueErrorcallable	TypeError
_processes_repopulate_poolrA   exitcode	terminater'   _get_sentinels	threadingThreadr   _handle_workersrY   _inqueue	_outqueue_wrap_exception_worker_handlerdaemonr
   start_handle_tasks
_quick_put_task_handler_handle_results
_quick_get_result_handlerr   Finalize_terminate_pool
_terminate)r   	processesrF   rG   maxtasksperchildcontextp	sentinelsr   r   r   r      s~    









zPool.__init__c                 C   s>   | j |kr:|d| t| d t| dd d ur:| jd  d S )Nz&unclosed running multiprocessing pool )sourcerf   )r`   ResourceWarninggetattrrf   r9   )r   _warnr
   r   r   r   __del__  s    

zPool.__del__c              	   C   s0   | j }d|j d|j d| j dt| j d	S )N<.z state=z pool_size=>)r1   r    r!   r`   lenr_   )r   clsr   r   r   r2     s    zPool.__repr__c                 C   s    | j jg}| jjg}g ||S r   )ry   r=   rf   )r   Ztask_queue_sentinelsZself_notifier_sentinelsr   r   r   rt     s    

zPool._get_sentinelsc                 C   s   dd | D S )Nc                 S   s   g | ]}t |d r|jqS )sentinel)r;   r   ).0rQ   r   r   r   
<listcomp>  s   
z.Pool._get_worker_sentinels.<locals>.<listcomp>r   workersr   r   r   _get_worker_sentinels  s    zPool._get_worker_sentinelsc                 C   sP   d}t tt| D ]6}| | }|jd urtd|  |  d}| |= q|S )NFcleaning up worker %dT)reversedranger   rr   r   r@   r'   )poolZcleanedrK   rQ   r   r   r   _join_exited_workers  s    
zPool._join_exited_workersc                 C   s0   |  | j| j| j| j| j| j| j| j| j	| j

S r   )_repopulate_pool_staticra   rY   rp   r_   rx   ry   ri   rj   rh   rz   r   r   r   r   rq   .  s    zPool._repopulate_poolc
              
   C   sf   t |t| D ]P}
|| t||||||	fd}|jdd|_d|_|  || t	d qd S )Nr[   rY   Z
PoolWorkerTzadded worker)
r   r   rQ   namereplacer|   r}   appendr   r@   )rZ   rY   r   r   rD   rE   rF   rG   r   rH   rK   wr   r   r   r   7  s    
zPool._repopulate_pool_staticc
           
      C   s*   t |r&t | |||||||||	
 d S r   )r   r   r   )
rZ   rY   r   r   rD   rE   rF   rG   r   rH   r   r   r   _maintain_poolJ  s    
zPool._maintain_poolc                 C   s4   | j  | _| j  | _| jjj| _| jjj| _	d S r   )
ra   rd   rx   ry   r4   sendr   r=   recvr   r   r   r   r   rb   V  s    zPool._setup_queuesc                 C   s   | j tkrtdd S )NzPool not running)r`   r
   rm   r   r   r   r   _check_running\  s    
zPool._check_runningc                 C   s   |  ||| S r   )apply_asyncr:   )r   rL   r   rM   r   r   r   apply`  s    z
Pool.applyc                 C   s   |  ||t| S r   )
_map_asyncr   r:   r   rL   iterable	chunksizer   r   r   r   g  s    zPool.mapc                 C   s   |  ||t| S r   )r   r   r:   r   r   r   r   r   n  s    zPool.starmapc                 C   s   |  ||t|||S r   )r   r   r   rL   r   r   callbackerror_callbackr   r   r   starmap_asyncv  s    zPool.starmap_asyncc              
   c   sl   z,d}t |D ]\}}||||fi fV  qW n: tyf } z"||d t|fi fV  W Y d }~n
d }~0 0 d S )Nr   )	enumeraterA   rB   )r   Z
result_jobrL   r   rK   xrO   r   r   r   _guarded_task_generation~  s    zPool._guarded_task_generationr   c                 C   s   |    |dkr:t| }| j| |j|||jf |S |dk rPtd|t	
|||}t| }| j| |jt||jf dd |D S d S )Nr   zChunksize must be 1+, not {0:n}c                 s   s   | ]}|D ]
}|V  q
qd S r   r   r   chunkrW   r   r   r   	<genexpr>      zPool.imap.<locals>.<genexpr>)r   IMapIteratorre   r9   r   _job_set_lengthrm   r8   r   
_get_tasksr   r   rL   r   r   rN   task_batchesr   r   r   imap  s4    z	Pool.imapc                 C   s   |    |dkr:t| }| j| |j|||jf |S |dk rPtd|t	
|||}t| }| j| |jt||jf dd |D S d S )Nr   zChunksize must be 1+, not {0!r}c                 s   s   | ]}|D ]
}|V  q
qd S r   r   r   r   r   r   r     r   z&Pool.imap_unordered.<locals>.<genexpr>)r   IMapUnorderedIteratorre   r9   r   r   r   rm   r8   r   r   r   r   r   r   r   imap_unordered  s0    zPool.imap_unorderedc                 C   s6   |    t| ||}| j|jd|||fgd f |S Nr   )r   ApplyResultre   r9   r   )r   rL   r   rM   r   r   rN   r   r   r   r     s    zPool.apply_asyncc                 C   s   |  ||t|||S r   )r   r   r   r   r   r   	map_async  s    zPool.map_asyncc           
      C   s   |    t|dst|}|d u rJtt|t| jd \}}|rJ|d7 }t|dkrZd}t|||}t| |t|||d}	| j	
| |	j||d f |	S )N__len__   r   r   r   )r   r;   r   divmodr   r_   r   r   	MapResultre   r9   r   r   )
r   rL   r   Zmapperr   r   r   extrar   rN   r   r   r   r     s,    
zPool._map_asyncc                 C   s"   t | |d | s|  qd S )N)timeout)r   emptyr:   )r   change_notifierr   r   r   r   _wait_for_updates  s    zPool._wait_for_updatesc                 C   st   t  }|jtks |r\|jtkr\| |||||||	|
||
 g | ||}| || q|d  t	
d d S )Nzworker handler exiting)ru   current_threadr`   r
   r   r   r   r   r9   r   r@   )r   cache	taskqueuerZ   rY   r   r   rD   rE   rF   rG   r   rH   r   r   threadZcurrent_sentinelsr   r   r   rw     s    
zPool._handle_workersc                 C   s  t  }t| jd D ]
\}}d }z|D ]}|jtkrDtd  qz|| W q( ty }	 zH|d d \}
}z||
 	|d|	f W n t
y   Y n0 W Y d }	~	q(d }	~	0 0 q(|rtd |r|d nd}||d  W d  } }}
qW d  } }}
 q,W d  } }}
qd  } }}
0 qtd z6td |d  td	 |D ]}|d  qPW n ty   td
 Y n0 td d S )Nz'task handler found thread._state != RUN   Fzdoing set_length()r   r   ztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exiting)ru   r   iterr:   r`   r
   r   r@   rA   _setKeyErrorr9   r?   )r   r9   rE   r   r   r   ZtaskseqZ
set_lengthrI   rO   rJ   idxr   r   r   r   r~     sJ    






zPool._handle_tasksc              	   C   s  t  }z
| }W n" ttfy4   td Y d S 0 |jtkrLtd q|d u r`td q|\}}}z|| || W n t	y   Y n0 d  } }}q|r6|jt
kr6z
| }W n" ttfy   td Y d S 0 |d u rtd q|\}}}z|| || W n t	y&   Y n0 d  } }}qt| drtd z,tdD ]}| j sl qv|  qVW n ttfy   Y n0 tdt||j d S )	Nz.result handler got EOFError/OSError -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelr=   z"ensuring that outqueue is not full
   z7result handler exiting: len(cache)=%s, thread._state=%s)ru   r   r?   r>   r   r@   r`   r
   r   r   r   r;   r   r=   pollr   )rE   r:   r   r   rI   rJ   rK   objr   r   r   r   :  sZ    











zPool._handle_resultsc                 c   s0   t |}tt||}|s d S | |fV  qd S r   )r   tupler   islice)rL   itsizer   r   r   r   r   v  s
    zPool._get_tasksc                 C   s   t dd S )Nz:pool objects cannot be passed between processes or pickled)NotImplementedErrorr   r   r   r   r*     s    zPool.__reduce__c                 C   s2   t d | jtkr.t| _t| j_| jd  d S )Nzclosing pool)r   r@   r`   r
   r   r{   rf   r9   r   r   r   r   r<     s
    

z
Pool.closec                 C   s   t d t| _|   d S )Nzterminating pool)r   r@   r   r`   r   r   r   r   r   rs     s    
zPool.terminatec                 C   sj   t d | jtkrtdn| jttfvr4td| j  | j	  | j
  | jD ]}|  qXd S )Nzjoining poolzPool is still runningzIn unknown state)r   r@   r`   r
   rm   r   r   r{   r'   r   r   r_   )r   r   r   r   r   r'     s    






z	Pool.joinc                 C   s@   t d | j  | r<| j r<| j  t	d qd S )Nz7removing tasks from inqueue until task handler finishedr   )
r   r@   _rlockacquireis_aliver=   r   r   timesleep)rD   task_handlerr   r   r   r   _help_stuff_finish  s
    


zPool._help_stuff_finishc
                 C   sX  t d t|_|d  t|_t d | ||t| | sXt|	dkrXtdt|_|d  |d  t d t	
 |ur|  |rt|d drt d |D ]}
|
jd u r|
  qt d t	
 |ur|  t d	 t	
 |ur|  |rTt|d drTt d
 |D ](}
|
 r*t d|
j  |
  q*d S )Nzfinalizing poolz&helping task handler/workers to finishr   z.Cannot have cache with result_hander not alivezjoining worker handlerrs   zterminating workerszjoining task handlerzjoining result handlerzjoining pool workersr   )r   r@   r   r`   r9   r   r   r   r7   ru   r   r'   r;   rr   rs   pid)r   r   rD   rE   r   r   Zworker_handlerr   Zresult_handlerr   r   r   r   r   r     sB    












zPool._terminate_poolc                 C   s   |    | S r   )r   r   r   r   r   	__enter__  s    zPool.__enter__c                 C   s   |    d S r   )rs   )r   exc_typeexc_valexc_tbr   r   r   __exit__  s    zPool.__exit__)NNr   NN)N)N)NNN)r   )r   )NNN)NNN)N),r   r    r!   rz   staticmethodrY   r   warningswarnr
   r   r2   rt   r   r   rq   r   r   rb   r   r   r   r   r   r   r   r   r   r   r   r   classmethodrw   r~   r   r   r*   r<   rs   r'   r   r   r   r   r   r   r   r   r      st   
  
P

	



  




  
  


-
;


5c                   @   sJ   e Zd Zdd Zdd Zdd Zddd	Zdd
dZdd Ze	e
jZdS )r   c                 C   s>   || _ t | _tt| _|j| _|| _|| _	| | j| j< d S r   )
r_   ru   Event_eventnextjob_counterr   rg   	_callback_error_callback)r   r   r   r   r   r   r   r     s    

zApplyResult.__init__c                 C   s
   | j  S r   )r   is_setr   r   r   r   ready  s    zApplyResult.readyc                 C   s   |   std| | jS )Nz{0!r} not ready)r   rm   r8   _successr   r   r   r   
successful  s    zApplyResult.successfulNc                 C   s   | j | d S r   )r   r   r   r   r   r   r   r     s    zApplyResult.waitc                 C   s,   |  | |  st| jr"| jS | jd S r   )r   r   r   r   _valuer   r   r   r   r:     s    
zApplyResult.getc                 C   sZ   |\| _ | _| jr$| j r$| | j | jr<| j s<| | j | j  | j| j= d | _d S r   )	r   r   r   r   r   setrg   r   r_   r   rK   r   r   r   r   r     s    

zApplyResult._set)N)N)r   r    r!   r   r   r   r   r:   r   r   typesGenericAlias__class_getitem__r   r   r   r   r     s   	

	
r   c                   @   s   e Zd Zdd Zdd ZdS )r   c                 C   sh   t j| |||d d| _d g| | _|| _|dkrNd| _| j  | j| j	= n|| t
||  | _d S )Nr   Tr   )r   r   r   r   
_chunksize_number_leftr   r   rg   r   bool)r   r   r   lengthr   r   r   r   r   r     s    

zMapResult.__init__c                 C   s   |  j d8  _ |\}}|rv| jrv|| j|| j |d | j < | j dkr| jrZ| | j | j| j= | j  d | _	nL|s| jrd| _|| _| j dkr| j
r| 
| j | j| j= | j  d | _	d S )Nr   r   F)r  r   r   r  r   rg   r   r   r   r_   r   )r   rK   Zsuccess_resultsuccessrN   r   r   r   r   &  s&    







zMapResult._setN)r   r    r!   r   r   r   r   r   r   r     s   r   c                   @   s:   e Zd Zdd Zdd ZdddZeZdd	 Zd
d ZdS )r   c                 C   sT   || _ tt | _tt| _|j| _t	
 | _d| _d | _i | _| | j| j< d S r   )r_   ru   	ConditionLock_condr   r   r   rg   collectionsdeque_items_index_length	_unsorted)r   r   r   r   r   r   D  s    

zIMapIterator.__init__c                 C   s   | S r   r   r   r   r   r   __iter__O  s    zIMapIterator.__iter__Nc                 C   s   | j  z| j }W nv ty   | j| jkr<d | _td | j | z| j }W n0 ty   | j| jkr|d | _td t	d Y n0 Y n0 W d    n1 s0    Y  |\}}|r|S |d S r   )
r  r  popleft
IndexErrorr  r  r_   StopIterationr   r   )r   r   rW   r	  r.   r   r   r   r   R  s&    0zIMapIterator.nextc                 C   s   | j  | j|krn| j| |  jd7  _| j| jv rb| j| j}| j| |  jd7  _q,| j   n
|| j|< | j| jkr| j| j	= d | _
W d    n1 s0    Y  d S Nr   )r  r  r  r   r  popnotifyr  rg   r   r_   r  r   r   r   r   j  s    


zIMapIterator._setc                 C   sV   | j < || _| j| jkr4| j   | j| j= d | _W d    n1 sH0    Y  d S r   )r  r  r  r  rg   r   r_   )r   r  r   r   r   r   {  s    

zIMapIterator._set_length)N)	r   r    r!   r   r  r   __next__r   r   r   r   r   r   r   B  s   
r   c                   @   s   e Zd Zdd ZdS )r   c                 C   sj   | j P | j| |  jd7  _| j   | j| jkrH| j| j= d | _W d    n1 s\0    Y  d S r  )	r  r  r   r  r  r  rg   r   r_   r  r   r   r   r     s    

zIMapUnorderedIterator._setN)r   r    r!   r   r   r   r   r   r     s   r   c                   @   sV   e Zd ZdZedd ZdddZdd	 Zd
d Zedd Z	edd Z
dd ZdS )r   Fc                 O   s   ddl m} ||i |S )Nr   rX   )dummyrY   )rZ   r   rM   rY   r   r   r   rY     s    zThreadPool.ProcessNr   c                 C   s   t | ||| d S r   )r   r   )r   r   rF   rG   r   r   r   r     s    zThreadPool.__init__c                 C   s,   t  | _t  | _| jj| _| jj| _d S r   )rc   rd   rx   ry   r9   r   r:   r   r   r   r   r   rb     s    


zThreadPool._setup_queuesc                 C   s
   | j jgS r   )rf   r=   r   r   r   r   rt     s    zThreadPool._get_sentinelsc                 C   s   g S r   r   r   r   r   r   r     s    z ThreadPool._get_worker_sentinelsc                 C   sD   z| j dd qW n tjy&   Y n0 t|D ]}| d  q0d S )NF)block)r:   rc   Emptyr   r9   )rD   r   r   rK   r   r   r   r     s    zThreadPool._help_stuff_finishc                 C   s   t | d S r   )r   r   )r   r   r   r   r   r   r   r     s    zThreadPool._wait_for_updates)NNr   )r   r    r!   rz   r   rY   r   rb   rt   r   r   r   r   r   r   r   r     s   




)Nr   NF))__all__r  r   rk   rc   ru   r   r$   r  r   r#   r   r   r   
connectionr   r	   r
   r   r   countr   r   r   rA   r   r"   r)   r,   rQ   rB   dictrS   objectr   r   ZAsyncResultr   r   r   r   r   r   r   r   <module>
   sN   	  
-    =++E