Viewing File: /opt/alt/python35/lib64/python3.5/multiprocessing/__pycache__/pool.cpython-35.pyc



Yfd@sddgZddlZddlZddlZddlZddlZddlZddlZddlm	Z	ddlm
Z
mZdZdZ
dZejZdd	Zd
dZGdd
d
eZGdddZddZGdddeZdfddddZddZGdddeZGdddeZeZGdddeZGdddeZGd d!d!eZ Gd"ddeZ!dS)#Pool
ThreadPoolN)util)get_contextTimeoutErrorcCstt|S)N)listmap)argsr9/opt/alt/python35/lib64/python3.5/multiprocessing/pool.pymapstar+srcCsttj|d|dS)Nrr)r		itertoolsstarmap)rrrr
starmapstar.src@s(eZdZddZddZdS)RemoteTracebackcCs
||_dS)N)tb)selfrrrr
__init__6szRemoteTraceback.__init__cCs|jS)N)r)rrrr
__str__8szRemoteTraceback.__str__N)__name__
__module____qualname__rrrrrr
r5src@s(eZdZddZddZdS)ExceptionWithTracebackcCsDtjt|||}dj|}||_d||_dS)Nz

"""
%s""")	tracebackformat_exceptiontypejoinexcr)rr rrrr
r<s	zExceptionWithTraceback.__init__cCst|j|jffS)N)rebuild_excr r)rrrr

__reduce__Asz!ExceptionWithTraceback.__reduce__N)rrrrr"rrrr
r;srcCst||_|S)N)r	__cause__)r rrrr
r!Dsr!cs@eZdZdZfddZddZddZS)MaybeEncodingErrorzVWraps possible unpickleable errors, so they can be
    safely sent through the socket.csAt||_t||_tt|j|j|jdS)N)reprr valuesuperr$r)rr r&)	__class__rr
rPszMaybeEncodingError.__init__cCsd|j|jfS)Nz(Error sending result: '%s'. Reason: '%s')r&r )rrrr
rUs	zMaybeEncodingError.__str__cCsd|jj|fS)Nz<%s: %s>)r(r)rrrr
__repr__YszMaybeEncodingError.__repr__)rrr__doc__rrr)rr)r(r
r$Lsr$Fc'Cs7|dks0t|tkr*|dks0t|j}|j}t|drk|jj|jj|dk	r||d}x|dks|r!||kr!y
|}	Wn&t	t
fk
rtjdPYnX|	dkrtjdP|	\}
}}}
}yd||
|f}WnUt
k
r}}z5|r_|tk	r_t||j}d|f}WYdd}~XnXy||
||fWnbt
k
r}zBt||d}tjd|||
|d|ffWYdd}~XnXd}	}
}}}
}|d7}qWtjd	|dS)
Nr_writerz)worker got EOFError or OSError -- exitingzworker got sentinel -- exitingTFrz0Possible encoding error while sending result: %szworker exiting after %d tasks)rintAssertionErrorputgethasattrr+close_readerEOFErrorOSErrorrdebug	Exception_helper_reraises_exceptionr
__traceback__r$)inqueueoutqueueinitializerinitargsZmaxtasksZwrap_exceptionr.r/Z	completedtaskjobifuncrkwdsresultewrappedrrr
worker]sF0		


!


	,rEcCs
|dS)z@Pickle-able helper function for use by _guarded_task_generation.Nr)Zexrrr
r7sr7c@seZdZdZdZddZddfddddZdd	Zd
dZdd
Z	ddZ
fiddZdddZdddZ
dddddZddZdddZdddZfidddd Zdddd!d"Zdddd#d$Zed%d&Zed'd(Zed)d*Zed+d,Zd-d.Zd/d0Zd1d2Zd3d4Zed5d6Zed7d8Z d9d:Z!d;d<Z"dS)=rzS
    Class which supports an async version of applying functions to arguments.
    TcOs|jj||S)N)_ctxProcess)rrrArrr
rGszPool.ProcessNcCs#|pt|_|jtj|_i|_t|_||_	||_
||_|dkrvtj
psd}|dkrtd|dk	rt|rtd||_g|_|jtjdtjd|f|_d|j_t|j_|jjtjdtjd|j|j|j|j|jf|_d|j_t|j_|jjtjdtjd|j|j |jf|_!d|j!_t|j!_|j!jt"j#||j$d|j|j%|j|j|j|j|j!|jfdd|_&dS)	Nrz&Number of processes must be at least 1zinitializer must be a callabletargetrTZexitpriority)'rrF
_setup_queuesqueueQueue
_taskqueue_cacheRUN_state_maxtasksperchild_initializer	_initargsos	cpu_count
ValueErrorcallable	TypeError
_processes_pool_repopulate_pool	threadingZThreadr_handle_workers_worker_handlerdaemonstart
_handle_tasks
_quick_put	_outqueue
_task_handler_handle_results
_quick_get_result_handlerrZFinalize_terminate_pool_inqueue
_terminate)r	processesr;r<Zmaxtasksperchildcontextrrr
rsT
							
		
		
		
z
Pool.__init__cCswd}xjttt|jD]M}|j|}|jdk	r"tjd||jd}|j|=q"W|S)zCleanup after any worker processes which have exited due to reaching
        their specified lifetime.  Returns True if any workers were cleaned up.
        FNzcleaning up worker %dT)reversedrangelenrZexitcoderr5r)rZcleanedr?rErrr
_join_exited_workerss"

zPool._join_exited_workerscCsxt|jt|jD]}|jdtd|j|j|j|j	|j
|jf}|jj||j
jdd|_
d|_|jtjdqWdS)zBring the number of pool processes up to the specified number,
        for use after reaping workers which have exited.
        rHrrGZ
PoolWorkerTzadded workerN)rnrYrorZrGrErircrRrSrQ_wrap_exceptionappendnamereplacer_r`rr5)rr?wrrr
r[s#	
zPool._repopulate_poolcCs|jr|jdS)zEClean up any exited workers and start replacements for them.
        N)rqr[)rrrr
_maintain_poolszPool._maintain_poolcCsL|jj|_|jj|_|jjj|_|jjj|_	dS)N)
rFZSimpleQueuerircr+sendrbr2recvrf)rrrr
rJszPool._setup_queuescCs.|jtkst|j|||jS)z6
        Equivalent of `func(*args, **kwds)`.
        )rPrOr-apply_asyncr/)rr@rrArrr
applysz
Pool.applycCs|j||t|jS)zx
        Apply `func` to each element in `iterable`, collecting the results
        in a list that is returned.
        )
_map_asyncrr/)rr@iterable	chunksizerrr
r
szPool.mapcCs|j||t|jS)z
        Like `map()` method but the elements of the `iterable` are expected to
        be iterables as well and will be unpacked as arguments. Hence
        `func` and (a, b) becomes func(a, b).
        )r|rr/)rr@r}r~rrr
rszPool.starmapcCs|j||t|||S)z=
        Asynchronous version of `starmap()` method.
        )r|r)rr@r}r~callbackerror_callbackrrr

starmap_asyncszPool.starmap_asyncccsy>d}x1t|D]#\}}||||fifVqWWn@tk
r}z ||dt|fifVWYdd}~XnXdS)zProvides a generator of tasks for imap and imap_unordered with
        appropriate handling for iterables which throw exceptions during
        iteration.rN)	enumerater6r7)rZ
result_jobr@r}r?xrCrrr
_guarded_task_generationszPool._guarded_task_generationrcCs|jtkrtd|dkret|j}|jj|j|j|||j	f|S|dkswt
tj|||}t|j}|jj|j|jt
||j	fdd|DSdS)zP
        Equivalent of `map()` -- can be MUCH slower than `Pool.map()`.
        zPool not runningrcss"|]}|D]}|Vq
qdS)Nr).0chunkitemrrr
	<genexpr>@szPool.imap.<locals>.<genexpr>N)rPrOrVIMapIteratorrNrMr.r_job_set_lengthr-r
_get_tasksr)rr@r}r~rBtask_batchesrrr
imap's"	
	
z	Pool.imapcCs|jtkrtd|dkret|j}|jj|j|j|||j	f|S|dkswt
tj|||}t|j}|jj|j|jt
||j	fdd|DSdS)zL
        Like `imap()` method but ordering of results is arbitrary.
        zPool not runningrcss"|]}|D]}|Vq
qdS)Nr)rrrrrr
r[sz&Pool.imap_unordered.<locals>.<genexpr>N)rPrOrVIMapUnorderedIteratorrNrMr.rrrr-rrr)rr@r}r~rBrrrr
imap_unorderedBs"	
	
zPool.imap_unorderedcCs_|jtkrtdt|j||}|jj|jd|||fgdf|S)z;
        Asynchronous version of `apply()` method.
        zPool not runningrN)rPrOrVApplyResultrNrMr.r)rr@rrArrrBrrr
rz]s
+zPool.apply_asynccCs|j||t|||S)z9
        Asynchronous version of `map()` method.
        )r|r)rr@r}r~rrrrr
	map_asynchszPool.map_asyncc
Cs|jtkrtdt|ds6t|}|dkrztt|t|jd\}}|rz|d7}t|dkrd}tj	|||}t
|j|t||d|}	|jj
|j|	j||df|	S)zY
        Helper function to implement map, starmap and their async counterparts.
        zPool not running__len__Nrrr)rPrOrVr0r	divmodrorZrr	MapResultrNrMr.rr)
rr@r}Zmapperr~rrZextrarrBrrr
r|ps&(
		
zPool._map_asynccCsrtj}xB|jtks6|jrP|jtkrP|jtjdqW|j	j
dtjddS)Ng?zworker handler exiting)
r\current_threadrPrOrN	TERMINATErwtimesleeprMr.rr5)poolthreadrrr
r]s*
zPool._handle_workersc
Cstj}x+t|jdD]
\}}d}zx|D]}|jrXtjdPy||Wq;tk
r}	zN|dd\}
}y||
j|d|	fWnt	k
rYnXWYdd}	~	Xq;Xq;W|rtjd|r|dnd}||dwPWdd}}}
XqWtjdyFtjd|j
dtjdx|D]}|dqkWWntk
rtjd	YnXtjd
dS)Nz'task handler found thread._state != RUNrFzdoing set_length()rztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exitingr)r\riterr/rPrr5r6_setKeyErrorr.r4)
	taskqueuer.r:rcacherZtaskseqZ
set_lengthr=rCr>idxprrr
rasB
	








zPool._handle_taskscCs tj}xy
|}Wn)ttfk
rGtjddSYnX|jrt|jtksfttjdP|dkrtjdP|\}}}y||j	||Wnt
k
rYnXd}}}qWx|r|jtkry
|}Wn)ttfk
r,tjddSYnX|dkrItjdq|\}}}y||j	||Wnt
k
rYnXd}}}qWt|drtjdy2x+tdD]}|j
jsP|qWWnttfk
rYnXtjdt||jdS)	Nz.result handler got EOFError/OSError -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelr2z"ensuring that outqueue is not full
z7result handler exiting: len(cache)=%s, thread._state=%s)r\rr4r3rr5rPrr-rrr0rnr2pollro)r:r/rrr=r>r?objrrr
res\

		




	


	zPool._handle_resultsccsDt|}x1ttj||}|s1dS||fVqWdS)N)rtuplerislice)r@itsizerrrr
rszPool._get_taskscCstddS)Nz:pool objects cannot be passed between processes or pickled)NotImplementedError)rrrr
r"szPool.__reduce__cCs5tjd|jtkr1t|_t|j_dS)Nzclosing pool)rr5rPrOCLOSEr^)rrrr
r1s
	z
Pool.closecCs0tjdt|_t|j_|jdS)Nzterminating pool)rr5rrPr^rj)rrrr
	terminates
	zPool.terminatecCsqtjd|jttfks(t|jj|jj|j	jx|j
D]}|jqYWdS)Nzjoining pool)rr5rPrrr-r^rrdrgrZ)rrrrr
rs



z	Pool.joincCsZtjd|jjx9|jrU|jjrU|jjtj	dqWdS)Nz7removing tasks from inqueue until task handler finishedr)
rr5Z_rlockacquireis_aliver2rryrr)r9task_handlerrrrr
_help_stuff_finish(s



zPool._help_stuff_finishc	
Cstjdt|_t|_tjd|j||t||jsit|dksitt|_|jdtjdt	j
|k	r|j|rt|ddrtjdx'|D]}	|	j
dkr|	jqWtjdt	j
|k	r!|jtjdt	j
|k	rJ|j|rt|ddrtjd	x8|D]0}	|	jrwtjd
|	j|	jqwWdS)Nzfinalizing poolz&helping task handler/workers to finishrzjoining worker handlerrzterminating workerszjoining task handlerzjoining result handlerzjoining pool workerszcleaning up worker %d)rr5rrPrrorr-r.r\rrr0rprpid)
clsrr9r:rZworker_handlerrZresult_handlerrrrrr
rh1s8
		
$	










zPool._terminate_poolcCs|S)Nr)rrrr
	__enter___szPool.__enter__cCs|jdS)N)r)rexc_typeZexc_valZexc_tbrrr
__exit__bsz
Pool.__exit__)#rrrr*rrrGrrqr[rwrJr{r
rrrrrrzrr|staticmethodr]rarerr"r1rrrclassmethodrhrrrrrr
rsF	8	

.<			.c@s^eZdZddZddZddZddd	Zdd
dZdd
ZdS)rcCsJtj|_tt|_||_||_||_|||j<dS)N)	r\ZEvent_eventnextjob_counterrrN	_callback_error_callback)rrrrrrr
rks			zApplyResult.__init__cCs
|jjS)N)rZis_set)rrrr
readysszApplyResult.readycCs|jst|jS)N)rr-_success)rrrr

successfulvszApplyResult.successfulNcCs|jj|dS)N)rwait)rtimeoutrrr
rzszApplyResult.waitcCs<|j||jst|jr/|jS|jdS)N)rrrr_value)rrrrr
r/}s
	zApplyResult.getcCsu|\|_|_|jr4|jr4|j|j|jrW|jrW|j|j|jj|j|j=dS)N)rrrrrsetrNr)rr?rrrr
rs
zApplyResult._set)	rrrrrrrr/rrrrr
ris	rc@s(eZdZddZddZdS)rcCstj|||d|d|_dg||_||_|dkrjd|_|jj||j=n||t	|||_dS)NrTr)
rrrr
_chunksize_number_leftrrrbool)rrr~lengthrrrrr
rs			

zMapResult.__init__cCs|\}}|r||j||j|d|j<|jd8_|jdkr|jrn|j|j|j|j=|jjnEd|_||_|j	r|j	|j|j|j=|jjdS)NrrF)
rrrrrNrrrrr)rr?Zsuccess_resultsuccessrBrrr
rs%	
			
zMapResult._setN)rrrrrrrrr
rs
rc@sUeZdZddZddZdddZeZdd	Zd
dZdS)rcCsktjtj|_tt|_||_tj	|_
d|_d|_i|_
|||j<dS)Nr)r\Z	ConditionZLock_condrrrrNcollectionsdeque_items_index_length	_unsorted)rrrrr
rs				zIMapIterator.__init__cCs|S)Nr)rrrr
__iter__szIMapIterator.__iter__NcCs|jy|jj}Wntk
r|j|jkrEt|jj|y|jj}Wn0tk
r|j|jkrttYnXYnXWdQRX|\}}|r|S|dS)N)	rrpopleft
IndexErrorrr
StopIterationrr)rrrrr&rrr
rs"


zIMapIterator.nextc
Cs|j|j|kr|jj||jd7_xJ|j|jkr|jj|j}|jj||jd7_q;W|jjn
||j|<|j|jkr|j|j	=WdQRXdS)Nr)
rrrrsrpopnotifyrrNr)rr?rrrr
rs

zIMapIterator._setc	CsJ|j:||_|j|jkr?|jj|j|j=WdQRXdS)N)rrrrrNr)rrrrr
rs

	
zIMapIterator._set_length)	rrrrrr__next__rrrrrr
rs
rc@seZdZddZdS)rc
Cs`|jP|jj||jd7_|jj|j|jkrU|j|j=WdQRXdS)Nr)rrrsrrrrNr)rr?rrrr
rs

zIMapUnorderedIterator._setN)rrrrrrrr
rsrc@s[eZdZdZeddZddfddZddZed	d
ZdS)rFcOsddlm}|||S)Nr)rG)ZdummyrG)rrArGrrr
rGszThreadPool.ProcessNcCstj||||dS)N)rr)rrkr;r<rrr
rszThreadPool.__init__cCs@tj|_tj|_|jj|_|jj|_dS)N)rKrLrircr.rbr/rf)rrrr
rJszThreadPool._setup_queuesc
CsF|j6|jj|jjdg||jjWdQRXdS)N)Z	not_emptyrKclearextendZ
notify_all)r9rrrrr
rs

zThreadPool._help_stuff_finish)	rrrrrrrGrrJrrrrr
rs
)"__all__r\rKrrrTrrrrrrrOrrcountrrrr6rrr!r$rEr7objectrrZAsyncResultrrrrrrrr
<module>
s<		*&%@
Back to Directory File Manager