]d @ s d d g Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l m Z d d l m
Z
m Z d Z d Z
d Z e j Z d d Z d
d Z Gd d
d
e Z Gd d d Z d d Z Gd d d e Z d f d d d d Z d d Z Gd d d e Z Gd d d e Z e Z Gd d d e Z Gd d d e Z Gd d! d! e Z Gd" d d e Z! d S)#Pool
ThreadPool N )util) get_context TimeoutError c C s t t | S)N)listmap)args r )/opt/alt/python35/lib64/python3.5/pool.pymapstar+ s r c C s t t j | d | d S)Nr r )r itertoolsstarmap)r r r r
starmapstar. s r c @ s( e Z d Z d d Z d d Z d S)RemoteTracebackc C s
| | _ d S)N)tb)selfr r r r
__init__6 s zRemoteTraceback.__init__c C s | j S)N)r )r r r r
__str__8 s zRemoteTraceback.__str__N)__name__
__module__ __qualname__r r r r r r
r 5 s r c @ s( e Z d Z d d Z d d Z d S)ExceptionWithTracebackc C sD t j t | | | } d j | } | | _ d | | _ d S)N z
"""
%s""") tracebackformat_exceptiontypejoinexcr )r r r r r r
r < s z ExceptionWithTraceback.__init__c C s t | j | j f f S)N) rebuild_excr r )r r r r
__reduce__A s z!ExceptionWithTraceback.__reduce__N)r r r r r" r r r r
r ; s r c C s t | | _ | S)N)r __cause__)r r r r r
r! D s r! c s: e Z d Z f d d Z d d Z d d Z S)MaybeEncodingErrorc sA t | | _ t | | _ t t | j | j | j d S)N)reprr valuesuperr$ r )r r r& ) __class__r r
r P s zMaybeEncodingError.__init__c C s d | j | j f S)Nz(Error sending result: '%s'. Reason: '%s')r& r )r r r r
r U s zMaybeEncodingError.__str__c C s d | j j | f S)Nz<%s: %s>)r( r )r r r r
__repr__Y s zMaybeEncodingError.__repr__)r r r r r r) r r )r( r
r$ L s r$ Fc ' C s | j } | j } t | d r; | j j | j j | d k rQ | | d } x| d k sx | r| | k ry
| } Wn& t t f k
r t j d PYn X| d k r t j d P| \ }
} } }
} y d | |
| f } WnU t
k
rM} z5 | r/| t k r/t | | j
} d | f } WYd d } ~ Xn Xy | |
| | f Wnb t
k
r} zB t | | d } t j d | | |
| d | f f WYd d } ~ Xn Xd } }
} } }
} | d 7} qZ Wt j d | d S)
N_writerr z)worker got EOFError or OSError -- exitingz worker got sentinel -- exitingTFr z0Possible encoding error while sending result: %sz worker exiting after %d tasks)putgethasattrr* close_readerEOFErrorOSErrorr debug Exception_helper_reraises_exceptionr
__traceback__r$ )inqueueoutqueue initializerinitargsZmaxtasksZwrap_exceptionr+ r, Z completedtaskjobifuncr kwdsresultewrappedr r r
worker] sD
!
,rB c C s
| d S)Nr )exr r r
r4 s r4 c @ s e Z d Z d Z d d Z d d f d d d d Z d d Z d d
Z d d Z d
d Z f i d d Z
d d d Z d d d Z d d d d d Z
d d Z d d d Z d d d Z f i d d d d Z d d d d d! Z d d d d" d# Z e d$ d% Z e d& d' Z e d( d) Z e d* d+ Z d, d- Z d. d/ Z d0 d1 Z d2 d3 Z e d4 d5 Z e d6 d7 Z d8 d9 Z d: d; Z! d S)<r Tc O s | j j | | S)N)_ctxProcess)r r r> r r r
rE s z Pool.ProcessNc C s# | p t | _ | j t j | _ i | _ t | _ | | _ | | _
| | _ | d k rv t j
ps d } | d k r t d | d k r t | r t d | | _ g | _ | j t j d t j d | f | _ d | j _ t | j _ | j j t j d t j d | j | j | j | j | j f | _ d | j _ t | j _ | j j t j d t j d | j | j | j f | _! d | j! _ t | j! _ | j! j t" j# | | j$ d | j | j% | j | j | j | j | j! | j f d d | _& d S) Nr z&Number of processes must be at least 1z initializer must be a callabletargetr T exitpriority )'r rD
_setup_queuesqueueQueue
_taskqueue_cacheRUN_state_maxtasksperchild _initializer _initargsos cpu_count
ValueErrorcallable TypeError
_processes_pool_repopulate_pool threadingThreadr _handle_workers_worker_handlerdaemonstart
_handle_tasks
_quick_put _outqueue
_task_handler_handle_results
_quick_get_result_handlerr Finalize_terminate_pool_inqueue
_terminate)r processesr8 r9 maxtasksperchildcontextr r r
r sT
z
Pool.__init__c C sw d } xj t t t | j D]M } | j | } | j d k r" t j d | | j d } | j | =q" W| S)NFzcleaning up worker %dT)reversedrangelenrY exitcoder r2 r )r Zcleanedr< rB r r r
_join_exited_workers s "
zPool._join_exited_workersc C s x t | j t | j D] } | j d t d | j | j | j | j | j
| j f } | j j | | j
j d d | _
d | _ | j t j d q Wd S)NrF r rE Z
PoolWorkerTz added worker)rp rX rq rY rE rB rj rc rQ rR rP _wrap_exceptionappendnamereplacer_ r` r r2 )r r< wr r r
rZ s #
zPool._repopulate_poolc C s | j r | j d S)N)rs rZ )r r r r
_maintain_pool s zPool._maintain_poolc C sL | j j | _ | j j | _ | j j j | _ | j j j | _ d S)N)
rD SimpleQueuerj rc r* sendrb r/ recvrf )r r r r
rI s zPool._setup_queuesc C s | j | | | j S)N) apply_asyncr, )r r= r r> r r r
apply s z
Pool.applyc C s | j | | t | j S)N)
_map_asyncr r, )r r= iterable chunksizer r r
r
s zPool.mapc C s | j | | t | j S)N)r r r, )r r= r r r r r
r s z Pool.starmapc C s | j | | t | | | S)N)r r )r r= r r callbackerror_callbackr r r
starmap_async s zPool.starmap_asyncc c s y> d } x1 t | D]# \ } } | | | | f i f Vq WWn@ t k
r } z | | d t | f i f VWYd d } ~ Xn Xd S)Nr ) enumerater3 r4 )r Z
result_jobr= r r< xr@ r r r
_guarded_task_generation s z Pool._guarded_task_generationr c C s | j t k r t d | d k re t | j } | j j | j | j | | | j f | St
j | | | } t | j } | j j | j | j t | | j f d d | D Sd S)NzPool not runningr c s s" | ] } | D] } | Vq
q d S)Nr ).0chunkitemr r r
<genexpr>@ s z Pool.imap.<locals>.<genexpr>)
rO rN rU IMapIteratorrM rL r+ r _job _set_lengthr
_get_tasksr )r r= r r r? task_batchesr r r
imap' s
z Pool.imapc C s | j t k r t d | d k re t | j } | j j | j | j | | | j f | St
j | | | } t | j } | j j | j | j t | | j f d d | D Sd S)NzPool not runningr c s s" | ] } | D] } | Vq
q d S)Nr )r r r r r r
r [ s z&Pool.imap_unordered.<locals>.<genexpr>)
rO rN rU IMapUnorderedIteratorrM rL r+ r r r r r r )r r= r r r? r r r r
imap_unorderedB s
zPool.imap_unorderedc C s_ | j t k r t d t | j | | } | j j | j d | | | f g d f | S)NzPool not runningr )rO rN rU ApplyResultrM rL r+ r )r r= r r> r r r? r r r
r} ] s
+zPool.apply_asyncc C s | j | | t | | | S)N)r r )r r= r r r r r r r
map_asynch s zPool.map_asyncc
C s | j t k r t d t | d s6 t | } | d k rz t t | t | j d \ } } | rz | d 7} t | d k r d } t j | | | } t
| j | t | | d | } | j j
| j | j | | d f | S)NzPool not running__len__ r r r )rO rN rU r- r divmodrq rY r r MapResultrM rL r+ r r )
r r= r Zmapperr r r extrar r? r r r
r p s& (
zPool._map_asyncc C sr t j } xB | j t k s6 | j rP | j t k rP | j t j d q W| j j
d t j d d S)Ng?zworker handler exiting)
r[ current_threadrO rN rM TERMINATEry timesleeprL r+ r r2 )poolthreadr r r
r] s *
zPool._handle_workersc
C s t j } x+t | j d D]
\ } } d } z x | D] } | j rX t j d Py | | Wq; t k
r } zN | d d \ }
} y | |
j | d | f Wn t k
r Yn XWYd d } ~ Xq; Xq; W| rt j d | r | d n d } | | d w PWd d } } }
Xq Wt j d yF t j d | j
d t j d x | D] } | d qkWWn t k
rt j d Yn Xt j d
d S) Nz'task handler found thread._state != RUNr Fzdoing set_length()r ztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exitingr ) r[ r iterr, rO r r2 r3 _setKeyErrorr+ r1 )
taskqueuer+ r7 r cacher ZtaskseqZ
set_lengthr: r@ r; idxpr r r
ra sB
zPool._handle_tasksc C s t j } x y
| } Wn) t t f k
rG t j d d SYn X| j r_ t j d P| d k ry t j d P| \ } } } y | | j | | Wn t k
r Yn Xd } } } q Wx | r| j t k ry
| } Wn) t t f k
rt j d d SYn X| d k r4t j d q | \ } } } y | | j | | Wn t k
roYn Xd } } } q Wt
| d rt j d y2 x+ t d D] } | j j
sP| qWWn t t f k
rYn Xt j d t | | j d S) Nz.result handler got EOFError/OSError -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelr/ z"ensuring that outqueue is not full
z7result handler exiting: len(cache)=%s, thread._state=%s)r[ r r1 r0 r r2 rO r r r r- rp r/ pollrq )r7 r, r r r: r; r< objr r r
re sZ
zPool._handle_resultsc c sD t | } x1 t t j | | } | s1 d S| | f Vq Wd S)N)r tupler islice)r= itsizer r r r
r s zPool._get_tasksc C s t d d S)Nz:pool objects cannot be passed between processes or pickled)NotImplementedError)r r r r
r" s zPool.__reduce__c C s5 t j d | j t k r1 t | _ t | j _ d S)Nz closing pool)r r2 rO rN CLOSEr^ )r r r r
r. s
z
Pool.closec C s0 t j d t | _ t | j _ | j d S)Nzterminating pool)r r2 r rO r^ rk )r r r r
terminate s
zPool.terminatec C sV t j d | j j | j j | j j x | j D] } | j q> Wd S)Nz joining pool)r r2 r^ r rd rg rY )r r r r r
r s
z Pool.joinc C sZ t j d | j j x9 | j rU | j j rU | j j t j d q Wd S)Nz7removing tasks from inqueue until task handler finishedr )
r r2 Z_rlockacquireis_aliver/ r r| r r )r6 task_handlerr r r r
_help_stuff_finish( s
zPool._help_stuff_finishc
C s t j d t | _ t | _ t j d | j | | t | t | _ | j d t j d t j | k r | j | r t
| d d r t j d x' | D] } | j d k r | j q Wt j d t j | k r | j t j d t j | k r&| j | rt
| d d rt j d x8 | D]0 } | j
rSt j d
| j | j qSWd S) Nzfinalizing poolz&helping task handler/workers to finishzjoining worker handlerr r zterminating workerszjoining task handlerzjoining result handlerzjoining pool workerszcleaning up worker %d)r r2 r rO r rq r+ r[ r r r- rr r r pid)
clsr r6 r7 r Zworker_handlerr Zresult_handlerr r r r r
ri 1 s6
zPool._terminate_poolc C s | S)Nr )r r r r
__enter___ s zPool.__enter__c C s | j d S)N)r )r exc_typeexc_valexc_tbr r r
__exit__b s z
Pool.__exit__)"r r r rt rE r rs rZ ry rI r~ r
r r r r r r} r r staticmethodr] ra re r r" r. r r r classmethodri r r r r r r
r sD 8
.< . c @ s^ e Z d Z d d Z d d Z d d Z d d d Z d d
d Z d d
Z d S)r c C sJ t j | _ t t | _ | | _ | | _ | | _ | | | j <d S)N) r[ Event_eventnext job_counterr rM _callback_error_callback)r r r r r r r
r k s zApplyResult.__init__c C s
| j j S)N)r is_set)r r r r
readys s zApplyResult.readyc C s | j S)N)_success)r r r r
successfulv s zApplyResult.successfulNc C s | j j | d S)N)r wait)r timeoutr r r
r z s zApplyResult.waitc C s< | j | | j s t | j r/ | j S| j d S)N)r r r r _value)r r r r r
r, } s
zApplyResult.getc C su | \ | _ | _ | j r4 | j r4 | j | j | j rW | j rW | j | j | j j | j | j =d S)N)r r r r r setrM r )r r< r r r r
r s
zApplyResult._set) r r r r r r r r, r r r r r
r i s r c @ s( e Z d Z d d Z d d Z d S)r c C s t j | | | d | d | _ d g | | _ | | _ | d k rj d | _ | j j | | j =n | | t | | | _ d S)Nr Tr )
r r r r
_chunksize _number_leftr r r bool)r r r lengthr r r r r
r s
zMapResult.__init__c C s | \ } } | r | | j | | j | d | j <| j d 8_ | j d k r | j rn | j | j | j | j =| j j nE d | _ | | _ | j r | j | j | j | j =| j j d S)Nr r F)
r r r r rM r r r r r )r r< Zsuccess_resultsuccessr? r r r
r s %
zMapResult._setN)r r r r r r r r r
r s
r c @ sU e Z d Z d d Z d d Z d d d Z e Z d d Z d
d Z d S) r c C sk t j t j | _ t t | _ | | _ t j | _
d | _ d | _ i | _
| | | j <d S)Nr )r[ ConditionLock_condr r r rM collectionsdeque_items_index_length _unsorted)r r r r r
r s zIMapIterator.__init__c C s | S)Nr )r r r r
__iter__ s zIMapIterator.__iter__Nc C s | j y | j j } Wn t k
r | j | j k rE t | j j | y | j j } Wn0 t k
r | j | j k r t t Yn XYn XWd QRX| \ } } | r | S| d S)N) r r popleft
IndexErrorr r
StopIterationr r )r r r r r& r r r
r s"
zIMapIterator.nextc
C s | j | j | k r | j j | | j d 7_ xJ | j | j k r | j j | j } | j j | | j d 7_ q; W| j j n
| | j | <| j | j k r | j | j =Wd QRXd S)Nr )
r r r ru r popnotifyr rM r )r r< r r r r
r s
zIMapIterator._setc C sJ | j : | | _ | j | j k r? | j j | j | j =Wd QRXd S)N)r r r r rM r )r r r r r
r s
zIMapIterator._set_length) r r r r r r __next__r r r r r r
r s
r c @ s e Z d Z d d Z d S)r c
C s` | j P | j j | | j d 7_ | j j | j | j k rU | j | j =Wd QRXd S)Nr )r r ru r r r rM r )r r< r r r r
r s
zIMapUnorderedIterator._setN)r r r r r r r r
r s r c @ s[ e Z d Z d Z e d d Z d d f d d Z d d Z e d d
Z d S) r Fc O s d d l m } | | | S)Nr )rE )dummyrE )r r> rE r r r
rE s zThreadPool.ProcessNc C s t j | | | | d S)N)r r )r rl r8 r9 r r r
r s zThreadPool.__init__c C s@ t j | _ t j | _ | j j | _ | j j | _ d S)N)rJ rK rj rc r+ rb r, rf )r r r r
rI s zThreadPool._setup_queuesc
C sF | j 6 | j j | j j d g | | j j Wd QRXd S)N) not_emptyrJ clearextend
notify_all)r6 r r r r r
r s
z ThreadPool._help_stuff_finish) r r r rt r rE r rI r r r r r
r s
)"__all__r[ rJ r r rS r r r r r r rN r r countr r r r3 r r r! r$ rB r4 objectr r Z AsyncResultr r r r r r r r
<module>
s< * &%@