]N @ s d Z d d l Z d d l Z d d l m Z d d l Z d d l m Z d d l Z d d l m Z d d l m
Z
d d l Z d d l Z d d l
m Z d d l Z d d l Z e j Z d a d d
Z d Z Gd d
d
e Z Gd d d Z d d Z Gd d d e Z Gd d d e Z Gd d d e Z d d Z d d Z d d Z d d Z! d d! Z" d a# d a$ d" d# Z% Gd$ d% d% e& Z' Gd& d' d' e j( Z) e j* e d S)(z"Brian Quinlan (brian@sweetapp.com) N)_base)Full) SimpleQueue)wait)partialFc C sa d a t t j } x! | D] \ } } | j d q Wx | D] \ } } | j qC Wd S)NT) _shutdownlist_threads_queuesitemsputjoin)r
tq r ,/opt/alt/python35/lib64/python3.5/process.py _python_exitO s r c @ s( e Z d Z d d Z d d Z d S)_RemoteTracebackc C s
| | _ d S)N)tb)selfr r r r __init__a s z_RemoteTraceback.__init__c C s | j S)N)r )r r r r __str__c s z_RemoteTraceback.__str__N)__name__
__module__ __qualname__r r r r r r r ` s r c @ s( e Z d Z d d Z d d Z d S)_ExceptionWithTracebackc C sD t j t | | | } d j | } | | _ d | | _ d S)N z
"""
%s""") tracebackformat_exceptiontyper excr )r r r r r r r g s z _ExceptionWithTraceback.__init__c C s t | j | j f f S)N) _rebuild_excr r )r r r r
__reduce__l s z"_ExceptionWithTraceback.__reduce__N)r r r r r" r r r r r f s r c C s t | | _ | S)N)r __cause__)r r r r r r! o s r! c @ s e Z d Z d d Z d S) _WorkItemc C s( | | _ | | _ | | _ | | _ d S)N)futurefnargskwargs)r r% r& r' r( r r r r t s z_WorkItem.__init__N)r r r r r r r r r$ s s r$ c @ s" e Z d Z d d d d Z d S) _ResultItemNc C s | | _ | | _ | | _ d S)N)work_id exceptionresult)r r* r+ r, r r r r { s z_ResultItem.__init__)r r r r r r r r r) z s r) c @ s e Z d Z d d Z d S) _CallItemc C s( | | _ | | _ | | _ | | _ d S)N)r* r& r' r( )r r* r& r' r( r r r r s z_CallItem.__init__N)r r r r r r r r r- s r- c g s> t | } x+ t t j | | } | s1 d S| Vq Wd S)N)ziptuple itertoolsislice) chunksize iterablesitchunkr r r _get_chunks s r6 c s f d d | D S)Nc s g | ] } | q Sr r ).0r' )r& r r
<listcomp> s z"_process_chunk.<locals>.<listcomp>r )r& r5 r )r& r _process_chunk s r9 c C s x | j d d } | d k r8 | j t j d Sy | j | j | j } WnS t k
r } z3 t | | j } | j t
| j d | WYd d } ~ Xq X| j t
| j d | q Wd S)NblockTr+ r, ) getr osgetpidr& r' r(
BaseExceptionr
__traceback__r) r* )
call_queue result_queue call_itemrer r r r _process_worker s
/rE c C s x | j r d Sy | j d d } Wn t j k
rD d SYq X| | } | j j r | j t | | j | j | j
d d q | | =q q Wd S)Nr: FT) fullr; queueEmptyr% set_running_or_notify_cancelr r- r& r' r( )pending_work_itemswork_idsr@ r* work_itemr r r _add_call_item_to_queue s
rM c s d f d d } f d d } | j } xt | | d d j D } t | g | }
| |
k r | j } n | d k r d _ d _ d x3 | j D]% \ } }
|
j j t
d ~
q W| j x j D] } | j q
W| d St
| t re j | } | j s| d Snb | d k r| j | j d }
|
d k r| j r|
j j | j n |
j j | j ~
| | ry | s| d SWn t k
rYn Xd q9 Wd S) Nc s t p d k p j S)N)r _shutdown_threadr )executorr r
shutting_down s z/_queue_management_worker.<locals>.shutting_downc su t d d j D } x$ t d | D] } j d q/ W j x j D] } | j q] Wd S)Nc s s | ] } | j Vq d S)N)is_alive)r7 pr r r <genexpr> s zD_queue_management_worker.<locals>.shutdown_worker.<locals>.<genexpr>r )sumvaluesrange
put_nowaitcloser )nb_children_aliveirR )r@ processesr r shutdown_worker s
z1_queue_management_worker.<locals>.shutdown_workerc S s g | ] } | j q Sr )sentinel)r7 rR r r r r8 s z,_queue_management_worker.<locals>.<listcomp>Tz^A process in the process pool was terminated abruptly while the future was running or pending.)_readerrM rU r recv_brokenrN r
r%
set_exceptionBrokenProcessPoolclear terminate
isinstanceintpopr r* r+
set_resultr, r )executor_referencer[ rJ work_ids_queuer@ rA rP r\ reader sentinelsready result_itemr* rL rR r )r@ rO r[ r _queue_management_worker sb
ro c C s t r t r t t d a y t j d } Wn t t f k
rO d SYn X| d k r` d S| d k rp d Sd | a t t d S)NTSC_SEM_NSEMS_MAXr z@system provides too few semaphores (%d available, 256 necessary))_system_limits_checked_system_limitedNotImplementedErrorr< sysconfAttributeError
ValueError) nsems_maxr r r _check_system_limitsQ s
rz c @ s e Z d Z d S)rb N)r r r r r r r rb h s rb c s e Z d Z d d d Z d d Z d d Z d d Z e j j j e _ d
d d d f d
d Z
d d d Z e j j j e _ S)ProcessPoolExecutorNc C s t | d k r+ t j p" d | _ n! | d k rC t d | | _ t j | j t | _ d | j _ t
| _ t j | _
d | _ i | _ d | _ t j | _ d | _ d | _ i | _ d S)Nr r z"max_workers must be greater than 0TF)rz r< cpu_count _max_workersrx multiprocessingQueueEXTRA_QUEUED_CALLS _call_queue
_ignore_epiper
_result_queuerG _work_ids_queue_management_thread
_processesrN threadingLock_shutdown_lockr` _queue_count_pending_work_items)r max_workersr r r r p s$
z ProcessPoolExecutor.__init__c
C s | j d d } | j d k r | j t j d t d t j | | | j | j | j
| j | j f | _ d | j _ | j j
| j t | j <d S)Nc S s | j d d S)N)r )_r r r r
weakref_cb s zFProcessPoolExecutor._start_queue_management_thread.<locals>.weakref_cbtargetr' T)r r _adjust_process_countr Threadro weakrefrefr r r r daemonstartr )r r r r r _start_queue_management_thread s
z2ProcessPoolExecutor._start_queue_management_threadc C sh xa t t | j | j D]D } t j d t d | j | j f } | j | | j | j
<q Wd S)Nr r' ) rV lenr r} r~ ProcessrE r r r pid)r r rR r r r r s "
z)ProcessPoolExecutor._adjust_process_countc O s | j | j r t d | j r4 t d t j } t | | | | } | | j | j <| j
j | j | j d 7_ | j j d | j
| SWd QRXd S)NzKA child process terminated abruptly, the process pool is not usable anymorez*cannot schedule new futures after shutdownr )r r` rb rN RuntimeErrorr Futurer$ r r r r r r )r r& r' r( fwr r r submit s
zProcessPoolExecutor.submittimeoutr2 r c sX | d k r t d t j t t | t | d | d | } t j j | S)Nr zchunksize must be >= 1.r2 r ) rx supermapr r9 r6 r0 chain
from_iterable)r r& r r2 r3 results) __class__r r r s zProcessPoolExecutor.mapTc C sn | j d | _ Wd QRX| j rF | j j d | rF | j j d | _ d | _ d | _ d | _ d S)NT)r rN r r r r r r )r r r r r shutdown s
z ProcessPoolExecutor.shutdown) r r r r r r r r Executor__doc__r r r r )r r r{ o s ( r{ )+
__author__atexitr< concurrent.futuresr rG r r~ r multiprocessing.connectionr r r functoolsr r0 r WeakKeyDictionaryr r r r Exceptionr r r! objectr$ r) r- r6 r9 rE rM ro rs rt rz r rb r r{ registerr r r r <module>. sD
% s