­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ 3 \ P@s^dZddlZddlZddlmZddlZddlmZddlZddlmZddl m Z ddl Z ddl Z ddl mZddlZddlZe jZdad d Zd ZGd d d eZGdddZddZGdddeZGdddeZGdddeZddZddZddZ ddZ!d d!Z"da#da$d"d#Z%d$d%Z&Gd&d'd'e'Z(Gd(d)d)ej)Z*ej+edS)*z"Brian Quinlan (brian@sweetapp.com)N)_base)Full) SimpleQueue)wait)partialFcCsJdattj}x|D]\}}|jdqWx|D]\}}|jq2WdS)NT) _shutdownlist_threads_queuesitemsputjoin)r tqr2/usr/lib64/python3.6/concurrent/futures/process.py _python_exitOs  rc@seZdZddZddZdS)_RemoteTracebackcCs ||_dS)N)tb)selfrrrr__init__asz_RemoteTraceback.__init__cCs|jS)N)r)rrrr__str__csz_RemoteTraceback.__str__N)__name__ __module__ __qualname__rrrrrrr`src@seZdZddZddZdS)_ExceptionWithTracebackcCs0tjt|||}dj|}||_d||_dS)Nz """ %s""") tracebackformat_exceptiontyper excr)rr rrrrrgs z _ExceptionWithTraceback.__init__cCst|j|jffS)N) _rebuild_excr r)rrrr __reduce__lsz"_ExceptionWithTraceback.__reduce__N)rrrrr"rrrrrfsrcCst||_|S)N)r __cause__)r rrrrr!os r!c@seZdZddZdS) _WorkItemcCs||_||_||_||_dS)N)futurefnargskwargs)rr%r&r'r(rrrrtsz_WorkItem.__init__N)rrrrrrrrr$ssr$c@seZdZdddZdS) _ResultItemNcCs||_||_||_dS)N)work_id exceptionresult)rr*r+r,rrrr{sz_ResultItem.__init__)NN)rrrrrrrrr)zsr)c@seZdZddZdS) _CallItemcCs||_||_||_||_dS)N)r*r&r'r()rr*r&r'r(rrrrsz_CallItem.__init__N)rrrrrrrrr-sr-cgs0t|}x"ttj||}|s"dS|Vq WdS)N)ziptuple itertoolsislice) chunksize iterablesitchunkrrr _get_chunkss r6csfdd|DS)Ncsg|] }|qSrr).0r')r&rr sz"_process_chunk..r)r&r5r)r&r_process_chunks r9cCsx|jdd}|dkr(|jtjdSy|j|j|j}WnBtk r~}z&t||j }|jt |j |dWYdd}~XqX|jt |j |dqWdS)NT)block)r+)r,) getr osgetpidr&r'r( BaseExceptionr __traceback__r)r*) call_queue result_queueZ call_itemrer rrr_process_workers   & rDcCsxxr|jrdSy|jdd}Wntjk r4dSX||}|jjrh|jt||j|j |j ddq||=qqWdS)NF)r:T) Zfullr;queueZEmptyr%Zset_running_or_notify_cancelr r-r&r'r()pending_work_itemsZwork_idsr@r* work_itemrrr_add_call_item_to_queues   rHc sdfdd}fdd}|j}xlt||ddjD} t|g| } || krf|j} nr|dk rd_d_dx&|jD]\} } | jj t d~ qW|j xjD] }|j qW|dSt | trj| }|jsR|dSnJ| dk rR|j| jd} | dk rR| jrB| jj | jn| jj| j~ ||ry|sr|dSWntk rYnXdq(WdS) NcstpdkpjS)N)r_shutdown_threadr)executorrr shutting_downsz/_queue_management_worker..shutting_downcsZtddjD}xtd|D]}jdq"WjxjD] }|jqFWdS)Ncss|]}|jVqdS)N)Zis_alive)r7prrr szD_queue_management_worker..shutdown_worker..r)sumvaluesrangeZ put_nowaitcloser )Znb_children_aliveirL)r@ processesrrshutdown_workers z1_queue_management_worker..shutdown_workercSsg|] }|jqSr)sentinel)r7rLrrrr8 sz,_queue_management_worker..Tz^A process in the process pool was terminated abruptly while the future was running or pending.)Z_readerrHrOrZrecv_brokenrIr r%Z set_exceptionBrokenProcessPoolclearZ terminate isinstanceintpopr r*r+Z set_resultr,r)Zexecutor_referencerSrFZwork_ids_queuer@rArKrTreaderZ sentinelsZreadyZ result_itemr*rGrLr)r@rJrSr_queue_management_workersb        r]c Cshtrtrttdaytjd}Wnttfk r:dSX|dkrHdS|dkrTdSd|attdS)NTSC_SEM_NSEMS_MAXrz@system provides too few semaphores (%d available, 256 necessary))_system_limits_checked_system_limitedNotImplementedErrorr<sysconfAttributeError ValueError)Z nsems_maxrrr_check_system_limitsQsrgccs.x(|D] }|jx|r$|jVqWqWdS)N)reverser[)iterableelementrrr_chain_from_iterable_of_listshs rkc@s eZdZdS)rWN)rrrrrrrrWtsrWcsheZdZdddZddZddZdd Zejjj e_ dd d fd d Z dddZ ejj j e _ Z S)ProcessPoolExecutorNcCst|dkrtjpd|_n|dkr.td||_tj|jt|_d|j_ t |_ t j|_ d|_i|_d|_tj|_d|_d|_i|_dS)Nrrz"max_workers must be greater than 0TF)rgr< cpu_count _max_workersrfmultiprocessingZQueueEXTRA_QUEUED_CALLS _call_queueZ _ignore_epiper _result_queuerE _work_ids_queue_management_thread _processesrI threadingZLock_shutdown_lockrV _queue_count_pending_work_items)rZ max_workersrrrr|s$   zProcessPoolExecutor.__init__cCsp|jfdd}|jdkrl|jtjttj|||j|j |j |j |jfd|_d|j_ |jj |jt|j<dS)NcSs|jddS)N)r )_rrrr weakref_cbszFProcessPoolExecutor._start_queue_management_thread..weakref_cb)targetr'T)rrrt_adjust_process_countrvZThreadr]weakrefrefruryrsrqZdaemonstartr )rr{rrr_start_queue_management_threads   z2ProcessPoolExecutor._start_queue_management_threadcCsJxDtt|j|jD].}tjt|j|jfd}|j ||j|j <qWdS)N)r|r') rPlenrurnroZProcessrDrqrrrpid)rrzrLrrrr}s z)ProcessPoolExecutor._adjust_process_countc Os|jt|jrtd|jr$tdtj}t||||}||j|j <|j j |j |j d7_ |j j d|j |SQRXdS)NzKA child process terminated abruptly, the process pool is not usable anymorez*cannot schedule new futures after shutdownr)rwrVrWrI RuntimeErrorrZFuturer$ryrxrsr rrr)rr&r'r(fwrrrsubmits  zProcessPoolExecutor.submitr)timeoutr2cs:|dkrtdtjtt|t|d|i|d}t|S)Nrzchunksize must be >= 1.r2)r)rfsupermaprr9r6rk)rr&rr2r3Zresults) __class__rrrs  zProcessPoolExecutor.mapTc CsT|j d|_WdQRX|jr8|jjd|r8|jjd|_d|_d|_d|_dS)NT)rwrIrtrrr r rqru)rrrrrshutdowns  zProcessPoolExecutor.shutdown)N)T) rrrrrr}rrExecutor__doc__rr __classcell__rr)rrrl{s (   rl), __author__atexitr<Zconcurrent.futuresrrErrorZmultiprocessing.connectionrrvr~ functoolsrr0rWeakKeyDictionaryr rrrp Exceptionrrr!objectr$r)r-r6r9rDrHr]rarbrgrkrrWrrlregisterrrrr.sH         %s