X
xeon Mailinglist
I am trying to debug my program that launch processes to run a function to copy data between hosts located really far away from each other. The end of my function are in the orders.py and mergedirs.py.
From this point onwards, in is python code. The problem is that this code hangs in the last instruction "forking()", instead of my program continue.
The call of my function were made with: outputs = parmap(mergeDirs, args). This is my parmap function.
It seems that the function mergeDirs ends, but the process does not join. I checked that the mergeDirs is working properly, because I ran the code without processes, and it works fine.
I don't know what is happening. How can I debug this problem?
def parmap(f, X):
pipe = [Pipe() for x in X]
# 2 - what is happening with the tuples (c,x) and (p, c)?
proc = [Process(target=spawn(f), args=(c, x))
for x, (p, c) in zip(X, pipe)]
#[p.start() for p in proc]
for p in proc:
print("Spawn")
p.start()
#[p.join() for p in proc]
for p in proc:
print("Joining")
p.join()
return [p.recv() for (p, c) in pipe]
Copy data time: 104.863273859
orders.py(99): return cluster
mergedirs.py(48): return (new_included, command, poutput)
spawn.py(7): pipe.close()
process.py(259): exitcode = 0
process.py(261): util._exit_function()
--- modulename: util, funcname: _exit_function
util.py(303): info('process shutting down')
--- modulename: util, funcname: info
util.py(77): if _logger:
util.py(304): debug('running all "atexit" finalizers with priority >= 0')
--- modulename: util, funcname: debug
util.py(73): if _logger:
util.py(305): _run_finalizers(0)
--- modulename: util, funcname: _run_finalizers
util.py(257): if _finalizer_registry is None:
util.py(263): if minpriority is None:
util.py(266): f = lambda p : p[0][0] is not None and p[0][0] >= minpriority
util.py(268): items = [x for x in _finalizer_registry.items() if f(x)]
util.py(269): items.sort(reverse=True)
util.py(271): for key, finalizer in items:
util.py(279): if minpriority is None:
util.py(307): if current_process() is not None:
--- modulename: process, funcname: current_process
process.py(63): return _current_process
util.py(318): for p in active_children():
--- modulename: process, funcname: active_children
process.py(69): _cleanup()
--- modulename: process, funcname: _cleanup
process.py(78): for p in list(_current_process._children):
process.py(70): return list(_current_process._children)
util.py(323): for p in active_children():
--- modulename: process, funcname: active_children
process.py(69): _cleanup()
--- modulename: process, funcname: _cleanup
process.py(78): for p in list(_current_process._children):
process.py(70): return list(_current_process._children)
util.py(327): debug('running the remaining "atexit" finalizers')
--- modulename: util, funcname: debug
util.py(73): if _logger:
util.py(328): _run_finalizers()
--- modulename: util, funcname: _run_finalizers
util.py(257): if _finalizer_registry is None:
util.py(263): if minpriority is None:
util.py(264): f = lambda p : p[0][0] is not None
util.py(268): items = [x for x in _finalizer_registry.items() if f(x)]
util.py(269): items.sort(reverse=True)
util.py(271): for key, finalizer in items:
util.py(279): if minpriority is None:
util.py(280): _finalizer_registry.clear()
process.py(278): util.info('process exiting with exitcode %d' % exitcode)
--- modulename: util, funcname: info
util.py(77): if _logger:
process.py(279): return exitcode
forking.py(127): sys.stdout.flush()
forking.py(128): sys.stderr.flush()
forking.py(129): os._exit(code)
From this point onwards, in is python code. The problem is that this code hangs in the last instruction "forking()", instead of my program continue.
The call of my function were made with: outputs = parmap(mergeDirs, args). This is my parmap function.
It seems that the function mergeDirs ends, but the process does not join. I checked that the mergeDirs is working properly, because I ran the code without processes, and it works fine.
I don't know what is happening. How can I debug this problem?
def parmap(f, X):
pipe = [Pipe() for x in X]
# 2 - what is happening with the tuples (c,x) and (p, c)?
proc = [Process(target=spawn(f), args=(c, x))
for x, (p, c) in zip(X, pipe)]
#[p.start() for p in proc]
for p in proc:
print("Spawn")
p.start()
#[p.join() for p in proc]
for p in proc:
print("Joining")
p.join()
return [p.recv() for (p, c) in pipe]
Copy data time: 104.863273859
orders.py(99): return cluster
mergedirs.py(48): return (new_included, command, poutput)
spawn.py(7): pipe.close()
process.py(259): exitcode = 0
process.py(261): util._exit_function()
--- modulename: util, funcname: _exit_function
util.py(303): info('process shutting down')
--- modulename: util, funcname: info
util.py(77): if _logger:
util.py(304): debug('running all "atexit" finalizers with priority >= 0')
--- modulename: util, funcname: debug
util.py(73): if _logger:
util.py(305): _run_finalizers(0)
--- modulename: util, funcname: _run_finalizers
util.py(257): if _finalizer_registry is None:
util.py(263): if minpriority is None:
util.py(266): f = lambda p : p[0][0] is not None and p[0][0] >= minpriority
util.py(268): items = [x for x in _finalizer_registry.items() if f(x)]
util.py(269): items.sort(reverse=True)
util.py(271): for key, finalizer in items:
util.py(279): if minpriority is None:
util.py(307): if current_process() is not None:
--- modulename: process, funcname: current_process
process.py(63): return _current_process
util.py(318): for p in active_children():
--- modulename: process, funcname: active_children
process.py(69): _cleanup()
--- modulename: process, funcname: _cleanup
process.py(78): for p in list(_current_process._children):
process.py(70): return list(_current_process._children)
util.py(323): for p in active_children():
--- modulename: process, funcname: active_children
process.py(69): _cleanup()
--- modulename: process, funcname: _cleanup
process.py(78): for p in list(_current_process._children):
process.py(70): return list(_current_process._children)
util.py(327): debug('running the remaining "atexit" finalizers')
--- modulename: util, funcname: debug
util.py(73): if _logger:
util.py(328): _run_finalizers()
--- modulename: util, funcname: _run_finalizers
util.py(257): if _finalizer_registry is None:
util.py(263): if minpriority is None:
util.py(264): f = lambda p : p[0][0] is not None
util.py(268): items = [x for x in _finalizer_registry.items() if f(x)]
util.py(269): items.sort(reverse=True)
util.py(271): for key, finalizer in items:
util.py(279): if minpriority is None:
util.py(280): _finalizer_registry.clear()
process.py(278): util.info('process exiting with exitcode %d' % exitcode)
--- modulename: util, funcname: info
util.py(77): if _logger:
process.py(279): return exitcode
forking.py(127): sys.stdout.flush()
forking.py(128): sys.stderr.flush()
forking.py(129): os._exit(code)