multithreading windows and embedding python

F

freesteel

I have posted about this problem before. SInce then I found a much
better article to help with embedding python in a multithreaded
application:

http://www.linuxjournal.com/article/3641

I found this article very good and it clarified for me what needs
doing. Now I have an example application that almost works, but it is
far from reliable. If I run this several times I get crashes telling me
that the heap is modified after deallocation. Can anybody else
reproduce this?

At the bottom of this file I left a debug dump, and a stack dump.

Here is my application, compile in windows using a standard windows
application project.




#include <afxmt.h>
#include <afxwin.h>
#include <stdio.h>
#include <Python.h>
#include <Windows.h>
#include <process.h>

static int threadnum = 0;




UINT MyThread(LPVOID lpParam)
{
ASSERT(Py_IsInitialized());
threadnum++;


PyThreadState* mainThreadState = (PyThreadState *)lpParam;

// get the global lock
PyEval_AcquireLock();
// get a reference to the PyInterpreterState
PyInterpreterState * mainInterpreterState = mainThreadState->interp;
PyThreadState_Swap(mainThreadState);

// create a thread state object for this thread
PyThreadState * myThreadState =
PyThreadState_New(mainInterpreterState);
// free the lock
PyEval_ReleaseLock();


// lock - swap in thread state - swap out thread state - unlock
PyEval_AcquireLock();
PyThreadState_Swap(myThreadState);

int num = 0;
int ret = 0;
ret = PyRun_SimpleString("x = []");
ret = PyRun_SimpleString("for i in range(10):\n x.append(i)");
char cmd[100];
sprintf(cmd, "f = open('c:/windows/temp/test%d.txt', 'w')",
threadnum);
ret = PyRun_SimpleString(cmd);
ret = PyRun_SimpleString("f.write('%s\\n' % x.__str__())");
sprintf(cmd, "f.write('0x%d\\n')", &myThreadState);
ret = PyRun_SimpleString(cmd);
ret = PyRun_SimpleString("f.close()");

PyThreadState_Swap(NULL);
PyEval_ReleaseLock();


// clean up
// grab the lock
PyEval_AcquireLock();
// swap my thread state out of the interpreter
PyThreadState_Swap(NULL);
// clear out any cruft from thread state object
PyThreadState_Clear(myThreadState);
// delete my thread state object
PyThreadState_Delete(myThreadState);
// release the lock
PyEval_ReleaseLock();

return 0;
}

class CMyWinApp : public CWinApp
{
public:
CMyWinApp() { }
BOOL InitInstance()
{
Py_Initialize();
PyEval_InitThreads();

// save a pointer to the main PyThreadState object
PyThreadState * mainThreadState = PyThreadState_Get();
// release the lock
PyEval_ReleaseLock();

const int nhandles = 100;
HANDLE hnd[nhandles];
CWinThread* pThread[nhandles];
for (int ih = 0; ih < nhandles; ih++)
{
pThread[ih] = AfxBeginThread(MyThread, mainThreadState,
THREAD_PRIORITY_NORMAL, CREATE_SUSPENDED);
pThread[ih]->m_bAutoDelete = false;
pThread[ih]->ResumeThread();

hnd[ih] = pThread[ih]->m_hThread;
}

int nwaits, nfails;
do
{
nwaits = 0;
nfails = 0;
for (int ih = 0; ih < nhandles; ih++)
{
DWORD ret = WaitForSingleObject(hnd[ih], INFINITE);
switch (ret)
{
case WAIT_OBJECT_0:
printf("WAIT_OBJECT_0\n");
break;

case WAIT_TIMEOUT:
++nwaits;
printf("WAIT_TIMEOUT\n");
break;

case WAIT_FAILED:
++nfails;
printf("WAIT_FAILED\n");
break;
}
}
}
while (nwaits > 0);
ASSERT(nfails == 0);

// delete all windows threads
for (int ih = 0; ih < nhandles; ++ih)
delete pThread[ih];

PyEval_AcquireLock();
PyThreadState_Swap(mainThreadState);
Py_Finalize();

return TRUE;
};
};

CMyWinApp app;




Debug dump:

'pyembed_test.exe': Loaded
'C:\mdunschen\pyembed_test\Debug\pyembed_test.exe', Symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\ntdll.dll', No symbols
loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\mscoree.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\kernel32.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\advapi32.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\rpcrt4.dll', No symbols
loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\user32.dll', No symbols
loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\gdi32.dll', No symbols
loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\comctl32.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\shlwapi.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\msvcrt.dll', No symbols
loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\oleacc.dll', No symbols
loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\msvcp60.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\ole32.dll', No symbols
loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\oleaut32.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\winspool.drv', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\comdlg32.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\shell32.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\python24.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\msvcr71.dll', Symbols
loaded.
'pyembed_test.exe': Loaded
'C:\WINDOWS\WinSxS\x86_Microsoft.Windows.Common-Controls_6595b64144ccf1df_6.0.2600.2180_x-ww_a84f1ff9\comctl32.dll',
No symbols loaded.
'pyembed_test.exe': Loaded
'C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\mscorwks.dll', No symbols
loaded.
'pyembed_test.exe': Loaded
'C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\fusion.dll', No symbols
loaded.
'pyembed_test.exe': Loaded
'C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\mscorlib.dll', No symbols
loaded.
'pyembed_test.exe': Loaded
'C:\WINDOWS\assembly\NativeImages1_v1.1.4322\mscorlib\1.0.5000.0__b77a5c561934e089_ca105f6f\mscorlib.dll',
No symbols loaded.
'pyembed_test.exe': Loaded
'C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\diasymreader.dll', No
symbols loaded.
'DefaultDomain': Loaded
'c:\windows\microsoft.net\framework\v1.1.4322\mscorlib.dll', No symbols
loaded.
'pyembed_test.exe': Loaded
'C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\mscorsn.dll', No symbols
loaded.
'pyembed_test': Loaded
'c:\mdunschen\pyembed_test\Debug\pyembed_test.exe', Symbols loaded.
'pyembed_test.exe': Loaded
'C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\mscorjit.dll', No symbols
loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\uxtheme.dll', No
symbols loaded.
The thread '_threadstartex' (0x500) has exited with code 0 (0x0).
The thread '_threadstartex' (0xadc) has exited with code 0 (0x0).
The thread '<No Name>' (0xadc) has exited with code 0 (0x0).
The thread '<No Name>' (0x500) has exited with code 0 (0x0).
The thread '_threadstartex' (0x928) has exited with code 0 (0x0).
The thread '<No Name>' (0x928) has exited with code 0 (0x0).
The thread '_threadstartex' (0xe48) has exited with code 0 (0x0).
The thread '<No Name>' (0xe48) has exited with code 0 (0x0).
The thread '_threadstartex' (0x984) has exited with code 0 (0x0).
The thread '<No Name>' (0x984) has exited with code 0 (0x0).
The thread '_threadstartex' (0x108) has exited with code 0 (0x0).
HEAP[pyembed_test.exe]: HEAP: Free Heap block eb5bd8 modified at eb5c24
after it was freed
Unhandled exception at 0x7c901230 in pyembed_test.exe: User breakpoint.




Stack dump:
pyembed_test.exe!_heap_alloc_dbg(unsigned int nSize=15400960, int nBlockUse=1073741920, const char * szFileName=0x00000064, int nLine=1242368) Line 359 + 0x1e C
Click to expand...
pyembed_test.exe!_heap_alloc_base(unsigned int size=100) Line 212 C
pyembed_test.exe!_heap_alloc_dbg(unsigned int nSize=64, int
nBlockUse=12582916, const char * szFileName=0x004c4dd4, int nLine=311)
Line 397 + 0x9 C
pyembed_test.exe!_nh_malloc_dbg(unsigned int nSize=64, int nhFlag=0,
int nBlockUse=12582916, const char * szFileName=0x004c4dd4, int
nLine=311) Line 260 + 0x15 C
pyembed_test.exe!_malloc_dbg(unsigned int nSize=64, int
nBlockUse=12582916, const char * szFileName=0x004c4dd4, int nLine=311)
Line 176 + 0x1b C
pyembed_test.exe!operator new(unsigned int nSize=64, int
nType=12582916, const char * lpszFileName=0x004c4dd4, int nLine=311)
Line 403 + 0x15 C++
pyembed_test.exe!CObject::eek:perator new(unsigned int nSize=64, const
char * lpszFileName=0x004c4dd4, int nLine=311) Line 93 + 0x16 C++
pyembed_test.exe!AfxBeginThread(unsigned int (void *)*
pfnThreadProc=0x00401b80, void * pParam=0x00df7b10, int nPriority=0,
unsigned int nStackSize=4, unsigned long dwCreateFlags=0,
_SECURITY_ATTRIBUTES * lpSecurityAttrs=0x00000000) Line 311 + 0x11 C++
00aaa43d()
pyembed_test.exe!CMyWinApp::InitInstance() Line 87 + 0x26 bytes C++
pyembed_test.exe!AfxWinMain(HINSTANCE__ * hInstance=0x00400000,
HINSTANCE__ * hPrevInstance=0x00000000, char * lpCmdLine=0x00141f12,
int nCmdShow=5) Line 39 + 0xb C++
pyembed_test.exe!WinMain(HINSTANCE__ * hInstance=0x00400000,
HINSTANCE__ * hPrevInstance=0x00000000, char * lpCmdLine=0x00141f12,
int nCmdShow=5) Line 25 C++
pyembed_test.exe!WinMainCRTStartup() Line 251 + 0x30 C
 
F

freesteel

freesteel wrote:
....
pThread[ih] = AfxBeginThread(MyThread, mainThreadState,
THREAD_PRIORITY_NORMAL, CREATE_SUSPENDED);
Click to expand...

....

Here the call to AfxBeginThread is wrong, there is one argument
missing, it should be:

pThread[ih] = AfxBeginThread(MyThread, mainThreadState,
THREAD_PRIORITY_NORMAL, 0, CREATE_SUSPENDED);

Because there are so many default arguments of similar types the
compiler did not notice that I passed 'CREATE_SUSPENDED' as a stack
size, and use the default 'creation' state of 'start right away' for
the thread.

Don't you love default args and types like 'void*' ?

Martin
 

Ask a Question

Want to reply to this thread or ask your own question?

You'll need to choose a username for the site, which only take a couple of moments. After that, you can post your question and our members will help you out.

Ask a Question

Similar Threads

embedded python and windows multi threading, can't get it to work 1
Compile C program in Visual Studio 2005 10
output 3
Difficulty debugging WCF service 0
Windows app client to XML Web Services 0
Project setting problem on Microsoft Visual C++ 2008 1
embedding: forcing an interpreter to end 1
Extending/Embedding python 4

Members online

No members online now.
Total: 47 (members: 0, guests: 47)
Robots: 282

Forum statistics

Threads
473,983
Messages
2,570,187
Members
46,747
Latest member
jojoBizaroo

Latest Threads

Top