An example showing how to use sub interpreters and threads in Python

CMakeLists.txt

cmake_minimum_required(VERSION 2.8.4)

project(py1)

find_package(PythonLibs REQUIRED)
include_directories(${PYTHON_INCLUDE_DIRS})

ADD_DEFINITIONS( -std=c++11 )

set(SOURCE_FILES main.cpp)

add_executable(py1 ${SOURCE_FILES})
target_link_libraries(py1 ${PYTHON_LIBRARIES})

main.cpp

// Author: Baruch Sterin <baruchs@gmail.com>

#include <thread>

#include <Python.h>

// initialize and clean up python
struct initialize
{
    initialize()
    {
        Py_InitializeEx(1);
        PyEval_InitThreads();
    }

    ~initialize()
    {
        Py_Finalize();
    }
};

// allow other threads to run
class enable_threads
{
public:
    enable_threads()
    {
        _state = PyEval_SaveThread();
    }

    ~enable_threads()
    {
        PyEval_RestoreThread(_state);
    }

private:
    PyThreadState* _state;
};

// acquire and release the GIL
struct gil_lock
{
    gil_lock()
    {
        PyEval_AcquireLock();
    }

    ~gil_lock()
    {
        PyEval_ReleaseLock();
    }
};

// restore the thread state when the object goes out of scope
class restore_tstate
{
public:

    restore_tstate()
    {
        _ts = PyThreadState_Get();
    }

    ~restore_tstate()
    {
        PyThreadState_Swap(_ts);
    }

private:
    PyThreadState* _ts;
};

// swap the current thread state with ts, restore when the object goes out of scope
class swap_tstate
{
public:

    swap_tstate(PyThreadState* ts)
    {
        _ts = PyThreadState_Swap(ts);
    }

    ~swap_tstate()
    {
        PyThreadState_Swap(_ts);
    }

private:
    PyThreadState* _ts;
};

// create new thread state for interpreter interp, clean up on destruction
class thread_state
{
public:

    thread_state(PyInterpreterState* interp)
    {
        _ts = PyThreadState_New(interp);
    }

    ~thread_state()
    {
        if( _ts )
        {
            PyThreadState_Clear(_ts);
            PyThreadState_Delete(_ts);

            _ts = nullptr;
        }
    }

    operator PyThreadState*()
    {
        return _ts;
    }

private:
    PyThreadState* _ts;
};

// represent a sub interpreter
class sub_interpreter
{
public:

    // perform the necessary setup and cleanup for a new thread running using a specific interpreter
    struct thread
    {
        gil_lock _lock;
        thread_state _state;
        swap_tstate _swap;

        thread(PyInterpreterState* interp) :
                _state(interp),
                _swap(_state)
        {
        }
    };

    sub_interpreter()
    {
        restore_tstate restore;

        _ts = Py_NewInterpreter();
    }

    ~sub_interpreter()
    {
        if( _ts )
        {
            swap_tstate sts(_ts);

            Py_EndInterpreter(_ts);
        }
    }

    PyInterpreterState* interp()
    {
        return _ts->interp;
    }

private:
    PyThreadState* _ts;
};

char program[] = "import sys; print \"ARGV: %s\"%hasattr(sys, \"argv\")";
char main_program[] = "import sys; sys.argv=sys.argv if hasattr(sys, \"argv\") else [\"abc\"] ; print \"ARGV: %d\"%id(sys.argv)";

// run in a new thread
void f(PyInterpreterState* interp)
{
    sub_interpreter::thread scope(interp);

    PyRun_SimpleString(program);
}

int main()
{
    initialize init;

    sub_interpreter s1;
    sub_interpreter s2;
    sub_interpreter s3;

    std::thread t1(f, s1.interp() );
    std::thread t2(f, s2.interp() );
    std::thread t3(f, s3.interp() );

    std::thread t4(f, s1.interp() );

    PyRun_SimpleString(main_program);

    enable_threads t;

    t1.join();
    t2.join();
    t3.join();
    t4.join();

    return 0;
}

Hi, how is it that this stops working if you comment the row 192 (PyRun_SimpleString(main_program);)? It seems like PyEval_AcquireLock() fails then...

Works for me (Python 2.7.10 on macOS). Could you provide some more details about the failure, such as platform, Python version, expected output, and actual output?

I have tested with Python 3.6.5 (changed the Python programs to something like print("Hello from Thread")) on windows 10, compiled with VS2019. If I remove the line 192 I get an FatalError in PyEval_AcquireLock.

Seems to work for me on both macOS and Windows 10, with Python 3.7, and 2.7.

I've updated the example for Python 3.3 at https://github.com/sterin/python-sub-interpreters-multiple-threads-example
Try that as well.

The linked example works like a charm. Thank you!