An example showing how to use sub interpreters and threads in Python

.dockerignore

/build/**
/venv*/**
/.idea/**
/repro.egg-info/**
/dist/
**/*.so

.gitignore

/build/**
/venv*/**
/.idea/**
/repro.egg-info/**
/dist/
*.so

build.sh

set -ex
find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf
rm -rf /Users/julius/.pyenv/versions/3.10.1/lib/python3.10/__pycache__/

python -m pip install --ignore-installed -e .
python --version
which python
python -c "import repro;[repro.f() for _ in range(1)];"
python -c "import repro;[repro.f2() for _ in range(1)];"

Dockerfile

FROM python:3.10.2

RUN rm -rf /usr/local/lib/python*/__pycache__/
RUN cat /usr/local/lib/python3.10/abc.py
COPY . .
RUN ls
RUN python3.10 --version
ARG CACHEBUST=1
RUN gcc $(python3.10-config --embed --cflags) subinterp_test.c $(python3.10-config --embed --ldflags) -o subinterp_test
#RUN ./build.sh
RUN ./subinterp_test

main.c

// based heavily on the instructions herein:
// "https://stackoverflow.com/questions/26061298/python-multi-thread-multi-interpreter-c-api"
#define PY_SSIZE_T_CLEAN
#include "Python.h"
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <pthread.h>

static void print_subinterp(void)
{
    /* Output information about the interpreter in the format
       expected in Lib/test/test_capi.py (test_subinterps). */
    PyThreadState *ts = PyThreadState_Get();
    PyInterpreterState *interp = ts->interp;
    int64_t id = PyInterpreterState_GetID(interp);
    printf("interp %" PRId64 " <0x%" PRIXPTR ">, thread state <0x%" PRIXPTR ">: ",
            id, (uintptr_t)interp, (uintptr_t)ts);
    fflush(stdout);
    PyRun_SimpleString(
        "import sys;"
        "print('id(modules) =', id(sys.modules));"
        "sys.stdout.flush()"
    );

}


void* worker(void* arg)
{
    PyInterpreterState* interp = (PyInterpreterState*)arg;
    // create a new thread state for the the sub interpreter interp
    PyThreadState* ts = PyThreadState_New(interp);
    // make it the current thread state and acquire the GIL
    PyEval_AcquireThread(ts);
    // at this point:
    // 1. You have the GIL
    // 2. You have the right thread state - a new thread state (this thread was not created by python) in the context of interp
    print_subinterp();
    fflush(stdout);
    PyObject* co = Py_CompileString(
                                    "import _random\n"
                                    "class repro(_random.Random):\n"
                                    "\tdef __init__(self):\n"
                                    "\t\tsuper().seed(10)\n",
                                    "subinterpreter.py",
                                    Py_file_input);
    PyImport_ExecCodeModuleEx("repro", co, "subinterpreter.py");
    if(PyErr_Occurred()){
        PyErr_Print();
    }
    PyThreadState_Clear(ts);
    PyThreadState_DeleteCurrent(); // clear the thread state we created and
                                   // release the GIL

    return (void*)NULL;
}


PyObject* f(PyObject *self, PyObject *args){
    Py_FinalizeEx();
    Py_SetProgramName(L"./_testembed");
    Py_DontWriteBytecodeFlag = 1;
    Py_Initialize();
    print_subinterp();
    PyThreadState* main_thread = PyThreadState_Get();

    Py_NewInterpreter(); /*automatically switch the thread
                                               state to ts */
    print_subinterp();
    PyThreadState* ts = PyEval_SaveThread(); /*discard the current thread state
                                               because we're  making a new one
                                               in worker() and also because this
                                               releases the GIL allowing our
                                               worker thread to run */
    pthread_t thread_id;
    pthread_create(&thread_id, NULL, worker, ts->interp);
    pthread_join(thread_id, NULL);
    PyInterpreterState_Delete(ts->interp); // delete the interpreter state
                                           // before re-acquiring the GIL
    PyEval_RestoreThread(main_thread);
    print_subinterp();
    Py_RETURN_NONE;
}
static PyThreadState* main_state;

void *on_thread(void *vargp){
    PyInterpreterState* interp = (PyInterpreterState*)vargp;
    PyEval_AcquireThread(main_state);
    PyThreadState* state = Py_NewInterpreter();
    print_subinterp();
    PyRun_SimpleString("from random import SystemRandom;");
    Py_EndInterpreter(state);
    PyThreadState_Swap(main_state);
    PyEval_ReleaseThread(main_state);
    return NULL;
}

PyObject* f2(PyObject *self, PyObject *args) {
    Py_SetProgramName(L"./_testembed");
    Py_DontWriteBytecodeFlag = 1;
    Py_Initialize();
    main_state = PyThreadState_Get();
    print_subinterp();
    Py_BEGIN_ALLOW_THREADS

    pthread_t thread_id;
    pthread_create(&thread_id, NULL, on_thread, NULL);
    pthread_join(thread_id, NULL);

    Py_END_ALLOW_THREADS
    print_subinterp();
    Py_Finalize();

    exit(0);
}

static PyMethodDef ReproMethods[] = {
    {"f",  f, METH_VARARGS,
     "Do the repro."},
     {"f2",  f2, METH_VARARGS,
     "Do the other repro."},
    {NULL, NULL, 0, NULL}        /* Sentinel */
};

static struct PyModuleDef repromodule = {
    PyModuleDef_HEAD_INIT,
    "repro",   /* name of module */
    NULL, /* module documentation, may be NULL */
    0,       /* size of per-interpreter state of the module,
                 or -1 if the module keeps state in global variables. */
    ReproMethods,
};
PyMODINIT_FUNC
PyInit_repro(void)
{
    return PyModuleDef_Init(&repromodule);
}

setup.py

from setuptools import setup, Extension

module1 = Extension('repro',
                    sources = ['main.c'])

setup (name = 'repro',
       version = '1.0',
       description = 'This is a demo package',
       ext_modules = [module1])

subinterp_test.c

#include "Python.h"
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <pthread.h>

static PyThreadState* main_state;
static void print_subinterp(void)
{
    /* Output information about the interpreter in the format
       expected in Lib/test/test_capi.py (test_subinterps). */
    PyThreadState *ts = PyThreadState_Get();
    PyInterpreterState *interp = ts->interp;
    int64_t id = PyInterpreterState_GetID(interp);
    printf("interp %" PRId64 " <0x%" PRIXPTR ">, thread state <0x%" PRIXPTR ">:\n",
            id, (uintptr_t)interp, (uintptr_t)ts);

}
void *on_thread(void *vargp){
    PyEval_AcquireThread(PyThreadState_New(main_state->interp));
    PyThreadState* state = Py_NewInterpreter();
    print_subinterp();
    PyRun_SimpleString("from random import SystemRandom; print(SystemRandom"
    "().random())\n");
    Py_EndInterpreter(state);
    PyThreadState_Swap(main_state);
    PyEval_ReleaseThread(main_state);
    return NULL;
}

int main() {
    Py_SetProgramName(L"./_testembed");
    Py_DontWriteBytecodeFlag = 1;
    Py_Initialize();
    main_state = PyThreadState_Get();
     print_subinterp();
    PyRun_SimpleString("print('Before creating sub interpreter')\n");
    Py_BEGIN_ALLOW_THREADS

    pthread_t thread_id;
    pthread_create(&thread_id, NULL, on_thread, NULL);
    pthread_join(thread_id, NULL);

    Py_END_ALLOW_THREADS
    print_subinterp();
    PyRun_SimpleString("print('After running sub interpreter')\n");
    Py_Finalize();

    exit(0);
}