pierreglaser

wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/.local/miniconda3
# Activate conda env by default
echo 'source $HOME/.local/miniconda3/etc/profile.d/conda.sh
conda activate
' >> $HOME/.bashrc
source $HOME/.bashrc

pierreglaser

import time
import numpy as np
from joblib import Parallel, delayed
from joblib.externals.loky.backend import resource_tracker

resource_tracker.VERBOSE = True
import os


print(f"PARENT PID IS : {os.getpid()}")

pierreglaser

# this will fetch no2 concentration for Paris, France.
curl https://api.openaq.org/v1/latest?city=Paris | jq -r '.results[0].measurements[0]'

pierreglaser

import multiprocessing as mp
from multiprocessing.reduction import AbstractReducer, AbstractPickler
from multiprocessing.context import SpawnContext, SpawnProcess, DefaultContext


class PicklerProtocol2(AbstractPickler):
    @classmethod
    def dumps(cls, obj, protocol=2):
        return super().dumps(obj, protocol=protocol)

pierreglaser

"""
Dump a function in Python3.6, and try to reconstruct and run it in Python3.5.

Cloudpickle will fail at this task because of bytecode incompatibility between
different Python versions.
However, if instead of pickling the bytecode, we pickle slightly higher level
abstraction such as instructions, and reconstruct the code object from the
depickled instructions, the reconstructed functions works fine.
"""
import pickle

pierreglaser

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Wall clock time measurement of pairwise_distances"""
import os
import csv
import time
import itertools

import numpy as np
import pandas as pd

pierreglaser

import time
import loky


def process_dict(x):
    return len(x)


if __name__ == "__main__":
    e = loky.get_reusable_executor(max_workers=2)

pierreglaser

import cloudpickle
import os
import pickle
import sys
import types


VERSION = cloudpickle.__version__

script = """