generate the samples one by one and save the metrics

raw_metric_samples.py

from tqdm import trange
DFLT_NBOOTS=500
def raw_metric_samples(metrics, *data_args, nboots=DFLT_NBOOTS, sort=False,
       **metric_kwargs):
    """Return dataframe containing metric(s) for nboots boot sample datasets.
    metrics is a metric func or iterable of funcs e.g. [m1, m2, m3]
    """
    if callable(metrics): metrics=[metrics] # single metric func to list
    metrics=list(metrics) # in case it is a generator
    dforig=pd.DataFrame\
    ( { arg.name if hasattr(arg,'name') else str(i) : np.array(arg)
          for i,arg in enumerate(data_args) 
      }# end of dict comprehen.; np.array() removes index
    ) # I like ( ) to be above one another
    res=pd.DataFrame\
    ( # dictionary comprehension:
      { b:[ m( *[col_as_arg for (colname,col_as_arg) in dfboot.items()],
               **_kws_this_metric(m,**metric_kwargs)
             ) for m in metrics # list comprehension ends w/following "]":
          ] for b,dfboot in # generator expr. avoids huge mem. of *list* of df's:
            ((b,make_boot_df(dforig)) for b in trange(nboots))
            if dfboot.iloc[:,0].nunique()>1 # >1 for log loss (no labels), roc
      }, index=[_metric_name(m) for m in metrics]
    ) # sorry but I like ( ) to be above #one #another:-)
    res.index.name="Metric (class 1 +ve)"
    return res.apply(lambda row: np.sort(row), axis=1) if sort else res
from sklearn import metrics
def specificity_score(true,pred, **kwargs):
    return metrics.recall_score(1-pd.Series(true), 1-pd.Series(pred), **kwargs)
specificity_score.__name__ = "Specificity (Recall of −ve)"
def _kws_this_metric(m,**metric_kwargs):
    # dict of just those metric_kwargs that this metric accepts
    return \
    { k: metric_kwargs[k] for k in metric_kwargs.keys()
           & m.__wrapped__.__kwdefaults__.keys() # intersect
    } if ( hasattr(m,"__wrapped__") and
               hasattr(m.__wrapped__,"__kwdefaults__") and
               isinstance(m.__wrapped__.__kwdefaults__, dict) 
         ) else dict() # no keywords if attribs aren't there
def _metric_name(m):
    name=re.sub(' score$','',m.__name__.replace('_',' '))
    return name.title() if name.lower()==name else name