Repartitioning of LSapp dataset

repartitioning.py

import pandas as pd


def pick_uniques(df: pd.DataFrame, size: int = 100):
    assert a.index.is_monotonic
    assert 'uid' in df
    index = []
    uids = set()
    for i, uid in df.uid.items():
        if uid in uids:
            continue
        uids.add(uid)
        index.append(i)
        if len(uids) == size:
            break
    head = df.loc[index]
    rest = df.loc[df.index.difference(index)]
    return head, rest


def convolve(frame: pd.DataFrame, size: int = 50):
    parts = []
    pbar = tqdm(total=len(frame) // size + 1)
    while not frame.empty:
        head, frame = pick_uniques(frame, size)
        head['subkey'] = len(parts)
        parts.append(head)
    return pd.concat(parts)