find_leak.py

# A list of lists of ordered column groups.
groups_40 = [['f190486d6', '58e2e02e6', 'eeb9cd3aa'...], ...]

total = pd.concat([train, test], sort=False)
X = (total[sum(groups_40, [])].values.copy().reshape(-1, len(groups_40), 40) * 100).astype(np.int64)

next_values = [{} for j in range(40 - 1)]
for i in range(X.shape[0]):
    for j in range(40 - 1):
        if (X[i, :, j + 1:] != 0).sum() == 0:
            continue

        key = X[i, :, j + 1:].tostring()
        if key in next_values[j]:
            if next_values[j][key] is None:
                continue
            elif (next_values[j][key] != X[i, :, j]).any():
                next_values[j][key] = None
        else:
            next_values[j][key] = X[i, :, j]

leak = -1 * np.ones(len(X), dtype=np.int64)
for i in range(X.shape[0]):
    next_value = None
    for j in range(40 - 1):
        key = X[i, :, :-1-j].tostring()
        if key in next_values[j] and next_values[j][key] is not None:
            next_value = next_values[j][key]
            break

    if next_value is not None:
        values = np.concatenate([next_value[:, None], X[i, :, :-1]], axis=1)
        assert values.shape == X.shape[1:]
        for j in range(40 - 1):
            key = values[:, :-1-j].tostring()
            if key in next_values[j] and next_values[j][key] is not None and next_values[j][key][0] != 0:
                leak[i] = next_values[j][key][0]
                break