Sparse csr scipy matrix

csr_matrix.py

row, col, data = [], [], []
row_idx = -1

y = []

feature_counter = Counter()
for file_path in data_path.glob(DATA_FILES_PATTERN):
    print(file_path)
    with open(Path(file_path).expanduser(), 'r') as data_file:
        class_idx = BLOCK_CLASSES[file_path.stem]

        for line in data_file:
            block = json.loads(line)

            row_idx += 1
            for k, v in block.items():
                if k not in ('0', '1'):  # (block_uuid, doc_uuid)
                    row.append(row_idx)
                    col.append(int(k))
                    data.append(v)

                    feature_counter[int(k)] += v

            y.append(class_idx)

X = csr_matrix((data, (row, col)), shape=(row_idx + 1, extractor.max_feature_index))