translating sparse arff files to sparse parquet files for OpenML

sparsedata.py

#Code for entire dataset to pyarrow table
import pyarrow as pa
import pyarrow.parquet as pq

# getting the dataset
did=39947

d = openml.datasets.get_dataset(did, download_qualities=False)
df , *_ = d.get_data(dataset_format="dataframe", include_row_id=True, include_ignore_attribute=True)
df = df[[f.name for f in d.features.values()]]
# converting every column to pyarrow array
parr_array = []
names = []
for i in range(len(df.columns)):
    arr = np.asarray(df[df.columns[i]])
    mask = []
    for val in arr:#Mask of null values
        if val==df[df.columns[i]].dtype._fill_value:
            mask.append(True)
        else:
            mask.append(False)
    mask = np.asarray(mask)
    parr = pyarrow.array(arr, from_pandas=True,mask=mask)
    parr_array.append(parr)
    names.append(df.columns[i])
table = pa.table(parr_array, names=names)
pq.write_table(table, 'df.parquet')

import pandas as pd
df_new = pd.read_parquet('df.parquet')