Batch Exporter for Pupilab Gaze data

gazedata_batch_exporter.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Tiziana's batch converter

Created on Mon Dec  4 19:45:22 2023

@author: giulio
"""

import os
import numpy as np
import pandas as pd
import msgpack 

SUBJECTPATH = 'E:\\NeurokineticAudioElectricResponse\\Sub9\\eye\\20231108' #This is the folder containing all other subfolders
OUTPUTPATH = 'E:\\NeurokineticAudioElectricResponse\\Sub9\\eye\\exports' #this is the path in which extracted CSV are saved

subjectID = SUBJECTPATH.split(os.path.sep)[-1] #get subject ID
if not os.path.exists(OUTPUTPATH):
    os.mkdir(OUTPUTPATH) #this creates the subject dir in OUTPUTPATH


for trial in os.listdir(SUBJECTPATH):
    #loop across trials
    #define the timestamps file and gazeData
    #timestamp is used only to check, not used in exported final file
    timestamps = SUBJECTPATH + os.path.sep + trial + os.path.sep  + 'gaze_timestamps.npy'
    gazeData =  SUBJECTPATH + os.path.sep + trial + os.path.sep  + 'gaze.pldata'
    dataTimeStamps = np.load(timestamps)

    #initialize the csv list
    csvData = []
    #open gaze data
    with open(gazeData, "rb") as fh:
        unpacker = msgpack.Unpacker(fh, raw=False, use_list=False)
        #black magic to unpack the pldata file
        for (topic, payload) in unpacker:
            datum = msgpack.unpackb(payload, raw=False, use_list=False)
            #check if topic is correct aka the line has data for both eyes
            if(datum['topic'] == 'gaze.3d.01.'):
                gaze_timestamp = datum['timestamp']
                confidence = datum['confidence']
                norm_pos_x = datum['norm_pos'][0]
                norm_pos_y =  datum['norm_pos'][1]
                gaze_point_3d_x = datum['gaze_point_3d'][0]
                gaze_point_3d_y = datum['gaze_point_3d'][1]
                gaze_point_3d_z = datum['gaze_point_3d'][2]
                eye_center0_3d_x = datum['eye_centers_3d']['0'][0]
                eye_center0_3d_y = datum['eye_centers_3d']['0'][1]
                eye_center0_3d_z = datum['eye_centers_3d']['0'][2]
                gaze_normal0_x = datum['gaze_normals_3d']['0'][0]
                gaze_normal0_y = datum['gaze_normals_3d']['0'][1]
                gaze_normal0_z = datum['gaze_normals_3d']['0'][2]
                eye_center1_3d_x = datum['eye_centers_3d']['1'][0]
                eye_center1_3d_y = datum['eye_centers_3d']['1'][1]
                eye_center1_3d_z = datum['eye_centers_3d']['1'][2]
                gaze_normal1_x = datum['gaze_normals_3d']['1'][0]
                gaze_normal1_y = datum['gaze_normals_3d']['1'][1]
                gaze_normal1_z = datum['gaze_normals_3d']['1'][2]
    
                row = [gaze_timestamp, confidence, norm_pos_x, norm_pos_y, gaze_point_3d_x, gaze_point_3d_y, gaze_point_3d_z, eye_center0_3d_x, eye_center0_3d_y, eye_center0_3d_z, gaze_normal0_x, gaze_normal0_y, gaze_normal0_z, eye_center1_3d_x, eye_center1_3d_y, eye_center1_3d_z, gaze_normal1_x, gaze_normal1_y, gaze_normal1_z]        
                csvData.append(row)
            else:
                #this can be improved to save data from a single eye if one is missing. 
                #Tricky but duable. We don't think is worth the effort rn as it concerns about 10 trials out of 3000.
                
                row = [datum['timestamp']]+[np.nan]*18
                csvData.append(row)
    
    #convert to pandas and save to csv
    df = pd.DataFrame(csvData, columns = ["gaze_timestamp","confidence","norm_pos_x","norm_pos_y","gaze_point_3d_x",
                                          "gaze_point_3d_y","gaze_point_3d_z","eye_center0_3d_x","eye_center0_3d_y","eye_center0_3d_z",
                                          "gaze_normal0_x","gaze_normal0_y","gaze_normal0_z","eye_center1_3d_x","eye_center1_3d_y",
                                          "eye_center1_3d_z","gaze_normal1_x","gaze_normal1_y","gaze_normal1_z"])
    
    df.to_csv(OUTPUTPATH + os.path.sep +'gaze_positions'+str(int(trial)+1)+'.csv') #the plus 1 is to use the files with MATLAB.