Example: Preprocessing of a full dataset with multiple subjects#
This is an example of a real dataset of a study from our cognitive neuroscience lab at the University of Tromsø.
The notebook cannot be run without the underlying data (which can be found at https://osf.io/7wcej/) but it can be used as an example to adapt for your own studies. To load and share the data of a study more easily, consider using a configuration file.
[ ]:
import glob
import pypillometry as pp
from pypillometry import u
import matplotlib.pyplot as plt
import pandas as pd
plt.rcParams.update({"figure.max_open_warning": 0}) # suppress a warning
[ ]:
# load all raw datasets
exclude = ["20_pav", "8_pav", "1_pav_pilot"] # these subjects did not have usable data
# here, data is already stored in pypillometry's own format as .pd files
pdfiles = glob.glob("data/eyedata/*.pd")
datasets = [pp.EyeData.from_file(fname) for fname in pdfiles]
# datasets are stored in a dict with the subject number as key
datasets = {d.name.split("_")[0]: d for d in datasets if d.name not in exclude}
[ ]:
# preprocessing parameters (durations in ms)
default_param = {"min_duration": 10, # min duration of a blink
"vel_onset_min_duration": 6, # duration of onset/offset for blink detection
"vel_offset_min_duration": 4,
"vel_onset": -5, # velocity thresholds for onset and offset
"vel_offset": 5,
"strategies": ["zero", "velocity"], # strategies for blink detection
"distance": 100, # minimum distance between two blinks
"margin": (50, 150), # margins for Mahot algorithm
"cutoff": 5} # lowpass-filter cutoff (Hz)
# all subjects get the same set of default parameters initially
params = {subj: default_param.copy() for subj in datasets.keys()}
[ ]:
# fine-tuning of the parameters per subject (this is done iteratively by
# inspecting the per-subject PDF files generated below and changing parameters)
params["4"].update(vel_onset=-10, min_duration=5)
params["6"].update(strategies=["zero"])
params["7"].update(vel_onset=-10)
params["9"].update(vel_onset=-10)
params["10"].update(vel_onset=-10)
params["12"].update(margin=(50, 200), vel_onset=-10)
params["13"].update(vel_onset=-10)
params["14"].update(margin=(50, 220), vel_onset=-10)
params["15"].update(margin=(50, 350), vel_onset=-10)
params["16"].update(margin=(50, 300), vel_onset=-10, min_duration=5)
params["17"].update(vel_onset=-10)
params["21"].update(vel_onset=-100, vel_offset=100,
vel_onset_min_duration=4, vel_offset_min_duration=4)
params["23"].update(vel_onset=-10, vel_onset_min_duration=4, min_duration=5)
params["24"].update(margin=(50, 320), vel_onset=-10, min_duration=1)
params["25"].update(margin=(50, 350), vel_onset=-10)
params["29"].update(margin=(100, 400), vel_onset=-10)
params["30"].update(margin=(100, 300), vel_onset=-20)
params["31"].update(margin=(100, 300), vel_onset=-10)
params["32"].update(vel_onset=-10)
params["34"].update(margin=(70, 350), vel_onset=-10)
params["35"].update(margin=(50, 250), vel_onset=-10, min_duration=5)
params["36"].update(margin=(100, 300), vel_onset=-10, min_duration=5)
params["37"].update(margin=(80, 250), vel_onset=-10, min_duration=1)
params["39"].update(vel_onset=-10, min_duration=5)
params["40"].update(margin=(60, 200), vel_onset=-10)
params["41"].update(margin=(60, 230), vel_onset=-20, min_duration=3)
params["42"].update(vel_onset=-10, min_duration=0)
params["43"].update(vel_onset=-10, min_duration=0)
params["46"].update(margin=(80, 220), vel_onset=-20, min_duration=0)
params["47"].update(vel_onset=-10)
params["49"].update(vel_onset=-10)
params["50"].update(vel_onset=-10, min_duration=0)
params["51"].update(margin=(50, 220), vel_onset=-10)
params["53"].update(vel_onset=-10, min_duration=0)
params["54"].update(margin=(50, 220), vel_onset=-10)
params["55"].update(margin=(50, 250), vel_onset=-10)
params["56"].update(vel_onset=-10, min_duration=0)
params["57"].update(margin=(80, 150), vel_onset=-10, min_duration=0)
[ ]:
# the preprocessing pipeline, with the parameters of one subject
def preproc(pars):
return (pp.pupil.blinks_detect(min_duration=pars["min_duration"],
strategies=pars["strategies"],
vel_onset=pars["vel_onset"], vel_offset=pars["vel_offset"],
vel_onset_min_duration=pars["vel_onset_min_duration"],
vel_offset_min_duration=pars["vel_offset_min_duration"])
| pp.pupil.blinks_merge_close(distance=pars["distance"])
| pp.pupil.blinks_interpolate(margin=pars["margin"],
vel_onset=pars["vel_onset"], vel_offset=pars["vel_offset"])
| pp.pupil.lowpass_filter(cutoff=pars["cutoff"]))
# run the preprocessing pipeline on all subjects and produce PDFs for inspection
preprocs = {}
for subj, d in datasets.items():
print(d.name)
dp = d | preproc(params[subj])
dp.plot.plot_pupil_segments(pdf_file="pics/%s.pdf" % d.name, interv=1 * u.min)
preprocs[subj] = dp
[ ]:
# write down notes about each subject when going through the PDFs
notes = {
"1": "last 5 mins missing; else quite ok, a few semi-blinks not properly detected",
"2": "many blinks after min 17; but reconstruction works very well",
"3": "fine",
"4": "fine (after tuning); a couple of short blinks that do not go all the way down to zero",
"5": "an awful lot of blinks; reconstruction seems to work but subj should probably be excluded",
"6": "fine (after tuning); very nice data with few blinks",
"7": "ok after tuning; weird stuff min 32-33",
"8": "missing data",
"9": "fine (after tuning); worse data in second half but ok",
"10": "many blinks, probably exclude",
"11": "usable but quite a few blinks (especially in the last part from min 33)",
"12": "fine after tuning but many blinks in second half",
"13": "usable but quite a few blinks; some segments are bad but should be ok due to exclusion crit",
"14": "fine",
"15": "exclude, many blinks and long opening-period after blinks",
"16": "fine; very nice data with few blinks",
"17": "usable but quite a few blinks",
"18": "missing",
"19": "usable (borderline), many blinks but nice reconstruction",
"20": "missing",
"21": "fine; except from min 37; incredibly fast transients!",
"22": "exclude; way to many blinks to do anything with this",
"23": "fine",
"24": "fine; very slow opening-transient",
"25": "borderline; a lot of blinks (more towards end) and a few weird artifacts in the second half",
"26": "fine; great, clean data!",
"27": "exclude",
"28": "missing",
"29": "usable; slow opening-transient",
"30": "fine",
"31": "fine",
"32": "fine",
"33": "usable; many blinks in second half",
"34": "fine",
"35": "borderline; lots of blinks during later segments",
"36": "borderline; esp near the end",
"37": "borderline; blinks consist of many consecutive, short-blinks",
"38": "missing",
"39": "fine (after tuning)",
"40": "borderline; many blinks throughout",
"41": "fine",
"42": "fine; has some of the weird triple-blinks",
"43": "fine; missing from min 34 (last block?)",
"44": "fine",
"45": "exclude; many blinks throughout",
"46": "usable",
"47": "fine",
"48": "missing",
"49": "fine",
"50": "fine",
"51": "fine",
"52": "fine",
"53": "fine",
"54": "fine",
"55": "usable; borderline towards the end",
"56": "usable, worse towards end",
"57": "exclude"
}
exclude_preproc = ["5", "10", "15", "22", "27", "45", "57"]
[ ]:
# save table with preprocessing parameters per subject
df = pd.DataFrame(params).T
df.index.name = "subj"
df = df.reset_index()
df["subj"] = pd.to_numeric(df.subj)
df.sort_values("subj").to_csv("preproc_params.csv")
[ ]:
# save table with notes
pd.DataFrame(notes.items(), columns=["subj", "notes"]).to_csv("preproc_notes.csv")
[ ]:
# write final preprocessed files for further analysis
preprocs = {subj: d for subj, d in preprocs.items() if subj not in exclude_preproc}
pp.write_file(preprocs, "data/export/eye_preproc.pd")
[ ]:
# save table with a final summary of the preprocessed data
def count_events(d, prefix):
return sum(lab.startswith(prefix) for lab in d.events.labels)
tab = pd.DataFrame([{"subj": int(subj),
"duration_minutes": d.get_duration(units="min"),
"nevents": d.nevents(),
"ncues": count_events(d, "C_"),
"nfb": count_events(d, "F_"),
"nresp": count_events(d, "R_"),
"perc_interpolated": 100 * d.data.get_layer("left_pupil", "interpolated").mean(),
"blinks_per_min": d.blink_stats()["left_pupil"]["n"] / d.get_duration(units="min"),
"note": notes[subj]}
for subj, d in preprocs.items()])
tab.sort_values("subj").to_csv("preproc_summary.csv")