-
-
Save skjerns/bc660ef59dca0dbd53f00ed38c42f6be to your computer and use it in GitHub Desktop.
# -*- coding: utf-8 -*- | |
""" | |
Created on Wed Dec 5 12:56:31 2018 | |
@author: skjerns | |
Gist to save a mne.io.Raw object to an EDF file using pyEDFlib | |
(https://github.com/holgern/pyedflib) | |
Disclaimer: | |
- Saving your data this way will result in slight | |
loss of precision (magnitude +-1e-09). | |
- It is assumed that the data is presented in Volt (V), | |
it will be internally converted to microvolt | |
- BDF or EDF+ is selected based on the filename extension | |
- Annotations preserved | |
""" | |
import pyedflib # pip install pyedflib | |
from pyedflib import highlevel # new high-level interface | |
from pyedflib import FILETYPE_BDF, FILETYPE_BDFPLUS, FILETYPE_EDF, FILETYPE_EDFPLUS | |
from datetime import datetime, timezone, timedelta | |
import mne | |
import os | |
def _stamp_to_dt(utc_stamp): | |
"""Convert timestamp to datetime object in Windows-friendly way.""" | |
if 'datetime' in str(type(utc_stamp)): return utc_stamp | |
# The min on windows is 86400 | |
stamp = [int(s) for s in utc_stamp] | |
if len(stamp) == 1: # In case there is no microseconds information | |
stamp.append(0) | |
return (datetime.fromtimestamp(0, tz=timezone.utc) + | |
timedelta(0, stamp[0], stamp[1])) # day, sec, μs | |
def write_mne_edf(mne_raw, fname, picks=None, tmin=0, tmax=None, | |
overwrite=False): | |
""" | |
Saves the raw content of an MNE.io.Raw and its subclasses to | |
a file using the EDF+/BDF filetype | |
pyEDFlib is used to save the raw contents of the RawArray to disk | |
Parameters | |
---------- | |
mne_raw : mne.io.Raw | |
An object with super class mne.io.Raw that contains the data | |
to save | |
fname : string | |
File name of the new dataset. This has to be a new filename | |
unless data have been preloaded. Filenames should end with .edf | |
picks : array-like of int | None | |
Indices of channels to include. If None all channels are kept. | |
tmin : float | None | |
Time in seconds of first sample to save. If None first sample | |
is used. | |
tmax : float | None | |
Time in seconds of last sample to save. If None last sample | |
is used. | |
overwrite : bool | |
If True, the destination file (if it exists) will be overwritten. | |
If False (default), an error will be raised if the file exists. | |
""" | |
if not issubclass(type(mne_raw), mne.io.BaseRaw): | |
raise TypeError('Must be mne.io.Raw type') | |
if not overwrite and os.path.exists(fname): | |
raise OSError('File already exists. No overwrite.') | |
# static settings | |
has_annotations = True if len(mne_raw.annotations)>0 else False | |
if os.path.splitext(fname)[-1] == '.edf': | |
file_type = FILETYPE_EDFPLUS if has_annotations else FILETYPE_EDF | |
dmin, dmax = -32768, 32767 | |
else: | |
file_type = FILETYPE_BDFPLUS if has_annotations else FILETYPE_BDF | |
dmin, dmax = -8388608, 8388607 | |
print('saving to {}, filetype {}'.format(fname, file_type)) | |
sfreq = mne_raw.info['sfreq'] | |
date = _stamp_to_dt(mne_raw.info['meas_date']) | |
if tmin: | |
date += timedelta(seconds=tmin) | |
# no conversion necessary, as pyedflib can handle datetime. | |
#date = date.strftime('%d %b %Y %H:%M:%S') | |
first_sample = int(sfreq*tmin) | |
last_sample = int(sfreq*tmax) if tmax is not None else None | |
# convert data | |
channels = mne_raw.get_data(picks, | |
start = first_sample, | |
stop = last_sample) | |
# convert to microvolts to scale up precision | |
channels *= 1e6 | |
# set conversion parameters | |
n_channels = len(channels) | |
# create channel from this | |
try: | |
f = pyedflib.EdfWriter(fname, | |
n_channels=n_channels, | |
file_type=file_type) | |
channel_info = [] | |
ch_idx = range(n_channels) if picks is None else picks | |
keys = list(mne_raw._orig_units.keys()) | |
for i in ch_idx: | |
try: | |
ch_dict = {'label': mne_raw.ch_names[i], | |
'dimension': mne_raw._orig_units[keys[i]], | |
'sample_rate': mne_raw._raw_extras[0]['n_samps'][i], | |
'physical_min': mne_raw._raw_extras[0]['physical_min'][i], | |
'physical_max': mne_raw._raw_extras[0]['physical_max'][i], | |
'digital_min': mne_raw._raw_extras[0]['digital_min'][i], | |
'digital_max': mne_raw._raw_extras[0]['digital_max'][i], | |
'transducer': '', | |
'prefilter': ''} | |
except: | |
ch_dict = {'label': mne_raw.ch_names[i], | |
'dimension': mne_raw._orig_units[keys[i]], | |
'sample_rate': sfreq, | |
'physical_min': channels.min(), | |
'physical_max': channels.max(), | |
'digital_min': dmin, | |
'digital_max': dmax, | |
'transducer': '', | |
'prefilter': ''} | |
channel_info.append(ch_dict) | |
f.setPatientCode(mne_raw._raw_extras[0]['subject_info'].get('id', '0')) | |
f.setPatientName(mne_raw._raw_extras[0]['subject_info'].get('name', 'noname')) | |
f.setTechnician('mne-gist-save-edf-skjerns') | |
f.setSignalHeaders(channel_info) | |
f.setStartdatetime(date) | |
f.writeSamples(channels) | |
for annotation in mne_raw.annotations: | |
onset = annotation['onset'] | |
duration = annotation['duration'] | |
description = annotation['description'] | |
f.writeAnnotation(onset, duration, description) | |
except Exception as e: | |
raise e | |
finally: | |
f.close() | |
return True |
Hi,
this gist is really great idea! Just the functionality, which I have been searched in mne and did not find. A great supplement. Big thanks to you!
I tried to save the preprocessed mne raw object into a new edf, and got the following error
...
line 27, in _stamp_to_dt
stamp = [int(s) for s in utc_stamp]
TypeError: 'datetime.datetime' object is not iterable
raw.info['meas_date'] in my script gives me:
datetime.datetime(2020, 3, 12, 16, 34, 55, tzinfo=datetime.timezone.utc)
Have you encoutered this problem?
Yes, I had the same problem. I solved it by changing line 76 to date = mne_raw.info['meas_date']
. However then I got a new problem and I actually gave up :P Let me know if you succed!
I've adapted the script.
@datalw can you check if it works now?
@skjerns Thanks a lot for the swift response! I tried the adapted version and got the following error:
----> 3 if 'datetime' in type(utc_stamp): return utc_stamp
4 # The min on windows is 86400
5 stamp = [int(s) for s in utc_stamp]
TypeError: argument of type 'type' is not iterable
So I changed if 'datetime' in type(utc_stamp):
to if 'datetime' in str(type(utc_stamp))
and it works :D Would you mind to adapt it again?
Thanks a lot for the great work!
@LunaHub it works for me so far, you could try it again : )
oops, yes didnt test it, thank's, I corrected it
I have used the gist for a while, it is really awesome, especially for saving the preprocessed data!
I am here again for a follow-up question. So far the files without annotations have been saved successfully. However for a file with annotations, the raw.times in the saved file was incorrect. I have looked into the code and found out the problem might be due to this line
'sample_rate': mne_raw._raw_extras[0]['n_samps'][i]
mne_raw.info['sfreq']
gives 256, but mne_raw._raw_extras[0]['n_samps'][i]
gives 2048 instead.
Since I have never dealt with _raw_extras, I tried to find some information about _raw_extras in mne documention, but failed.
Does anyone know how to solve this problem, or how I can proceed? Or is this a bug in _raw_extras?
I am not familiar with raw._raw_extras
you might be able to reach a wider audience that wrote that part of the code on Gitter unless someone on this thread knows https://gitter.im/mne-tools/mne-python
@datalw did you resample the data? was one of original sample frequencies 2048?
to be honest, I just hacked this snippet together and changed it often over time, I'm also not very familiar with the mne
internals. If you find a solution, let me know! I'm still in favor of integrating edf writing compatibility inside mne
, but currently they don't want to introduce optional dependencies. maybe that changes as a python-native implementation of edflib
has been published last month :)
So this still isn't incorporated into mne-python, any chance it ever will? Been a few years.
Feel free to ask this directly at MNE: https://github.com/mne-tools/mne-python/issues or the gitter https://gitter.im/mne-tools/mne-python
The problem was the additional dependency on pyedflib
and that pyedflib
uses Cython to compile some C libraries, which they did not want to include (afaik, and which I somehow understand).
However, we could ask if it would be possible to include an optional dependency on pyedflib. Additionally, EDDlib has just released a Python-only-version which would solve some problems :) but its quite slow (it's Python, nevertheless)
@alexrockhill @skjerns Thanks a lot for the reply and sorry for my delayed answer. I plan to come back to this issue in the next couple of weeks and if I find a solution I will let you @skjerns know ; )
Hello, thanks a lot for the great work! I am having one problem when using this script. When I try to open the resulted edf file this message came up "Error, number of datarecords is 0, expected >0. You can fix this problem with the header editor, check the manual for details. File is not a valid EDF or BDF file.". I am not quite sure why is this the case and I wonder if anyone is having the same problem or has any solution.
Thanks a lot!
That's a bit hard to diagnose what's going on without sharing a minimally reproducible example that someone else can run on their machine.
It sounds like maybe your mne.io.Raw
object didn't have any data in it or any data of type eeg
, grad
, mag
or seeg
. See https://mne.tools/stable/generated/mne.io.Raw.html#mne.io.Raw.set_channel_types.
That's a bit hard to diagnose what's going on without sharing a minimally reproducible example that someone else can run on their machine.
It sounds like maybe your
mne.io.Raw
object didn't have any data in it or any data of typeeeg
,grad
,mag
orseeg
. See https://mne.tools/stable/generated/mne.io.Raw.html#mne.io.Raw.set_channel_types.
@alexrockhill Hello, thanks a lot for the reply. This is the piece of code I used when creating the mne.io.Raw
object.
sample_data_raw_file = os.path.join("filename.edf")
raw = mne.io.read_raw_edf(sample_data_raw_file)
My original document is already in .edf, I put the file through some processing in mne-python and I would want to save the end-product as .edf using your script. Thanks a lot for your help!
From my experience reading in edf files, often the data types are not correctly set by mne by default. I would try setting all your channel types to eeg
I assume but whatever type they are and saving again. I'd be interested to know if that works.
Also, I helped a bit but the thanks definitely goes to @skjerns for this gist.
Maybe this will help
import mne
raw_fname = 'filename.edf'
raw = mne.io.read_raw_edf(raw_fname)
raw = raw.set_channel_types({ch: 'eeg' for ch in raw.ch_names})
@alexrockhill
Thanks for the reply. I have tried setting the channel types to eeg
using raw = raw.set_channel_types({ch: 'eeg' for ch in raw.ch_names})
, but the resulting file still can't be opened.
I am having the same. My guess is that there is some unexpected bug occurs while you reading the raw data from edf. Is there any method to sure I read the edf correctly?
Ok, that was just an informed guess but I couldn't tell you what's wrong because I don't have access to your data.
I realized that the smaller the original file I use the smaller the resulting file that is saved. But nomatter how big my original file is, the resulting file is always too small to be opened. When I called the function I called it this way write_mne_edf(raw,'testfile.edf', overwrite=True)
. I wonder if I am not aware of some details from the script resulting in this error of getting an extremely small file.
It is difficult to help you without having the original file to work with. Feel free to upload the file somewhere (eg wetransfer) and post the link here. Alternatively, if the data is sensitive, send it privately to tm2v0apq1o2@temp.mailbox.org
It is difficult to help you without having the original file to work with. Feel free to upload the file somewhere (eg wetransfer) and post the link here. Alternatively, if the data is sensitive, send it privately to tm2v0apq1o2@temp.mailbox.org
Hello, thank you very much for your reply. I have sent an email to the address provided. Thanks!
There was an Exception raised during the file you sent, as the date
wasn't set correctly, so the resulting file was just a leftover of an incomplete write. I changed the gist to raise an Exception explicitly and not just print it. It should work now. This was due to a bug in pyedflib, which I fixed now.
There was an Exception raised during the file you sent, as the
date
wasn't set correctly, so the resulting file was just a leftover of an incomplete write. I changed the gist to raise an Exception explicitly and not just print it. It should work now. This was due to a bug in pyedflib, which I fixed now.
Thank you very much for your help, it works perfectly now!
Hi @skjerns! Thank you for this great gist, we've been using it a lot in my lab. If the option of integrating it directly into MNE ever comes back to the table (now that there is a native Python implementation), I'd be happy to vouch for it!
Just a quick comment on the code, wouldn't it make more sense to update the date
so that it starts at tmin
, and not meas_date
? Concretely, just adding a timedelta(seconds=tmin) to the date
datetime?
Thanks,
Raphael
The Python native implementation of edflib
is unfortunately extremely slow (I posted a benchmark here), up to 20x slower. So I don't think implementation in mne is going to come soon. But I see what I can do w.r.t this issue, as it seems to be a major feature lacking in mne
. It should be possible to speed up quite a bit, with knowing a bit of Python internals and bottlenecks.
good one for the tmin! I'll implement that. I actually never use this gist myself, so I rely on users to improve it :)
Hello, and thank you for your great job!
I have a little problem because when I convert a file using your method, it goes from 1h duration to 8h duration (probably related to the sample rate, which is transformed from 2048 to 256Hz somewhere in the process). However, I change absolutely nothing, I just load my .edf data (duration = 1h), use write_edf, load the new .edf data (duration = 8h).
Edit : I solved the problem by changing the following code. Basically I just changed the 'sample_rate' parameter into the ch_dict.
try:
ch_dict = {'label': mne_raw.ch_names[i],
'dimension': mne_raw._orig_units[keys[i]],
#'sample_rate': mne_raw._raw_extras[0]['n_samps'][i],
'sample_rate': mne_raw.info["sfreq"],
'physical_min': mne_raw._raw_extras[0]['physical_min'][i],
'physical_max': mne_raw._raw_extras[0]['physical_max'][i],
'digital_min': mne_raw._raw_extras[0]['digital_min'][i],
'digital_max': mne_raw._raw_extras[0]['digital_max'][i],
'transducer': '',
'prefilter': ''}
except:
ch_dict = {'label': mne_raw.ch_names[i],
'dimension': mne_raw._orig_units[keys[i]],
#'sample_rate': sfreq,
'sample_rate': mne_raw.info["sfreq"],
'physical_min': channels.min(),
'physical_max': channels.max(),
'digital_min': dmin,
'digital_max': dmax,
'transducer': '',
'prefilter': ''}
Hello, thanks a lot for this amazing script. I realize that the resulting file saved from this script will be in the folder which the original data is located. I wonder if there is any way we can change where the output file will be within the script. Thank you.
Hello, thanks a lot for this amazing script. I realize that the resulting file saved from this script will be in the folder which the original data is located. I wonder if there is any way we can change where the output file will be within the script. Thank you.
Hi, you can just set "fname" to the full path you want (folderpath/filename.edf) when you call the write_mne_edf() function.
Hi, thanks for your great work.
I'm trying to fit this code into mff
files, but I'm not sure what does channels *= 1e6
do. Can someone give more explanation? Thanks.
It is assumed that the data is passed from MNE is in Volts (see disclaimer of the function, this is default in MNE afaik), even though most values recorded with EEG are in the mV or uV range. However, as EDF+ is only using 16-bit precision, such small values easily lose much precision. Therefore values are converted from Volt to Microvolt, where less precision is lost. The physical dimension is then added to the EDF+ header, so it should load correctly. To be honest, now that I'm re-reading the code I'm struggeling myself to trust that this also works in cases where data is not EEG (e.g. with data in nT
like MEG). I haven't used MNE ever since, so I don't really know.
just use pyedflib
see here for use case examples https://github.com/holgern/pyedflib#highlevel-interface
Thanks @skjerns , but the labels? How could I introduce them? Thanks again!
you can add them as annotations
signals = np.random.rand(5, 256*300)*200 # 5 minutes of random signal
channel_names = ['ch1', 'ch2', 'ch3', 'ch4', 'ch5']
signal_headers = highlevel.make_signal_headers(channel_names, sample_frequency=256)
header = highlevel.make_header(patientname='patient_x', gender='Female')
annotations = [[0, 0, "Wake"], [30, 0, "S2"]] # format [onset, duration, description], I think time is in seconds or ms, I don't remember
header['annotations'] = annotations
highlevel.write_edf('edf_file.edf', signals, signal_headers, header)
likely some wrong values for digital min/max and physical min/max in EDF. Else just a display problem in EDFBrowser, try pressing '+' to increase the signals. However, this is not the place to get support for that - it has little to do with this blob.
Hi
I'm trying to open an edf in mne, edit it, and save result in a new file using your gist. Unfortunately, when I do:
write_mne_edf(EEG_raw, fname, picks=None, tmin=0, tmax=None,
overwrite=False)
console-->
saving to EDF_test, filetype 2
'name'
the resulting file is very small and its content is :
Error! C:\Python\Python38\EDF_test is not UTF-8 encoded
Saving disabled.
See Console for more details.
edf original:
https://drive.google.com/file/d/1uuiZ4hswH2i4CLZq5BjqHAvwhhR9xARJ/view?usp=sharing
Could you help me . Thanks in advance
header:
{'technician': '', 'recording_additional': '', 'patientname': 'RLSR', 'patient_additional': 'righthanded 4 Kg', 'patientcode': '00000', 'equipment': 'EMSA Equipamentos Medicos S.A.', 'admincode': '0000001', 'gender': 'Female', 'startdate': datetime.datetime(2004, 11, 1, 8, 19, 53), 'birthdate': '02 nov 1995', 'annotations': []}
I edited the gist, it should work now. (I also removed some patient-sensitive data from your post, please also remove the file from your google drive as it contains patient specific data that you're probably not allowed to share)
Dear Mr Kern
I updated the gist
original file without sensitive info:
https://drive.google.com/file/d/1kJKHQRE5FslZkm7RkOoZkVPZ1b7P_5bB/view?usp=sharing
but the result doesnot copy the data:
https://drive.google.com/file/d/14Sog6jkKrfLIOwFgNGAGlorYz2KHRhrs/view?usp=sharing
I cannot reproduce the problem.
raw = mne.io.read_raw('0651701_Copy (1).edf')
write_mne_edf(raw, 'test.edf')
I can read and save the file with no problems. Sorry, I will not be able to give individual support for specific problems that are not directly related to the gist.
Sure, I do it for fun and I'll find a way, thank you for your time, have a great day.
PKanda
Brazil
I am getting an os error when trying to save the preprocessed eeg data in EDF format. Earlier it worked. I am now unable to save the epoched eeg data into a folder in EDF format. Could you tell me how to solve this issue? I'm getting the error shown below
"OSError: The filename (/Users/sreelakshmiraveendran/Desktop/Research papersMAC/Python programming/out_data) for file type raw must end with .fif or .fif.gz"
You are not using the script, but probably raw.save()
, which is MNE buitlin. Also I have no idea what you are doing, please provide a context with code that reproduces the issue. This does not seem like an error in the script, but rather a general Python programming mistake.
The original file is edF file, now want to convert to BDF file format, can this library be used successfully?Thank you very much,My email is 1738388008@qq.com
You can do so easily with pyedflib
. However, you will gain nothing from it, as your precision will be obviously capped by the EDF file.
EDF and BDF are the same file format with the difference of having 16 and 24 bits of precision.
import pyedflib
signals, sheads, header = pyedflib.highlevel.read_edf(filename)
pyedflib.highlevel.write_edf('out.bdf', signals, sheads, header)
you might need to manually alter the physical min/max in the signal headers of some channels. Sorry, cannot give any further help than this :-/
I'm encountering an error with the current gist here and BDF files, using the latest versions of mne and pyedflib (1.3.1 and 0.1.32 respectively)
Using the files directly from BioSemi I mentioned in an earlier message here (https://gist.github.com/skjerns/bc660ef59dca0dbd53f00ed38c42f6be?permalink_comment_id=3187890#gistcomment-3187890), I'm running this script:
import mne, save_edf, os
if __name__ == "__main__":
path = '~~~/Downloads/BDFtestfiles'
files = [x for x in os.listdir(path) if 'mod' not in x]
for file in files:
minus = '.'.join(file.split('.')[:-1])
nneww = '{}/{}_mod.bdf'.format(path, minus)
dat = mne.io.read_raw_bdf('{}/{}'.format(path, file))
save_edf.write_mne_edf(mne_raw=dat, fname=nneww, overwrite=True)
There is another file named save_edf.py that is the gist on this page. I get the following error:
I'm not sure but I suspect this is related to BioSemi's default units--I think their devices save in micro-Volts. The only field I see than mentions that (in the mne.io.Raw
object) is the _orig_units
field; however the "original" in that name makes me hesitant to treat it as a ground truth for the current state of the file. Any suggestions for handling this?
MNE has an mne.io.export_raw now too which might be helpful if that's all handled within that function
I think the only thing that you can try is trial and error: downscale the data and change dimension accordingly and check with EdfBrowser if the signal is loaded correctly. If they save in microvolts I'm surprised that there are values of 1e13, which would be death for most organisms-----------Sent from mobileAm 27.04.2023 um 13:52 schrieb GABowers @.>:Re: @. commented on this gist.I'm encountering an error with the current gist here and BDF files, using the latest versions of mne and pyedflib (1.3.1 and 0.1.32 respectively)Using the files directly from BioSemi I mentioned in an earlier message here (https://gist.github.com/skjerns/bc660ef59dca0dbd53f00ed38c42f6be?permalink_comment_id=3187890#gistcomment-3187890), I'm running this script:import mne, save_edf, os if name == "main": path = '~~~/Downloads/BDFtestfiles' files = [x for x in os.listdir(path) if 'mod' not in x] for file in files: minus = '.'.join(file.split('.')[:-1]) nneww = '{}/{}_mod.bdf'.format(path, minus) dat = mne.io.read_raw_bdf('{}/{}'.format(path, file)) save_edf.write_mne_edf(mne_raw=dat, fname=nneww, overwrite=True) There is another file named save_edf.py that is the gist on this page. I get the following error:I'm not sure but I suspect this is related to BioSemi's default units--I think their devices save in micro-Volts. The only field I see than mentions that (in the mne.io.Raw object) is the _orig_units field; however the "original" in that name makes me hesitant to treat it as a ground truth for the current state of the file. Any suggestions for handling this?—Reply to this email directly, view it on GitHub or unsubscribe.You are receiving this email because you authored the thread.Triage notifications on the go with GitHub Mobile for iOS or Android.
Your comment led me to look a bit deeper. It seems that the mne_raw._raw_extras
dictionary is missing some keys, which causes the code to use values from the data for the various channel infos--the "except" in the following:
try:
ch_dict = {'label': mne_raw.ch_names[i],
'dimension': mne_raw._orig_units[keys[i]],
'sample_rate': mne_raw._raw_extras[0]['n_samps'][i],
'physical_min': mne_raw._raw_extras[0]['physical_min'][i],
'physical_max': mne_raw._raw_extras[0]['physical_max'][i],
'digital_min': mne_raw._raw_extras[0]['digital_min'][i],
'digital_max': mne_raw._raw_extras[0]['digital_max'][i],
'transducer': '',
'prefilter': ''}
except:
ch_dict = {'label': mne_raw.ch_names[i],
'dimension': mne_raw._orig_units[keys[i]],
'sample_rate': sfreq,
'physical_min': channels.min(),
'physical_max': channels.max(),
'digital_min': dmin,
'digital_max': dmax,
'transducer': '',
'prefilter': ''}
And it's getting those values from, surprise surprise, the BioSemi Status channel.
I think the simplest solution is to get that physical_min
and max from all channels except any named Status--I'll write something up for that in a few hours when I have some time.
Edit: See below. Simple change.
except:
physical = channels if 'Status' not in mne_raw.ch_names[-1] else channels[:-1]
ch_dict = {'label': mne_raw.ch_names[i],
'dimension': mne_raw._orig_units[keys[i]],
'sample_rate': sfreq,
'physical_min': physical.min(),
'physical_max': physical.max(),
'digital_min': dmin,
'digital_max': dmax,
'transducer': '',
'prefilter': ''}
That doesn't have any tests either haha
It seems like there is a lot of use of the gist and like people would probably like it to be officially supported by
mne
and @skjerns said he would write the initial PR but I understand the hesitance and I get the message. I thought it was worth floating the idea.