Skip to content

Instantly share code, notes, and snippets.

@theXYZT
Last active April 29, 2020 06:40
Show Gist options
  • Save theXYZT/9bd177424cdff9e90fb2b43afaf28e1e to your computer and use it in GitHub Desktop.
Save theXYZT/9bd177424cdff9e90fb2b43afaf28e1e to your computer and use it in GitHub Desktop.
Baseband Windows Test Failures
>>> baseband.test()
=========================================================================== test session starts ===========================================================================
platform win32 -- Python 3.8.1, pytest-5.3.4, py-1.8.1, pluggy-0.13.1
rootdir: C:\
plugins: hypothesis-4.54.2, arraydiff-0.2, astropy-header-0.1.1, doctestplus-0.5.0, openfiles-0.4.0, remotedata-0.3.2
collected 397 items
..\..\..\..\..\ProgramData\Anaconda3\envs\pkgdev\Lib\site-packages\baseband\dada\tests\test_dada.py .....FF.FFF.. [ 3%]
..\..\..\..\..\ProgramData\Anaconda3\envs\pkgdev\Lib\site-packages\baseband\gsb\tests\test_gsb.py ................... [ 8%]
..\..\..\..\..\ProgramData\Anaconda3\envs\pkgdev\Lib\site-packages\baseband\guppi\tests\test_guppi.py ......FF.F..... [ 11%]
..\..\..\..\..\ProgramData\Anaconda3\envs\pkgdev\Lib\site-packages\baseband\helpers\tests\test_sequentialfile.py .................... [ 16%]
..\..\..\..\..\ProgramData\Anaconda3\envs\pkgdev\Lib\site-packages\baseband\mark4\tests\test_corrupt_files.py ............... [ 20%]
..\..\..\..\..\ProgramData\Anaconda3\envs\pkgdev\Lib\site-packages\baseband\mark4\tests\test_mark4.py ................................................ [ 32%]
..\..\..\..\..\ProgramData\Anaconda3\envs\pkgdev\Lib\site-packages\baseband\mark5b\tests\test_corrupt_files.py ............................... [ 40%]
..\..\..\..\..\ProgramData\Anaconda3\envs\pkgdev\Lib\site-packages\baseband\mark5b\tests\test_mark5b.py ................................. [ 48%]
..\..\..\..\..\ProgramData\Anaconda3\envs\pkgdev\Lib\site-packages\baseband\tests\test_conversion.py ............... [ 52%]
..\..\..\..\..\ProgramData\Anaconda3\envs\pkgdev\Lib\site-packages\baseband\tests\test_core.py ........ [ 54%]
..\..\..\..\..\ProgramData\Anaconda3\envs\pkgdev\Lib\site-packages\baseband\tests\test_file_info.py ................. [ 58%]
..\..\..\..\..\ProgramData\Anaconda3\envs\pkgdev\Lib\site-packages\baseband\tests\test_sequential_baseband.py . [ 59%]
..\..\..\..\..\ProgramData\Anaconda3\envs\pkgdev\Lib\site-packages\baseband\vdif\tests\test_corrupt_files.py ........................................... [ 70%]
..\..\..\..\..\ProgramData\Anaconda3\envs\pkgdev\Lib\site-packages\baseband\vdif\tests\test_vdif.py ................................................. [ 82%]
..\..\..\..\..\ProgramData\Anaconda3\envs\pkgdev\Lib\site-packages\baseband\vlbi_base\tests\test_utils.py ...................... [ 87%]
..\..\..\..\..\ProgramData\Anaconda3\envs\pkgdev\Lib\site-packages\baseband\vlbi_base\tests\test_vlbi_base.py ................................................ [100%]
================================================================================ FAILURES =================================================================================
_______________________________________________________________________ TestDADA.test_frame_memmap ________________________________________________________________________
self = <baseband.dada.tests.test_dada.TestDADA object at 0x0000018E5A9AEF70>
tmpdir = local('C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_frame_memmap0')
def test_frame_memmap(self, tmpdir):
# Get frame regular way.
with dada.open(SAMPLE_FILE, 'rb') as fr:
frame = fr.read_frame(memmap=False)
assert not isinstance(frame.payload.words, np.memmap)
# Check that if we map it instead, we get the same result.
with dada.open(SAMPLE_FILE, 'rb') as fh:
frame2 = fh.read_frame(memmap=True)
assert frame2 == frame
assert isinstance(frame2.payload.words, np.memmap)
# Bit superfluous perhaps, but check decoding as well.
assert np.all(frame2[:3] == np.array(
[[[-38.-38.j], [-38.-38.j]],
[[-38.-38.j], [-40.+0.j]],
[[-105.+60.j], [85.-15.j]]], dtype=np.complex64))
assert np.all(frame2.data == frame.data)
# Now check writing. First, without memmap, just ensuring writing
# to file works as well as to BytesIO done above.
filename = str(tmpdir.join('a.dada'))
with dada.open(filename, 'wb') as fw:
fw.write_frame(frame)
with dada.open(filename, 'rb') as fw:
frame3 = fw.read_frame()
assert frame3 == frame
# Now memmap file to be written to.
> with dada.open(filename, 'wb') as fw:
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\dada\tests\test_dada.py:260:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\dada\base.py:508: in open
return opener(name, mode, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
name = 'C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_frame_memmap0\\a.dada', mode = 'wb', kwargs = {}, cls_type = 'FileWriter', got_fh = False
def open(name, mode='rs', **kwargs):
# If sequentialfile object, check that it's opened properly.
if isinstance(name, sf.SequentialFileBase):
assert (('r' in mode and name.mode == 'rb')
or ('w' in mode and name.mode == 'w+b')), (
"open only accepts sequential files opened in 'rb' "
"mode for reading or 'w+b' mode for writing.")
# If passed some kind of list, open a sequentialfile object.
if isinstance(name, (tuple, list, sf.FileNameSequencer)):
if 'r' in mode:
name = sf.open(name, 'rb')
else:
file_size = kwargs.pop('file_size', None)
name = sf.open(name, 'w+b', file_size=file_size)
# Select FileReader/Writer for binary, StreamReader/Writer for stream.
if 'b' in mode:
cls_type = 'File'
else:
cls_type = 'Stream'
# Select reading or writing. Check if ``name`` is a filehandle, and
# open it if not.
if 'w' in mode:
cls_type += 'Writer'
got_fh = hasattr(name, 'write')
if not got_fh:
> name = io.open(name, 'w+b')
E OSError: [Errno 22] Invalid argument: 'C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_frame_memmap0\\a.dada'
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\vlbi_base\base.py:1112: OSError
_______________________________________________________________________ TestDADA.test_filestreamer ________________________________________________________________________
self = <baseband.dada.tests.test_dada.TestDADA object at 0x0000018E5A8D7F40>
tmpdir = local('C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_filestreamer0')
def test_filestreamer(self, tmpdir):
start_time = self.header.time
with dada.open(SAMPLE_FILE, 'rs') as fh:
assert fh.header0 == self.header
assert fh.sample_shape == (2,)
assert fh.shape == (16000,) + fh.sample_shape
assert fh.size == np.prod(fh.shape)
assert fh.ndim == len(fh.shape)
assert fh.start_time == start_time
assert fh.sample_rate == 16 * u.MHz
record1 = fh.read(12)
assert fh.tell() == 12
fh.seek(10000)
record2 = np.zeros((2, 2), dtype=np.complex64)
record2 = fh.read(out=record2)
assert fh.tell() == 10002
assert fh.time == fh.tell(unit='time')
assert (np.abs(fh.time - (start_time + 10002 / (16 * u.MHz)))
< 1. * u.ns)
fh.seek(fh.start_time + 1000 / (16*u.MHz))
assert fh.tell() == 1000
assert fh._last_header == fh.header0
assert np.abs(fh.stop_time
- (start_time + 16000 / (16.*u.MHz))) < 1.*u.ns
# Test seeker works with both int and str values for whence.
assert fh.seek(13, 0) == fh.seek(13, 'start')
assert fh.seek(-13, 2) == fh.seek(-13, 'end')
fhseek_int = fh.seek(17, 1)
fh.seek(-17, 'current')
fhseek_str = fh.seek(17, 'current')
assert fhseek_int == fhseek_str
with pytest.raises(ValueError):
fh.seek(0, 'last')
fh.seek(1, 'end')
with pytest.raises(EOFError):
fh.read()
assert record1.shape == (12, 2)
assert np.all(record1[:3] == np.array(
[[-38.-38.j, -38.-38.j],
[-38.-38.j, -40.+0.j],
[-105.+60.j, 85.-15.j]], dtype=np.complex64))
assert record1.shape == (12, 2) and record1.dtype == np.complex64
assert np.all(record1 == self.payload[:12].squeeze())
assert record2.shape == (2, 2)
assert np.all(record2 == self.payload[10000:10002].squeeze())
filename = str(tmpdir.join('a.dada'))
with dada.open(filename, 'ws', header0=self.header,
squeeze=False) as fw:
assert fw.sample_rate == 16 * u.MHz
fw.write(self.payload.data)
assert fw.start_time == start_time
assert (np.abs(fw.time - (start_time + 16000 / (16. * u.MHz)))
< 1. * u.ns)
with dada.open(filename, 'rs') as fh:
data = fh.read()
assert fh.start_time == start_time
assert (np.abs(fh.time - (start_time + 16000 / (16. * u.MHz)))
< 1. * u.ns)
assert fh.stop_time == fh.time
assert fh.sample_rate == 16 * u.MHz
assert np.all(data == self.payload.data.squeeze())
# Try single polarisation, and check initialisation by header keywords.
h = self.header
> with dada.open(filename, 'ws', time=h.time, bps=h.bps,
complex_data=h.complex_data, sample_rate=h.sample_rate,
payload_nbytes=32000, npol=1, nchan=1) as fw:
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\dada\tests\test_dada.py:366:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\dada\base.py:508: in open
return opener(name, mode, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
name = 'C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_filestreamer0\\a.dada', mode = 'ws'
kwargs = {'header0': DADAHeader("""HEADER DADA
HDR_VERSION 1.0
HDR_SIZE 4096
DADA_VERSION 1.0
OBS_ID unset
PRIMARY unset
SECOND...OL 1
NCHAN 1
RESOLUTION 1
DSB 1
TSAMP 0.0625
BW 16.0
UTC_START 2013-07-02-01:39:20
MJD_START 56475.068981481481481""")}
cls_type = 'StreamWriter', got_fh = False
def open(name, mode='rs', **kwargs):
# If sequentialfile object, check that it's opened properly.
if isinstance(name, sf.SequentialFileBase):
assert (('r' in mode and name.mode == 'rb')
or ('w' in mode and name.mode == 'w+b')), (
"open only accepts sequential files opened in 'rb' "
"mode for reading or 'w+b' mode for writing.")
# If passed some kind of list, open a sequentialfile object.
if isinstance(name, (tuple, list, sf.FileNameSequencer)):
if 'r' in mode:
name = sf.open(name, 'rb')
else:
file_size = kwargs.pop('file_size', None)
name = sf.open(name, 'w+b', file_size=file_size)
# Select FileReader/Writer for binary, StreamReader/Writer for stream.
if 'b' in mode:
cls_type = 'File'
else:
cls_type = 'Stream'
# Select reading or writing. Check if ``name`` is a filehandle, and
# open it if not.
if 'w' in mode:
cls_type += 'Writer'
got_fh = hasattr(name, 'write')
if not got_fh:
> name = io.open(name, 'w+b')
E OSError: [Errno 22] Invalid argument: 'C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_filestreamer0\\a.dada'
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\vlbi_base\base.py:1112: OSError
___________________________________________________________________ TestDADA.test_multiple_files_stream ___________________________________________________________________
self = <baseband.dada.tests.test_dada.TestDADA object at 0x0000018E5A9F0760>
tmpdir = local('C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_multiple_files_stream0')
def test_multiple_files_stream(self, tmpdir):
start_time = self.header.time
data = self.payload.data.squeeze()
header = self.header.copy()
header.payload_nbytes = self.header.payload_nbytes // 2
filenames = (str(tmpdir.join('a.dada')),
str(tmpdir.join('b.dada')))
with dada.open(filenames, 'ws', **header) as fw:
start_time = fw.start_time
fw.write(data[:1000])
time1000 = fw.time
fw.write(data[1000:])
stop_time = fw.time
assert start_time == header.time
assert np.abs(time1000 - (start_time + 1000 / (16.*u.MHz))) < 1.*u.ns
assert np.abs(stop_time - (start_time + 16000 / (16.*u.MHz))) < 1.*u.ns
with dada.open(filenames[1], 'rs') as fr:
assert (np.abs(fr.time - (start_time + 8000 / (16. * u.MHz)))
< 1. * u.ns)
data1 = fr.read()
assert np.all(data1 == data[8000:])
with dada.open(filenames, 'rs') as fr:
assert fr.start_time == start_time
assert fr.time == start_time
assert np.abs(fr.stop_time
- (start_time + 16000 / (16. * u.MHz))) < 1. * u.ns
data2 = fr.read()
assert fr.time == fr.stop_time
assert np.all(data2 == data)
# Pass sequentialfile objects to reader.
with sf.open(filenames, 'w+b',
file_size=(header.payload_nbytes + 4096)) as fraw, \
dada.open(fraw, 'ws', header0=header) as fw:
> fw.write(data)
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\dada\tests\test_dada.py:513:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\vlbi_base\base.py:1005: in write
self._frame = self._make_frame(frame_index)
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\dada\base.py:362: in _make_frame
return self.fh_raw.memmap_frame(header)
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\dada\base.py:204: in memmap_frame
header.tofile(self.fh_raw)
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\dada\header.py:217: in tofile
fh.write(s.read())
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\helpers\sequentialfile.py:338: in write
self._open(self.file_nr + 1)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <[TypeError('list indices must be integers or slices, not NoneType') raised in repr()] SequentialFileWriter object at 0x18e5a9f0fd0>, file_nr = 1
def _open(self, file_nr):
"""Open the ``file_nr``th file of the list of underlying files.
If a different file was already open, it is closed. Nothing is done
if the requested file is already open.
"""
if file_nr != self.file_nr:
try:
> fh = self.opener(self.files[file_nr], mode=self.mode)
E OSError: [Errno 22] Invalid argument: 'C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_multiple_files_stream0\\b.dada'
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\helpers\sequentialfile.py:119: OSError
____________________________________________________________________ TestDADA.test_partial_last_frame _____________________________________________________________________
self = <baseband.dada.tests.test_dada.TestDADA object at 0x0000018E5A4579D0>
tmpdir = local('C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_partial_last_frame0')
def test_partial_last_frame(self, tmpdir):
"""Test reading an incomplete frame from one or a sequence of files."""
# Prepare file sequence.
header = self.header.copy()
data = self.payload.data.squeeze()
data = np.r_[data, data, data]
filenames = [str(tmpdir.join('a.dada')),
str(tmpdir.join('b.dada')),
str(tmpdir.join('c.dada'))]
with dada.open(filenames, 'ws', header0=header) as fw:
fw.write(data)
# Replace c.dada with partially complete file.
with dada.open(str(tmpdir.join('c.dada')), 'rb') as fh, \
dada.open(str(tmpdir.join('c_partial.dada')), 'wb') as fw:
full_filesize = fh.seek(0, 2)
fh.seek(0)
fw.write(fh.read(full_filesize // 2 - 37))
filenames[-1] = str(tmpdir.join('c_partial.dada'))
# Check reading single partial frame.
with dada.open(filenames[-1], 'rs') as fh:
# Check that we've written the right number of bytes to file.
filesize = fh.fh_raw.seek(0, 2)
fh.fh_raw.seek(0)
assert filesize == full_filesize // 2 - 37
# Payload truncates 3 bytes so there is an integer number of
# complete samples.
assert fh.header0.frame_nbytes == filesize - 3
assert fh.header0.nbytes == self.header.nbytes
assert fh.samples_per_frame == (
(filesize - self.header.nbytes) * 8 // fh.header0.bps // 2
// np.prod(fh.header0.sample_shape))
assert fh.header0 is fh._last_header
assert np.abs(fh.stop_time - fh.start_time
- 7478 / fh.sample_rate) < 1 * u.ns
assert fh.shape == (7478, 2)
# Taking advantage of data being repeated 3 times.
assert np.all(fh.read() == data[:7478])
# Check reading sequence of files.
with dada.open(filenames) as fh:
assert fh.samples_per_frame == header.samples_per_frame
assert np.abs(fh.stop_time
- fh.start_time - 39478 / fh.sample_rate) < 1 * u.ns
assert fh.shape == (39478, 2)
assert np.all(fh.read() == data[:39478])
fh.seek(-29, 2)
assert np.all(fh.read() == data[7478 - 29:7478])
assert fh.tell() == 39478
# Replace c.dada with only the header (and no payload).
with dada.open(str(tmpdir.join('c.dada')), 'rb') as fh, \
dada.open(str(tmpdir.join('c_header_only.dada')), 'wb') as fw:
fw.write(fh.read(4096))
fh.seek(0)
header_c = fh.read_header()
filenames[-1] = str(tmpdir.join('c_header_only.dada'))
# Check that reading the frame gives payload of zero size.
with dada.open(str(tmpdir.join('c_header_only.dada')), 'rb') as fp:
assert fp.read_header() == header_c
fp.seek(0)
with pytest.raises(ValueError) as excinfo:
> fp.read_frame()
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\dada\tests\test_dada.py:594:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\dada\base.py:137: in read_frame
return DADAFrame.fromfile(self.fh_raw, memmap=memmap, verify=verify)
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\dada\frame.py:73: in fromfile
payload = cls._payload_class.fromfile(fh, header=header, memmap=memmap)
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\dada\payload.py:94: in fromfile
words = np.memmap(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
subtype = <class 'numpy.memmap'>
filename = <_io.BufferedReader name='C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_partial_last_frame0\\c_header_only.dada'>
dtype = dtype('uint32'), mode = 'r', offset = 4096, shape = (16000,), order = 'C'
def __new__(subtype, filename, dtype=uint8, mode='r+', offset=0,
shape=None, order='C'):
# Import here to minimize 'import numpy' overhead
import mmap
import os.path
try:
mode = mode_equivalents[mode]
except KeyError:
if mode not in valid_filemodes:
raise ValueError("mode must be one of %s" %
(valid_filemodes + list(mode_equivalents.keys())))
if mode == 'w+' and shape is None:
raise ValueError("shape must be given")
if hasattr(filename, 'read'):
f_ctx = contextlib_nullcontext(filename)
else:
f_ctx = open(os_fspath(filename), ('r' if mode == 'c' else mode)+'b')
with f_ctx as fid:
fid.seek(0, 2)
flen = fid.tell()
descr = dtypedescr(dtype)
_dbytes = descr.itemsize
if shape is None:
bytes = flen - offset
if bytes % _dbytes:
raise ValueError("Size of available data is not a "
"multiple of the data-type size.")
size = bytes // _dbytes
shape = (size,)
else:
if not isinstance(shape, tuple):
shape = (shape,)
size = np.intp(1) # avoid default choice of np.int_, which might overflow
for k in shape:
size *= k
bytes = long(offset + size*_dbytes)
if mode in ('w+', 'r+') and flen < bytes:
fid.seek(bytes - 1, 0)
fid.write(b'\0')
fid.flush()
if mode == 'c':
acc = mmap.ACCESS_COPY
elif mode == 'r':
acc = mmap.ACCESS_READ
else:
acc = mmap.ACCESS_WRITE
start = offset - offset % mmap.ALLOCATIONGRANULARITY
bytes -= start
array_offset = offset - start
> mm = mmap.mmap(fid.fileno(), bytes, access=acc, offset=start)
E OSError: [WinError 8] Not enough memory resources are available to process this command
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\numpy\core\memmap.py:264: OSError
______________________________________________________________________ TestDADA.test_template_stream ______________________________________________________________________
self = <baseband.dada.tests.test_dada.TestDADA object at 0x0000018E58AC7C70>
tmpdir = local('C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_template_stream0')
def test_template_stream(self, tmpdir):
start_time = self.header.time
data = self.payload.data.squeeze()
header = self.header.copy()
header.payload_nbytes = self.header.payload_nbytes // 4
template = str(tmpdir.join('a{frame_nr}.dada'))
with dada.open(template, 'ws', header0=header) as fw:
fw.write(data[:1000])
time1000 = fw.time
fw.write(data[1000:])
stop_time = fw.time
assert np.abs(time1000 - (header.time + 1000 / (16.*u.MHz))) < 1.*u.ns
assert np.abs(stop_time
- (header.time + 16000 / (16. * u.MHz))) < 1. * u.ns
with dada.open(template.format(frame_nr=1), 'rs') as fr:
data1 = fr.read()
assert fr.time == fr.stop_time
assert np.abs(fr.start_time
- (start_time + 4000 / (16.*u.MHz))) < 1.*u.ns
assert np.abs(fr.stop_time
- (start_time + 8000 / (16.*u.MHz))) < 1.*u.ns
assert np.all(data1 == data[4000:8000])
with dada.open(template, 'rs') as fr:
assert fr.time == start_time
data2 = fr.read()
assert fr.stop_time == fr.time
assert np.abs(fr.stop_time
- (header.time + 16000 / (16. * u.MHz))) < 1. * u.ns
assert np.all(data2 == data)
# More complicated template, 8 files.
header.payload_nbytes = self.header.payload_nbytes // 8
template = str(tmpdir
.join('{utc_start}_{obs_offset:016d}.000000.dada'))
> with dada.open(template, 'ws', header0=header) as fw:
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\dada\tests\test_dada.py:644:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\dada\base.py:508: in open
return opener(name, mode, **kwargs)
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\vlbi_base\base.py:1098: in open
name = sf.open(name, 'w+b', file_size=file_size)
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\helpers\sequentialfile.py:390: in open
return SequentialFileWriter(files, mode, file_size=file_size,
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\helpers\sequentialfile.py:327: in __init__
super().__init__(files, mode, opener)
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\helpers\sequentialfile.py:100: in __init__
self._open(0)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <[TypeError('list indices must be integers or slices, not NoneType') raised in repr()] SequentialFileWriter object at 0x18e5a5459a0>, file_nr = 0
def _open(self, file_nr):
"""Open the ``file_nr``th file of the list of underlying files.
If a different file was already open, it is closed. Nothing is done
if the requested file is already open.
"""
if file_nr != self.file_nr:
try:
> fh = self.opener(self.files[file_nr], mode=self.mode)
E OSError: [Errno 22] Invalid argument: 'C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_template_stream0\\2013-07-02-01:37:40_0000006400000000.000000.dada'
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\helpers\sequentialfile.py:119: OSError
_______________________________________________________________________ TestGUPPI.test_frame_memmap _______________________________________________________________________
self = <baseband.guppi.tests.test_guppi.TestGUPPI object at 0x0000018E58ACD220>
tmpdir = local('C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_frame_memmap1')
def test_frame_memmap(self, tmpdir):
# Get frame the regular way.
with guppi.open(SAMPLE_FILE, 'rb') as fr:
frame = fr.read_frame(memmap=False)
assert not isinstance(frame.payload.words, np.memmap)
# Check that if we map it instead, we get the same result.
with guppi.open(SAMPLE_FILE, 'rb') as fh:
frame2 = fh.read_frame(memmap=True)
assert frame2 == frame
assert isinstance(frame2.payload.words, np.memmap)
# Bit superfluous perhaps, but check decoding as well.
assert np.all(frame2[337:340] == np.array(
[[[2.-25.j, 31.+2.j, -10.+1.j, -29.+14.j],
[24.+6.j, -23.-16.j, -22.-20.j, -11.-6.j]],
[[11.+10.j, -2.-1.j, -6.+9.j, 19.+16.j],
[10.-25.j, -33.-5.j, 14.+0.j, 3.-3.j]],
[[22.-7.j, 5.+11.j, -21.+4.j, 2.+0.j],
[-4.-12.j, 1.+1.j, 13.+6.j, -31.-4.j]]], dtype=np.complex64))
assert np.all(frame2.data == frame.data)
# Now check writing. First, without memmap, just ensuring writing
# to file works as well as to BytesIO done above.
filename = str(tmpdir.join('testguppi.raw'))
with guppi.open(filename, 'wb') as fw:
fw.write_frame(frame)
with guppi.open(filename, 'rb') as fw:
frame3 = fw.read_frame()
assert frame3 == frame
# Now memmap file to be written to.
> with guppi.open(filename, 'wb') as fw:
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\guppi\tests\test_guppi.py:355:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\guppi\base.py:441: in open
return opener(name, mode, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
name = 'C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_frame_memmap1\\testguppi.raw', mode = 'wb', kwargs = {}, cls_type = 'FileWriter'
got_fh = False
def open(name, mode='rs', **kwargs):
# If sequentialfile object, check that it's opened properly.
if isinstance(name, sf.SequentialFileBase):
assert (('r' in mode and name.mode == 'rb')
or ('w' in mode and name.mode == 'w+b')), (
"open only accepts sequential files opened in 'rb' "
"mode for reading or 'w+b' mode for writing.")
# If passed some kind of list, open a sequentialfile object.
if isinstance(name, (tuple, list, sf.FileNameSequencer)):
if 'r' in mode:
name = sf.open(name, 'rb')
else:
file_size = kwargs.pop('file_size', None)
name = sf.open(name, 'w+b', file_size=file_size)
# Select FileReader/Writer for binary, StreamReader/Writer for stream.
if 'b' in mode:
cls_type = 'File'
else:
cls_type = 'Stream'
# Select reading or writing. Check if ``name`` is a filehandle, and
# open it if not.
if 'w' in mode:
cls_type += 'Writer'
got_fh = hasattr(name, 'write')
if not got_fh:
> name = io.open(name, 'w+b')
E OSError: [Errno 22] Invalid argument: 'C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_frame_memmap1\\testguppi.raw'
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\vlbi_base\base.py:1112: OSError
_______________________________________________________________________ TestGUPPI.test_filestreamer _______________________________________________________________________
self = <baseband.guppi.tests.test_guppi.TestGUPPI object at 0x0000018E5A4612E0>
tmpdir = local('C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_filestreamer1')
def test_filestreamer(self, tmpdir):
start_time = self.header.time
with guppi.open(SAMPLE_FILE, 'rs') as fh:
assert fh.header0 == self.header
assert fh.sample_shape == (2, 4)
assert fh.shape == (3840,) + fh.sample_shape
assert fh.size == np.prod(fh.shape)
assert fh.ndim == len(fh.shape)
assert fh.start_time == start_time
assert fh.sample_rate == 250 * u.Hz
record = fh.read()
fh.seek(0)
record1 = fh.read(12)
assert fh.tell() == 12
fh.seek(1523)
record2 = np.zeros((2, 2, 4), dtype=np.complex64)
record2 = fh.read(out=record2)
# Check that stream properly skips overlap.
assert np.all(record2 == fh._frame[563:565])
assert fh.tell() == 1525
assert fh.time == fh.tell(unit='time')
assert (np.abs(fh.time - (start_time + 1525 / (250 * u.Hz)))
< 1. * u.ns)
fh.seek(fh.start_time + 100 / (250 * u.Hz))
assert fh.tell() == 100
assert (np.abs(fh.stop_time
- (start_time + 3840 / (250 * u.Hz))) < 1. * u.ns)
fh.seek(1, 'end')
with pytest.raises(EOFError):
fh.read()
assert record1.shape == (12, 2, 4)
assert np.all(record1[:3] == np.array(
[[[-7.+12.j, -32.-10.j, -17.+25.j, 16.-5.j],
[14.+21.j, -5.-7.j, 19.-8.j, 7.+7.j]],
[[5.-3.j, -15.-14.j, -8.+14.j, -6.-18.j],
[21.-1.j, 22.+6.j, -30.-13.j, 12.+23.j]],
[[11.+2.j, 9.-13.j, 9.-15.j, -21.-6.j],
[10.-12.j, -3.-10.j, -12.-8.j, 4.-27.j]]], dtype=np.complex64))
assert record1.dtype == np.complex64
assert np.all(record1 == self.payload[:12].squeeze())
assert record2.shape == (2, 2, 4)
filename = str(tmpdir.join('testguppi.raw'))
spf = self.header.samples_per_frame - self.header.overlap
# Need to remove overlap.
with guppi.open(filename, 'ws', header0=self.header_w,
squeeze=False) as fw:
assert fw.sample_rate == 250 * u.Hz
fw.write(self.payload[:spf])
assert fw.start_time == start_time
assert (np.abs(fw.time - (start_time + spf / (250 * u.Hz)))
< 1. * u.ns)
with guppi.open(filename, 'rs') as fh:
data = fh.read()
assert fh.start_time == start_time
assert (np.abs(fh.time - (start_time + spf / (250 * u.Hz)))
< 1. * u.ns)
assert fh.stop_time == fh.time
assert fh.sample_rate == 250 * u.Hz
assert np.all(data == self.payload[:spf].squeeze())
# Try single polarisation (two channels to keep it complex), and check
# initialisation by header keywords. This also tests writing
# with squeeze=True.
h = self.header
> with guppi.open(filename, 'ws', time=h.time, bps=h.bps,
sample_rate=h.sample_rate,
pktsize=h['PKTSIZE'], overlap=0,
payload_nbytes=self.header_w.payload_nbytes // 4,
nchan=2, npol=1) as fw:
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\guppi\tests\test_guppi.py:461:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\guppi\base.py:441: in open
return opener(name, mode, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
name = 'C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_filestreamer1\\testguppi.raw', mode = 'ws'
kwargs = {'header0': <GUPPIHeader BACKEND = 'GUPPI '
BLOCSIZE= 3840
STT_OFFS= ...BSBW = 0.0005
STT_IMJD= 58132
STT_SMJD= 51093>}
cls_type = 'StreamWriter', got_fh = False
def open(name, mode='rs', **kwargs):
# If sequentialfile object, check that it's opened properly.
if isinstance(name, sf.SequentialFileBase):
assert (('r' in mode and name.mode == 'rb')
or ('w' in mode and name.mode == 'w+b')), (
"open only accepts sequential files opened in 'rb' "
"mode for reading or 'w+b' mode for writing.")
# If passed some kind of list, open a sequentialfile object.
if isinstance(name, (tuple, list, sf.FileNameSequencer)):
if 'r' in mode:
name = sf.open(name, 'rb')
else:
file_size = kwargs.pop('file_size', None)
name = sf.open(name, 'w+b', file_size=file_size)
# Select FileReader/Writer for binary, StreamReader/Writer for stream.
if 'b' in mode:
cls_type = 'File'
else:
cls_type = 'Stream'
# Select reading or writing. Check if ``name`` is a filehandle, and
# open it if not.
if 'w' in mode:
cls_type += 'Writer'
got_fh = hasattr(name, 'write')
if not got_fh:
> name = io.open(name, 'w+b')
E OSError: [Errno 22] Invalid argument: 'C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_filestreamer1\\testguppi.raw'
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\vlbi_base\base.py:1112: OSError
__________________________________________________________________ TestGUPPI.test_multiple_files_stream ___________________________________________________________________
self = <baseband.guppi.tests.test_guppi.TestGUPPI object at 0x0000018E58EB2190>
tmpdir = local('C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_multiple_files_stream1')
def test_multiple_files_stream(self, tmpdir):
start_time = self.header_w.time
with guppi.open(SAMPLE_FILE, 'rs') as fh:
data = fh.read()
filenames = (str(tmpdir.join('guppi_1.raw')),
str(tmpdir.join('guppi_2.raw')))
with guppi.open(filenames, 'ws', header0=self.header_w,
frames_per_file=2) as fw:
start_time = fw.start_time
fw.write(data[:1000])
time1000 = fw.time
fw.write(data[1000:])
stop_time = fw.time
assert start_time == self.header_w.time
assert np.abs(time1000
- (start_time + 1000 / (250 * u.Hz))) < 1.*u.ns
assert np.abs(stop_time
- (start_time + 3840 / (250 * u.Hz))) < 1.*u.ns
with guppi.open(filenames[1], 'rs') as fr:
assert (np.abs(fr.time - (start_time + 1920 / (250 * u.Hz)))
< 1. * u.ns)
data1 = fr.read()
assert np.all(data1 == data[1920:])
with guppi.open(filenames, 'rs') as fr:
assert fr.start_time == start_time
assert fr.time == start_time
assert np.abs(fr.stop_time
- (start_time + 3840 / (250 * u.Hz))) < 1. * u.ns
data2 = fr.read()
assert np.abs(fr.time - fr.stop_time) < 1. * u.ns
assert np.all(data2 == data)
# Pass sequentialfile objects to reader.
with sf.open(filenames, 'w+b',
file_size=2*self.header_w.frame_nbytes) as fraw, \
guppi.open(fraw, 'ws', header0=self.header_w) as fw:
> fw.write(data)
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\guppi\tests\test_guppi.py:573:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\vlbi_base\base.py:1005: in write
self._frame = self._make_frame(frame_index)
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\guppi\base.py:301: in _make_frame
return self.fh_raw.memmap_frame(header)
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\guppi\base.py:190: in memmap_frame
header.tofile(self.fh_raw)
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\guppi\header.py:147: in tofile
fh.write(self.tostring(padding=False).encode('ascii'))
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\helpers\sequentialfile.py:338: in write
self._open(self.file_nr + 1)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <[TypeError('list indices must be integers or slices, not NoneType') raised in repr()] SequentialFileWriter object at 0x18e5a457910>, file_nr = 1
def _open(self, file_nr):
"""Open the ``file_nr``th file of the list of underlying files.
If a different file was already open, it is closed. Nothing is done
if the requested file is already open.
"""
if file_nr != self.file_nr:
try:
> fh = self.opener(self.files[file_nr], mode=self.mode)
E OSError: [Errno 22] Invalid argument: 'C:\\Users\\Nikhil\\AppData\\Local\\Temp\\pytest-of-Nikhil\\pytest-0\\test_multiple_files_stream1\\guppi_2.raw'
C:\ProgramData\Anaconda3\envs\pkgdev\lib\site-packages\baseband\helpers\sequentialfile.py:119: OSError
===================================================================== 8 failed, 389 passed in 39.90s ======================================================================
<ExitCode.TESTS_FAILED: 1>
>>>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment