Skip to content

Instantly share code, notes, and snippets.

@mattjj
Last active August 24, 2021 15:11
Show Gist options
  • Save mattjj/5213172 to your computer and use it in GitHub Desktop.
Save mattjj/5213172 to your computer and use it in GitHub Desktop.
data chunking with overlapping windows! uses stride tricks so no data is copied; it just produces a view!
from __future__ import division
import numpy as np
from numpy.lib.stride_tricks import as_strided as ast
def chunk_data(data,window_size,overlap_size=0,flatten_inside_window=True):
assert data.ndim == 1 or data.ndim == 2
if data.ndim == 1:
data = data.reshape((-1,1))
# get the number of overlapping windows that fit into the data
num_windows = (data.shape[0] - window_size) // (window_size - overlap_size) + 1
overhang = data.shape[0] - (num_windows*window_size - (num_windows-1)*overlap_size)
# if there's overhang, need an extra window and a zero pad on the data
# (numpy 1.7 has a nice pad function I'm not using here)
if overhang != 0:
num_windows += 1
newdata = np.zeros((num_windows*window_size - (num_windows-1)*overlap_size,data.shape[1]))
newdata[:data.shape[0]] = data
data = newdata
sz = data.dtype.itemsize
ret = ast(
data,
shape=(num_windows,window_size*data.shape[1]),
strides=((window_size-overlap_size)*data.shape[1]*sz,sz)
)
if flatten_inside_window:
return ret
else:
return ret.reshape((num_windows,-1,data.shape[1]))
@mattjj
Copy link
Author

mattjj commented Mar 21, 2013

In [53]: chunk_data.chunk_data(arange(5),3,1)
Out[53]:
array([[0, 1, 2],
       [2, 3, 4]])

In [54]: chunk_data.chunk_data(arange(5),3,2)
Out[54]:
array([[0, 1, 2],
       [1, 2, 3],
       [2, 3, 4]])
In [55]: chunk_data.chunk_data(arange(12).reshape((-1,2)),3,2)
Out[55]:
array([[ 0,  1,  2,  3,  4,  5],
       [ 2,  3,  4,  5,  6,  7],
       [ 4,  5,  6,  7,  8,  9],
       [ 6,  7,  8,  9, 10, 11]])

In the last one, the first row contains the first three vectors (which are pairs), etc.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment