Last active
August 29, 2015 14:18
-
-
Save saxbophone/ff15b190695f8d0792a2 to your computer and use it in GitHub Desktop.
Python Data Padding Template
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def flatten_list(l, ignore_iterables=True): | |
""" | |
Flatten out nested lists and return a non-nested list. | |
By default, ignores other iterables. | |
""" | |
if ignore_iterables: | |
arr = [] | |
if isinstance(l, list): | |
for item in l: | |
if isinstance(item, list): | |
subs = flatten_list(item, | |
ignore_iterables=ignore_iterables) | |
for sub_item in subs: | |
arr.append(sub_item) | |
else: | |
arr.append(item) | |
return arr | |
else: | |
return [l] | |
else: | |
arr = [] | |
if hasattr(l, '__iter__'): | |
for item in l: | |
if hasattr(item, '__iter__'): | |
subs = flatten_list(item, | |
ignore_iterables=ignore_iterables) | |
for sub_item in subs: | |
arr.append(sub_item) | |
else: | |
arr.append(item) | |
return arr | |
else: | |
return [l] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from math import log, ceil, floor | |
def serialise_arbitrary(integer, arr=[(8,)]): | |
""" | |
For the given int or long, serialise it to an arbitrary encoded | |
byte slug using certain encoding techniques for efficiency. | |
Integers are always interpreted as Big-Endian (for now). | |
""" | |
arr_l = len(arr) | |
arr = flatten_list(arr) | |
if isinstance(arr[0], tuple): | |
""" | |
first item is tuple, meaning there can be unlimited numbers | |
of integers of this byte width used up to this point | |
""" | |
if arr_l > 1: | |
# calculate max possible value of fixed bit widths: | |
max_fixed = arr[1] | |
for i in arr[2:]: | |
max_fixed *= i | |
leftover = int(integer / max_fixed) | |
else: | |
max_fixed = 0 | |
leftover = integer | |
if max_fixed >= integer: | |
# we only need one of the unlimited numbers: | |
arr = flatten_list(arr, ignore_iterables=False) | |
else: | |
num_iter = int(floor(log(leftover, 2 ** arr[0][0]) + 1)) | |
arr[0] = [arr[0][0]] * num_iter | |
arr = flatten_list(arr) | |
return arr | |
elif isinstance(arr[arr_l - 1], tuple): | |
""" | |
last item is tuple, meaning there can be unlimited numbers | |
of integers of this byte width used from this point onwards | |
""" | |
max_fixed = arr[arr_l-2] | |
for i in arr[:2]: | |
max_fixed *= i | |
if max_fixed > integer: | |
# we only need one of the unlimited numbers: | |
arr = flatten_list(arr, ignore_iterables=False) | |
else: | |
leftover = integer - max_fixed | |
num_iter = int(floor(log(leftover, 2 ** arr[arr_l-1][0]) + 1)) | |
arr[arr_l-1] = [arr[arr_l-1][0]] * num_iter | |
arr = flatten_list(arr) | |
return arr | |
else: | |
""" | |
neither first nor last items are tuples, so this slug has a | |
fixed maximmum size. | |
""" | |
pass |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from bitstring import BitArray | |
def pad_data(data, size, separate_headers=False): | |
""" | |
For the given array of bytes (data), return one or more arrays that | |
are (size) elements long, using various means to ensure that the padded | |
data can be decoded to the original after. If separate_headers is set to | |
True, then each bytearray will have a length header, otherwise only the | |
first shall have a header. | |
""" | |
return [bytearray()] | |
def unpad_data(padded): | |
""" | |
For the given array of padded data, return the original data (pre-padding). | |
If (padded) is an array, assume that each item in it is a bytearray and | |
decode these separately, else if it is a bytearray then decode this one | |
item. | |
""" | |
return bytearray() | |
assert (pad_data(bytearray('Cabbage'), 8) == bytearray('\xffCabbage')) | |
pad_data(bytearray('Cabbages!'), 8) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment