Skip to content

Instantly share code, notes, and snippets.

@jspaezp

jspaezp/complex_yaml_001.yaml

Created July 12, 2020 01:51
Show Gist options
  • Save jspaezp/6152733a644b8f6fe9b985b58c856997 to your computer and use it in GitHub Desktop.
Save jspaezp/6152733a644b8f6fe9b985b58c856997 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
complex_yaml_001.yaml
Sequential:
children:
- Fliplr:
args: 0.5 # horizontally flip 50% of all images
- Flipud:
args: 0.2 # vertically flip 20% of all images
- sometimes:
Crop:
percent: (0, 0.1)
- sometimes:
Affine:
scale: # This section is tricky for the progrem due to the dictionary argument
x: (0.8, 1.2)
y: (0.8, 1.2)
translate_percent:
x: (-0.2, 0.2)
y: (-0.2, 0.2)
rotate: (-45, 45)
shear: (-16, 16)
order: [0, 1]
cval: (0, 255)
mode: ia.ALL
- SomeOf:
args: (0, 5)
children:
- sometimes:
- Superpixels:
p_replace: (0, 1.0)
n_segments: (20, 200)
- OneOf:
-
- GaussianBlur: (0, 3.0)
- AverageBlur:
k: (2, 7)
- MedianBlur:
k: (3, 11)
- Sharpen:
alpha: (0, 1.0)
lightness: (0.75, 1.5)
- Emboss:
alpha: (0, 1.0)
strength: (0, 2.0)
- sometimes:
OneOf:
-
- EdgeDetect:
alpha: (0, 0.7)
- DirectedEdgeDetect:
alpha: (0, 0.7)
direction: (0.0, 1.0)
- AdditiveGaussianNoise:
loc: 0
scale: (0.0, 17.0) # originally (0.0, 0.05*255)
per_channel: 0.5
- OneOf:
-
- Dropout:
args: (0.01, 0.1)
per_channel: 0.5
- CoarseDropout:
args: (0.03, 0.15)
size_percent: (0.02, 0.05)
per_channel: 0.2
- Invert:
args: 0.05
per_channel: True
- Add:
args: (-10, 10)
per_channel: 0.5
- Multiply:
args: (0.5, 1.5)
per_channel: 0.5
- LinearContrast:
args: (0.5, 2.0)
per_channel: 0.5
- Grayscale:
alpha: (0.0, 1.0)
- sometimes:
ElasticTransformation:
alpha: (0.5, 3.5)
sigma: 0.25
- sometimes:
PiecewiseAffine:
scale: (0.01, 0.05)
random_order: True
random_order: True
Raw
simple_yaml_001.yaml
max:
args:
- 1
- 2
Raw
simple_yaml_002.yaml
min:
max:
args:
- 1
- 2
args:
- 1
Raw
simple_yaml_003.yaml
Sequential:
-
- Fliplr:
args:
- 0.5
- Flipud:
args:
- 0.2
Raw
simple_yaml_004.yaml
Sequential:
-
- Fliplr:
args:
- 0.5
- Flipud:
args:
- 0.2
- Sometimes:
- 0.1
-
- CropAndPad:
kwargs:
percent: (-0.05, 0.1)
pad_mode: wrap
pad_cval: (0, 255)
Raw
simple_yaml_005.yaml
Sequential:
children:
- Fliplr:
args:
- 0.5 # horizontally flip 50% of all images
- Flipud:
args:
- 0.2 # vertically flip 20% of all images
# Adding a seed, this is ON PURPOSE done in the opposite order
# as the poistionl in the function to test the correct dispatchment
# of the argument
seed: 44
# do all of the above augmentations in random order
random_order: True
Raw
simple_yaml_006.yaml
Sequential:
children:
- Fliplr:
args: [0.5] # horizontally flip 50% of all images
- Flipud:
args: [0.2] # vertically flip 20% of all images
- sometimes:
Crop:
percent: (0, 0.1)
- SomeOf:
args:
- (0, 5)
children:
- sometimes:
- Superpixels:
p_replace: (0, 1.0)
n_segments: (20, 200)
- OneOf:
-
- GaussianBlur:
args:
- (1.0, 3.0)
# TODO Fix issue that makes this argument be interpredes sometimes as an int
- AverageBlur:
k: (2, 7)
- MedianBlur:
k: (3, 11)
- Sharpen:
alpha: (0, 1.0)
lightness: (0.75, 1.5)
# - Emboss:
# alpha: (0, 1.0)
# strength: (0, 2.0)
# TODO This section is commented out because the memmory address
# in the output makes the test fail
- sometimes:
OneOf:
-
- EdgeDetect:
alpha: (0, 0.7)
- DirectedEdgeDetect:
alpha: (0, 0.7)
direction: (0.0, 1.0)
- AdditiveGaussianNoise:
loc: 0
scale: (0.0, 12.75)
# TODO This lane was originally written as (0.0, 0.05*255)
# bit it makes ast.eval_literal fail
per_channel: 0.5
- OneOf:
-
- Dropout:
args:
- (0.01, 0.1)
per_channel: 0.5
- CoarseDropout:
args:
- (0.03, 0.15)
size_percent: (0.02, 0.05)
per_channel: 0.2
- Invert:
args:
- 0.05
per_channel: True
- Add:
args:
- (-10, 10)
per_channel: 0.5
- Multiply:
args:
- (0.5, 1.5)
per_channel: 0.5
- LinearContrast:
args:
- (0.5, 2.0)
per_channel: 0.5
- Grayscale:
alpha: (0.0, 1.0)
- sometimes:
ElasticTransformation:
alpha: (0.5, 3.5)
sigma: 0.25
- sometimes:
PiecewiseAffine:
scale: (0.01, 0.05)
random_order: True
random_order: True
Raw
tests_parser.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from yaml_parser import *
def recursive_check_names(out, expected_out, checked = 0):
for x, y in zip(out, expected_out):
assert x.name == y.name
checked += 0
if hasattr(y, "__iter__"):
recursive_check_names(x, y)
print(str(checked), " Names match")
def test_simple_yaml_001():
LOCAL_SEARCH_PATH = [__builtins__]
out = traverse_yaml_dict(
get_yaml("simple_yaml_001.yaml"),
LOCAL_SEARCH_PATH)
assert out == 2
def test_simple_yaml_002():
LOCAL_SEARCH_PATH = [__builtins__]
out = traverse_yaml_dict(
get_yaml("simple_yaml_002.yaml"),
LOCAL_SEARCH_PATH)
assert out == 1
def test_simple_yaml_003():
import imgaug.augmenters as iaa
expected_out = iaa.Sequential(
[iaa.Fliplr(0.5), iaa.Flipud(0.2)])
LOCAL_SEARCH_PATH = [__builtins__, iaa]
out = traverse_yaml_dict(
get_yaml("simple_yaml_003.yaml"),
LOCAL_SEARCH_PATH)
print(expected_out)
print(out)
assert str(out) == str(expected_out)
def test_simple_yaml_with_kwargs():
"""
Tests that dictionary arguments are interpreted as such
foo:
scale:
X: 1
Y: 2
"""
def foo(scale):
return scale
LOCAL_SEARCH_PATH = [__builtins__, locals()]
out = traverse_yaml_dict(
get_yaml("simple_dict_kwargs_yaml_001.yaml"),
LOCAL_SEARCH_PATH)
expected_out = {"X": 1, "Y": 2}
assert str(out) == str(expected_out)
def test_simple_yaml_004():
import imgaug.augmenters as iaa
expected_out = iaa.Sequential(
[iaa.Fliplr(0.5),
iaa.Flipud(0.2),
iaa.Sometimes(
0.1,
[iaa.CropAndPad(
percent=(-0.05, 0.1),
pad_mode='wrap',
pad_cval=(0, 255))]
)])
LOCAL_SEARCH_PATH = [__builtins__, iaa]
yml = get_yaml("simple_yaml_004.yaml")
out = traverse_yaml_dict(yml, LOCAL_SEARCH_PATH)
print(expected_out)
print(out)
assert str(out) == str(expected_out)
def test_simple_yaml_005():
import imgaug.augmenters as iaa
expected_out = iaa.Sequential(
[iaa.Fliplr(0.5),
iaa.Flipud(0.2)],
random_order=True,
seed=44)
LOCAL_SEARCH_PATH = [__builtins__, iaa]
yml = get_yaml("simple_yaml_005.yaml")
out = traverse_yaml_dict(yml, LOCAL_SEARCH_PATH)
print(expected_out)
print(out)
assert str(out) == str(expected_out)
def test_simple_yaml_006():
import imgaug.augmenters as iaa
import imgaug as ia
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
expected_out = iaa.Sequential(
[
#
# Apply the following augmenters to most images.
#
iaa.Fliplr(0.5), # horizontally flip 50% of all images
iaa.Flipud(0.2), # vertically flip 20% of all images
# crop some of the images by 0-10% of their height/width
sometimes(iaa.Crop(percent=(0, 0.1))),
#
# Execute 0 to 5 of the following (less important) augmenters per
# image. Don't execute all of them, as that would often be way too
# strong.
#
iaa.SomeOf((0, 5),
[
# Convert some images into their superpixel representation,
# sample between 20 and 200 superpixels per image, but do
# not replace all superpixels with their average, only
# some of them (p_replace).
sometimes(
iaa.Superpixels(
p_replace=(0, 1.0),
n_segments=(20, 200)
)
),
# Blur each image with varying strength using
# gaussian blur (sigma between 0 and 3.0),
# average/uniform blur (kernel size between 2x2 and 7x7)
# median blur (kernel size between 3x3 and 11x11).
iaa.OneOf([
iaa.GaussianBlur((1.0, 3.0)),
# TODO This is also a source of error in the tests,
# sometimes the parsing leads to a float and not an integer
# or the other way around
iaa.AverageBlur(k=(2, 7)),
iaa.MedianBlur(k=(3, 11)),
]),
# Sharpen each image, overlay the result with the original
# image using an alpha between 0 (no sharpening) and 1
# (full sharpening effect).
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),
# Same as sharpen, but for an embossing effect.
# TODO commented out because it makes the tet fail
# due to an explicit memmory address
# iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),
# Search in some images either for all edges or for
# directed edges. These edges are then marked in a black
# and white image and overlayed with the original image
# using an alpha of 0 to 0.7.
sometimes(iaa.OneOf([
iaa.EdgeDetect(alpha=(0, 0.7)),
iaa.DirectedEdgeDetect(
alpha=(0, 0.7), direction=(0.0, 1.0)
),
])),
# Add gaussian noise to some images.
# In 50% of these cases, the noise is randomly sampled per
# channel and pixel.
# In the other 50% of all cases it is sampled once per
# pixel (i.e. brightness change).
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05*255), per_channel=0.5
),
# Either drop randomly 1 to 10% of all pixels (i.e. set
# them to black) or drop them on an image with 2-5% percent
# of the original size, leading to large dropped
# rectangles.
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5),
iaa.CoarseDropout(
(0.03, 0.15), size_percent=(0.02, 0.05),
per_channel=0.2
),
]),
# Invert each image's channel with 5% probability.
# This sets each pixel value v to 255-v.
iaa.Invert(0.05, per_channel=True), # invert color channels
# Add a value of -10 to 10 to each pixel.
iaa.Add((-10, 10), per_channel=0.5),
# Change brightness of images (50-150% of original value).
iaa.Multiply((0.5, 1.5), per_channel=0.5),
# Improve or worsen the contrast of images.
iaa.LinearContrast((0.5, 2.0), per_channel=0.5),
# Convert each image to grayscale and then overlay the
# result with the original with random alpha. I.e. remove
# colors with varying strengths.
iaa.Grayscale(alpha=(0.0, 1.0)),
# In some images move pixels locally around (with random
# strengths).
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)
),
# In some images distort local areas with varying strength.
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05)))
],
# do all of the above augmentations in random order
random_order=True
)
],
# do all of the above augmentations in random order
random_order=True
)
LOCAL_SEARCH_PATH = [__builtins__, iaa, locals()]
yml = get_yaml("simple_yaml_006.yaml")
out = traverse_yaml_dict(yml, LOCAL_SEARCH_PATH)
print("\n\n\n\n")
print(str(expected_out))
print(str(out))
print("\n\n\n\n")
assert len(out) == len(expected_out)
for x, y in zip(out, expected_out):
assert x.name == y.name
if hasattr(y, "__iter__"):
for x2, y2 in zip(x, y):
assert x2.name == y2.name
print(len(out))
print(len(expected_out))
# Literal representation of memmory addresses in generators prevent this from
# Successfully checking the equality
# assert str(out) == str(expected_out)
def test_complex_yaml_001():
import imgaug.augmenters as iaa
import imgaug as ia
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
expected_out = iaa.Sequential(
[
#
# Apply the following augmenters to most images.
#
iaa.Fliplr(0.5), # horizontally flip 50% of all images
iaa.Flipud(0.2), # vertically flip 20% of all images
# crop some of the images by 0-10% of their height/width
sometimes(iaa.Crop(percent=(0, 0.1))),
# Apply affine transformations to some of the images
# - scale to 80-120% of image height/width (each axis independently)
# - translate by -20 to +20 relative to height/width (per axis)
# - rotate by -45 to +45 degrees
# - shear by -16 to +16 degrees
# - order: use nearest neighbour or bilinear interpolation (fast)
# - mode: use any available mode to fill newly created pixels
# see API or scikit-image for which modes are available
# - cval: if the mode is constant, then use a random brightness
# for the newly created pixels (e.g. sometimes black,
# sometimes white)
sometimes(iaa.Affine(
scale={"x": (0.8, 1.2), "y": (0.8, 1.2)},
translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)},
rotate=(-45, 45),
shear=(-16, 16),
order=[0, 1],
cval=(0, 255),
mode=ia.ALL
)),
#
# Execute 0 to 5 of the following (less important) augmenters per
# image. Don't execute all of them, as that would often be way too
# strong.
#
iaa.SomeOf((0, 5),
[
# Convert some images into their superpixel representation,
# sample between 20 and 200 superpixels per image, but do
# not replace all superpixels with their average, only
# some of them (p_replace).
sometimes(
iaa.Superpixels(
p_replace=(0, 1.0),
n_segments=(20, 200)
)
),
# Blur each image with varying strength using
# gaussian blur (sigma between 0 and 3.0),
# average/uniform blur (kernel size between 2x2 and 7x7)
# median blur (kernel size between 3x3 and 11x11).
iaa.OneOf([
iaa.GaussianBlur((0, 3.0)),
iaa.AverageBlur(k=(2, 7)),
iaa.MedianBlur(k=(3, 11)),
]),
# Sharpen each image, overlay the result with the original
# image using an alpha between 0 (no sharpening) and 1
# (full sharpening effect).
iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),
# Same as sharpen, but for an embossing effect.
iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),
# Search in some images either for all edges or for
# directed edges. These edges are then marked in a black
# and white image and overlayed with the original image
# using an alpha of 0 to 0.7.
sometimes(iaa.OneOf([
iaa.EdgeDetect(alpha=(0, 0.7)),
iaa.DirectedEdgeDetect(
alpha=(0, 0.7), direction=(0.0, 1.0)
),
])),
# Add gaussian noise to some images.
# In 50% of these cases, the noise is randomly sampled per
# channel and pixel.
# In the other 50% of all cases it is sampled once per
# pixel (i.e. brightness change).
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05*255), per_channel=0.5
),
# Either drop randomly 1 to 10% of all pixels (i.e. set
# them to black) or drop them on an image with 2-5% percent
# of the original size, leading to large dropped
# rectangles.
iaa.OneOf([
iaa.Dropout((0.01, 0.1), per_channel=0.5),
iaa.CoarseDropout(
(0.03, 0.15), size_percent=(0.02, 0.05),
per_channel=0.2
),
]),
# Invert each image's channel with 5% probability.
# This sets each pixel value v to 255-v.
iaa.Invert(0.05, per_channel=True), # invert color channels
# Add a value of -10 to 10 to each pixel.
iaa.Add((-10, 10), per_channel=0.5),
# Change brightness of images (50-150% of original value).
iaa.Multiply((0.5, 1.5), per_channel=0.5),
# Improve or worsen the contrast of images.
iaa.LinearContrast((0.5, 2.0), per_channel=0.5),
# Convert each image to grayscale and then overlay the
# result with the original with random alpha. I.e. remove
# colors with varying strengths.
iaa.Grayscale(alpha=(0.0, 1.0)),
# In some images move pixels locally around (with random
# strengths).
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)
),
# In some images distort local areas with varying strength.
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05)))
],
# do all of the above augmentations in random order
random_order=True
)
],
# do all of the above augmentations in random order
random_order=True
)
LOCAL_SEARCH_PATH = [__builtins__, iaa, locals()]
yml = get_yaml("complex_yaml_001.yaml")
out = traverse_yaml_dict(yml, LOCAL_SEARCH_PATH)
assert len(out) == len(expected_out)
recursive_check_names(out, expected_out)
print(expected_out)
print(out)
# assert str(out) == str(expected_out)
Raw
yaml_parser.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import ast
from inspect import getouterframes, currentframe
import yaml
SEARCH_PATH = [__builtins__]
def safe_eval(val):
try:
val = ast.literal_eval(val)
except ValueError:
pass
return val
def safe_eval_dict(in_dict):
out_dict = {}
for k, v in in_dict.items():
v = safe_eval(v)
out_dict[k] = v
return out_dict
def get_arguments(function):
fun_args = None
try:
fun_args = function.__init__.__code__.co_varnames
except AttributeError:
try:
fun_args = function.__code__.co_varnames
except AttributeError:
pass
if fun_args is None:
print("Unable to retrieve arguments")
return fun_args
def process_callable_elements(callable_elem, callable_name,
children_elems, search_path):
fun_args = get_arguments(callable_elem)
if isinstance(children_elems, dict):
args, kwargs = process_callable_args_dict(
children_elems,
fun_args,
search_path)
elif isinstance(children_elems, list):
args = traverse_list_yaml(children_elems, search_path)
kwargs = {}
elif isinstance(children_elems, str):
args = [safe_eval(children_elems)]
kwargs = {}
try:
call_args = [x for x in args if x is not None]
except UnboundLocalError:
print(children_elems)
raise UnboundLocalError
print("Calling '{}' with args >> {} <<".format(
callable_name, str(call_args)))
print("Calling '{}' with kwargs >> {} <<".format(
callable_name, str(kwargs)))
fun_out = callable_elem(*call_args, **kwargs)
return fun_out
def process_callable_args_dict(val: dict, arg_names, search_path):
args = []
kwargs = {}
tmp_val = val.copy()
if "kwargs" in tmp_val:
tmp_kwargs = tmp_val.pop('kwargs')
out_kwargs = safe_eval_dict(tmp_kwargs)
kwargs.update(out_kwargs)
print("Kwargs are: {}".format(str(out_kwargs)))
if "args" in tmp_val:
tmp_args = tmp_val.pop('args')
if isinstance(tmp_args, list):
out_args = [safe_eval(x) for x in tmp_args]
[args.append(x) for x in out_args]
else:
out_args = safe_eval(tmp_args)
args.append(out_args)
print("Args are: {}".format(str(args)))
leftover_keys = list(tmp_val.keys())
if len(leftover_keys) > 0 and arg_names is not None:
for valname in leftover_keys:
if valname in arg_names:
argument_val = tmp_val.pop(valname)
argument_val = safe_eval(argument_val)
if isinstance(argument_val, dict):
argument_val = traverse_yaml_dict(
argument_val, search_path)
elif isinstance(argument_val, list):
argument_val = traverse_list_yaml(
argument_val, search_path)
print("Detected value >> {} << for argument {}".format(
str(argument_val), valname))
kwargs[valname] = argument_val
args.append(traverse_yaml_dict(tmp_val, search_path))
return args, kwargs
def traverse_yaml_dict(yaml_dict, search_path):
level = len(getouterframes(currentframe()))
print(">> Recursion level {}".format(str(level)))
# Used only if the args are not a function or callable
return_dict = {}
for nam, val in yaml_dict.items():
# if the name is not in locals/globals or ia
# call the name with the args parsed as a python
# expression
found_function = None
for loc in search_path:
if hasattr(loc, "__dict__"):
loc_dict = loc.__dict__
else:
loc_dict = loc
if nam in loc_dict:
print("Detected '{}' as a function".format(nam))
found_function = loc_dict[nam]
return process_callable_elements(
found_function, nam, val, search_path)
assert found_function is None
if isinstance(val, dict):
ret = traverse_yaml_dict(val, search_path)
return ret
elif isinstance(val, list):
ret = traverse_list_yaml(val, search_path)
return ret
elif nam == "kwargs":
raise ValueError("reached a kwargs argument ... should not have")
elif nam == "args":
raise ValueError("reached an args argument ... should not have")
else:
print("Unhandled instance of type ",
type(val), "and value: ", str(val))
val = safe_eval(val)
return_dict.update({nam: val})
if len(return_dict) > 0:
return return_dict
def traverse_list_yaml(val, search_path):
ret = []
for elem in val:
if hasattr(elem, 'items') and not isinstance(elem, str):
elem_ret = traverse_yaml_dict(elem, search_path)
elif isinstance(elem, list):
elem_ret = traverse_list_yaml(elem, search_path)
else:
elem_ret = safe_eval(elem)
ret.append(elem_ret)
return ret
def get_yaml(yml_path):
with open(yml_path, 'r') as stream:
try:
loaded_yaml = yaml.safe_load(stream)
except yaml.YAMLError as exc:
print(exc)
return loaded_yaml
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment