dask_bag_filter_issue.py

import sys

import dask.bag as db


def gt(x):
    return x > 3

def even(x):
    return x % 2 == 0



## WORKS
bag = db.from_sequence([1,2,3,4,5,6])
bag = bag.filter(lambda x: gt(x))
bag = bag.filter(lambda x: even(x))

res = bag.compute()

print (res)




## DOES NOT WORK
bag = db.from_sequence([1,2,3,4,5,6])

for tform in ['gt', 'even']:
    transform_func = getattr(sys.modules[__name__], tform)
    bag = bag.filter(transform_func)

res = bag.compute()

print (res)

## DOES NOT WORK
bag = db.from_sequence([1,2,3,4,5,6])

transform_func = getattr(sys.modules[__name__], 'gt')
lambda_func = lambda x: transform_func(x)
bag = bag.filter(lambda_func)

transform_func = getattr(sys.modules[__name__], 'even')
lambda_func = lambda x: transform_func(x)
bag = bag.filter(lambda_func)

res = bag.compute()

print (res)



## DOES NOT WORK
bag = db.from_sequence([1,2,3,4,5,6])

for tform in ['gt', 'even']:
    lambda_func = lambda x: getattr(sys.modules[__name__], tform)(x)
    bag = bag.filter(lambda_func)

res = bag.compute()

print (res)




## WORKS
bag = db.from_sequence([1,2,3,4,5,6])

lambda_func = lambda x: getattr(sys.modules[__name__], 'gt')(x)
bag = bag.filter(lambda_func)

lambda_func = lambda x: getattr(sys.modules[__name__], 'even')(x)
bag = bag.filter(lambda_func)

res = bag.compute()

print (res)

Filters will get overridden in certain scenarios....

In general, want the ability to pass kwargs to a filter function like you can with dask.map