Skip to content

Instantly share code, notes, and snippets.

@ShopifyEng

ShopifyEng/random_airflow_schedule.py Secret

Created May 23, 2022 13:18
Download ZIP

Revisions

  1. ShopifyEng created this gist May 23, 2022.
    27 changes: 27 additions & 0 deletions random_airflow_schedule.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,27 @@
    from hashlib import md5
    from random import randint, seed

    def compute_schedule(dag_id, interval):

    # create deterministic randomness by seeding PRNG with a hash of the table name:
    seed(md5(dag_id.encode()).hexdigest())

    if interval.endswith("h"):
    val = int(interval[:-1])
    if 24 % val != 0:
    raise ValueError("Must use a number which evenly divides 24.")
    offset = randint(0, val-1)
    minutes = randint(0, 59)
    return f"{minutes} {offset}/{val} * * *"

    elif interval.endswith("m"):
    val = int(interval[:-1])
    if 60 % val != 0:
    raise ValueError("Minutes must use a number which evenly divides 60.")
    offset = randint(0, val-1)
    return f"{offset}/{val} * * * *"

    elif interval == "1d":
    return f"{randint(0, 59)} {randint(0, 23)} * * *"

    raise ValueError("Interval must be (1, 2, 3, 4, 5, 6, 10, 12, 15, 20, 30)m, (1, 2, 3, 4, 6, 12)h or 1d")