Skip to content

Instantly share code, notes, and snippets.

@anna-geller
Created December 19, 2021 23:57
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save anna-geller/e1bc1f3e22236ede9deafe7cfc2b13c4 to your computer and use it in GitHub Desktop.
Save anna-geller/e1bc1f3e22236ede9deafe7cfc2b13c4 to your computer and use it in GitHub Desktop.
import pendulum
from prefect import task, Flow, Parameter
@task
def get_start_date(param_start_date: str) -> str:
if param_start_date is None:
param_start_date = pendulum.yesterday(tz="America/New_York").isoformat()
return param_start_date
@task
def get_end_date(param_end_date: str) -> str:
if param_end_date is None:
param_end_date = pendulum.today(tz="America/New_York").isoformat()
return param_end_date
@task(log_stdout=True)
def extract_data(start_date: str, end_date: str):
# your task logic based on those dates
print(f"Backloading data for the interval {start_date} - {end_date}")
with Flow("backfilling_flow") as flow:
custom_start_date = Parameter("start_date", default=None)
custom_end_date = Parameter("end_date", default=None)
start_date = get_start_date(custom_start_date)
end_date = get_end_date(custom_end_date)
extract_data(start_date=start_date, end_date=end_date)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment