lauralorenz

"""
Run a daily backup of all our GraphStory instances using the GraphStory alpha API.
Run every day at 23:00 UTC
"""
import admin  # noqa
from airflow import DAG
import datetime
from datadive import settings

from datadive.neo4j_backups.sense import GraphStoryExportSensor

lauralorenz

from random import randint
from time import sleep

import prefect
from prefect import Task, task


class AddTask(Task):
    def __init__(self, default: int, *args, **kwargs):
        super().__init__(*args, **kwargs)

lauralorenz

import requests
import json
from collections import namedtuple
from contextlib import closing
import sqlite3

from prefect import task, Flow

## extract
@task

lauralorenz

import time
from prefect import task, Flow
from prefect.engine.executors import DaskExecutor


@task
def times():
  return [1, 5, 10]

lauralorenz

from prefect import Flow, task, Parameter


@task
def get_s3_keys(bucket: str):
    # get a list_of_keys from your s3 bucket with boto3
    return list_of_keys

@task
def process_s3_key(key: str):

lauralorenz

from prefect import Flow, task, Parameter


@task
def get_s3_keys(bucket: str):
    # get a list_of_keys from your s3 bucket with boto3
    return list_of_keys

@task
def process_s3_key(key: str):

lauralorenz

from prefect import task, Flow
import datetime
import random


@task
def inc(x):
    return x + 1

lauralorenz

from dask_flow import flow

from prefect.engine.executors import DaskExecutor

executor = DaskExecutor(address="10.255.253.2:8786") # change this to yours

flow.run(executor=executor)

lauralorenz

worker:
  replicas: 4
  resources:
    limits:
      cpu: 1
      memory: 3G
    requests:
      cpu: 1
      memory: 3G
  env:

lauralorenz

from dask_flow import flow

from prefect.environments.storage import Docker


flow.storage = Docker(registry_url="gcr.io/execution-layer-demo/flows")
flow.register('execution-layer-demo')