Wittline

{
    "InstanceFleets": [
        {
            "Name": "fleetmaster",
            "InstanceFleetType": "MASTER",
            "TargetOnDemandCapacity": 1,
            "InstanceTypeConfigs": [
                {"InstanceType":"m5.xlarge"}       
            ]
        },

Wittline

{
"steps":
    [
        {            
            "executor_memory": "18G",
            "executor_cores": "4",
            "description" : "Reading from dataset data and filtering",
            "name": "step_0",
            "guiid": "0",
            "ActionOnFailure": "CANCEL_AND_WAIT",

Wittline

from bs4 import BeautifulSoup
import re
import dateutil.parser as parser
from dateutil import tz
from datetime import datetime
import csv
import s3fs
import pickle
from airflow.hooks.base_hook import BaseHook

Wittline

import logging
import datetime
from airflow import DAG
from airflow.models import Variable
from airflow.models.connection import Connection
from airflow.contrib.hooks.aws_hook import AwsHook
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
from airflow.hooks.postgres_hook import PostgresHook
from airflow.operators.dummy_operator import DummyOperator
from airflow.operators.postgres_operator import PostgresOperator