nickefy

# airflow related
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
# other packages
from datetime import datetime, timedelta
from os import environ
import csv

class DataSourceToCsv(BaseOperator):
    """

nickefy

# airflow related
from airflow import models
from airflow import DAG
# other packages
from datetime import datetime, timedelta
# import operators from the 'operators' file
from operators import DataSourceToCsv

default_dag_args = {
    # Setting start date as yesterday starts the DAG immediately       when it is

nickefy

from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
import os
from os import environ
from datetime import timedelta
import getpass, imaplib
import sys
import string

nickefy

from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
from os import environ
from datetime import timedelta
from google.cloud import bigquery
import pandas as pd
import logging
import os

nickefy

from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
from os import environ
from google.cloud import bigquery
from datetime import datetime, timedelta
import logging
import time


class CheckBQDuplication(BaseOperator):

nickefy

from airflow.exceptions import AirflowException
from airflow.operators import ShortCircuitOperator
from airflow import models
from airflow import DAG
from operators import GmailToGCS
from operators import StorageToBQ
from operators import CheckDupBQ
from operators import WriteLogs
from operators import SendEmail
from airflow.utils.email import send_email

nickefy

function scan_files() {
  var folder = DriveApp.getFolderById(folderID);
  var processedfolder = DriveApp.getFolderById(processedfolderId);
  var files = folder.getFiles();
  while (files.hasNext()) {
    var file = files.next();
    if (file.getName().substring(0, 10) != 'processed_' || file.getName().substring(0, 7) != 'loaded_') {
      loadCSVfromdrive(file);
    }
  }

nickefy

from airflow.exceptions import AirflowException
from airflow import models
from airflow import DAG
from datetime import datetime, timedelta
from airflow.utils.email import send_email
from dateutil.relativedelta import relativedelta
import os

schedule_interval_dag = timedelta(days=1)

nickefy

import dash
import dash_core_components as dcc
import dash_html_components as html
import plotly.graph_objs as go
import pandas as pd
from dash.dependencies import Input, Output
from numpy import random

app = dash.Dash()

nickefy

import dash
import dash_html_components as html
import dash_core_components as dcc
from dash.dependencies import Input, Output
import plotly.graph_objs as go
from numpy import random

app = dash.Dash()

# initiate dataframe