flolas

{
	context : {
		session : <int>,
		id: <int>
	},
	timestamp: <timestamp(6)>,
	event: <str>,
	subevent: <str>,
	fields: [
	{

flolas

FROM ubuntu

ENV LANGUAGE en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LC_ALL en_US.UTF-8
ENV LC_CTYPE en_US.UTF-8
ENV LC_MESSAGES en_US.UTF-8
ENV LC_ALL  en_US.UTF-8

RUN apt update && \

flolas

#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# multiproc_trim.py
"""A program that reads integer values from a CSV file and writes out their
sums to another CSV file, using multiple processes if desired.
"""

import csv
import multiprocessing
import optparse

flolas

"""
Code that goes along with the Airflow tutorial located at:
https://github.com/airbnb/airflow/blob/master/airflow/example_dags/tutorial.py
"""
from airflow import DAG

from datetime import datetime, timedelta
from airflow.operators.docker_operator import DockerOperator

start = datetime.combine(datetime.today() - timedelta(2), datetime.min.time())

flolas

def unpack_col(col_to_unpack, df_to_append = None, header = 'col', sep=',', na_value=''):
    import pandas as pd
    unpacked_cols = col_to_unpack.fillna(na_value).apply(lambda x: pd.Series(x.split(','))).fillna(na_value)
    #add dynamic columns names based on # of rows and parameter header passed for prefix (header_#)
    col_names = []
    for i in unpacked_cols.columns:
        col_names.append(header + '_' + str(i))
    unpacked_cols.columns = col_names
    if isinstance(df_to_append, pd.DataFrame):
        #return df concatenated with previously unpacked columns