I want to do ETL and I want to do it in python. What tools do I have access to?
Orca is a lightweight pandas centric library for building simple data pipelines
Luigi - seems to be for very heavy weight projects
Airflow from Airbnb - not used this yet
night-shift - lightweight data flow.
| class SendEmail(luigi.Task): | |
| def run(self): | |
| fromaddr = EMAIL | |
| toaddr = EMAIL | |
| msg = MIMEMultipart() | |
| msg['From'] = fromaddr | |
| msg['To'] = toaddr | |
| msg['Subject'] = "Daily Update of Cohorts" | |
| body = "Here is the cohort update" | |
| msg.attach(MIMEText(body, 'plain')) |
| object Factorial2 { | |
| def main(args: Array[String]) | |
| { for (i <- 1 to 30) | |
| println("Factorial of " + i + ": = " + factorial(i)) | |
| } | |
| def factorial(n: BigInt): BigInt = { | |
| if (n <= 1) | |
| 1 | |
| else |
| --------------------------------------------------------------------------- | |
| TypeError Traceback (most recent call last) | |
| <ipython-input-34-180a07193cfb> in <module>() | |
| 21 for eval_name, evaluation in evaluations.iteritems(): | |
| 22 for year in range(2002, 2013): | |
| ---> 23 metric = evaluation(ranker, year) | |
| 24 results.append({ | |
| 25 'rank': name, | |
| <ipython-input-34-180a07193cfb> in <lambda>(rank, year) |
| # these modules are needed for the task | |
| import luigi | |
| import smtplib | |
| from email.mime.multipart import MIMEMultipart | |
| from email.mime.text import MIMEText | |
| from email.mime.base import MIMEBase | |
| from email import encoders | |
| from dataextract import Manager | |
| import pandas as pd | |
| import json |