mtustin-handy

  <configuration>
    
    <property>
      <!-- Maximum resources to allocate to application masters
           If this is too high application masters can crowd out actual work -->
      <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
      <value>0.25</value>
    </property>
    
    <property>

mtustin-handy

    <property>
      <name>yarn.resourcemanager.scheduler.class</name>
      <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
    </property>

mtustin-handy

lognames = list(
    hdfs.list_filenames(conf.get('incoming_log_path'), full_path=False))


for logname in lognames:
    # TODO use a proper regex to filter out bad lognames
    # Airflow is particular about which characters exist in task names
    if logname not in excluded_logs and '%' not in logname and '@' not in logname:

        ingest = LogIngesterOperator(

mtustin-handy

def make_spooq_exporter(table, schema, task_id, dag):
     return SpooqExportOperator(
        jdbc_url=('jdbc:mysql://%s/%s?user=user&password=pasta'
                    % (TARGET_DB_HOST,TARGET_DB_NAME)),
        target_table=table,
        hive_table='%s.%s' % (schema, table),
        dag=dag,
        on_retry_callback=truncate_db,
        task_id=task_id)
    

mtustin-handy

from airflow.models import DAG
from airflow.operators import PythonOperator, SubDagOperator
from good_dags.subdag import hive_dag
from datetime import timedelta, datetime

main_dag = DAG(
    dag_id='main_dag',
    schedule_interval=timedelta(hours=1),
    start_date=datetime(2015, 9, 18, 21)
)

mtustin-handy

from airflow.models import DAG
from airflow.operators import PythonOperator, SubDagOperator
from bad_dags.subdag import hive_dag
from datetime import timedelta, datetime

main_dag = DAG(
    dag_id='main_dag',
    schedule_interval=timedelta(hours=1),
    start_date=datetime(2015, 9, 18, 21)
)

mtustin-handy

val hex = area.split(" ").map(Integer.parseInt(_, 16).toByte)

mtustin-handy

sqoop-import --connect jdbc:mysql://<hostname>:3306/handy \
--username <user> --table geodata_table \
--target-dir /path/to/tables/geodata_table\
--fetch-size -2147483648 --null-string \\\\N --null-non-string \\\\N\
--map-column-hive area=STRING --delete-target-dir --hive-import\
--hive-database handy_db --hive-drop-import-delims --hive-overwrite