fozziethebeat

import numpy as np
import pandas as pd
import prefect

from os import listdir
from os.path import isfile, join
from prefect import Flow, apply_map, case, task
from prefect.engine.results import LocalResult
from prefect.tasks.control_flow import merge

fozziethebeat

import numpy as np
import pandas as pd
import prefect

from os import listdir
from os.path import isfile, join
from prefect import Flow, apply_map, case, task
from prefect.tasks.control_flow import merge

INPUT_BASE_PATH = './data/input'

fozziethebeat

# Requires
#  pip install apache-beam
#  pip install apache-beam[dataframe]
#
# Associated documentation
#   Beam Dataframe API: https://beam.apache.org/releases/pydoc/2.34.0/apache_beam.dataframe.html
#   Beam Dataframe Overview: https://beam.apache.org/documentation/dsls/dataframes/overview/
#   Beam Dataframe Differences: https://beam.apache.org/documentation/dsls/dataframes/differences-from-pandas/

fozziethebeat

import apache_beam as beam

from apache_beam.options.pipeline_options import PipelineOptions

from libs.datasets import combined_datasets
from pyseir.run import OneRegionPipeline


class ProcessOneRegion(beam.DoFn):
    def process(self, one_region):

fozziethebeat

from collections import defaultdict
from airflow.decorators import dag, task
from airflow.utils.dates import days_ago

from pyseir.run import OneRegionPipeline

from libs.datasets import combined_datasets

default_args = {
    'owner': 'airflow',

fozziethebeat

from collections import defaultdict
from airflow import DAG
from airflow.decorators import dag, task
from airflow.utils.dates import days_ago
from airflow.operators.python import PythonOperator

from libs.datasets import combined_datasets

default_args = {
    'owner': 'airflow',

fozziethebeat

import edu.stanford.nlp.ie.crf.CRFClassifier
import edu.stanford.nlp.ling.CoreLabel
import edu.stanford.nlp.ling.Word
import edu.stanford.nlp.util.StringUtils
import edu.stanford.nlp.sequences.PlainTextDocumentReaderAndWriter
import edu.stanford.nlp.sequences.PlainTextDocumentReaderAndWriter.OutputStyle

import scala.collection.JavaConversions.collectionAsScalaIterable
import scala.collection.JavaConversions.seqAsJavaList
import scala.io.Source

fozziethebeat

/*
 * Copyright (c) 2011, Lawrence Livermore National Security, LLC. Produced at
 * the Lawrence Livermore National Laboratory. Written by Keith Stevens,
 * kstevens@cs.ucla.edu OCEC-10-073 All rights reserved. 
 *
 * This file is part of the S-Space package and is covered under the terms and
 * conditions therein.
 *
 * The S-Space package is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as published

fozziethebeat

import scala.collection.mutable.HashMap
import scala.collection.mutable.HashSet
import scala.collection.mutable.Stack

class NeighborChainAgglomerativeClustering {

    /**
     * Clusters the elements represented as symmetric adjacency matrix.  Values in
     * {@code adj} represent the similarity between any two points using a
     * symmetric similarity metric.  This returns sets of points assigned to the

fozziethebeat

/** 
 * Sample code for using the <a
 * href="http://wiki.apertium.org/wiki/Lttoolbox-java">lttoolbox-java</a> code
 * from within Scala.  This morhpologically analyzes a simple sentence using
 * this <a
 * href="http://sourceforge.net/projects/apertium/files/apertium-mk-en/apertium-mk-en-0.1.0.tar.gz/download">english
 * dictionary</a>.  To run this code, first do two steps:
 * 
 * <ul>
 *  <li> download and compile lttoolbox-java as per <a href="http://wiki.apertium.org/wiki/Lttoolbox-java">these instructions</a>.</li>