This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xml | |
index 70f66f973d4..5063da640cc 100644 | |
--- a/hadoop-tools/hadoop-pipes/pom.xml | |
+++ b/hadoop-tools/hadoop-pipes/pom.xml | |
@@ -54,6 +54,9 @@ | |
<source>${basedir}/src</source> | |
<vars> | |
<JVM_ARCH_DATA_MODEL>${sun.arch.data.model}</JVM_ARCH_DATA_MODEL> | |
+ <OPENSSL_INCLUDE_DIR>/usr/local/Cellar/openssl/1.0.2t/include</OPENSSL_INCLUDE_DIR> | |
+ <OPENSSL_SSL_LIBRARY>/usr/local/Cellar/openssl/1.0.2t/lib/libssl.dylib</OPENSSL_SSL_LIBRARY> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# example_airflow.py | |
from dagster import lambda_solid, pipeline | |
@lambda_solid | |
def hello_world(): | |
return "hello, world" | |
@pipeline | |
def basic_pipeline(): | |
hello_world() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import datetime | |
import os | |
from airflow.contrib.operators import dataproc_operator | |
from airflow.contrib.operators.bigquery_operator import BigQueryOperator | |
from airflow.contrib.operators.gcs_to_bq import GoogleCloudStorageToBigQueryOperator | |
from airflow.models import DAG | |
from airflow.utils import trigger_rule | |
PROJECT_ID = os.getenv('GCP_PROJECT_ID') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import dask | |
import dask.distributed | |
from tornado import gen | |
PUBSUB_NAME = 'test' | |
class Task: | |
def __init__(self, key, num, upstream_tasks): |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class DataProcSparkOperator(BaseOperator): | |
""" | |
Start a Cloud DataProc cluster, run a Spark job, then shut down the Spark cluster. | |
""" | |
template_fields = ['arguments'] | |
ui_color = '#0273d4' | |
@apply_defaults | |
def __init__( | |
self, |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import multiprocessing | |
import impala.dbapi | |
class Impala(obect): | |
def __init__(self, host, database, port, username, password, lock=None): | |
self.conn = impala.dbapi.connect(host=host, | |
port=port, | |
database=database, | |
user=username, | |
password=password, |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
# -*- coding: utf-8 -*- | |
# | |
# Terminal visualization of 2D numpy arrays | |
# Copyright (c) 2009 Nicolas P. Rougier | |
# | |
# This program is free software: you can redistribute it and/or modify it under | |
# the terms of the GNU General Public License as published by the Free Software | |
# Foundation, either version 3 of the License, or (at your option) any later | |
# version. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
try: | |
# start http server | |
self.p_http = subprocess.Popen( | |
["python", "-m", "SimpleHTTPServer"], | |
preexec_fn = sigint_replace, | |
cwd = self.resource_dir | |
) | |
# start node.js server | |
self.p_node = subprocess.Popen( |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
python waf clean | |
export PYTHONPATH=/Library/Frameworks/Python.framework/Versions/2.7/ | |
export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/2.7/:$LD_LIBRARY_PATH | |
export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/2.7/lib:$LD_LIBRARY_PATH | |
export LINKFLAGS='-search_dylibs_first -L /Library/Frameworks/Python.framework/Versions/2.7/lib/' | |
export ARCHFLAGS='-arch x86_64' | |
export CC=/usr/bin/gcc-4.2 | |
export PKG_CONFIG_PATH=/usr/local/Cellar/cairo/1.10.2/lib/pkgconfig/ | |
python waf configure --prefix=$PYTHONPATH |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def buildmodel(): | |
EPS = 0.001 | |
data = pandas.read_csv('iris.csv') | |
xtrain = data.ix[:50,0:2] | |
xtest = data.ix[51:100,0:2] | |
ytrain = data.ix[:50,2] | |
ytest = data.ix[51:100,2] | |
# Use pandas.DataFrame | |
models = ffx.run(xtrain, ytrain, xtest, ytest) |
NewerOlder