willfuks WillianFuks

## example_dataproc_utils.sh
#!/usr/bin/env bash

function_exists () {
    declare -f -F $1 > /dev/null
    return $?
}

throw () {
    echo "$*" >&2
    echo

## rpi.ipynb

      
              1 file
            
          
              0 forks
            
          
              0 comments
            
          
              0 stars
            
          
                WillianFuks
                / rpi.ipynb
            
            
              Last active
              November 22, 2017 00:45
            
          
      Sorry, something went wrong. Reload?
      Sorry, we cannot display this file.
      Sorry, this file is invalid so it cannot be displayed.
      
          Viewer requires iframe.
      
    
## example_dataproc_twitter_naive.py
import operator
import math
import time

from base import JobsBase
from pyspark.sql import SparkSession
from pyspark.sql import types as stypes

class NaiveJob(JobsBase):
    def run(self, sc, args):

## example_dataproc_twitter_worker_bq.py
import utils
from flask import Flask, request
from config import config
from connector.gcp import GCPService
from scheduler import SchedulerJob


app = Flask(__name__)
gcp_service = GCPService()
scheduler = SchedulerJob()

## example_dataproc_twitter_main_handler.py
import utils
import base_utils
from config import config
from flask import Flask, request, jsonify
from factory import JobsFactory
from google.appengine.ext import ndb
import time


app = Flask(__name__)

## example_dataproc_twitter_worker.py
import utils
from flask import Flask, request
from config import config
from connector.gcp import GCPService
from scheduler import SchedulerJob


app = Flask(__name__)
gcp_service = GCPService()
scheduler = SchedulerJob()

## example_dataproc_twitter_main.yaml
service: dataproc-twitter
runtime: python27
api_version: 1
threadsafe: true

handlers:
- url: /.*/
  script: main.app
  login: admin

## example_dataproc_twitter_worker.yaml
runtime: python27
api_version: 1
threadsafe: true
service: worker

handlers:
- url: /.*
  script: worker.app
  login: admin

## queue.yaml
queue:
- name: default
  rate: 1/m

## example_dataproc_twitter_cron.yaml
cron:
- description: daily export job from BigQuery to Cloud Storage
  url: /run_job/export_customers_from_bq/?url=/export_customers&target=worker
  target: dataproc-twitter
  schedule: every day 01:05
- description: Runs DIMSUM job in Dataproc
  url: /run_job/run_dimsum/?url=/dataproc_dimsum&target=worker&extended_args=--days_init=30,--days_end=1,--threshold=0.1&force=no
  target: dataproc-twitter
  schedule: every day 01:15
	#!/usr/bin/env bash

	function_exists () {
	declare -f -F $1 > /dev/null
	return $?
	}

	throw () {
	echo "$*" >&2
	echo
	import operator
	import math
	import time

	from base import JobsBase
	from pyspark.sql import SparkSession
	from pyspark.sql import types as stypes

	class NaiveJob(JobsBase):
	def run(self, sc, args):
	import utils
	from flask import Flask, request
	from config import config
	from connector.gcp import GCPService
	from scheduler import SchedulerJob


	app = Flask(__name__)
	gcp_service = GCPService()
	scheduler = SchedulerJob()
	import utils
	import base_utils
	from config import config
	from flask import Flask, request, jsonify
	from factory import JobsFactory
	from google.appengine.ext import ndb
	import time


	app = Flask(__name__)
	service: dataproc-twitter
	runtime: python27
	api_version: 1
	threadsafe: true

	handlers:
	- url: /.*/
	script: main.app
	login: admin
	cron:
	- description: daily export job from BigQuery to Cloud Storage
	url: /run_job/export_customers_from_bq/?url=/export_customers&target=worker
	target: dataproc-twitter
	schedule: every day 01:05
	- description: Runs DIMSUM job in Dataproc
	url: /run_job/run_dimsum/?url=/dataproc_dimsum&target=worker&extended_args=--days_init=30,--days_end=1,--threshold=0.1&force=no
	target: dataproc-twitter
	schedule: every day 01:15