I will print the markdown file to the console for you.
This comprehensive guide covers the core concepts, common transformations, and advanced operations for building robust and scalable streaming applications with PySpark.
The life of a streaming query follows a simple pattern:
| create table oauth_client_details ( | |
| client_id VARCHAR(256) PRIMARY KEY, | |
| resource_ids VARCHAR(256), | |
| client_secret VARCHAR(256), | |
| scope VARCHAR(256), | |
| authorized_grant_types VARCHAR(256), | |
| web_server_redirect_uri VARCHAR(256), | |
| authorities VARCHAR(256), | |
| access_token_validity INTEGER, | |
| refresh_token_validity INTEGER, |
| .metadata | |
| bin/ | |
| tmp/ | |
| *.tmp | |
| *.bak | |
| *.swp | |
| *~.nib | |
| local.properties | |
| .settings/ |
| import org.apache.hadoop.fs._ | |
| import org.apache.hadoop.conf.Configuration | |
| import scala.collection.mutable.ListBuffer | |
| object HdfsUtil { | |
| val pf = new PathFilter { | |
| override def accept(path: Path): Boolean = { |
| /** | |
| * This file contains the core idea of wrapping an underlying OutputFormat with an OutputFormat | |
| * with an augmented key that writes to partitions using MultipleOutputs (or something similar) | |
| */ | |
| package model.hadoop | |
| import model.hadoop.HadoopIO.MultipleOutputer | |
| import model.hadoop.HadoopIO.MultipleOutputer._ | |
| import org.apache.hadoop.io.{DataInputBuffer, NullWritable} |
| ADD JAR s3://<s3-bucket>/jars/hive_contrib-0.5.jar; | |
| CREATE TEMPORARY FUNCTION now as 'com.mt.utils.udf.Now'; | |
| CREATE TEMPORARY FUNCTION user_agent_f as 'com.mt.utils.UserAgent'; | |
| set hive.merge.mapredfiles=true; | |
| set hive.merge.mapfiles=true; | |
| set hive.merge.size.per.task=500000000; | |
| CREATE EXTERNAL TABLE data |
| public class SortReducerByValues { | |
| public static final String INPUT = "/tmp/data_in"; | |
| public static final String OUTPUT = "/tmp/data_out"; | |
| public static void main(String[] args) throws IOException { | |
| new SortReducerByValues().run(); | |
| } | |
| public void run() throws IOException { | |
| JobConf conf = new JobConf(); |
| #!/usr/bin/env bash | |
| # | |
| # Copyright 2012 Zemian Deng | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # |