imatespl

#!/usr/bin/env bash

SPARK_JAR=./assembly/target/scala-2.10/spark-assembly_2.10-0.9.0-incubating-hadoop2.2.0.jar \
    ./bin/spark-class org.apache.spark.deploy.yarn.Client \
      --jar ~/spark_wordcount/target/scala-2.10/wordcount_2.10-1.0.0.jar \
      --class  spark.example.WordCount \
      --args yarn-standalone \
      --args /testWordCount.txt \
      --args /resultWordCount \
      --num-workers 3 \

imatespl

package spark.example

import org.apache.spark._
import SparkContext._

object WordCount {
  def main(args: Array[String]) {
    //命令行参数个数检查
    if (args.length == 0) {
      System.err.println("Usage: spark.example.WordCount <input> <output>")

imatespl

#!/usr/bin/env bash

SPARK_JAR=./assembly/target/scala-2.10/spark-assembly_2.10-0.9.0-incubating-hadoop2.2.0.jar \
    ./bin/spark-class org.apache.spark.deploy.yarn.Client \
      --jar ./examples/target/spark-examples_2.10-0.9.0-incubating.jar  \
      --class  org.apache.spark.examples.SparkPi  \
      --args yarn-standalone \
      --num-workers 3 \
      --master-memory 4g \
      --worker-memory 2g \

imatespl

<configuration>

<!-- Site specific YARN configuration properties -->
  <property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
  </property>
  <property>
    <name>yarn.resourcemanager.address</name>
    <value>bd001:8032</value>

imatespl

<configuration>
  <property>
     <name>mapreduce.framework.name</name>
     <value>yarn</value>
  </property>
  <property>
     <name>mapreduce.jobhistory.address</name>
     <value>bd001:10020</value>
  </property>
  <property>

imatespl

<configuration>
  <property>
     <name>dfs.namenode.name.dir</name>
     <value>file:/disk1/hadoopdata/hfs/name/</value>
  </property>
  <property>
     <name>dfs.datanode.data.dir</name>
     <value>file:/disk1/hadoopdata/hfs/data/</value>
  </property>
</configuration>

imatespl

<configuration>
  <property>
    <name>fs.defaultFS</name>
    <value>hdfs://bd001:9000/</value>
  </property>
  <property>
    <name>dfs.replication</name>
    <value>3</value>
  </property>
  <property>