Apache Zeppelinでデータ分析を分散処理する - Part 5: Ambari on DockerとsbtのScalaビルド環境 ref: http://qiita.com/masato/items/49c0d7911f3ba0baa340

build.sbt

name := "Sample app"

version := "1.0"

scalaVersion := "2.10.5"

libraryDependencies += "org.apache.spark" %% "spark-core" % "1.4.1"

docker-compose.yml

sbt:
  build: .
  volumes:
    - .:/app
    - /temp/.sbt:/root/.sbt
    - /temp/.ivy2:/root/.ivy2
  dns:
    - $AMB_CONSUL
    - 172.17.0.1
    - 8.8.8.8
  dns_search:
    - service.consul

Dockerfile

FROM java:openjdk-7-jdk
MAINTAINER Masato Shimizu <ma6ato@gmail.com>

ENV SBT_VERSION 0.13.9
ENV PATH ${PATH}:/usr/local/sbt/bin

WORKDIR /app

RUN wget -O- http://dl.bintray.com/sbt/native-packages/sbt/$SBT_VERSION/sbt-$SBT_VERSION.tgz | tar xz -C /usr/local

VOLUME ["/root/.ivy2", "/root/.sbt", "/app"]

file0.txt

$ mkdir -p ~/scala_apps/spike
$ cd !$

file10.txt

$ docker exec amb1 hadoop fs -ls /user
Found 5 items
drwxrwx---   - ambari-qa hdfs          0 2015-12-25 17:08 /user/ambari-qa
drwxr-xr-x   - hcat      hdfs          0 2015-12-25 17:10 /user/hcat
drwx------   - hive      hdfs          0 2015-12-25 17:10 /user/hive
drwxr-xr-x   - root      hdfs          0 2015-12-26 02:40 /user/root
drwxrwxr-x   - spark     hdfs          0 2015-12-25 17:09 /user/spark

file11.txt

$ su hdfs 
$ hadoop fs -mkdir /user/root
$ hadoop fs -chown root /user/root

file12.txt

/usr/hdp/2.3.4.0-3485/spark/LICENSE

file13.txt

$ docker exec amb1 hadoop fs -put /usr/hdp/2.3.4.0-3485/spark/LICENSE /user/root
$ docker exec amb1 hadoop fs -ls /user/root/LICENSE
-rw-r--r--   3 root hdfs      17356 2015-12-25 17:20 LICENSE

file14.txt

$ docker-compose run --rm sbt \
  curl -L -X PUT -T /app/target/scala-2.10/sample-app_2.10-1.0.jar "http://amb1:50070/webhdfs/v1/user/root/test/sample-app_2.10-1.0.jar?user.name=root&op=CREATE&overwrite=true"

file15.txt

$ docker exec amb1 hadoop fs -ls ./test/sample-app_2.10-1.0.jar
-rwxr-xr-x   3 root hdfs       1862 2015-12-26 04:16 test/sample-app_2.10-1.0.jar

file16.txt

$ docker exec  amb1 \
 spark-submit \
  --class SampleApp \
  --master yarn-cluster \
  --driver-memory 1g \
  --executor-memory 1g \
  --executor-cores 1 \
  hdfs://amb1/user/root/test/sample-app_2.10-1.0.jar
...
15/12/26 02:54:18 INFO Client:
     client token: N/A
     diagnostics: N/A
     ApplicationMaster host: 172.17.0.5
     ApplicationMaster RPC port: 0
     queue: default
     start time: 1451098443955
     final status: SUCCEEDED
     tracking URL: http://amb1.service.consul:8088/proxy/application_1451063385654_0011/
     user: root
15/12/26 02:54:18 INFO ShutdownHookManager: Shutdown hook called
15/12/26 02:54:18 INFO ShutdownHookManager: Deleting directory /tmp/spark-c30cdca1-6f2b-4432-a866-c63391e619b2

file17.txt

$ docker exec amb1 \
  yarn logs -applicationId application_1451063385654_0011
...
LogType:stdout
Log Upload Time:Sat Dec 26 02:54:19 +0000 2015
LogLength:10
Log Contents:
Lines 294
End of LogType:stdout

file3.txt

$ docker-compose build

file4.txt

$ export AMB_CONSUL=$(docker inspect --format="{{ .NetworkSettings.IPAddress }}" amb-consul)
$ docker-compose run --rm sbt sbt console
[info] Set current project to Sample app (in build file:/app/)
[info] Starting scala interpreter...
[info]
Welcome to Scala version 2.10.5 (OpenJDK 64-Bit Server VM, Java 1.7.0_91).
Type in expressions to have them evaluated.
Type :help for more information.

scala> :quit

file5.txt

$ cd ~/scala_apps/spike
$ mkdir -p src/main/scala/

file8.txt

$ tree
.
├── Dockerfile
├── build.sbt
├── docker-compose.yml
└── src
    └── main
        └── scala
            └── SampleApp.scala

file9.txt

$ docker-compose run --rm sbt sbt package
[info] Set current project to Sample app (in build file:/app/)
[info] Packaging /app/target/scala-2.10/sample-app_2.10-1.0.jar ...
[info] Done packaging.
[success] Total time: 1 s, completed Dec 26, 2015 4:13:52 AM

SampleApp.scala

import org.apache.spark.{SparkContext, SparkConf}

object SampleApp {
  def main(args: Array[String]) {
    val conf = new SparkConf().setAppName("App")
    val sc = new SparkContext(conf)

    val src = sc.textFile("LICENSE")
    val lines = src.count()

    println("Lines %,d".format(lines))
  }
}