Skip to content

Instantly share code, notes, and snippets.

@tomfaulhaber
Created April 24, 2014 17:07
Show Gist options
  • Save tomfaulhaber/11261998 to your computer and use it in GitHub Desktop.
Save tomfaulhaber/11261998 to your computer and use it in GitHub Desktop.
Is this using :extends correctly?
(ns hadoop-int.common
(:require [pallet.api :refer [server-spec plan-fn]]
[pallet.actions :refer [package package-manager]]
[pallet.configure :as configure]
[pallet.core.session :as session]
[pallet.crate :as crate]
[pallet.node :as node]
[com.palletops.jclouds.ec2.security-group2 :as sg2]))
(def aws (configure/compute-service :sc-aws))
(def tags [["Client" "Internal"]
["Project" "HadoopIntegration"]])
(def with-tags
(server-spec
:phases {:configure
(plan-fn (doseq [pair tags]
(apply node/tag! (crate/target-node) pair)))}))
(def with-mosh
(server-spec
:phases {:configure (plan-fn
(let [compute (-> (session/session) :environment :compute)
sg-name (str "jclouds#" (.getGroup (crate/target-node)))]
(try
(sg2/authorize compute sg-name [60000 61000] :protocol :udp)
(catch Exception e
(println "Message is" (.getMessage e))
(when (neg? (.indexOf (.getMessage e) "has already been authorized"))
(throw e))))
(package-manager :update)
(package "mosh")))}))
(ns hadoop-int.hadoop
(:require
[pallet.api :as api]
[palletops.cluster.hadoop :as h]
[hadoop-int.common :as common]))
(def our-node
{:hardware {:hardware-id "m3.large"}
:image {:os-family :ubuntu
:os-version-matches "12.04"
:os-64-bit true}})
(def our-hadoop-cluster
(h/hadoop-cluster
{:cluster-prefix "hi-hadoop"
:groups {:nn {:node-spec our-node
:count 1
:extends [common/with-tags common/with-mosh]
:roles #{:namenode :jobtracker}}
:slave {:node-spec our-node
:count 1
:extends [common/with-tags]
:roles #{:datanode :tasktracker}}}
:hadoop-config
{:io.file.buffer.size 65536
:config {
:namenode {:jmx-port 3000}
:secondary-namenode {:jmx-port 3001}
:jobtracker {:jmx-port 3002}
:datanode {:jmx-port 3003}
:tasktracker {:jmx-port 3004}}}}))
(defn start-cluster
[& {:keys [cluster] :or {cluster our-hadoop-cluster}}]
(api/converge
(:groups cluster)
:compute common/aws
:phase [:install
:collect-ssh-keys
:configure
:restart-collectd
:run :init
:install-test
:configure-test]
:async true))
(defn stop-cluster
[& {:keys [cluster] :or {cluster our-hadoop-cluster}}]
(let [count0 (update-in cluster [:groups]
(fn [gs] (map #(assoc % :count 0) gs)))]
(api/converge count0 :compute common/aws :async true)))
(ns hadoop-int.vanilla
(:require [hadoop-int.common :as common]
[pallet.api :as api]
[pallet.crate.automated-admin-user :refer [automated-admin-user]]))
;;; Make a regular old compute node with no Hadoop or SCDB
(def vanilla-group
(api/group-spec "tom-testnode"
:extends [common/with-tags common/with-mosh]
:node-spec (pallet.api/node-spec
:image {:os-family :ubuntu
:image-id "us-east-1/ami-3c994355"})
:phases {:bootstrap automated-admin-user}))
(defn start-vanilla []
(api/converge {vanilla-group 1} :compute common/aws))
(defn stop-vanilla []
(api/converge {vanilla-group 0} :compute common/aws))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment