Created
April 24, 2014 17:07
-
-
Save tomfaulhaber/11261998 to your computer and use it in GitHub Desktop.
Is this using :extends correctly?
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
(ns hadoop-int.common | |
(:require [pallet.api :refer [server-spec plan-fn]] | |
[pallet.actions :refer [package package-manager]] | |
[pallet.configure :as configure] | |
[pallet.core.session :as session] | |
[pallet.crate :as crate] | |
[pallet.node :as node] | |
[com.palletops.jclouds.ec2.security-group2 :as sg2])) | |
(def aws (configure/compute-service :sc-aws)) | |
(def tags [["Client" "Internal"] | |
["Project" "HadoopIntegration"]]) | |
(def with-tags | |
(server-spec | |
:phases {:configure | |
(plan-fn (doseq [pair tags] | |
(apply node/tag! (crate/target-node) pair)))})) | |
(def with-mosh | |
(server-spec | |
:phases {:configure (plan-fn | |
(let [compute (-> (session/session) :environment :compute) | |
sg-name (str "jclouds#" (.getGroup (crate/target-node)))] | |
(try | |
(sg2/authorize compute sg-name [60000 61000] :protocol :udp) | |
(catch Exception e | |
(println "Message is" (.getMessage e)) | |
(when (neg? (.indexOf (.getMessage e) "has already been authorized")) | |
(throw e)))) | |
(package-manager :update) | |
(package "mosh")))})) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
(ns hadoop-int.hadoop | |
(:require | |
[pallet.api :as api] | |
[palletops.cluster.hadoop :as h] | |
[hadoop-int.common :as common])) | |
(def our-node | |
{:hardware {:hardware-id "m3.large"} | |
:image {:os-family :ubuntu | |
:os-version-matches "12.04" | |
:os-64-bit true}}) | |
(def our-hadoop-cluster | |
(h/hadoop-cluster | |
{:cluster-prefix "hi-hadoop" | |
:groups {:nn {:node-spec our-node | |
:count 1 | |
:extends [common/with-tags common/with-mosh] | |
:roles #{:namenode :jobtracker}} | |
:slave {:node-spec our-node | |
:count 1 | |
:extends [common/with-tags] | |
:roles #{:datanode :tasktracker}}} | |
:hadoop-config | |
{:io.file.buffer.size 65536 | |
:config { | |
:namenode {:jmx-port 3000} | |
:secondary-namenode {:jmx-port 3001} | |
:jobtracker {:jmx-port 3002} | |
:datanode {:jmx-port 3003} | |
:tasktracker {:jmx-port 3004}}}})) | |
(defn start-cluster | |
[& {:keys [cluster] :or {cluster our-hadoop-cluster}}] | |
(api/converge | |
(:groups cluster) | |
:compute common/aws | |
:phase [:install | |
:collect-ssh-keys | |
:configure | |
:restart-collectd | |
:run :init | |
:install-test | |
:configure-test] | |
:async true)) | |
(defn stop-cluster | |
[& {:keys [cluster] :or {cluster our-hadoop-cluster}}] | |
(let [count0 (update-in cluster [:groups] | |
(fn [gs] (map #(assoc % :count 0) gs)))] | |
(api/converge count0 :compute common/aws :async true))) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
(ns hadoop-int.vanilla | |
(:require [hadoop-int.common :as common] | |
[pallet.api :as api] | |
[pallet.crate.automated-admin-user :refer [automated-admin-user]])) | |
;;; Make a regular old compute node with no Hadoop or SCDB | |
(def vanilla-group | |
(api/group-spec "tom-testnode" | |
:extends [common/with-tags common/with-mosh] | |
:node-spec (pallet.api/node-spec | |
:image {:os-family :ubuntu | |
:image-id "us-east-1/ami-3c994355"}) | |
:phases {:bootstrap automated-admin-user})) | |
(defn start-vanilla [] | |
(api/converge {vanilla-group 1} :compute common/aws)) | |
(defn stop-vanilla [] | |
(api/converge {vanilla-group 0} :compute common/aws)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment