Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
(ns ansible.core
(:require [clojure.java.shell :as sh]
[clojure.string :as str]
[cheshire.core :as cheshire]
[clojure.java.io :as io])
(:import (java.io File)
(com.fasterxml.jackson.core JsonGenerator)
(java.util Base64)
(java.security MessageDigest MessageDigest$Delegate)))
(defmacro with-temp-files
"Create a block where `varname` is a temporary `File` containing `content`.
The tempfile is deleted right after the body is run!"
[bindings & body]
(let [bindings (partition 2 bindings)]
`(let ~(into []
(mapcat
(fn [[bind-name content]]
`[~bind-name (File/createTempFile ~(str bind-name) ".json")]))
bindings)
~@(mapv
(fn [[bind-name content]]
`(io/copy ~content ~bind-name))
bindings)
(let [rv# (do ~@body)]
~@(mapv
(fn [[bind-name _]]
`(.delete ^File ~bind-name))
bindings)
rv#))))
(def ^MessageDigest$Delegate md5-encoder (MessageDigest/getInstance "MD5"))
(defn hash-for-string
"Returns a hash for a given string"
[^String s]
(.encodeToString (Base64/getUrlEncoder) (.digest md5-encoder (.getBytes s))))
(defn persistent-file
"Creates a temporary file that won't be deleted and returns the filename string"
[content]
(let [hash (hash-for-string content)
file (io/file (str (System/getProperty "java.io.tmpdir") "/ansible-" hash))]
(when-not (.exists file)
(io/copy content file))
(str file)))
(defn emit-ini-value
[v]
(cond (sequential? v) (str/join " " (mapv emit-ini-value v))
(keyword? v) (name v)
:else v))
(defn emit-ini-section
[m]
(reduce
(fn [s [k v]]
(reduce
(fn [s v]
(str s \newline (name k) \= (emit-ini-value v)))
s (cond (sequential? v) v
(map? v) (mapv (fn [[k v]] (str (name k) \= (emit-ini-value v)))
(sort-by first v))
:else [v])))
"" (if (map? m) (sort-by first m) m)))
(defn ini-str
"Creates an ini-like string for the given data.
Works well with systemd service configs"
([d] (ini-str d nil))
([d note]
{:pre [(map? d)]}
(reduce-kv
(fn [s sec m]
(str s \newline \[ (name sec) \] (emit-ini-section m) \newline))
(if note (str "# " note) "")
d)))
(defn kvs
"Generates a string like 'k0=v0 k1=v1' that ansible sometimes wants."
[m]
(str/join " "
(mapv (fn [[k v]]
(str (name k) "=" (if (keyword? v) (name v) v)))
m)))
(defn json
"Generates a json string where any keywords are converted with a simple (name x)"
[x]
(let [orig (find-protocol-impl cheshire.generate/JSONable :key)
_ (extend clojure.lang.Keyword
cheshire.generate/JSONable
{:to-json (fn encode-named
[^clojure.lang.Keyword k ^JsonGenerator jg]
(.writeString jg (name k)))})
res (cheshire/generate-string x {:pretty true
:key-fn name})]
(extend clojure.lang.Keyword cheshire.generate/JSONable orig)
res))
#_(json {:a/b :b/c})
(defn nlsv
"New line separated values"
[& args]
(str/join "\n" args))
(defn ssv
"Space separted values"
[& vals]
(str/join " " vals))
(defn path-join
"A robust path joiner, adds a slash between any argument"
[& args]
(let [sep (File/separator)
sws? #(.startsWith ^String % sep)
ews? #(.endsWith ^String % sep)
join (fn [a b]
(if (ews? a)
(if (sws? b) (str a (subs b 1)) (str a b))
(if (sws? b) (str a b) (str a sep b))))]
(reduce join args)))
#_(path-join "a/" "/b/" "/c/" "/foo.bar")
(defn dir-exists? [dir] (.exists (io/file dir)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ANSIBLE
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn lookup-file
"A {{lookup('file', ...}} for usage in playbook values"
[f]
(str "{{ lookup('file', '" f "') }}"))
(defn parse-out
"Parses ansible output. Could be made smarter by finding valid json within
output and parsing it..."
[out]
(try
(cheshire/parse-string out)
(catch Exception _
(mapv str/trim (str/split out #"\n")))))
(defn gen-ansible-files
[work-dir inventory config]
{:pre [(dir-exists? work-dir)]}
(spit (path-join work-dir "hosts.json") (json inventory))
(spit (path-join work-dir "ansible.cfg") (ini-str config)))
(defn ansible-sh!
[& args]
(let [start (System/currentTimeMillis)
res (apply sh/sh args)
end (System/currentTimeMillis)]
(-> res
(assoc :took (/ (- end start) 1000.0))
(update :out parse-out)
(with-meta {:cmd (vec args)}))))
(defn ansible
"Runs ansible with the given arguments"
[{:keys [work-dir inventory config vars exec]} host-pattern & args]
(gen-ansible-files work-dir inventory config)
(apply ansible-sh! (or exec "ansible") host-pattern
"-i" "hosts.json"
"-e" (json vars)
(concat args [:dir work-dir])))
(defn ansible-play
"Runs ansible-playbook with the given arguments"
[{:keys [work-dir inventory config vars exec
play-file]} playbook & args]
(gen-ansible-files work-dir inventory config)
(let [run (fn [playb-file]
(apply ansible-sh! (or exec "ansible-playbook")
"-i" "hosts.json"
"-e" (json vars)
(concat args [(str playb-file)] [:dir work-dir])))]
(if (some? play-file)
(let [play-file (str (name play-file) ".yaml")]
(spit (path-join work-dir play-file) (json playbook))
(run play-file))
(with-temp-files [playb-file (json playbook)]
(run playb-file)))))
(ns ansible.your.example
(:require
[clojure.java.io :as io]
[clojure.java.shell :as sh]
[ansible.core :refer [ansible ansible-play ssv path-join kvs ini-str nlsv json
lookup-file]]
[nomad :as nomad]))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; MY ANSIBLE CFG
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn systemd-file [s] (str "/etc/systemd/system/" (name s) ".service"))
(defn datomic-dir [append]
(str "/usr/local/datomic/" append))
(defn prod-cfg
[& kork]
(get-in
(nomad/with-location-override
{:environment "prod"
:hostname "example.com"}
(nomad/read-config (io/resource "config/main.edn")))
kork))
;; Fill in PW here and run in repl to get PW into memory:
#_(def sudo-password "")
(defonce sudo-password "")
(def ssh-port 2291)
(def ansible-common-args
["--become"
"--become-user" "root"
"--forks" "10"
"--ssh-common-args" (ssv "-o" "ControlMaster=auto"
"-o" "ControlPath=\"/home/rauh/.ssh/%l-%h-%p-%r\""
"-o" "ControlPersist=24h")])
(def inventory {:hosts/openresty {:hosts {"8.8.8.8" ""}}
:hosts/app {:hosts {"8.9.223.999" ""}}
:hosts/cassandra {:hosts {"cassandra0" ""
"cassandra1" ""}}})
(def ansible-cfg {"defaults" {;"stdout_callback" "json";; VERY verbose output!
"remote_port" ssh-port
"fact_caching" "jsonfile"
"fact_caching_connection" "/tmp/ansible-fact-cache/"}
"ssh_connection" {"pipelining" "True"}})
(def vars {"ansible_become_pass" sudo-password})
(def ansible-work-dir "ansible-cfg")
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Create wrappers around the two main command:
(defn ansible*
[hosts & args]
(apply ansible {:work-dir ansible-work-dir
:inventory inventory
:config ansible-cfg
:vars vars}
(name hosts)
(concat ansible-common-args args)))
(defn ansible-play*
[playbook & [maybe-playbook & actual-args :as args]]
(let [playfile? (or (keyword? playbook)
(string? playbook))]
(apply ansible-play {:work-dir ansible-work-dir
:inventory inventory
:config ansible-cfg
:vars vars
:play-file (when playfile? playbook)}
(mapv (partial merge {:gather_facts false}) (if playfile? maybe-playbook playbook))
(concat ansible-common-args (if playfile? actual-args args)))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ANSIBLE COMMANDS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(comment
(ansible* "all" "--list-hosts")
(ansible* :hosts/openresty "-a" "/usr/bin/whoami")
(ansible* :hosts/openresty "-a" "which lein")
(ansible* :hosts/openresty "-m" "setup")
(ansible* :hosts/openresty "-m" "ping")
(ansible* :hosts/openresty "-m" "systemd" "-a" (kvs {:daemon_reload true}))
;; Start app:
(ansible* :hosts/app "-m" "systemd" "-a" (kvs {:name :systemd/clj-app :state :started}))
;; Datomic license info:
(ansible* :hosts/datomic "-a" (ssv (datomic-dir "bin/datomic") "describe-license"
(datomic-config-file)))
(ansible* :hosts/elasticsearch "-a" (ssv "bash" "-c" "'netstat -ln | grep :9200'"))
(ansible* :hosts/elasticsearch "-a" (ssv "systemctl" "status" "elasticsearch"))
;; Reload openresty:
(ansible* :hosts/openresty "-m" "systemd" "-a" (kvs {:name :systemd/openresty :state :reloaded})))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; PLAYBOOKS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def start-stop-daemon "-/sbin/start-stop-daemon")
(def ansbile-managed "Ansible managed")
(defn rsync-copy
"A :block which rsync's in two steps since rsync task cant sudo w/ password:
1. Rsync to the given temp dir (retained so rsync is still fast)
2. Rsync within the server to dest"
[{:keys [name dest temp src]} & [opt-to-remote opt-on-remote]]
{:block [{:name (str name ": To remote")
:synchronize (merge {:delete true
:dest temp
:src src
:group false
:owner false
:links false}
opt-to-remote)
;; We can't sudo w/ rsync:
:become false}
{:name (str name ": Within remote")
:synchronize (merge {:dest dest
:group false
:owner false
:perms false
:src temp
:links false}
opt-on-remote)
;; To rsync within the remote host
:delegate_to "{{ inventory_hostname }}"}]})
(defn std-file-perms
"Recursively sets the perms to 0775/0664 on the given dir.
Changes all files to user/group"
[dir user group]
{:block [{:name (str "Set perms to 0775/0664 for " dir)
:command (ssv "chmod"
"-c" ;; output changed files
"-R" ;; recursive
"ug=rw,o=r,a-x+X"
dir)
:register "chmod_status"
:changed_when "chmod_status.stdout != \"\""}
{:name (str "chown: " user ":" group)
:file {:group group
:owner user
:path dir
:state "directory"
:recurse true}}]})
(defn enable-and-start-service
([svc] (enable-and-start-service svc :state/started))
([svc state]
{:name (str "Enabling " (name svc) " service")
:service {:name svc
:state state
:enabled true}}))
(defn enable-and-start-serviced
([svc] (enable-and-start-serviced svc :state/started))
([svc state]
{:name (str "Enabling " (name svc) " service")
:systemd {:name svc
:state state
:enabled true}}))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; MY PLAYBOOKS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def all-handlers
(let [restart (fn [handler service]
{:name handler
:systemd {:name service
:state :state/restarted}})]
[{:name :handler/reload-systemd
:systemd {:daemon_reload true}}
{:name :handler/reload-openresty
:systemd {:name :systemd/openresty
:state :state/reloaded}}
(restart :handler/restart-redis-stats :systemd/redis-stats)
(restart :handler/restart-redis-volatile :systemd/redis-volatile)
(restart :handler/restart-datadog :systemd/datadog-agent)]))
(defn copy-openresty-systemd
[]
{:block [{:name "Copy OpenResty systemd service file"
:copy {:content (ini-str
(let [bin "/usr/local/openresty/bin/openresty"
pid "/run/nginx.pid"
g-conf "'daemon on; master_process on;'"]
{:Unit {:Description "Openresty server"
:After :network.target}
:Service {:Type :type/forking
;; Note: User/group is set by the config file
#_#_:ExecStartPre [[bin "-t" "-q" "-g" g-conf]]
:PIDFile pid
:ExecStart [[bin "-g" g-conf]]
:ExecReload [[bin "-g" g-conf "-s" "reload"]]
:ExecStop [[start-stop-daemon "--quiet" "--stop"
"--retry" "QUIT/5" "--pidfile" pid]]
:TimeoutStopSec 5
:PrivateTmp true
:KillMode "mixed"}
:Install {:WantedBy :multi-user.target}})
ansbile-managed)
:dest (systemd-file :systemd/openresty)
:mode "0644"}
:notify [:handler/reload-systemd]}
{:name "Enabling openresty service"
:systemd {:name :systemd/openresty
:enabled true}}]})
(defn datomic-dir [append]
(str "/usr/local/datomic/" append))
(defn datomic-config-file []
(datomic-dir "transactor-cass-prod.properties"))
(defn install-datomic-systemd
[]
{:block [{:name "Copy Datomic systemd service file:"
:copy {:content (ini-str
(let [bin "/usr/local/datomic/bin/transactor"
data-dir (datomic-dir "data/")
ug "datomic:datomic"
log-dir (datomic-dir "log/")]
{:Unit {:Description "SRS Datomic"
:After [[:network.target "cassandra.service"]]}
:Service {:Type :type/simple
:User "datomic"
:Group "datomic"
:PermissionsStartOnly true
:ExecStartPre [["-/bin/mkdir" "-p" data-dir]
["-/bin/chown" "-R" ug data-dir]
["-/bin/mkdir" "-p" log-dir]
["-/bin/chown" ug log-dir]]
:ExecStart [[bin
"-Xms2048m" "-Xmx2048m"
(datomic-config-file)]]
:Restart "always"
:RestartSecs 3}
:Install {:WantedBy :multi-user.target}})
ansbile-managed)
:dest (systemd-file :systemd/datomic)
:mode "0644"}
:notify [:handler/reload-systemd]}
{:name "Enabling datomic service systemd"
:systemd {:name :systemd/datomic
:enabled true}}]})
(defn create-backup-user
[]
{:name "Creating backup user"
:user {:name "backup-user"
:generate_ssh_key true}})
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; RUN PLAYBOOKS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(comment
(ansible-play*
:playbook/backup
[{:hosts :hosts/backup
:tasks [(create-backup-user)]
:handlers all-handlers}])
;; CASSANDRA ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(ansible-play*
[{:hosts :hosts/cassandra
:tasks [(enable-and-start-service :service/cassandra)]
:handlers all-handlers}])
;; DATOMIC ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(ansible-play*
[{:hosts :hosts/datomic
:tasks [(install-datomic-systemd)]
:handlers all-handlers}]
#_"--check")
;; OPENRESTY ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(ansible-play*
[{:hosts :hosts/openresty
:tasks [(copy-openresty-systemd)]
:handlers all-handlers}]
#_"--check"))
@atrakic

This comment has been minimized.

Copy link

@atrakic atrakic commented Feb 6, 2018

+1

@xingzheone

This comment has been minimized.

Copy link

@xingzheone xingzheone commented Jun 18, 2019

good

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment