net-flaky-nemesis.clj

; Network latency injection!

(defn fast!
  "Clears all tc rules."
  [dev]
  (c/su
    ;(info "fast!")
    (try+ (c/exec :tc :qdisc :del :dev dev :root)
          (catch [:exit 2] e
            ; Ah, there weren't any rules to clear.
            nil))))

(defn slow-to!
  "Slows outbound traffic to the given node via the given device (e.g. eth0)."
  [dev node]
  (let [ip (cn/ip node)]
    ;(info "slow" ip)
    ; Attach a filter to the top-level queue which matches IP traffic to the
    ; given node, and routes it to flow 1:4, which is our high-latency
    ; prioqueue.
    (c/exec :tc :filter :add :dev dev :protocol :ip :parent "1:0"
            :u32 :match :ip :dst ip :flowid "1:4")))

(defn slow-to-all!
  "Slows outbound traffic to all given nodes."
  [dev nodes mean-delay-ms rate-kbit]
  (fast! dev)
  ; See https://wiki.linuxfoundation.org/networking/netem
  (c/su ; Add a new root qdisc to the interface which is a prioqueue. The first
        ; 3 priorities will be used for standard traffic; the 4th we use as our
        ; EVIL tier of service.
        ; prioqueue?
        (info "add prioqueue")
        (c/exec :tc :qdisc :add :dev dev :root :handle "1:" :prio :bands 4
                ; This maps combinations of packet TOS flags to priority
                ; queues; this is the standard mapping for 3 queues.
                :priomap 1 2 2 2 1 2 0 0 1 1 1 1 1 1 1 1)
        ; Add another qdisc to that prioqueue--specifically on the fourth
        ; queue, which is our special target for slow traffic. This TBF node
        ; performs rate limiting.
        (info "add tbf" rate-kbit "kbit")
        (c/exec :tc :qdisc :add :dev dev :parent "1:4" :handle "40:"
                :tbf :rate (str rate-kbit "kbit")
                :buffer 10000 ; How do we tune this?
                :limit 20000) ; How do we tune this?
        ; To THAT we attach a network emulator (netem), introducing some packet
        ; delay.
        (info "add netem")
        (c/exec :tc :qdisc :add :dev dev :parent "40:1" :handle "50:"
                :netem :delay
                (str mean-delay-ms "ms")
                "1ms" ; variance
                :distribution :normal)
        ; Apply filters routing those particular nodes to the netem qdisc node
        (doseq [node nodes]
          (slow-to! dev node))))

(defn net-device
  "Returns the network interface that we should set tc queuing disciplines on"
  []
  (let [choices (c/su (c/exec :ls "/sys/class/net/"))
        iface (->> choices
                   (str/split-lines)
                   (remove (fn [iface] (re-find #"^lo" iface)))
                   first)]
    (assert iface
            (str "Couldn't determine network interface!\n" choices))
    iface))

(defn slow-net-nemesis
  "Slows down the network on all nodes. Operations are either :slow-net, whose
  value is an option map passed to net/slow!, or :fast-net, which brings
  everything back to as-fast-as-possible."
  []
  (reify n/Nemesis
    (setup! [this test]
      this)

    (invoke! [this test op]
      (case (:f op)
        :slow-net (c/on-nodes test (fn [test node]
                                     (slow-to-all! (net-device)
                                                   (remove #{node} (:nodes test))
                                                   (:mean (:value op))
                                                   (:rate (:value op)))))
        :fast-net (c/with-test-nodes test
                    (fast! (net-device))))
      op)

  (teardown! [this test]
             (c/with-test-nodes test
               (fast! (net-device))))

  n/Reflection
  (fs [this]
      #{:slow-net :fast-net})))

(defn slow-net-final-gen
  "Final generator for slow networks."
  [opts]
  (assert (set? (:faults opts)))
  (when (:slow-net (:faults opts))
    {:type :info, :f :fast-net}))

(defn slow-net-gen
  "Normal generator for slow networks."
  [opts]
  (when (:slow-net (:faults opts))
    ; Alternatively: just do a 500ms delay one time.
    ;[(gen/sleep 10)
    ; {:type :info, :f :slow-net, :value {:mean 500}}]
    (fn gen [test context]
      ; For every combination of network latency and rate limit:
      (->> (for [rate    (:net-rates test)
                 latency (:net-latencies test)]
             {:mean latency
              :rate rate})
           ; Endlessly...
           cycle
           (mapcat (fn [slow-params]
                     [{:type :info, :f :fast-net}
                      {:type :info, :f :slow-net, :value slow-params}
                      ; sleep a while between changes
                      (gen/sleep (:interval opts))]))))))

(def slow-net-perf
  "Helps the performance graph system understand and render slow-net nemesis
  activity."
  #{{:name  "slow-net"
     :start #{:slow-net}
     :stop  #{:fast-net}
     :color "#873D9D"}})

(defn slow-net-package
  "Constructs a nemesis package for slowing the network."
  [opts]
  {:nemesis          (slow-net-nemesis)
   :generator        (slow-net-gen opts)
   :final-generator  (slow-net-final-gen opts)
   :perf             slow-net-perf})