analyze.clj

(ns geoip.analyze
  (:use clojure.contrib.str-utils
        [clojure.contrib.seq-utils :exclude [group-by]]
        [clojure.contrib.duck-streams :exclude [copy]]
        [clojure.set :only (union)]
        (incanter core datasets io charts stats)
        (geoip core))
    (:import [com.maxmind.geoip Country
                              Location
                              Region
                              regionName
                              timeZone
                              LookupService
                              DatabaseInfo])
  (:gen-class))

;; TODO
;; - Add -main [customer-name logfile top-n]
;; - Create filename of graph from args to -main

;; Initial Configuration
(def *logfile* (str *project-root* "/resources/" "nunya-business.log"))

;; (set! *print-length* 50)
;; (set! *warn-on-reflection* true)

(defn find-lines [#^String text file]
  (filter
   (fn [#^String line]
     (< 0 (.indexOf line text)))
   (line-seq (reader file))))

(defn of-IPs [coll]
  (flatten
   (remove empty?
     (pmap #(rest
             (re-find
              #"Deny tcp src outside:(\d+\.\d+\.\d+\.\d+)" %))
           coll))))

(defn count-ips [logfile]
  (remove empty?
   (reduce
    #(assoc %1 %2 (inc (get %1 %2 0)))
    {}
    (of-IPs
     (find-lines "Deny tcp src outside" logfile)))))

(defn add-country-codes [logfile]
  (reduce
   (fn [m [ip cnt]]
     (assoc m ip {:count cnt
                  :country (try
                            (country-name ip)
                            (catch Exception e "NO COUNTRY"))}))
   {}
   (count-ips logfile)))

;; Serialization
(defn serialize-to-file [filename logfile]
  (spit
   (java.io.File. (str *project-root* "serialized/" filename ".sclj"))
   (pr-str (add-country-codes logfile))))

(defn to-vec-of-maps [logfile]
  (vec
   (vals
    (add-country-codes logfile))))

(defn reduce-country-counts [logfile]
  (reduce
   (fn [a {:keys [country count]}]
     (merge-with + a {country count}))
   {}
   (to-vec-of-maps logfile)))

;; (defn to-incanter-structure [logfile]
;;   (vec
;;    (map
;;     (fn [[country count]]
;;       {country count})
;;     (reduce-country-counts logfile))))

(defn to-incanter-structure [logfile]
  (vec (map (fn [[country count]]
              {:country country :count count})
            (reduce-country-counts logfile))))

(defn save-graph [company, filename, date, countries, values]
  (save
   (bar-chart
    countries values
    :legend false
    :title (str company ": Firewall Denials for " date)
    :y-label "Number of Denials"
    :x-label "Countries"
    :vertical false)
   filename
   :width 2560
   :height 2560))

(defn save-graph-top-n [company, filename, n, date, countries, values]
  (save
   (bar-chart
    (take-last n countries)
    (take-last n values)
    :legend false
    :title (str company ": Firewall Denials for " date)
    :y-label "Number of Denials"
    :x-label "Countries"
    :vertical false)
   filename
   :width 800
   :height 800))

;; For Testing -- Remove for release...
;; (incanter.core/view (incanter.charts/bar-chart ["a" "b" "c"] [1 2 3]))

(defn -main [& args]
  "Usage: java -jar -Xms512m -Xmx2048m geoip.jar \"Customer\" \"/path/to/logfile.log\" 50 \"03/01/2010\""
 
  (let [client-name (str (nth args 0))
        logfile (str (nth args 1))
        top-n (Integer/valueOf (nth args 2))
        date (str (nth args 3))]
    (if (=(nth args 0) "help")
      (print-doc ))
    (println "Parsing log...")
    (def log-data
         (time (sort-by :count (to-incanter-structure logfile))))
    (println "Dividing schema...")
    (def countries
         (time (vec (map first (map vals log-data)))))
    (println "Dividing schema...")
    (def values
         (time (vec (flatten (vec (map rest (map vals log-data)))))))
    
    (println "Generating graphs...")
    (save-graph client-name (str logfile ".png") date countries values)
    (save-graph-top-n client-name (str logfile "-top" (str top-n) ".png") top-n date countries values)))