Collects github status.

gh-stats.lisp

(defpackage #:gh-stats/stats
  (:use #:cl)
  (:import-from :dexador)
  (:import-from :log4cl)
  (:import-from :local-time)
  (:import-from :cl-arrows
                :->)
  (:import-from :link-header)
  (:import-from :jonathan)
  (:import-from :cl-strings))
(in-package gh-stats/stats)


(defvar *data-output* t)
(defvar *number-of-gets* 0)
(defvar *token* INSERT OAUTH TOKEN HERE)


(defun dump (obj)
  (format *data-output*
          "~A~%"
          (jonathan:to-json obj)))


(defun get-current-timestamp ()
  (local-time:timestamp-to-unix (local-time:now)))


(defun get-with-retries (url)
  (tagbody
   retry
     (handler-bind ((usocket:ns-host-not-found-error
                      (lambda (condition)
                        (declare (ignorable condition))
                        (log:warn "Unable to resolve host sleeping 1 second before next try.")
                        (sleep 1)
                        (go retry)))
                    (dexador.error:http-request-forbidden
                      (lambda (condition)
                        (let* ((headers (dex:response-headers condition))
                               (remaining (gethash "x-ratelimit-remaining" headers))
                               (reset-at (gethash "x-ratelimit-reset" headers))
                               (now (get-current-timestamp)))
                          (when (= remaining 0)
                            ;; We hit a rate limit
                            (let ((delay (max (- reset-at now)
                                              0)))
                              (when (> delay 0)
                                (log:warn "Sleeping" delay "seconds because of rate limit")
                                (sleep delay)
                                (go retry))))))))
       (incf *number-of-gets*)
       (return-from get-with-retries
         (dex:get url
                  :headers
                  (list (cons "Authorization"
                              (format nil "token ~A"
                                      *token*))))))))

(defun get-data (url)
  (link-header:with-links (response next-link)
                          (get-with-retries url)
                          (values (jonathan:parse response)
                                  next-link)))


(defun %with-all-data (url func)
  (loop
    do (multiple-value-bind (items next-url)
           (progn
             (log:info "Fetching" url)
             (get-data url))
         (setf url next-url)
         (mapc func items)
         (unless url
           (return)))))


(defmacro with-all-data ((url item-var) &body body)
  "Iterates over all items accessable via url, using GitHub's pagination."
  `(%with-all-data ,url
                   (lambda (,item-var)
                     ,@body)))


(defun get-all-data (url)
  (let (items)
    (loop
      do (multiple-value-bind (data next-url)
             (get-data url)
           (setf url next-url
                 items (nconc data items))
           (unless url
             (return items))))))


(defun format-url (url)
  ;; a HACK to string {/issues}
  (format nil
          "~A?state=all"
          (first (cl-strings:split url "{"))))


(defun process-comments (items &optional (type "comment"))
  (loop for item in items
        for user = (-> item
                       (getf :|user|)
                       (getf :|login|))
        do (format *data-output* "~A ~A~%" type user)))


(defun get-repo-name-from-url (url)
  "Processes urls like this: https://api.github.com/repos/mojombo/grit returns mojombo/grit."
  (let* ((splitted (cl-strings:split url #\/))
         (pos (position "repos" splitted :test #'string=))
         (parts (subseq splitted
                        (+ pos 1)
                        (+ pos 3))))
    (cl-strings:join parts :separator "/")))


(defun process-issue (data &key (type "issue"))
  ;; :|user| :|login|
  ;; интересные ключи
  ;; :|requested_teams|
  ;; :|requested_reviewers|
  ;; :|assignees|
  ;; :|assignee|
  ;; :|review_comments|
  ;; :|comments|
  ;; :|merged_by|
  ;; :|mergeable|
  ;; :|merged|

  (let* ((number (getf data :|number|))
         (author (-> data
                     (getf :|user|)
                     (getf :|login|)))
         (name (or (-> data
                       (getf :|base|)
                       (getf :|repo|)
                       (getf :|full_name|))
                   (-> data
                       (getf :|repository_url|)
                       (get-repo-name-from-url))))
         (requested-teams (getf data :|requested_teams|))
         (requested-reviewers (getf data :|requested_reviewers|))
         (assignees (getf data :|assignees|))
         (assignee (getf data :|assignee|))
         (review-comments (getf data :|review_comments|))
         (comments (getf data :|comments|))
         (merged-by (getf data :|merged_by|))
         (mergeable (getf data :|mergeable|)))
    (unless name
      (break))
    (format *data-output* "~A ~A ~A~%" type name number)
    (format *data-output* "author ~A~%" author)
    
    ;; (when requested-teams
    ;;   (break))
    ;; (when requested-reviewers
    ;;   (break))
    (when assignees
      (loop for assinee in assignees
            for login = (getf assignee :|login|)
            do (format *data-output*
                       "assignee ~A~%"
                       login)))
    (when review-comments
      (process-comments (get-all-data (getf data :|comments_url|))
                        "review-comment"))
    (when comments
      (process-comments (get-all-data (getf data :|comments_url|))))
    (when merged-by
      (format *data-output*
              "merged-by ~A~%"
              (getf merged-by
                    :|login|)))
    ;; (when mergeable
    ;;   (break))
    )
  )


(defun process-issues (url &key (type "issue"))
  (loop for item in (get-all-data url)
        do (process-issue item :type type)))



(defun process-repository (data)
  (format *data-output* "repository ~A ~A num-gets: ~A~%"
          (getf data :|full_name|)
          (getf data :|id|)
          *number-of-gets*)
  (process-issues (format-url (getf data :|pulls_url|))
                  :type "pull")
  (process-issues (format-url (getf data :|issues_url|))))

;; (defun process-repositories (&optional (url "https://api.github.com/repositories"))
;;   (uiop:with-output-file (*data-output* "data.log" :if-exists :append)
;;     (format *data-output* "url ~A~%" url)
;;     (loop for repository in (get-data url)
;;           do (process-repository repository))))


(defun process-repositories (&optional (url "https://api.github.com/repositories"))
  (uiop:with-output-file (*data-output* "data.log" :if-exists :append)
    (format *data-output* "url ~A~%" url)
    (with-all-data (url repository)
      (process-repository repository))))


(defun process-user-repository (data)
  (let ((pulls (get-all-data (format-url (getf data :|pulls_url|))))
        (issues (get-all-data (format-url (getf data :|issues_url|)))))
    (dump (list :type "repository"
                :name (getf data :|full_name|)
                :id (getf data :|id|)
                :updated-at (getf data :|updated_at|)
                :num-pulls (length pulls)
                :open-pulls (length (remove-if-not (lambda (p)
                                                     (string-equal (getf p :|state|)
                                                                   "open"))
                                                   pulls))
                :num-issues (length issues)
                :open-issues (length (remove-if-not (lambda (p)
                                                      (string-equal (getf p :|state|)
                                                                    "open"))
                                                    issues))))))

(defun make-events-dict (events)
  (loop with result = nil
        for item in events
        for type = (-> item
                       (getf :|type|)
                       (alexandria:make-keyword))
        do (push item
                 (getf result type))
        finally (return result)))


(defun process-user (login)
  (let ((events (make-events-dict
                 (get-all-data (format nil
                                       "https://api.github.com/users/~A/events"
                                       login)))))
    (dump (list :type "user"
                :login login
                :events events)))
  

  
  (with-all-data ((format nil
                          "https://api.github.com/users/~A/repos?type=all"
                          login)
                  repository)
    (process-user-repository repository))
  
  (with-all-data ((format nil
                          "https://api.github.com/users/~A/orgs"
                          login)
                  organization)
    (let ((org-name (getf organization :|login|)))
      (dump (list :type "organization"
                  :name org-name))
      (with-all-data ((format nil
                              "https://api.github.com/orgs/~A/repos"
                              org-name)
                      repository)
        (process-user-repository repository)))))


(defun process-users ()
  ;; https://api.github.com/users?since=48000000
  ;; TODO: select n users randomly, and dump their data
  )