Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Collects github status.
(defpackage #:gh-stats/stats
(:use #:cl)
(:import-from :dexador)
(:import-from :log4cl)
(:import-from :local-time)
(:import-from :cl-arrows
:->)
(:import-from :link-header)
(:import-from :jonathan)
(:import-from :cl-strings))
(in-package gh-stats/stats)
(defvar *data-output* t)
(defvar *number-of-gets* 0)
(defvar *token* INSERT OAUTH TOKEN HERE)
(defun dump (obj)
(format *data-output*
"~A~%"
(jonathan:to-json obj)))
(defun get-current-timestamp ()
(local-time:timestamp-to-unix (local-time:now)))
(defun get-with-retries (url)
(tagbody
retry
(handler-bind ((usocket:ns-host-not-found-error
(lambda (condition)
(declare (ignorable condition))
(log:warn "Unable to resolve host sleeping 1 second before next try.")
(sleep 1)
(go retry)))
(dexador.error:http-request-forbidden
(lambda (condition)
(let* ((headers (dex:response-headers condition))
(remaining (gethash "x-ratelimit-remaining" headers))
(reset-at (gethash "x-ratelimit-reset" headers))
(now (get-current-timestamp)))
(when (= remaining 0)
;; We hit a rate limit
(let ((delay (max (- reset-at now)
0)))
(when (> delay 0)
(log:warn "Sleeping" delay "seconds because of rate limit")
(sleep delay)
(go retry))))))))
(incf *number-of-gets*)
(return-from get-with-retries
(dex:get url
:headers
(list (cons "Authorization"
(format nil "token ~A"
*token*))))))))
(defun get-data (url)
(link-header:with-links (response next-link)
(get-with-retries url)
(values (jonathan:parse response)
next-link)))
(defun %with-all-data (url func)
(loop
do (multiple-value-bind (items next-url)
(progn
(log:info "Fetching" url)
(get-data url))
(setf url next-url)
(mapc func items)
(unless url
(return)))))
(defmacro with-all-data ((url item-var) &body body)
"Iterates over all items accessable via url, using GitHub's pagination."
`(%with-all-data ,url
(lambda (,item-var)
,@body)))
(defun get-all-data (url)
(let (items)
(loop
do (multiple-value-bind (data next-url)
(get-data url)
(setf url next-url
items (nconc data items))
(unless url
(return items))))))
(defun format-url (url)
;; a HACK to string {/issues}
(format nil
"~A?state=all"
(first (cl-strings:split url "{"))))
(defun process-comments (items &optional (type "comment"))
(loop for item in items
for user = (-> item
(getf :|user|)
(getf :|login|))
do (format *data-output* "~A ~A~%" type user)))
(defun get-repo-name-from-url (url)
"Processes urls like this: https://api.github.com/repos/mojombo/grit returns mojombo/grit."
(let* ((splitted (cl-strings:split url #\/))
(pos (position "repos" splitted :test #'string=))
(parts (subseq splitted
(+ pos 1)
(+ pos 3))))
(cl-strings:join parts :separator "/")))
(defun process-issue (data &key (type "issue"))
;; :|user| :|login|
;; интересные ключи
;; :|requested_teams|
;; :|requested_reviewers|
;; :|assignees|
;; :|assignee|
;; :|review_comments|
;; :|comments|
;; :|merged_by|
;; :|mergeable|
;; :|merged|
(let* ((number (getf data :|number|))
(author (-> data
(getf :|user|)
(getf :|login|)))
(name (or (-> data
(getf :|base|)
(getf :|repo|)
(getf :|full_name|))
(-> data
(getf :|repository_url|)
(get-repo-name-from-url))))
(requested-teams (getf data :|requested_teams|))
(requested-reviewers (getf data :|requested_reviewers|))
(assignees (getf data :|assignees|))
(assignee (getf data :|assignee|))
(review-comments (getf data :|review_comments|))
(comments (getf data :|comments|))
(merged-by (getf data :|merged_by|))
(mergeable (getf data :|mergeable|)))
(unless name
(break))
(format *data-output* "~A ~A ~A~%" type name number)
(format *data-output* "author ~A~%" author)
;; (when requested-teams
;; (break))
;; (when requested-reviewers
;; (break))
(when assignees
(loop for assinee in assignees
for login = (getf assignee :|login|)
do (format *data-output*
"assignee ~A~%"
login)))
(when review-comments
(process-comments (get-all-data (getf data :|comments_url|))
"review-comment"))
(when comments
(process-comments (get-all-data (getf data :|comments_url|))))
(when merged-by
(format *data-output*
"merged-by ~A~%"
(getf merged-by
:|login|)))
;; (when mergeable
;; (break))
)
)
(defun process-issues (url &key (type "issue"))
(loop for item in (get-all-data url)
do (process-issue item :type type)))
(defun process-repository (data)
(format *data-output* "repository ~A ~A num-gets: ~A~%"
(getf data :|full_name|)
(getf data :|id|)
*number-of-gets*)
(process-issues (format-url (getf data :|pulls_url|))
:type "pull")
(process-issues (format-url (getf data :|issues_url|))))
;; (defun process-repositories (&optional (url "https://api.github.com/repositories"))
;; (uiop:with-output-file (*data-output* "data.log" :if-exists :append)
;; (format *data-output* "url ~A~%" url)
;; (loop for repository in (get-data url)
;; do (process-repository repository))))
(defun process-repositories (&optional (url "https://api.github.com/repositories"))
(uiop:with-output-file (*data-output* "data.log" :if-exists :append)
(format *data-output* "url ~A~%" url)
(with-all-data (url repository)
(process-repository repository))))
(defun process-user-repository (data)
(let ((pulls (get-all-data (format-url (getf data :|pulls_url|))))
(issues (get-all-data (format-url (getf data :|issues_url|)))))
(dump (list :type "repository"
:name (getf data :|full_name|)
:id (getf data :|id|)
:updated-at (getf data :|updated_at|)
:num-pulls (length pulls)
:open-pulls (length (remove-if-not (lambda (p)
(string-equal (getf p :|state|)
"open"))
pulls))
:num-issues (length issues)
:open-issues (length (remove-if-not (lambda (p)
(string-equal (getf p :|state|)
"open"))
issues))))))
(defun make-events-dict (events)
(loop with result = nil
for item in events
for type = (-> item
(getf :|type|)
(alexandria:make-keyword))
do (push item
(getf result type))
finally (return result)))
(defun process-user (login)
(let ((events (make-events-dict
(get-all-data (format nil
"https://api.github.com/users/~A/events"
login)))))
(dump (list :type "user"
:login login
:events events)))
(with-all-data ((format nil
"https://api.github.com/users/~A/repos?type=all"
login)
repository)
(process-user-repository repository))
(with-all-data ((format nil
"https://api.github.com/users/~A/orgs"
login)
organization)
(let ((org-name (getf organization :|login|)))
(dump (list :type "organization"
:name org-name))
(with-all-data ((format nil
"https://api.github.com/orgs/~A/repos"
org-name)
repository)
(process-user-repository repository)))))
(defun process-users ()
;; https://api.github.com/users?since=48000000
;; TODO: select n users randomly, and dump their data
)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment