package-catalog-analytics.rkt

#lang racket

(require net/url
         rebellion/base/immutable-string
         rebellion/binary/immutable-bytes
         rebellion/collection/entry
         rebellion/collection/list
         rebellion/collection/multidict
         rebellion/collection/multiset
         rebellion/type/record
         rebellion/type/tuple)

(define-record-type package-details
  (source
   checksum
   name
   authors
   description
   tags
   dependencies
   modules
   versions
   ring))

(define/contract (hash->package-details details-hash)
  (-> hash? package-details?)
  (package-details
   #:source (hash-ref details-hash 'source)
   #:checksum (hash-ref details-hash 'checksum)
   #:name (hash-ref details-hash 'name)
   #:authors (list->set (string-split (hash-ref details-hash 'author)))
   #:description (hash-ref details-hash 'description #f)
   #:tags (list->set (hash-ref details-hash 'tags))
   #:dependencies (list->set (hash-ref details-hash 'dependencies))
   #:modules (list->set (hash-ref details-hash 'modules))
   #:versions (hash-ref details-hash 'versions)
   #:ring (hash-ref details-hash 'ring)))

(define/contract (get-pkgs-all-bytes)
  (-> immutable-bytes?)
  (bytes->immutable-bytes
   (call/input-url (string->url "https://pkgs.racket-lang.org/pkgs-all")
                   (λ (resource) (get-pure-port resource #:redirections 5))
                   port->bytes)))

(define/contract (get-pkgs-all)
  (-> hash?)
  (with-input-from-bytes (get-pkgs-all-bytes) read))

(define/contract (get-package-details)
  (-> (set/c package-details?))
  (for/set ([details-hash (in-hash-values (get-pkgs-all))])
    (hash->package-details details-hash)))

(define/contract (get-authorship-graph)
  (-> multidict?)
  (for*/multidict ([pkg (in-immutable-set (get-package-details))]
                   [author (in-immutable-set (package-details-authors pkg))])
    (entry author (package-details-name pkg))))

(define/contract (get-tag-graph)
  (-> multidict?)
  (for*/multidict ([pkg (in-immutable-set (get-package-details))]
                   [tag (in-immutable-set (package-details-tags pkg))])
    (entry tag (package-details-name pkg))))

(define/contract (get-dependency-graph)
  (-> multidict?)
  (for*/multidict ([(name details) (in-hash (get-pkgs-all))]
                   [dep (in-list (hash-ref details 'dependencies))])
    (define dep-name
      (if (string? dep)
          dep
          (list-first dep)))
    (entry (string->immutable-string name)
           (string->immutable-string dep-name))))

(define/contract (get-clients pkg)
  (-> immutable-string? (set/c immutable-string?))
  (multidict-ref (multidict-inverse (get-dependency-graph)) pkg))

(define/contract (get-rebellion-clients)
  (-> (set/c immutable-string?))
  (multidict-ref (multidict-inverse (get-dependency-graph)) "rebellion"))

(define/contract (depends-on? details pkg)
  (-> hash? immutable-string? boolean?)
  (for/or ([dep (in-list (hash-ref details 'dependencies))])
    (equal? dep pkg)))

(define/contract (get-transitive-deps pkg
                                      #:graph [deps (get-dependency-graph)])
  (->* (immutable-string?) (#:graph multidict?)
       (set/c immutable-string?))
  (let loop ([unanalyzed-packages (set pkg)]
             [analyzed-packages (set)]
             [known-dependencies (set)])
    (cond
      [(set-empty? unanalyzed-packages) known-dependencies]
      [else
       (define next (set-first unanalyzed-packages))
       (define rest (set-rest unanalyzed-packages))
       (define new-direct-deps
         (set-subtract (multidict-ref deps next)
                       analyzed-packages
                       known-dependencies))
       (loop (set-union rest new-direct-deps)
             (set-add analyzed-packages next)
             (set-union known-dependencies new-direct-deps))])))

(define-record-type package-statistics
  (package
   direct-dependency-count
   direct-client-count
   transitive-dependency-count
   transitive-client-count))

(define-record-type package-relationships
  (package
   direct-dependencies
   direct-clients
   transitive-dependencies
   transitive-clients))

(define (get-package-relationships
         pkg
         #:dep-graph [graph (get-dependency-graph)]
         #:reverse-dep-graph [reverse-graph (multidict-inverse graph)])
  (define direct-deps (multidict-ref graph pkg))
  (define direct-clients (multidict-ref reverse-graph pkg))
  (define trans-deps (get-transitive-deps pkg #:graph graph))
  (define trans-clients
    (get-transitive-deps pkg #:graph reverse-graph))
  (package-statistics
   #:package pkg
   #:direct-dependency-count direct-deps
   #:direct-client-count direct-clients
   #:transitive-dependency-count trans-deps
   #:transitive-client-count trans-clients))

(define (get-package-statistics
         pkg
         #:dep-graph [graph (get-dependency-graph)]
         #:reverse-dep-graph [reverse-graph (multidict-inverse graph)])
  (define rels
    (get-package-relationships pkg
                               #:dep-graph graph
                               #:reverse-dep-graph reverse-graph))
  (package-statistics
   #:package pkg
   #:direct-dependency-count
   (set-count (package-relationships-direct-dependencies rels))
   #:direct-client-count
   (set-count (package-relationships-direct-clients rels))
   #:transitive-dependency-count
   (set-count (package-relationships-transitive-dependencies rels))
   #:transitive-client-count
   (set-count (package-relationships-transitive-clients rels))))

(define (top-packages-by-transitive-dep-count)
  (define deps (get-dependency-graph))
  (define reverse-deps (multidict-inverse deps))
  (define all-pkg-stats
    (for/list ([pkg (in-immutable-set (multidict-unique-keys deps))])
      (get-package-statistics pkg
                              #:dep-graph deps
                              #:reverse-dep-graph reverse-deps)))
  (define sorted-pkg-stats
    (sort all-pkg-stats > #:key package-statistics-transitive-dependency-count))
  (take sorted-pkg-stats 20))