Parses a Hackage package index and outputs a JSON description of dependencies. { package: { version: { dependency: bounds } } }

Dependencies.hs

{-
  stack
  --resolver lts-7
  --install-ghc
  runghc
  --package aeson
  --package bytestring
  --package containers
  --
  -Wall
-}

-- Usage:
-- $ stack Dependencies.hs latest.json > dependencies.json

import Data.Function ((&))

import qualified Data.Aeson as Aeson
import qualified Data.ByteString.Lazy as ByteString
import qualified Data.Map as Map
import qualified System.Environment as Environment

main :: IO ()
main = do
  [file] <- Environment.getArgs
  contents <- ByteString.readFile file
  let Right json = Aeson.eitherDecode contents
  (json :: Map.Map String (Map.Map String String))
    & Map.map Map.keys
    & Aeson.encode
    & ByteString.putStr

Latest.hs

{-
  stack
  --resolver lts-7
  --install-ghc
  runghc
  --package aeson
  --package bytestring
  --package containers
  --
  -Wall
-}

-- Usage:
-- $ stack Latest.hs packages.json > latest.json

import Data.Function ((&))

import qualified Data.Aeson as Aeson
import qualified Data.ByteString.Lazy as ByteString
import qualified Data.List as List
import qualified Data.Map as Map
import qualified Data.Maybe as Maybe
import qualified Data.Ord as Ord
import qualified Data.Version as Version
import qualified System.Environment as Environment
import qualified Text.ParserCombinators.ReadP as ReadP

main :: IO ()
main = do
  [file] <- Environment.getArgs
  contents <- ByteString.readFile file
  let Right json = Aeson.eitherDecode contents
  (json :: Map.Map String (Map.Map String (Map.Map String String)))
    & Map.mapMaybe (\ versions -> versions
      & Map.toList
      & Maybe.mapMaybe (\ (rawVersion, dependencies) -> do
        version <- readVersion rawVersion
        pure (version, dependencies))
      & safeMaximumBy (Ord.comparing fst)
      & fmap snd)
    & Aeson.encode
    & ByteString.putStr

readVersion :: String -> Maybe Version.Version
readVersion rawVersion = do
  let parses = ReadP.readP_to_S Version.parseVersion rawVersion
  parse <- safeLast parses
  case parse of
    (version, "") -> Just version
    _ -> Nothing

safeLast :: [a] -> Maybe a
safeLast xs =
  case xs of
    [] -> Nothing
    _ -> Just (last xs)

safeMaximumBy :: (a -> a -> Ordering) -> [a] -> Maybe a
safeMaximumBy f xs = case xs of
  [] -> Nothing
  _ -> Just (List.maximumBy f xs)

Main.hs

{-
  stack
  --resolver lts-7
  --install-ghc
  runghc
  --package aeson
  --package bytestring
  --package Cabal
  --package containers
  --package string-conv
  --package tar
  --package zlib
  --
  -Wall
-}

-- Usage:
-- $ wget https://hackage.haskell.org/packages/index.tar.gz
-- $ stack Main.hs index.tar.gz > packages.json

import Data.Function ((&))

import qualified Codec.Archive.Tar as Tar
import qualified Codec.Compression.GZip as GZip
import qualified Data.Aeson as Aeson
import qualified Data.ByteString.Lazy as ByteString
import qualified Data.Map as Map
import qualified Data.Maybe as Maybe
import qualified Data.String.Conv as Conv
import qualified Data.Version as Version
import qualified Distribution.Package as Cabal
import qualified Distribution.PackageDescription as Cabal
import qualified Distribution.PackageDescription.Parse as Cabal
import qualified Distribution.Version as Cabal
import qualified System.Environment as Environment

main :: IO ()
main = do
  [file] <- Environment.getArgs
  contents <- ByteString.readFile file
  let archive = GZip.decompress contents
  let entries = Tar.read archive
  let packages = Tar.foldEntries foldEntry Map.empty (\_ -> Map.empty) entries
  let json = Aeson.encode packages
  ByteString.putStr json

foldEntry :: Tar.Entry -> Packages -> Packages
foldEntry entry packages =
  case Tar.entryContent entry of
    Tar.NormalFile contents _ ->
      case parsePackage contents of
        Just package -> insertPackage package packages
        _ -> packages
    _ -> packages

type Packages = Map.Map Name (Map.Map Version Dependencies)

type Name = String

type Version = String

type Dependencies = Map.Map Name Bounds

type Bounds = String

parsePackage :: ByteString.ByteString -> Maybe Package
parsePackage bytes = do
  let contents = Conv.toSL bytes
  case Cabal.parsePackageDescription contents of
    Cabal.ParseOk _ package -> Just package
    _ -> Nothing

type Package = Cabal.GenericPackageDescription

insertPackage :: Package -> Packages -> Packages
insertPackage package packages =
  let name = getName package
      version = getVersion package
      dependencies = getDependencies package
      alter maybeElement =
        let newElement =
              case maybeElement of
                Nothing -> Map.singleton version dependencies
                Just element -> Map.insert version dependencies element
        in Just newElement
  in Map.alter alter name packages

getName :: Package -> Name
getName package =
  package & Cabal.packageDescription & Cabal.package & Cabal.pkgName &
  Cabal.unPackageName

getVersion :: Package -> Version
getVersion package =
  package & Cabal.packageDescription & Cabal.package & Cabal.pkgVersion &
  Version.showVersion

getDependencies :: Package -> Dependencies
getDependencies package =
  package & Cabal.condLibrary & fmap Cabal.condTreeConstraints &
  Maybe.fromMaybe [] &
  map formatDependency &
  Map.fromList

formatDependency :: Cabal.Dependency -> (Name, Bounds)
formatDependency dependency =
  case dependency of
    Cabal.Dependency name range ->
      (Cabal.unPackageName name, formatRange range)

formatRange :: Cabal.VersionRange -> Bounds
formatRange range =
  Cabal.foldVersionRange
    ""
    (\version -> "== " ++ Version.showVersion version)
    (\version -> "> " ++ Version.showVersion version)
    (\version -> "< " ++ Version.showVersion version)
    (\this that -> this ++ " || " ++ that)
    (\this that -> this ++ " && " ++ that)
    range

Reverse.hs

{-
  stack
  --resolver lts-7
  --install-ghc
  runghc
  --package aeson
  --package bytestring
  --package containers
  --
  -Wall
-}

-- Usage:
-- $ stack Reverse.hs dependencies.json > reverse.json

import Data.Function ((&))

import qualified Data.Aeson as Aeson
import qualified Data.ByteString.Lazy as ByteString
import qualified Data.List as List
import qualified Data.Map as Map
import qualified System.Environment as Environment

main :: IO ()
main = do
  [file] <- Environment.getArgs
  contents <- ByteString.readFile file
  let Right json = Aeson.eitherDecode contents
  (json :: Map.Map String [String])
    & Map.toList
    & concatMap (\ (package, dependencies) -> dependencies
      & map (\ dependency -> (dependency, [package])))
    & Map.fromListWith (++)
    & Map.map List.sort
    & Aeson.encode
    & ByteString.putStr

run.sh

set -o errexit -o xtrace

# Get the package index.
# Only download it once because it takes a while.
# Result is about 14 MB.
if ! test -f index.tar.gz
then
  wget https://hackage.haskell.org/packages/index.tar.gz
fi

# Get dependency information from the index.
# Parsing all the package descriptions is really slow.
# That's why this compiles instead of using runghc.
# Result is about 23 MB.
stack --resolver lts-7 --install-ghc ghc \
  --package aeson \
  --package bytestring \
  --package Cabal \
  --package containers \
  --package string-conv \
  --package tar \
  --package zlib \
  -- -O2 -threaded -Wall Main.hs
time ./Main index.tar.gz +RTS -N > packages.min.json
jq . < packages.min.json > packages.json
rm packages.min.json

# Get dependency information for the latest version of each package.
# Result is about 2.5 MB.
stack Latest.hs packages.json | jq . > latest.json

# Strip the bounds information from the dependency list.
# Result is about 1.3 MB.
stack Dependencies.hs latest.json | jq . > dependencies.json

# Generate reverse dependency information.
# Result is about 1.5 MB.
stack Reverse.hs dependencies.json | jq . > reverse.json