Twitter async web services with F# and ServiceStack running on Windows/OSX/Linux

FTweetStack.fs

(*  Stand-alone Twitter API ServiceStack Web Service with F# on Mono/OSX hosted by HttpListener

    Instructions:
      1) download https://github.com/ServiceStack/ServiceStack/downloads
      2) fsharpc -r:ServiceStack.Common.dll -r:ServiceStack.Interfaces.dll -r:ServiceStack.Text.dll -r:ServiceStack.dll FTweetStack.fs
      3) sudo mono FTweetStack.exe
*)

open System
open ServiceStack.ServiceHost
open ServiceStack.WebHost.Endpoints

type System.Net.WebRequest with
    member x.GetResponseAsync() = Async.FromBeginEnd(x.BeginGetResponse, x.EndGetResponse)
                
let asyncHttp (url:string) =
    async {
        //printfn "downloading: %s" url
        let req = System.Net.WebRequest.Create(url)
        let! rsp = req.GetResponseAsync()
        use stream = rsp.GetResponseStream()
        use reader = new System.IO.StreamReader(stream)
        return reader.ReadToEnd() }

let asyncJson url =
    async {
        try
            return! asyncHttp url
        with 
            | ex -> printfn "Error downloading: %s => %s" url ex.Message; return "[]" }

let batchesOf size (s: seq<'v>) =
    seq {
        let en = s.GetEnumerator ()
        let more = ref true
        while !more do
        let group = [
            let i = ref 0
            while !i < size && en.MoveNext () do
                yield en.Current
                i := !i + 1 ]
        if List.isEmpty group then
            more := false
        else
            yield group }

let joinWith delim seq =
    let sb = new System.Text.StringBuilder()
    seq |> Seq.iter (fun x -> sb.Append((if sb.Length > 0 then delim else "") + x.ToString()) |> ignore)
    sb.ToString()
let join seq = joinWith "," seq

let jsonTo<'a> json = ServiceStack.Text.JsonSerializer.DeserializeFromString<'a>(json)

module FTweetStack =

    type UserStat = {
        mutable id: uint64;
        mutable screen_name: string;
        mutable name: string;
        mutable friends_count: int;
        mutable followers_count: int;
        mutable listed_count: int;
        mutable statuses_count: int }

    type TwitterIds = { mutable ids: uint64[]; }    

    let asyncUsers      screenNames = asyncJson("http://api.twitter.com/1/users/lookup.json?screen_name=" + (screenNames |> join)) 
    let asyncFollowerIds screenName = asyncJson("http://api.twitter.com/1/followers/ids.json?screen_name=" + screenName)
    let asyncFriendIds   screenName = asyncJson("http://api.twitter.com/1/friends/ids.json?screen_name=" + screenName)
    let asyncUserIds        userIds = asyncJson("http://api.twitter.com/1/users/lookup.json?user_id=" + (userIds |> join)) 
    
    let userCache = System.Collections.Concurrent.ConcurrentDictionary<uint64,UserStat>()

    let usersByIds userIds = 
        let cachedIds, missingIds = userIds |> List.partition userCache.ContainsKey
        missingIds |> batchesOf 100 
            |> Seq.map asyncUserIds
            |> Async.Parallel |> Async.RunSynchronously
            |> Seq.map jsonTo<UserStat[]>
            |> Seq.collect (fun xs -> xs |> Seq.map (fun x -> userCache.TryAdd(x.id, x) |> ignore; x)) |> Seq.toList  
            |> List.append (cachedIds |> List.map (fun x -> userCache.TryGetValue x |> snd))

    let followerIds   screenName = (asyncFollowerIds screenName |> Async.RunSynchronously |> jsonTo<TwitterIds>).ids |> Array.toList
    let friendIds     screenName = (asyncFriendIds screenName   |> Async.RunSynchronously |> jsonTo<TwitterIds>).ids |> Array.toList        
    let followers     screenName = followerIds screenName |> usersByIds
    let friends       screenName = friendIds screenName   |> Seq.toList |> usersByIds
    let usersByNames screenNames = asyncUsers screenNames |> Async.RunSynchronously |> jsonTo<UserStat[]> |> Seq.toList

    //Define Web Services
    type UserStats = { mutable ScreenNames: string; }
    type UserStatsService() =
        interface IService<UserStats> with
            member this.Execute (req:UserStats) = req.ScreenNames.Split(',') |> usersByNames :> Object
    
    type FollowerStats = { mutable ScreenName: string; }
    type FollowerStatsService() =
        interface IService<FollowerStats> with
            member this.Execute (req:FollowerStats) = followers req.ScreenName :> Object
    
    type FriendsStats = { mutable ScreenName: string; }
    type FriendsStatsService() =
        interface IService<FriendsStats> with
            member this.Execute (req:FriendsStats) = friends req.ScreenName :> Object

open FTweetStack

//Define the Web Services AppHost
type AppHost =
    inherit AppHostHttpListenerBase
    new() = { inherit AppHostHttpListenerBase("FTweetStack Services", typeof<UserStatsService>.Assembly ) }
    override this.Configure container =
        base.Routes
            .Add<UserStats>("/users/{ScreenNames}") 
            .Add<FollowerStats>("/followers/{ScreenName}") 
            .Add<FriendsStats>("/friends/{ScreenName}") 
                |> ignore

[<EntryPoint>]
let main args =
    let host = if args.Length = 0 then "http://*:8080/" else args.[0]
    printfn "listening on %s ..." host
    let appHost = new AppHost()
    appHost.Init()
    appHost.Start host
    Console.ReadLine() |> ignore
    0