Last active
September 27, 2015 08:38
-
-
Save mythz/1242208 to your computer and use it in GitHub Desktop.
Twitter async web services with F# and ServiceStack running on Windows/OSX/Linux
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
(* Stand-alone Twitter API ServiceStack Web Service with F# on Mono/OSX hosted by HttpListener | |
Instructions: | |
1) download https://github.com/ServiceStack/ServiceStack/downloads | |
2) fsharpc -r:ServiceStack.Common.dll -r:ServiceStack.Interfaces.dll -r:ServiceStack.Text.dll -r:ServiceStack.dll FTweetStack.fs | |
3) sudo mono FTweetStack.exe | |
*) | |
open System | |
open ServiceStack.ServiceHost | |
open ServiceStack.WebHost.Endpoints | |
type System.Net.WebRequest with | |
member x.GetResponseAsync() = Async.FromBeginEnd(x.BeginGetResponse, x.EndGetResponse) | |
let asyncHttp (url:string) = | |
async { | |
//printfn "downloading: %s" url | |
let req = System.Net.WebRequest.Create(url) | |
let! rsp = req.GetResponseAsync() | |
use stream = rsp.GetResponseStream() | |
use reader = new System.IO.StreamReader(stream) | |
return reader.ReadToEnd() } | |
let asyncJson url = | |
async { | |
try | |
return! asyncHttp url | |
with | |
| ex -> printfn "Error downloading: %s => %s" url ex.Message; return "[]" } | |
let batchesOf size (s: seq<'v>) = | |
seq { | |
let en = s.GetEnumerator () | |
let more = ref true | |
while !more do | |
let group = [ | |
let i = ref 0 | |
while !i < size && en.MoveNext () do | |
yield en.Current | |
i := !i + 1 ] | |
if List.isEmpty group then | |
more := false | |
else | |
yield group } | |
let joinWith delim seq = | |
let sb = new System.Text.StringBuilder() | |
seq |> Seq.iter (fun x -> sb.Append((if sb.Length > 0 then delim else "") + x.ToString()) |> ignore) | |
sb.ToString() | |
let join seq = joinWith "," seq | |
let jsonTo<'a> json = ServiceStack.Text.JsonSerializer.DeserializeFromString<'a>(json) | |
module FTweetStack = | |
type UserStat = { | |
mutable id: uint64; | |
mutable screen_name: string; | |
mutable name: string; | |
mutable friends_count: int; | |
mutable followers_count: int; | |
mutable listed_count: int; | |
mutable statuses_count: int } | |
type TwitterIds = { mutable ids: uint64[]; } | |
let asyncUsers screenNames = asyncJson("http://api.twitter.com/1/users/lookup.json?screen_name=" + (screenNames |> join)) | |
let asyncFollowerIds screenName = asyncJson("http://api.twitter.com/1/followers/ids.json?screen_name=" + screenName) | |
let asyncFriendIds screenName = asyncJson("http://api.twitter.com/1/friends/ids.json?screen_name=" + screenName) | |
let asyncUserIds userIds = asyncJson("http://api.twitter.com/1/users/lookup.json?user_id=" + (userIds |> join)) | |
let userCache = System.Collections.Concurrent.ConcurrentDictionary<uint64,UserStat>() | |
let usersByIds userIds = | |
let cachedIds, missingIds = userIds |> List.partition userCache.ContainsKey | |
missingIds |> batchesOf 100 | |
|> Seq.map asyncUserIds | |
|> Async.Parallel |> Async.RunSynchronously | |
|> Seq.map jsonTo<UserStat[]> | |
|> Seq.collect (fun xs -> xs |> Seq.map (fun x -> userCache.TryAdd(x.id, x) |> ignore; x)) |> Seq.toList | |
|> List.append (cachedIds |> List.map (fun x -> userCache.TryGetValue x |> snd)) | |
let followerIds screenName = (asyncFollowerIds screenName |> Async.RunSynchronously |> jsonTo<TwitterIds>).ids |> Array.toList | |
let friendIds screenName = (asyncFriendIds screenName |> Async.RunSynchronously |> jsonTo<TwitterIds>).ids |> Array.toList | |
let followers screenName = followerIds screenName |> usersByIds | |
let friends screenName = friendIds screenName |> Seq.toList |> usersByIds | |
let usersByNames screenNames = asyncUsers screenNames |> Async.RunSynchronously |> jsonTo<UserStat[]> |> Seq.toList | |
//Define Web Services | |
type UserStats = { mutable ScreenNames: string; } | |
type UserStatsService() = | |
interface IService<UserStats> with | |
member this.Execute (req:UserStats) = req.ScreenNames.Split(',') |> usersByNames :> Object | |
type FollowerStats = { mutable ScreenName: string; } | |
type FollowerStatsService() = | |
interface IService<FollowerStats> with | |
member this.Execute (req:FollowerStats) = followers req.ScreenName :> Object | |
type FriendsStats = { mutable ScreenName: string; } | |
type FriendsStatsService() = | |
interface IService<FriendsStats> with | |
member this.Execute (req:FriendsStats) = friends req.ScreenName :> Object | |
open FTweetStack | |
//Define the Web Services AppHost | |
type AppHost = | |
inherit AppHostHttpListenerBase | |
new() = { inherit AppHostHttpListenerBase("FTweetStack Services", typeof<UserStatsService>.Assembly ) } | |
override this.Configure container = | |
base.Routes | |
.Add<UserStats>("/users/{ScreenNames}") | |
.Add<FollowerStats>("/followers/{ScreenName}") | |
.Add<FriendsStats>("/friends/{ScreenName}") | |
|> ignore | |
[<EntryPoint>] | |
let main args = | |
let host = if args.Length = 0 then "http://*:8080/" else args.[0] | |
printfn "listening on %s ..." host | |
let appHost = new AppHost() | |
appHost.Init() | |
appHost.Start host | |
Console.ReadLine() |> ignore | |
0 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment