タイムライン取得 (Twitter API v1.1)

getTimeline.fsx

// get timeline
//
// run:
//   fsi getTimeline.fsx
//

open System
open System.IO
open System.Net

do
    System.Net.ServicePointManager.SecurityProtocol
        <- System.Net.ServicePointManager.SecurityProtocol
       ||| System.Net.SecurityProtocolType.Tls12


let apiUrl = "https://api.twitter.com/1.1/statuses/user_timeline.json"

let userAgent = "User Agent Name here"

let accessToken = "Access Token here" // https://api.twitter.com/oauth2/token

let doIt screen_name count max_id =

    let url = apiUrl
            + "?screen_name=" + screen_name
            + (if count > 0 then (sprintf "&count=%d" (min 200 count)) else "")
            + (if max_id <> "" then (sprintf "&max_id=%s" max_id) else "")
            + "&tweet_mode=extended"

    let req = WebRequest.Create(url) :?> HttpWebRequest

    req.UserAgent <- userAgent
    req.Headers.Add("Accept-Encoding", "gzip")
    req.Headers.Add("Authorization: Bearer " + accessToken)
    req.AutomaticDecompression <- DecompressionMethods.GZip

    let res = req.GetResponse() :?> HttpWebResponse
    let mutable ret = None
    if res.StatusCode <> HttpStatusCode.OK then
        eprintfn "Network Error: %A" res.StatusCode
        eprintfn "%s" res.StatusDescription
    else
        let rstm = res.GetResponseStream()
        let reader = new StreamReader(rstm)
        let text = reader.ReadToEnd()
        ret <- Some text
        reader.Close()
        rstm.Close()

    res.Close()
    ret


let getMinId text =
    let split (sep: string) (s: string) = s.Split([|sep|], System.StringSplitOptions.None)
    let f (s: string) = s.EndsWith("\"")
    let trim (s: string) = s.Trim('"')
    text
    |> split ",\"full_text\""
    |> Array.filter f
    |> Array.map (split "\"id_str\":")
    |> Array.filter (fun a -> Array.length a > 1)
    |> Array.map Array.last
    |> Array.sort
    |> fun a -> Array.get a (Array.length a / 4)
    |> trim


let downloads screen_name =
    let rec download tc mid =
        eprintfn "download... %d %s" tc mid
        System.Threading.Thread.Sleep(5000)
        if tc >= 3000 then
            ()
        else
            match doIt screen_name 200 mid with
            | None -> eprintfn "error in %d %s" tc mid
            | Some text ->
                let f = sprintf "timeline_%s_%04d.json" screen_name tc
                File.WriteAllText(f, text)
                let i = getMinId text
                download (tc + 150) i
    download 0 ""

do
    downloads "neetsdkasu"

取得した4分の1の位置のツイートIDから次のツイートを取得する
理由としては、このIDの抽出方法だとリツイートや引用ツイートのIDまで取得してしまい
それらのID次第では正しく取得できない・・・

ところで通信エラーが例外がスローされてて res.StatusCodeで取れないくさいんだけど？

例外スローのしんどさを感じた

&tweet_mode=extendedをつけないと、省略されたツイートの取得になる・・・

罠すぎる・・・

参考資料
https://stackoverflow.com/questions/46654189/twitter-api-truncating-tweet-by-adding-url