Twitter Web Clientでツイートデータ収集用スクリプト(メンマさんお誕生日ツイート取得用スクリプト付き)

search.ts

var csv;
function searchMenmaAll(){
    csv = "\"id\",\"tweet\",\"time\",\"user id\",\"screen name\",\"name\",\"protected\",\"reply to menmanist\"\n";
    var callback = function(data: Array<Tweet>){
            if(data.length){
                for(var i=0;i<data.length;i++){
                    csv +=  "\""+ data[i].id + "\",\"" +
                            data[i].tweet + "\",\"" +
                            data[i].time_ms + "\",\"" +
                            data[i].user.id + "\",\"" +
                            data[i].user.screenname + "\",\"" +
                            data[i].user.name + "\",\"" +
                            data[i].user.isProtected + "\",\"" +
                            /staring_inferno|menmanist/.test(data[i].mentions) +
                            "\"\n";
                }
                console.log(new Date(data[data.length-1].time_ms));
                searchPrevious("メンマ氏 OR menmanist OR staring_inferno OR メンマさん  誕生日 OR お誕生日 -from:menmanist -from:staring_inferno since:2013-05-17",
                    callback, data[data.length-1].id);
            } else {
                open('data:text/csv;charset=utf-8,' + encodeURI(csv));
            }
        };
    searchRecent("メンマ氏 OR menmanist OR staring_inferno OR メンマさん  誕生日 OR お誕生日 -from:menmanist -from:staring_inferno since:2013-05-17", callback);
}

function searchMenmaSince(tweetId: string) {
    csv = "\"id\",\"tweet\",\"time\",\"user id\",\"screen name\",\"name\",\"protected\",\"reply to menmanist\"\n";
    var callback = function(data: Array<Tweet>){
            if(data.length && Long.compare(new Long(data[data.length - 1].id), new Long(tweetId)) == 1){
                for(var i=0;i<data.length;i++){
                    csv +=  "\""+ data[i].id + "\",\"" +
                            data[i].tweet + "\",\"" +
                            data[i].time_ms + "\",\"" +
                            data[i].user.id + "\",\"" +
                            data[i].user.screenname + "\",\"" +
                            data[i].user.name + "\",\"" +
                            data[i].user.isProtected + "\",\"" +
                            /staring_inferno|menmanist/.test(data[i].mentions) +
                            "\"\n";
                }
                console.log(new Date(data[data.length-1].time_ms));
                searchPrevious("メンマ氏 OR menmanist OR staring_inferno OR メンマさん  誕生日 OR お誕生日 -from:menmanist -from:staring_inferno since:2013-05-17",
                    callback, data[data.length-1].id);
            } else {
                if(data.length){
                    for(var i=0; Long.compare(new Long(data[i].id), new Long(tweetId)) == 1; i++){
                        csv +=  "\""+ data[i].id + "\",\"" +
                            data[i].tweet + "\",\"" +
                            data[i].time_ms + "\",\"" +
                            data[i].user.id + "\",\"" +
                            data[i].user.screenname + "\",\"" +
                            data[i].user.name + "\",\"" +
                            data[i].user.isProtected + "\",\"" +
                            /staring_inferno|menmanist/.test(data[i].mentions) +
                            "\"\n";
                    }
                }
                open('data:text/csv;charset=utf-8,' + encodeURI(csv));
            }
        };
    searchRecent("メンマ氏 OR menmanist OR staring_inferno OR メンマさん  誕生日 OR お誕生日 -from:menmanist -from:staring_inferno since:2013-05-17", callback);
}

class Long{
    downer: number;
    upper: number;
    constructor(num: string){
        if(num.length>16){
            this.downer = Number(num.slice(-15));
            this.upper = Number(num.slice(0,-15));
        }
    }

    increment(): Long{
        this.downer++;
        if(String(this.downer).length==16){
            this.downer = Number(String(this.downer).slice(-15));
            this.upper++;
        }
        return this;
    }

    decrement(): Long{
        if(this.downer==0){
            this.downer = 999999999999999;
            this.upper--;
        } else {
            this.downer--;
        }
        return this;
    }

    static compare(a: Long, b: Long): number {
        return (a.upper > b.upper) ? 1 : (a.upper < b.upper) ? -1 : (a.downer > b.downer) ? 1 : (a.downer < b.downer) ? -1 : 0;
    }

    toString(): string{

        return String(this.upper)+("000000000000000"+String(this.downer)).slice(-15);
    }
}

class Tweet{
    id: string;
    tweet: string;
    time_ms: number;
    user: User;
    mentions: string;
}

class User{
    name: string;
    screenname: string;
    id: string;
    isProtected: boolean;
}

function searchRecent(word: string, callback: Function){
    var url = "https://twitter.com/i/search/timeline?f=realtime&q="
            +encodeURI(word)
            +"&src=typd";
    search(url, callback);
}

function searchPrevious(word: string, callback: Function, id: string){
    var url = "https://twitter.com/i/search/timeline?f=realtime&q="
            +encodeURI(word)
            +"&src=typd&include_available_features=1&include_entities=1&last_note_ts="
            +parseInt((Number(new Date())/1000).toString(10))
            +"&scroll_cursor=TWEET-"
            +id
            +"-"
            +new Long(id).increment().toString()
            +"-BD1UO2FFu9QAAAAAAAAETAAAAAcAAAASAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
    console.log(url);
    search(url, callback);
}

function searchFollowing(word: string, callback: Function, id: string){
    var url = "https://twitter.com/i/search/timeline?f=realtime&q="
            +encodeURI(word)
            +"&src=typd&include_available_features=1&include_entities=1&include_new_items_bar=true&last_note_ts="
            +parseInt((new Date().valueOf()/1000).toString(10))
            +"&refresh_cursor=TWEET-"
            +new Long(id).decrement().toString()
            +"-"
            +id
            +"-BD1UO2FFu9QAAAAAAAAETAAAAAcAAAASAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
    console.log(url);
    search(url, callback);
}

function search(url: string, callback: Function){
    delete console.log;
    var xhr = new XMLHttpRequest();
    xhr.open("GET", url, true);
    xhr.responseType = "json";
    xhr.addEventListener("load", function (ev: any) {
        if (ev.target.status == 200) {
            var div: HTMLDivElement = document.createElement("div");
            div.innerHTML = xhr.response.items_html;
            var array = new Array();
            for(var i=0;i<div.children.length;i++){
                var child: HTMLElement = <HTMLElement>div.children[i];
                if(child.getAttribute("data-item-type")!="tweet"){
                    continue;
                }
                var tweet: Tweet = new Tweet();
                tweet.id = child.getAttribute("data-item-id");
                tweet.tweet = (<HTMLElement>child.getElementsByClassName("js-tweet-text")[0]).innerText.replace(/\n/g," ").replace(/&lt/g,"<").replace(/&gt;/g,">").replace(/&quot;|"/g,"\"\"").replace(/&#039;/g,"'").replace(/&amp;/g,"&");
                tweet.time_ms = Number((<HTMLElement>child.getElementsByClassName("_timestamp")[0]).getAttribute("data-time-ms"));
                var user: User = new User();
                user.name = (<HTMLElement>child.getElementsByClassName("tweet")[0]).getAttribute("data-name").replace(/\n/g," ").replace(/&lt/g,"<").replace(/&gt;/g,">").replace(/&quot;|"/g,"\"\"").replace(/&#039;/g,"'").replace(/&amp;/g,"&");
                user.id = (<HTMLElement>child.getElementsByClassName("tweet")[0]).getAttribute("data-user-id");
                user.screenname = (<HTMLElement>child.getElementsByClassName("tweet")[0]).getAttribute("data-screen-name");
                user.isProtected = (<HTMLElement>child.getElementsByClassName("tweet")[0]).getAttribute("data-protected")=="true";
                tweet.mentions = (<HTMLElement>child.getElementsByClassName("tweet")[0]).getAttribute("data-mentions");
                tweet.user = user;
                array.push(tweet);
            }
            delete div;
            callback(array);
        } else {
            alert("There was a problem with the URL.");
        }
        delete xhr;
    });
    xhr.send(null);
}