twitter_media_collector.gs

// 自分がTwitterにアップロードしたmediaを収集していく奴

var settings = {
    // oauth_* : https://dev.twitter.com/ 見て作る
    oauth_consumer_token: '*************************',
    oauth_consumer_secret: '**************************************************',
    oauth_access_token: '*******-******************************************',
    oauth_access_secret: '*********************************************',
    spread_sheet_id: '1UuGwEJZmaP1********************_WKpBD0Iyc7o', // スプレッドシートのURLに付いてる長い奴
    spread_sheet_meta: 'meta', // (データ管理用)↑にシートを追加し、付けたシート名を書いておく
    spread_sheet_data: 'data', // (収集先)↑にシートを追加し、付けたシート名を書いておく
    target_screen_name: 'dolpen', // 対象スクリーン名、自分か自分が見える相手
    read_count: 200 // 収集単位 <=200
};

var Objects = (function () {
    var u = function () {
    };
    u.prototype = {
        sort: function (opt) {
            var keys = Object.keys(opt);
            var result = {};
            keys.sort();
            keys.forEach(function (key) {
                result[key] = opt[key];
            });
            return result;
        },
        merge: function (o1, o2) {
            var result = {};
            Object.keys(o1).forEach(function (key) {
                result[key] = o1[key];
            });
            Object.keys(o2).forEach(function (key) {
                result[key] = o2[key];
            });
            return result;
        }
    };
    return new u();
})();

var SpreadSheet = (function () { // 完成した内部クラスの提供
    var c = function (sid, sname) {
        this.inner = SpreadsheetApp.openById(sid).getSheetByName(sname);
    };
    c.prototype = {
        addRows: function (rows) {
            var height = rows.length;
            if (height <= 0)return;
            var width = rows[0].length;
            if (width <= 0)return;
            var headRow = this.inner.getLastRow() + 1;
            Logger.log([headRow, 1, headRow + height - 1, width]);
            this.inner.getRange(headRow, 1, height, width).setValues(rows);
        },
        addRow: function (row) {
            this.addRows([row]);
        },
        setRow: function (row, toRow) {
            var width = row.length;
            if (width <= 0)return;
            this.inner.getRange(toRow, 1, 1, width).setValues([row]);
        },
        addCell: function (cell) {
            this.addRows([[cell]]);
        },
        getDataFromRow: function (targetRow) {
            return this.inner.getRange(targetRow, 1, 1, this.inner.getLastColumn()).getDisplayValues()[0];
        },
        getAllRows: function () {
            return this.inner.getRange(1, 1, this.inner.getLastRow(), this.inner.getLastColumn()).getDisplayValues();
        }
    };
    return c;
})();


var KVS = (function () {
    var e = function (k, v, r, s) {
        this.r = r;
        this.k = k;
        this.v = v;
        this.s = s;
    };
    e.prototype = {
        commit: function () {
            this.s.setRow([this.k, '' + this.v], this.r);
        },
        update: function (v) {
            this.v = v;
        },
        updateWithCommit: function (v) {
            this.v = v;
            this.s.setRow([this.k, '' + this.v], this.r);
        },
        get: function () {
            return this.v;
        },
    };
    var u = function (ss) {
        this.inner = ss;
        this.entities = null;
        this.next = 0;
        this._readEntities();
    };
    u.prototype = {
        _readEntities: function () {
            var rows = this.inner.getAllRows();
            var indexes = {};
            for (var i = 1; i < rows.length; i++) {
                indexes[rows[i][0]] = new e(
                    rows[i][0],
                    rows[i][1],
                    i + 1,
                    this.inner
                );
            }
            this.entities = indexes;
            this.next = rows.length;
        },
        getEntity: function (key) {
            var r = this.entities[key];
            if (!r) r = new e(
                key,
                '',
                ++this.next,
                this.inner
            );
            return r;
        }
    };
    return u;
})();


var Twitter = (function (ck, cs, at, as) {
    var nonce_chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
    var nonce_length = 32;
    var signature_method = 'HMAC-SHA1';
    var version = '1.0';
    // request object
    var r = function (url, method, data) {
        this.method = method;
        this.url = url;
        this.data = data || {};
    };
    r.prototype = {
        pickParam: function () { // pick paload
            var tmp = this.url.split('?');
            if (tmp.length === 1)
                return this.data;
            var arr = tmp[1].replace(/\+/g, ' ').split('&');
            var data = {};
            for (var i = 0; i < arr.length; i++) {
                var item = arr[i].split('=');
                data[item[0]] = decodeURIComponent(item[1]);
            }
            return Objects.merge(data, this.data);
        },
        pickEndpoint: function () { // pick base url
            return this.url.split('?')[0];
        },
        pickMethod: function () { // fix method name
            return this.method.toUpperCase();
        }
    };


    var u = function (ck, cs, at, as) {
        this.ck = ck;
        this.cs = cs;
        this.at = at;
        this.as = as;
    };
    u.prototype = {
        getData: function () {
            return {
                oauth_consumer_key: this.ck,
                oauth_token: this.at,
                oauth_nonce: this._getNonce(),
                oauth_signature_method: signature_method,
                oauth_timestamp: this._getTimeStamp(),
                oauth_version: version
            };
        },
        _getNonce: function () {
            var result = '';
            for (var i = 0; i < nonce_length; i++)
                result += nonce_chars[parseInt(Math.random() * nonce_chars.length, 10)];
            return result;

        },
        _getTimeStamp: function () {
            return Math.round(new Date().getTime() / 1000);
        },
        _objToString: function (obj) {
            var self = this;
            return Object.keys(Objects.sort(obj)).map(function (key) {
                return key + '=' + self._pEncode(obj[key]);
            }).join('&');
        },
        _objToHeader: function (obj) {
            var self = this;
            return Object.keys(Objects.sort(obj)).filter(function (key) {
                return key.indexOf('oauth_') == 0; // startsWith
            }).map(function (key) {
                return key + '="' + self._pEncode(obj[key]) + '"';
            }).join(', ');
        },
        _pEncode: function (str) {
            return encodeURIComponent(str)
                .replace(/!/g, '%21')
                .replace(/\*/g, '%2A')
                .replace(/'/g, '%27')
                .replace(/\(/g, '%28')
                .replace(/\)/g, '%29');
        },
        _getBaseString: function (req, data) {
            return req.pickMethod()
                + '&'
                + this._pEncode(req.pickEndpoint())
                + '&'
                + this._pEncode(this._objToString(
                    Objects.merge(
                        req.pickParam(),
                        data
                    )
                ));
        },
        _getSignKey: function () {
            return this._pEncode(this.cs) + '&' + this._pEncode(this.as);
        },
        _getSignature: function (baseString, key) {
            var sign = Utilities.computeHmacSignature(Utilities.MacAlgorithm.HMAC_SHA_1, baseString, key);
            return Utilities.base64Encode(sign);
        },
        _getAuthorizationHeader: function (req) {
            var data = this.getData();
            var sign = this._getSignature(
                this._getBaseString(req, data),
                this._getSignKey()
            );
            return 'OAuth ' + this._objToHeader(Objects.merge(data, {oauth_signature: sign}));
        },
        makeRequest: function (url, method, data) {
            return new r(url, method, data);
        },
        execRequest: function (url, method, data) {
            var req = this.makeRequest(url, method, data);
            Logger.log(this._getAuthorizationHeader(req));
            Logger.log(req.url);
            //return "{}";
            var resp = UrlFetchApp.fetch(req.url, {
                'method': req.method,
                'headers': {
                    'Authorization': this._getAuthorizationHeader(req)
                },
                'payload': req.data
            });
            var data = JSON.parse(resp.getContentText());
            return data;
        }

    };
    return new u(ck, cs, at, as);
})(
    settings.oauth_consumer_token,
    settings.oauth_consumer_secret,
    settings.oauth_access_token,
    settings.oauth_access_secret
);


function main() {
    var meta = new KVS(new SpreadSheet(settings.spread_sheet_id, settings.spread_sheet_meta));
    var data = new SpreadSheet(settings.spread_sheet_id, settings.spread_sheet_data);
    var since_id_cache = meta.getEntity('since_id');
    var since_id = since_id_cache.get();
    var since_id_query = (since_id == '' || since_id == '0') ? '' : ('&since_id=' + since_id);
    Logger.log(since_id);
    var url = 'https://api.twitter.com/1.1/statuses/user_timeline.json?trim_user=1&screen_name='
        + settings.target_screen_name
        + '&count='
        + settings.read_count
        + since_id_query;
    Logger.log(url);
    var resp = Twitter.execRequest(url, 'get', {});
    resp.reverse();
    var cache = [];
    resp.forEach(function (res) {
        if (!res.entities.media)return true;
        res.entities.media.forEach(function (med) {
            cache.push([res.id_str, med.media_url_https]);
        });
    });
    Logger.log(cache);
    data.addRows(cache);
    if (resp.length > 0) since_id_cache.updateWithCommit(resp[resp.length - 1].id_str);
}