Skip to content

Instantly share code, notes, and snippets.

@geotheory
Created December 8, 2019 11:51
Show Gist options
  • Save geotheory/d1bce1734ad27906221ed143ff3a6219 to your computer and use it in GitHub Desktop.
Save geotheory/d1bce1734ad27906221ed143ff3a6219 to your computer and use it in GitHub Desktop.
// Manually export Twitter followers/following data from browser as a CSV
// 1. navigate to followers/following page
// 2. change "target_count" variable value below to desired number
// 3. paste script into Javascript terminal and execute
// 4. once collection is complete and you see download dialogue box - confirm
// x. to stop manually at any time just execute "collect = false"
var collect = true;
var dat = {}
var sleep = 798;
// functions
function getElementByXpath(path, doc = null) {
if (doc == null){ doc = document; }
return document.evaluate(path, doc, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
}
function grabList(){
s = getElementByXpath("//section/div/div/*");
sc = s.children;
for(var i=0; i<sc.length; i++){
var n = sc[i];
var nt = n.textContent;
var handle = nt.substring(nt.indexOf('@'), nt.indexOf('Follow')).replace('@','');
if(!(handle in dat) && handle != ''){
var name = '"' + nt.substring(0, nt.indexOf('@')).replace(/"/g, '\'') + '"';
var N = 6;
if(nt.search('Following([a-zA-Z0-9]|$)') > -1){ N = 9 }
if(nt.search('Follows you') > -1) N += 11
var desc = '"' + nt.substring(nt.indexOf('Follow') + N).replace(/"/g, '\'') + '"';
dat[handle] = [handle, name, desc];
}
}
if(keys(dat).length >= target_count){saveOutput();}
return (speed_multiplier/100) * Math.floor(normalRandomScaled(1, .2, .6) * 500);
}
function saveOutput(){
collect = false;
var u = window.location.toString();
var fn = u.substring(20).replace('/','-') + '.csv';
dat2 = [];
datkeys = keys(dat);
for(var i=0; i<datkeys.length; i++){ dat2.push(dat[datkeys[i]].join(',')) }
dat3 = ['handle,name,desc'].concat(dat2).join('\n')
function download(data, filename, type) {
var file = new Blob([data], {type: type});
if (window.navigator.msSaveOrOpenBlob) // IE10+
window.navigator.msSaveOrOpenBlob(file, filename);
else { // Others
var a = document.createElement("a"),
url = URL.createObjectURL(file);
a.href = url;
a.download = filename;
document.body.appendChild(a);
a.click();
setTimeout(function() {
document.body.removeChild(a);
window.URL.revokeObjectURL(url);
}, 0);
}
}
download(dat3, fn, 'csv')
}
var spareRandom = null;
function normalRandom(){
var val, u, v, s, mul;
if(spareRandom !== null){
val = spareRandom;
spareRandom = null;
}
else { do
{
u = Math.random()*2-1;
v = Math.random()*2-1;
s = u*u+v*v;
} while(s === 0 || s >= 1);
mul = Math.sqrt(-2 * Math.log(s) / s);
val = u * mul;
spareRandom = v * mul;
}
return val;
}
function normalRandomScaled(mean, stddev, min){
var r = normalRandom();
r = r * stddev + mean;
return r > min? r:normalRandomScaled(mean*5, stddev*2, min*5)
}
//------------------------------------------------------------------------------
var target_count = 100;
var speed_multiplier = 100; // increase to slow collection for slower internet connections
// runs average 1 per second
window.setInterval(function(){
if(collect){
sleep = grabList();
window.scrollBy(0, 580);
}
}, sleep);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment