about.md

1. Download your Twitter archive

You can find this feature in Settings > Download Twitter Archive. It might take 24 hours to receive. Unzip the file and open the data folder in your terminal:

cd ~/Downloads/twitter-archive-zip-you-downloaded/data

(I have seen reports that this function may no longer be working, so this guide is mostly useful to those who were lucky enough to already have downloaded their archive.)

2. Setup Twitter API

Go to Developer Portal and create a new App + Project with OAuth2 configuration. Save the API Keys into a .env file in the same folder as data (which has all your downloaded Twitter data and .js files).

TWITTER_CONSUMER_KEY="..."
TWITTER_CONSUMER_SECRET="..."

3. Run script

Copy the scrape_followers.js file into the data folder and then install deps:

npm init -y
npm install oauth axios dotenv

Then:

node scrape_followers.js

This will only save your following (people you follow) which hopefully is small enough that you can capture the whole thing without getting rate limited. If you see an error in terminal it might be that you have hit a rate limit (max 400-500 requests per day or something, and each request in this script grabs 100 followers at once).

It will save it to a file called scraped_following.json and scraped_follower.json which includes more details about each account.

scrape_followers.js

require('dotenv/config')

const axios = require('axios')
const fs = require('fs/promises');
const path = require('path');

const OAuth = require('oauth')
const { promisify } = require('util')

global.window = {
  YTD: {
    following: {
      part0: []
    },
    follower: {
      part0: []
    }
  }
}
require('./following.js');
require('./follower.js');
const following = window.YTD.following.part0.map(d => d.following)
const follower = window.YTD.follower.part0.map(d => d.follower);

async function fetchUserIds (opts = {}) {
  const {
    ids = []
  } = opts;
  var oauth2 = new OAuth.OAuth2(
    process.env.TWITTER_CONSUMER_KEY,
    process.env.TWITTER_CONSUMER_SECRET,
    'https://api.twitter.com/', null, 'oauth2/token', null
  )
  const getOAuthAccessToken = promisify(oauth2.getOAuthAccessToken.bind(oauth2))
  const accessToken = await getOAuthAccessToken('', { grant_type: 'client_credentials' })

  return axios.get(`https://api.twitter.com/2/users?ids=${ids.join(',')}&user.fields=created_at,description,id,location,name,url,username`, {
    headers: {
      Authorization: `Bearer ${accessToken}`
    }
  })
}

async function fetchChunked (list) {
  const followChunks = [];
  let chunk = [];
  list.forEach(k => {
    if (chunk.length >= 100) {
      followChunks.push(chunk);
      chunk = [];
    }
    chunk.push(k);
  })
  if (chunk.length > 0) followChunks.push(chunk)
  
  console.log('Total:', list.length);
  console.log('Pages:', followChunks.length);
  const result = [];
  for (let chunk of followChunks) {
    const ids = chunk.map(c => c.accountId);
    console.log('Expected #', ids.length)
    const userData = await fetchUserIds({ ids })
    console.log('Got #', userData.data.data.length)
    userData.data.data.forEach(d => {
      result.push(d);
    });
  }
  return result;
}

(async() => {
  let r = await fetchChunked(following)
  let file = path.resolve(__dirname, 'scraped_following.json')
  await fs.writeFile(file, JSON.stringify(r, null, 2))
  
  r = await fetchChunked(follower)
  file = path.resolve(__dirname, 'scraped_follower.json')
  await fs.writeFile(file, JSON.stringify(r, null, 2))
})();