get_twitter_following_ids.py

import requests
import json
from bs4 import BeautifulSoup as bs

def get_following_ids(user_id, username, cursor=-1):
    headers = {
        'authority': 'api.twitter.com',
        'origin': 'https://twitter.com',
        'x-twitter-client-language': 'ko',
        'x-csrf-token': '[TESTTOKEN]',
        'authorization': 'Bearer [TESTTOKEN]',
        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36',
        'x-twitter-auth-type': 'OAuth2Session',
        'x-twitter-active-user': 'yes',
        'dnt': '1',
        'accept': '*/*',
        'sec-fetch-site': 'same-site',
        'sec-fetch-mode': 'cors',
        'referer': f'https://twitter.com/{username}/following',
        'accept-encoding': 'gzip, deflate, br',
        'accept-language': 'ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7',
        'cookie': '[TESTTOKEN]',
    }

    params = (
        ('include_profile_interstitial_type', '1'),
        ('include_blocking', '1'),
        ('include_blocked_by', '1'),
        ('include_followed_by', '1'),
        ('include_want_retweets', '1'),
        ('include_mute_edge', '1'),
        ('include_can_dm', '1'),
        ('include_can_media_tag', '1'),
        ('skip_status', '1'),
        ('cursor', cursor),
        ('user_id', user_id),
        ('count', '5000'),
    )

    response = requests.get('https://api.twitter.com/1.1/friends/ids.json', headers=headers, params=params)
    return response.json()

def get_following_all(user_id, username, user_following_count=None):    
    cursor = -1
    dataset = []
    
    while cursor != 0: # 커서가 0이면 다음페이지가 없다.
        data = get_following_ids(user_id, username, cursor)
        try:
            cursor = data['next_cursor']
        except KeyError as e:
            print(data)
            if data.get('error') and data['error'] == 'Not authorized.':
                open(f"local_following_ids/protected/{user_id}", 'w').close()
                print(username, "Protected")
                return
            if data.get('errors') and data['errors'][0]['code'] == 34:
                open(f"local_following_ids/deleted/{user_id}", 'w').close()
                print(username, "Does Not exist")
                return
            print('='*50)
            print('Username:', username, 'User_id', user_id)
            print(data)
            print('='*50)
            raise e
        dataset += data['ids']
        print(f'{username}({user_id}): {len(set(dataset))} / {user_following_count}', end='\r')
    dataset = list(set(dataset))
    json.dump(dataset, open(f'local_following_ids/success/{user_id}.json', 'w+'))
    print(f'# Final for USER [{username}({user_id})]: {len(dataset)} / {user_following_count}')