beautifulSoup.py

beautifulSoup.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
##=========================================================
##  beautifulSoup.py                               Dec 2018
##  Eli Innis   @Doyousketch2      Doyousketch2 @ yahoo.com
##  GNU AGPLv3            gnu.org/licenses/agpl-3.0.en.html

""" required  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""

##  sudo pip3 install requests beautifulsoup4 html5lib lxml
##  (or)
##  sudo python3 -m pip install requests beautifulsoup4 html5lib lxml
##  (or)
##  easy_install requests beautifulsoup4 html5lib lxml

"""  libs  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""

import re
import requests
from bs4 import BeautifulSoup

"""  vars  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""

name  = 'YAHOONAME'
domain  = '@yahoo.com'
email_address  = name +domain
password = 'PASSWORD'

parsers = ('html.parser', 'lxml', 'lxml-xml', 'html5lib')
parser = parsers[3]

b_url = 'https://www.yahoo.com/'
login_url = 'https://login.yahoo.com/'

pw_url = 'https://login.yahoo.com/account/challenge/password'
init_params = '?authMechanism=primary&display=login&browser-fp-data&passwordContext=normal&verifyPassword=Sign%20in&done=https%3A%2F%2Fwww.yahoo.com%2F&yid='

url = 'https://football.fantasysports.yahoo.com'

headers = { 'DNT': '1',
           'Host': 'yahoo.com',
          'Connection': 'keep-alive',
         'Upgrade-Insecure-Requests': '1',
        'Accept-Encoding': 'gzip, deflate, br',
       'Accept-Language':'en-US,en;q=0.5',
      'Content-Type': 'application/x-www-form-urlencoded',
     'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'  }

"""  get part of the cookie from main webpage  ~~~~~~~~~"""

with requests.Session() as s:
    page_1 = s.get( b_url, headers = headers, allow_redirects = False )
    soup = BeautifulSoup( page_1.content, parser )

    B = page_1.cookies['B']
    headers['Host'] = 'login.yahoo.com'

    """  username webpage  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""

    page_2 = s.get( login_url, headers = headers )
    soup = BeautifulSoup( page_2.content, parser )

    sessionIndex = soup.form.find( 'input', attrs={'name': 'sessionIndex'} )
    acrumb = soup.form.find( 'input', attrs={'name': 'acrumb'} )

    sess = '&sessionIndex=' +str( sessionIndex['value'] )
    acr = '&acrumb=' +str( acrumb['value'] )

    fail_url = pw_url +init_params +name +sess +acr
    headers['Referer'] = login_url

    AS = page_2.cookies['AS']

    """  fail the password webpage to get crumb  ~~~~~~~"""

    page_3 = s.get(  fail_url,
                     headers = headers,
                     cookies = page_1.cookies )
    soup = BeautifulSoup( page_3.content, parser )

    top, filling = re.split( 'crumb=', str(soup.noscript.img) )
    crumb, bottom = re.split( '&', filling )

    params = '&crumb=' +crumb +'&passwd=' +password +'&displayName=' +email_address
    password_url = fail_url +sess +acr +params

    """  password webpage  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""

    ## headers['Content-Length'] = str( len(password_url) -len(pw_url) )

    page_4 = s.get(  password_url,
                     headers = headers,
                     cookies = {'AS': AS, 'B': B}  )
    soup = BeautifulSoup( page_4.content, parser )

    print( soup, '\n' )
    print( 'Status code, Reason:', page_4.status_code, page_4.reason )
    ## print( 'Content-Length:', headers['Content-Length'] )