Crawls Bloc Alliances for Economy and Military info.

Bloc-Alliance-Crawler.py

from bs4 import BeautifulSoup
import aiohttp
import asyncio
from datetime import datetime
import csv

loop = asyncio.get_event_loop()
client = aiohttp.ClientSession(loop=loop)
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:51.0) Gecko/20100101 Firefox/51.0'}
BLOC = 'http://blocgame.com'
DATA = {'username': 'USERNAME', 'password': 'PASSWORD', 'login': ''}


class Nation:
    def __init__(self, soup):
        self.name = soup.find(id='nationtitle').text
        self.gdp = int(soup.find(text='Gross Domestic Product:').parent
                           .next_sibling.next_sibling.contents[0]
                           .string[1:][:4])

        factoryString = soup.find(text='Industry:').parent.next_sibling\
                            .next_sibling.contents[0].string

        uniString = soup.find(text='Universities:').parent.next_sibling\
                        .next_sibling.contents[0].string

        if uniString == 'None':
            self.universities = 0
        elif uniString == '1 university':
            self.universities = 1
        else:
            self.universities = int(uniString[:-13])

        # And Literacy

        if factoryString == 'None':
            self.factories = 0
        elif factoryString == '1 factory':
            self.factories = 1
        else:
            self.factories = int(factoryString[:-10])

        if soup.find(text='Oil Production:').parent.next_sibling.next_sibling\
                                            .contents[0].string == 'None':
            self.wells = 0
        else:
            self.wells = int(soup.find(text='Oil Production:').parent
                                 .next_sibling.next_sibling.contents[0]
                                 .string[:-15])

        if soup.find(text='Raw Material Production:').parent.next_sibling\
                                                     .next_sibling\
                                                     .contents[0].string\
           == 'None':
            self.mines = 0
        else:
            self.mines = int(soup.find(text='Raw Material Production:')
                                 .parent.next_sibling.next_sibling
                                 .contents[0].string[:-23])

        self.region = (soup.find(text='Region:').parent.next_sibling
                       .next_sibling.contents[0].string)
        self.troops = int(soup.find(text='Army Size:').parent.next_sibling
                          .next_sibling.contents[0].string[:-18])
        self.tech = (soup.find(text='Equipment:').parent.next_sibling
                         .next_sibling.contents[0].string)

        for i in soup.find(text='Airforce:').parent.next_sibling.next_sibling\
                .contents[0].contents[1].strings:
            self.airforce = i

        for i in soup.find(text='Navy:').parent.next_sibling.next_sibling\
                .contents[0].contents[1].strings:
            self.navy = i


class Alliance:
    def __init__(self, name, nations):
        self.name = name
        self.nations = nations
        self.gdp = 0
        self.avggdp = 0
        self.factories = 0
        self.universities = 0
        self.avgfactories = 0
        self.wells = 0
        self.avgwells = 0
        self.mines = 0
        self.avgmines = 0

        self.troopLevels = {
            'mex': 0,
            'car': 0,
            'gra': 0,
            'ama': 0,
            'con': 0,
            'waf': 0,
            'gui': 0,
            'eaf': 0,
            'cng': 0,
            'saf': 0,
            'atl': 0,
            'egy': 0,
            'mes': 0,
            'ara': 0,
            'per': 0,
            'sub': 0,
            'chi': 0,
            'pac': 0,
            'sea': 0,
            'ind': 0,
            'tot': 0,
            'avg': 0
        }
        self.techLevels = {
            'sta': 0,
            'fin': 0,
            'ww1': 0,
            'ww2': 0,
            'kor': 0,
            'vie': 0,
            'alm': 0,
            'pgw': 0,
            'adv': 0,
        }
        self.airforceLevels = {
            'non': 0,
            'mea': 0,
            'sma': 0,
            'med': 0,
            'som': 0,
            'lar': 0,
            'pow': 0,
            'vpo': 0
        }
        self.navyLevels = {
            'non': 0,
            'dng': 0,
            'cst': 0,
            'grn': 0,
            'blu': 0,
            'pbl': 0,
        }

        print('Indexing %s...' % self.name)

        for i in self.nations:
            self.gdp += i.gdp
            self.factories += i.factories
            self.universities += i.universities
            self.wells += i.wells
            self.mines += i.mines

            if i.region == 'Mesoamerica':
                self.troopLevels['mex'] += i.troops
            elif i.region == 'Caribbean':
                self.troopLevels['car'] += i.troops
            elif i.region == 'Gran Colombia':
                self.troopLevels['gra'] += i.troops
            elif i.region == 'Amazonia':
                self.troopLevels['ama'] += i.troops
            elif i.region == 'Southern Cone':
                self.troopLevels['con'] += i.troops
            elif i.region == 'West Africa':
                self.troopLevels['waf'] += i.troops
            elif i.region == 'Guinea':
                self.troopLevels['gui'] += i.troops
            elif i.region == 'East Africa':
                self.troopLevels['eaf'] += i.troops
            elif i.region == 'Congo':
                self.troopLevels['cng'] += i.troops
            elif i.region == 'Southern Africa':
                self.troopLevels['saf'] += i.troops
            elif i.region == 'Atlas':
                self.troopLevels['atl'] += i.troops
            elif i.region == 'Egypt':
                self.troopLevels['egy'] += i.troops
            elif i.region == 'Mesopotamia':
                self.troopLevels['mes'] += i.troops
            elif i.region == 'Arabia':
                self.troopLevels['ara'] += i.troops
            elif i.region == 'Persia':
                self.troopLevels['per'] += i.troops
            elif i.region == 'The Subcontinent':
                self.troopLevels['sub'] += i.troops
            elif i.region == 'China':
                self.troopLevels['chi'] += i.troops
            elif i.region == 'Pacific Rim':
                self.troopLevels['pac'] += i.troops
            elif i.region == 'Indochina':
                self.troopLevels['sea'] += i.troops
            else:
                self.troopLevels['ind'] += i.troops

            self.troopLevels['tot'] += i.troops

            if i.tech == 'Stone Age':
                self.techLevels['sta'] += 1
            elif i.tech == 'Finest of the 19th century':
                self.techLevels['fin'] += 1
            elif i.tech == 'First World War surplus':
                self.techLevels['ww1'] += 1
            elif i.tech == 'Second World War surplus':
                self.techLevels['ww2'] += 1
            elif i.tech == 'Korean War surplus':
                self.techLevels['kor'] += 1
            elif i.tech == 'Vietnam War surplus':
                self.techLevels['vie'] += 1
            elif i.tech == 'Almost Modern':
                self.techLevels['alm'] += 1
            elif i.tech == 'Persian Gulf War surplus':
                self.techLevels['pgw'] += 1
            else:
                self.techLevels['adv'] += 1

            if i.airforce == ' None':
                self.airforceLevels['non'] += 1
            elif i.airforce == ' Meagre':
                self.airforceLevels['mea'] += 1
            elif i.airforce == ' Small':
                self.airforceLevels['sma'] += 1
            elif i.airforce == ' Mediocre':
                self.airforceLevels['med'] += 1
            elif i.airforce == ' Somewhat Large':
                self.airforceLevels['som'] += 1
            elif i.airforce == ' Large':
                self.airforceLevels['lar'] += 1
            elif i.airforce == ' Powerful':
                self.airforceLevels['pow'] += 1
            elif i.airforce == ' Very Powerful':
                self.airforceLevels['vpo'] += 1

            if i.navy == ' None':
                self.navyLevels['non'] += 1
            elif i.navy == ' Some Dinghies':
                self.navyLevels['dng'] += 1
            elif i.navy == ' Coast Guard':
                self.navyLevels['cst'] += 1
            elif i.navy == ' Green Water':
                self.navyLevels['grn'] += 1
            elif i.navy == ' Blue Water':
                self.navyLevels['blu'] += 1
            else:
                self.navyLevels['pbl'] += 1

        self.members = len(self.nations)
        self.avggdp = round(self.gdp / float(len(self.nations)), 1)
        self.avgfactories = round(self.factories / float(len(self.nations)),
                                  1)
        self.avguniversities = round(self.universities / float(len(self.nations)),
                                     1)
        self.avgwells = round(self.wells / float(len(self.nations)), 1)
        self.avgmines = round(self.mines / float(len(self.nations)), 1)
        self.troopLevels['avg'] = round(self.troopLevels['tot']
                                        / float(len(self.nations)), 1)
        print("Finished Indexing {}\n".format(self.name))


async def getSoup(url):
    try:
        async with client.get(url) as response:
            html = await response.text()
            status = response.status
            '''
            if "alliance" in url:
                print("{0}: {1}".format(url, response.status))
            '''
        soup = BeautifulSoup(html, 'html.parser')
        ht = "<html>{}</html>"
        
        if "alliance" not in url.lower():
            info = soup.find_all("table", attrs={"class":
                                                 "table table-striped table-condensed table-hover table-bordered"})
            name = soup.find(id='nationtitle')
            name = name if name else ""
            new_html = ht.format(str(name) + "\n".join([str(i) for i in info]))
            soup = BeautifulSoup(new_html, 'html.parser')
        
        if "alliancestats" in url.lower():
            info = soup.find_all('table', attrs={'class' : 'table table-striped '})
            name = soup.find('h1', text=True)
            new_html = ht.format(str(name) + "\n".join([str(i) for i in info]))
            soup = BeautifulSoup(new_html, 'html.parser')
            
        if status == 404:
            try:
                soup = await getSoup(url[20:])
            except:
                return
    except Exception as e:
        print("getSoup| {0}:{1} on {2}".format(type(e).__name__, str(e), url))
        return

    return soup


async def scrape(pages):
    allianceLinks = []

    for i in range(1, 1 + pages):
        soup = await getSoup(BLOC + '/alliancerankings.php?page={}'.format(i))

        for link in soup.find_all('a'):
            if link.get('href') in allianceLinks:
                continue
            elif 'alliancestats.php?allianceid=' in str(link.get('href')):
                allianceLinks.append(link.get('href'))

    alliances = []

    for i in allianceLinks:
        nations = []
        nationLinks = []
        soup = await getSoup(BLOC + "/" + i)
        name = soup.find('h1', text=True).text

        for link in soup.find_all('a'):
            if 'stats.php?id=' in str(link.get('href')):
                if link.get('href') not in nationLinks:
                    nationLinks.append(link.get('href'))
        try:
            for n in nationLinks:
                nation = Nation(await getSoup(BLOC + '/{}'.format(n)))
                nations.append(nation)

            alliance = Alliance(name, nations)
            alliances.append(alliance)
        except Exception as e:
            print("scrape| {0}:{1} on {2}".format(type(e).__name__, str(e), i))
            continue

    return alliances


def writeCSV(name, alliances):
    with open(name, 'w') as index:
        writer = csv.writer(index, quoting=csv.QUOTE_MINIMAL)

        writer.writerow(['Alliance'] + [i.name for i in alliances])
        writer.writerow(['Members'] + [i.members for i in alliances])
        writer.writerow(['GDP'] + [i.gdp for i in alliances])
        writer.writerow(['Average GDP'] + [i.avggdp for i in alliances])
        writer.writerow(['Factories']
                        + [i.factories for i in alliances])
        writer.writerow(['Average Factories']
                        + [i.avgfactories for i in alliances])
        writer.writerow(['Universities']
                        + [i.universities for i in alliances])
        writer.writerow(['Average Universities']
                        + [i.avguniversities for i in alliances])
        writer.writerow(['Oil Wells']
                        + [i.wells for i in alliances])
        writer.writerow(['Average Oil Wells']
                        + [i.avgwells for i in alliances])
        writer.writerow(['Mines']
                        + [i.mines for i in alliances])
        writer.writerow(['Average Mines']
                        + [i.avgmines for i in alliances])

        writer.writerow(['Troops'])
        writer.writerow(['Mesoamerica']
                        + [i.troopLevels['mex'] for i in alliances])
        writer.writerow(['Caribbean']
                        + [i.troopLevels['car'] for i in alliances])
        writer.writerow(['Gran Colombia']
                        + [i.troopLevels['gra'] for i in alliances])
        writer.writerow(['Amazonia']
                        + [i.troopLevels['ama'] for i in alliances])
        writer.writerow(['Southern Cone']
                        + [i.troopLevels['con'] for i in alliances])
        writer.writerow(['West Africa']
                        + [i.troopLevels['waf'] for i in alliances])
        writer.writerow(['Guinea']
                        + [i.troopLevels['gui'] for i in alliances])
        writer.writerow(['East Africa']
                        + [i.troopLevels['eaf'] for i in alliances])
        writer.writerow(['Congo']
                        + [i.troopLevels['cng'] for i in alliances])
        writer.writerow(['Southern Africa']
                        + [i.troopLevels['saf'] for i in alliances])
        writer.writerow(['Atlas']
                        + [i.troopLevels['atl'] for i in alliances])
        writer.writerow(['Egypt']
                        + [i.troopLevels['egy'] for i in alliances])
        writer.writerow(['Mesopotamia']
                        + [i.troopLevels['mes'] for i in alliances])
        writer.writerow(['Arabia']
                        + [i.troopLevels['ara'] for i in alliances])
        writer.writerow(['Persia']
                        + [i.troopLevels['per'] for i in alliances])
        writer.writerow(['The Subcontinent']
                        + [i.troopLevels['sub'] for i in alliances])
        writer.writerow(['China']
                        + [i.troopLevels['chi'] for i in alliances])
        writer.writerow(['Pacific Rim']
                        + [i.troopLevels['pac'] for i in alliances])
        writer.writerow(['Indochina']
                        + [i.troopLevels['sea'] for i in alliances])
        writer.writerow(['East Indies']
                        + [i.troopLevels['ind'] for i in alliances])
        writer.writerow(['Total']
                        + [i.troopLevels['tot'] for i in alliances])
        writer.writerow(['Average']
                        + [i.troopLevels['avg'] for i in alliances])

        writer.writerow(['Tech'])
        writer.writerow(['Stone Age']
                        + [i.techLevels['sta'] for i in alliances])
        writer.writerow(['Finest of the 19th century']
                        + [i.techLevels['fin'] for i in alliances])
        writer.writerow(['First World War surplus']
                        + [i.techLevels['ww1'] for i in alliances])
        writer.writerow(['Second World War surplus']
                        + [i.techLevels['ww2'] for i in alliances])
        writer.writerow(['Korean War surplus']
                        + [i.techLevels['kor'] for i in alliances])
        writer.writerow(['Vietnam War surplus']
                        + [i.techLevels['vie'] for i in alliances])
        writer.writerow(['Almost Modern']
                        + [i.techLevels['alm'] for i in alliances])
        writer.writerow(['Persian Gulf War surplus']
                        + [i.techLevels['pgw'] for i in alliances])
        writer.writerow(['Advanced']
                        + [i.techLevels['adv'] for i in alliances])

        writer.writerow(['Airforce'])
        writer.writerow(['None']
                        + [i.airforceLevels['non'] for i in alliances])
        writer.writerow(['Meagre']
                        + [i.airforceLevels['mea'] for i in alliances])
        writer.writerow(['Small']
                        + [i.airforceLevels['sma'] for i in alliances])
        writer.writerow(['Mediocre']
                        + [i.airforceLevels['med'] for i in alliances])
        writer.writerow(['Somewhat Large']
                        + [i.airforceLevels['som'] for i in alliances])
        writer.writerow(['Large']
                        + [i.airforceLevels['lar'] for i in alliances])
        writer.writerow(['Powerful']
                        + [i.airforceLevels['pow'] for i in alliances])
        writer.writerow(['Very Powerful']
                        + [i.airforceLevels['vpo'] for i in alliances])

        writer.writerow(['Navy'])
        writer.writerow(['None']
                        + [i.navyLevels['non'] for i in alliances])
        writer.writerow(['Some Dinghies']
                        + [i.navyLevels['dng'] for i in alliances])
        writer.writerow(['Coast Guard']
                        + [i.navyLevels['cst'] for i in alliances])
        writer.writerow(['Green Water']
                        + [i.navyLevels['grn'] for i in alliances])
        writer.writerow(['Blue Water']
                        + [i.navyLevels['blu'] for i in alliances])
        writer.writerow(['Powerful Blue Water']
                        + [i.navyLevels['pbl'] for i in alliances])


async def main():
    async with client.post(BLOC+'/login.php', headers=HEADERS, data=DATA) as login:
        l = await login.text()
    async with client.get(BLOC+'/main.php', headers=HEADERS) as main:
        lg = await main.text()

    if l:
        pass

    name = BeautifulSoup(lg, 'html.parser').find(id='nationtitle')

    if name:
        print("Logged in as {}\n".format(name.text))
    else:
        print("Not logged in")
        return

    s = await scrape(2)
    csv_name = "bloc-alliance-data" + datetime.now().strftime('%Y-%m-%d-%H-%M-%S') + ".csv"
    writeCSV(csv_name, s)
    print('Done!1')


if __name__ == '__main__':
    loop.run_until_complete(main())
    client.close()
    loop.close()

SAMPLE_bloc-alliance-data-2017-02-01-19-37-55.csv