scrapping.py

results = {}
print("Progress scrapping")
for i, code in enumerate(tqdm(list(country_codes.values()))):
    tmp_data = {}
    webpage = 'https://www.exchange-rates.org/history/{}/USD/T'.format(code)
    webpage = requests.get(webpage)
    webpage = webpage.text
    soup = BeautifulSoup(webpage, 'html.parser')
    data = soup.find_all('div', attrs={'class':'table-responsive'})
    
    days = []
    currencys = []
    dates = []
    x = data[0].text[3:-3].split(" ")
    for i, y in enumerate(x[:-1]):
        if i % 5 == 0:
            tmp = []
            for i, x in enumerate(list(y.split('/2018')[-1])):
                if is_number(x) == True:
                    tmp.append(i)
            days.append((y.split('/2018')[-1])[:tmp[0]])
            tmp_curr = (y.split('/2018')[-1])[tmp[0]:tmp[-1]]
            if ',' in list(tmp_curr):
                tmp_curr = list(tmp_curr)
                tmp_curr.remove(',')
                tmp_curr = "".join(tmp_curr)
            currencys.append(float(tmp_curr))
            tmp_dates = y[:-12].split('2018')[-2].split('/')
            tmp_dates[-1] = '2018'
            tmp_dates = "/".join(tmp_dates)
            dates.append(tmp_dates)
    if len(days) != len(currencys) and len(days) != len(dates):
        print("Length days, currencys and days lists must same. got length days {}, currencys {}, dates {}".format(len(days), len(currencys), len(dates)))
        break
    else:
        tmp_data['day'] = days
        tmp_data['date'] = dates
        tmp_data['currency'] = currencys
    
    results[code] = tmp_data

with open('results.json', 'w') as f:
    json.dump(results, f)