philshem

philshem

import requests
from bs4 import BeautifulSoup
import json
from collections import defaultdict
from random import randint
from time import sleep

'''
scraper for counts of each make of car found at autoscout.ch
https://twitter.com/philshem/status/1295371670321070080

philshem

philshem

philshem

# made by @philshem
# https://twitter.com/philshem/status/1256535940442591232
# the unlicense https://unlicense.org/
# requires youtube-dl and ffmpeg (easily installed with homebrew or conda)

# download the video
youtube-dl jS0bjDbWydw --output raw.mp4

# get dimensions
# 1920 × 1080 in this case

philshem

import os
import pandas as pd
from datetime import datetime
	
url='https://www.swissgrid.ch/dam/dataimport/energy-statistic/EnergieUebersichtCH-{YEAR}.xls'

dfs = []
for y in range(2009,datetime.now().year+1):
	# generate url with year in it
	url_y = url.format(YEAR=str(y))

philshem

# fetches raw data from https://shopsensor-272815.appspot.com/
#
# requires multiple queries for each city's bounding box
#
# if you actually run this, pipe each curl statement to a seperate file

# bern, 27 results
curl 'https://shopsensor-272815.appspot.com/searchresults' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:75.0) Gecko/20100101 Firefox/75.0' -H 'Accept: application/json, text/javascript, */*; q=0.01' -H 'Accept-Language: en-US,en;q=0.5' --compressed -H 'Content-Type: application/json; charset=utf-8' -H 'X-Requested-With: XMLHttpRequest' -H 'Origin: https://shopsensor-272815.appspot.com' -H 'DNT: 1' -H 'Connection: keep-alive' -H 'Referer: https://shopsensor-272815.appspot.com/' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' --data '{"_southWest":{"lat":46.92687938388263,"lng":7.421092987060548},"_northEast":{"lat":46.96941802389087,"lng":7.471647262573243}}'
# > bern.json

philshem

#!/usr/bin/env python

'''
Post processing the scrape job
requires `pip install pandas gcsfs`
'''

import pandas as pd
import numpy as np
from datetime import datetime

philshem

import pandas as pd
import numpy as np
from datetime import datetime

url = 'https://raw.githubusercontent.com/openZH/covid_19/master/COVID19_Fallzahlen_Kanton_ZH_alter_geschlecht.csv'

# define date parsing format
dateparse = lambda dates: [datetime.strptime(d, '%d.%m.%Y') for d in dates]

# read csv from web

philshem

#!/usr/bin/python3

# pip3 install pandas matplotlib
import pandas as pd
from matplotlib import pyplot as plt

# this gets the worldwide confirmed cases from JHU
url = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Confirmed.csv'

# read/download csv