Skip to content

Instantly share code, notes, and snippets.

@eddie-knight
Last active May 11, 2020 18:02
Show Gist options
  • Save eddie-knight/16e2c8c1fd02b66bd608e6741a09d579 to your computer and use it in GitHub Desktop.
Save eddie-knight/16e2c8c1fd02b66bd608e6741a09d579 to your computer and use it in GitHub Desktop.
Simple request helper
import requests
from bs4 import BeautifulSoup, element
class Fetcher():
def __init__(self, base, endpoint=None):
self.base = base
self.endpoint = endpoint
def url(self):
return f'{self.base}/{self.endpoint}' if self.endpoint else self.base
def get_content(self, name):
target = f'{self.url()}?page={name}'
page = requests.get(target)
return BeautifulSoup(page.content, 'html.parser')
def get_id(self, name, _id):
content = self.get_content(name)
return content.find(id=_id)
def get_class(self, name, tag, classname, _id='body'):
body = self.get_id(name, _id)
return body.find_all(tag, class_=classname)
def get(self, name, tag, _id='body'):
body = self.get_id(name, _id)
return body.find_all(tag)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment