Skip to content

Instantly share code, notes, and snippets.

👀
Uh-oh, something went wrong! Recalibrating...

Satwik Kansal satwikkansal

View GitHub Profile
@satwikkansal
satwikkansal / scrape.py
Created Jun 27, 2016
Scraping top Stackoverflow posts using Scrapy
View scrape.py
import scrapy
class Stackoverflowspider(scrapy.spider):
name = 'stackoverflow'
start_urls = ['http://stackoverflow.com/questions?sort=votes']
def parse(self, response):
for href in response.css('.question-summary h3 a::attr(href)'):
full_url = response.urljoin(href.extract())
yield scrapy.Request(full_url, callback=self.parse_question)
def parse_question(self, response):
yield {
@satwikkansal
satwikkansal / scrape.py
Last active Jun 18, 2016
Fetching the content
View scrape.py
import requests
response = requests.get('https://in.pycon.org/cfp/2016/proposals/')
if response.status_code == 200:
print "Fetched the page sucessfully"
print response.content
@satwikkansal
satwikkansal / request_ranking.py
Last active Mar 22, 2016
Ranking of exploration request for oppia
View request_ranking.py
from datetime import datetime, timedelta
from math import log
epoch = datetime(1970, 1, 1)
def epoch_seconds(date):
td = date - epoch
return td.days * 86400 + td.seconds + (float(td.microseconds) / 1000000)
You can’t perform that action at this time.