opennet_scraper.py

#!/usr/bin/env python3

import requests
from pyquery import PyQuery

FIRST_ARTICLE=1
LAST_ARTICLE=58340

def main():
  f = open("opennet.txt", "a")
  for article in range(FIRST_ARTICLE, (LAST_ARTICLE + 1)):
    url = "https://www.opennet.ru/opennews/art.shtml?num={}".format(article)
    r = requests.get(url)
    html = r.text
    pq = PyQuery(html)
    tag = pq("#r_memo > p:nth-child(1)")
    text = tag.text()
    if text:
      percentage = round((article / LAST_ARTICLE) * 100, 2)
      print("  Working on {} of {}... {}%".format(article, LAST_ARTICLE, percentage), end="\r")
      f.write("{}\n".format(text))
      f.flush()

  f.close()

if __name__ == "__main__":
  main()