BNMP Scraping Detail (by Felipe Koblinger)

bnmp_scraping_detail.py

import pymongo
import requests

headers = {
    'Host': 'www.cnj.jus.br',
    'Connection': 'keep-alive',
    'Accept': 'application/json, text/plain, */*',
    'Origin': 'http://www.cnj.jus.br',
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
    'Content-Type': 'application/json;charset=UTF-8',
    'Referer': 'http://www.cnj.jus.br/bnmp/',
    'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7,es;q=0.6'
}

def request(url, payload, headers):
    post = requests.post(url, json = payload, headers = headers)
    return post.json()

def download_court_order_detail(court_order_id):
    payload_json = { "id": court_order_id }
    response = request('http://www.cnj.jus.br/bnmp/rest/detalhar',
                       payload_json,
                       headers)
    connection = pymongo.MongoClient('mongodb://localhost')
    db = connection.bnmp
    details = db.mandados.detalhes
    
    detail = response['mandado']
    detail['_id'] = detail.pop('id')
    details.save(detail)