request_text.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

""" Python3 only """

import requests
import http

http.client.HTTPConnection._http_vsn = 10
http.client.HTTPConnection._http_vsn_str = "HTTP/1.0"

headers = {
    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36"
}

def process_response(resp):
    buf = b""
    size = 0
    for chunk in resp.iter_content(chunk_size=1024*100):
        if chunk:
            buf += chunk
            size += 1024*100
        else:
            break
        if size >= 1024 * 1024 * 3:
            break

    text = None
    try:
        text = buf.decode(resp.encoding)
    except:
        try:
            text = buf.decode("utf-8")
        except:
            text = str(buf)

    return text

def get(url, params=None, headers=headers, timeout=10, **kwargs):
    # sometimes keep-alive will cause parse error.
    s = requests.session()
    s.keep_alive = False

    try:
        resp = s.get(url, params=params, stream=True, timeout=timeout, headers=headers, **kwargs)
    except:
        return ""

    return process_response(resp)

def post(url, data=None, json=None, headers=headers, timeout=10, **kwargs):
    # sometimes keep-alive will cause parse error.
    s = requests.session()
    s.keep_alive = False

    try:
        resp = s.post(url, data=data, json=json, stream=True, timeout=timeout, headers=headers, **kwargs)
    except:
        return ""

    return process_response(resp)