golang http proxy

demo.go

package main
import (
    "flag"
    "log"
    "net/http"
    "strings"
    "unicode"
    "io/ioutil"
    //"os"
)
var (
    listen = flag.String("listen", "localhost:1080", "listen on address")
    logp = flag.Bool("log", true, "enable logging")
)
func main() {
    flag.Parse()
    proxyHandler := http.HandlerFunc(proxyHandlerFunc)
    log.Println("listen on ", *listen)
   log.Fatal(http.ListenAndServe(*listen, proxyHandler))
}
func proxyHandlerFunc(w http.ResponseWriter, r *http.Request) {
    if *logp {
        log.Println(r.URL)
    }
    log.Println("request is")
    //log.Println(r)

    client := &http.Client{}
    r.RequestURI = ""
    r.URL.Scheme = strings.Map(unicode.ToLower, r.URL.Scheme)
    resp, err := client.Do(r)
    //defer resp.Body.Close()
    if err != nil {
        log.Fatal(err)
    }

    for k, v := range resp.Header {
        for _, vv := range v {
            w.Header().Add(k, vv)
        }
    }
    log.Println("Header")
    //log.Println(resp.Header)
    /*
    for _, c := range resp.SetCookie {
        w.Header().Add("Set-Cookie", c.Raw)
    }
    */

    w.WriteHeader(resp.StatusCode)
    result, err := ioutil.ReadAll(resp.Body)
    if err != nil  {log.Fatal(err)}

    //log.Println("Body")
    //log.Println(result)
    w.Write(result)

    
    /*
    log.Println(resp)
    resp.Write(w)
    */
}

local.go

package main
import (
    "flag"
    "log"
    "net/http"
    "strings"
    "unicode"
    "io/ioutil"
    "io"
    "bytes"
    //"os"
)
var (
    listen = flag.String("listen", "localhost:1080", "listen on address")
    logp = flag.Bool("log", true, "enable logging")
)
func main() {
    flag.Parse()
    proxyHandler := http.HandlerFunc(proxyHandlerFunc)
    log.Println("listen on ", *listen)
   log.Fatal(http.ListenAndServe(*listen, proxyHandler))
}



func proxyHandlerFunc(w http.ResponseWriter, r *http.Request) {
    REPSRC := "sohuu"
    REPTAR := "baidu"

    log.Println("url")
    if *logp {
        log.Println(r.URL)
    }
    //replace request url 
    log.Println(r.URL)
    log.Println("host is")
    log.Println(r.URL.Host)
    r.URL.Host =  strings.Replace(r.URL.Host, REPSRC, REPTAR, -1)

    //r.URL := "http://www.baidu.com/"
    log.Println("request is")
    log.Println(r)

    header := r.Header
    bodyReq, err := ioutil.ReadAll(r.Body)
    //defer resp.Body.Close()

    log.Println("header")
    log.Println(header)
    log.Println("body is")
    log.Println(bodyReq)

    bodyReq = bytes.Replace(bodyReq, []byte(REPSRC), []byte(REPTAR), -1)
    log.Println("new body")
    log.Println(bodyReq)
    r.Body = ioutil.NopCloser(bytes.NewReader(bodyReq))
    //r.Body = bytes.NewReadCloser(bodyReq)


    for k, v := range r.Header {
        for k1, v1 := range v {
            log.Println(k, k1, v1)
            v[k1] = strings.Replace(v1, REPSRC, REPTAR, -1)
        }
    }
    r.Host = strings.Replace(r.Host, REPSRC, REPTAR, -1)
    r.RemoteAddr = strings.Replace(r.Host, REPSRC, REPTAR, -1)



    
    client := &http.Client{}
    r.RequestURI = ""
    r.URL.Scheme = strings.Map(unicode.ToLower, r.URL.Scheme)

    log.Println("after change")
    log.Println(r)

    log.Println("before do client")

    resp, err := client.Do(r)
    if err != nil && err != io.EOF {
        log.Println("error")
        log.Fatal(err)
    }

    for k, v := range resp.Header {
        for _, vv := range v {
            w.Header().Add(k, vv)
        }
    }
    //log.Println(resp.Header)
    /*
    for _, c := range resp.SetCookie {
        w.Header().Add("Set-Cookie", c.Raw)
    }
    */

    w.WriteHeader(resp.StatusCode)
    result, err := ioutil.ReadAll(resp.Body)
    if err != nil  {log.Fatal(err)}

    //log.Println("Body")
    //log.Println(result)
    w.Write(result)

    
    /*
    log.Println(resp)
    resp.Write(w)
    */
}

local.py

#coding:utf8
import os, sys, thread
#, socket
import gevent
from gevent import socket
 
 
BACKLOG = 50
MAX_DATA_RECV = 4096
DEBUG = False
 
REPSRC = 'baidu'
REPTAR = 'sohuu'

allThreads = {}
def main():
    host = ''
    port = int(sys.argv[1])
 
    try:
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 
        #s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        s.bind((host, port))
        s.listen(BACKLOG)
        print("start recv")
    except socket.error, (value, message):
        if s:
            s.close()
        print 'could not open socket', message
        sys.exit(1)
    
    gevent.spawn(checkThread)

    while 1:
        conn, client_addr = s.accept()
        print 'accept connection'
        #thread.start_new_thread(proxy_thread, (conn, client_addr))
        allThreads[conn] = True
        print 'start new thread', len(allThreads)

        gevent.spawn(proxy_thread, conn, client_addr)
        print("finish spawn")
    s.close()
 
def checkThread():
    while True:
        print "allThread", allThreads
        gevent.sleep(2)

#send data to golang server
#http server implement not very good
import base64
def proxy_thread(conn, client_addr):
    print 'start proxy'
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    #s.connect(('23.226.77.8', 1080))
    s.connect(('localhost', 1080))

    while True:
        #read request from connection then send to middle server
        #read request
        conn.settimeout(10)
        print 'wait for browser information'
        request = ''
        try:
            request = conn.recv(MAX_DATA_RECV)
        except:
            print 'browser connect over'
            conn.close()
            break
        print 'request'
        print request
        print 'end'
        #resend request as base64 to server
        #handle request
        print type(request)
        print 'wait request'
        request = request.replace(REPSRC, REPTAR)

        #connection to server
        
        #s.connect(('localhost', 1080))
        
        if len(request) > 0:
            #send request
            st = s.send(request)
            print 'send number', st, len(request)

            #虚拟代理http 服务器
            noData = False
            while 1:
                print conn, 'waitData'
                s.settimeout(5)
                data = ''
                try:
                    data = s.recv(MAX_DATA_RECV)
                except:
                    print 'time out'
                    break

                print conn, 'read data', len(data)
                print data
                if len(data) > 0:
                    while 1:
                        num = conn.send(data)
                        print('send is', len(data), num)
                        if num < len(data):
                            data = data[num:]
                        else:
                            break
                    
                else:
                    #noData
                    break
        else:
            print 'no request'
            conn.close()
            break

    s.close()
    print 'finish thread'
    #allThreads[conn] = None
    allThreads.pop(conn)
    print 'over thread', len(allThreads)

    '''
    first_line = request.split('\n')[0]
    url = first_line.split(' ')[1]
 
    print 'first line'
    print first_line
    print 'url'
    print url
 
    http_pos = url.find('://')
    if http_pos == -1:
        temp = url
    else:
        temp = url[(http_pos+3):]
    
    port_pos = temp.find(":")
 
    webserver_pos = temp.find("/")
    if webserver_pos == -1:
        webserver_pos = len(temp)
    
    webserver = ""
    port = -1
 
    if port_pos == -1 or webserver_pos < port_pos:
        port = 80
        webserver = temp[:webserver_pos]
    else:
        port = int(temp[(port_pos+1):])[:webserver_pos-port_pos-1]
        webserver = temp[:port_pos]
    
    print 'Connect to', webserver, port
 
    try:
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.connect((webserver, port))
        s.send(request)
 
        while 1:
            data = s.recv(MAX_DATA_RECV)
            if len(data) > 0:
                conn.send(data)
            else:
                break
        s.close()
        conn.close()
    except socket.error, (value, message):
        if s:
            s.close()
        if conn:
            conn.close()
        print 'runtime error', message
        sys.exit(1)
    '''
 
 
if __name__ == '__main__':
    main()

localProxy.py

#coding:utf8
import os, sys, thread
#, socket
import gevent
from gevent import socket
 
 
BACKLOG = 50
MAX_DATA_RECV = 4096
DEBUG = False
 
REPSRC = 'baidu'
REPTAR = 'sohu'

allThreads = {}
def main():
    host = ''
    port = int(sys.argv[1])
 
    try:
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 
        #s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        s.bind((host, port))
        s.listen(BACKLOG)
        print("start recv")
    except socket.error, (value, message):
        if s:
            s.close()
        print 'could not open socket', message
        sys.exit(1)
    
    gevent.spawn(checkThread)

    while 1:
        conn, client_addr = s.accept()
        print 'accept connection'
        #thread.start_new_thread(proxy_thread, (conn, client_addr))
        allThreads[conn] = True
        print 'start new thread', len(allThreads)

        gevent.spawn(proxy_thread, conn, client_addr)
        print("finish spawn")
    s.close()
 
def checkThread():
    while True:
        print "allThread", allThreads
        gevent.sleep(2)

#send data to golang server
#http server implement not very good
import base64
def proxy_thread(conn, client_addr):
    print 'start proxy'
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    #s.connect(('23.226.77.8', 1080))
    s.connect(('localhost', 1080))

    while True:
        #read request from connection then send to middle server
        #read request
        conn.settimeout(10)
        print 'wait for browser information'
        request = ''
        try:
            request = conn.recv(MAX_DATA_RECV)
        except:
            print 'browser connect over'
            conn.close()
            break
        print 'request'
        print request
        print 'end'
        #resend request as base64 to server
        #handle request
        print type(request)
        print 'wait request'
        request = request.replace(REPSRC, REPTAR)

        #connection to server
        
        #s.connect(('localhost', 1080))
        
        if len(request) > 0:
            #send request
            st = s.send(request)
            print 'send number', st, len(request)

            #虚拟代理http 服务器
            noData = False
            while 1:
                print conn, 'waitData'
                s.settimeout(5)
                data = ''
                try:
                    data = s.recv(MAX_DATA_RECV)
                except:
                    print 'time out'
                    break

                print conn, 'read data', len(data)
                print data
                if len(data) > 0:
                    while 1:
                        num = conn.send(data)
                        print('send is', len(data), num)
                        if num < len(data):
                            data = data[num:]
                        else:
                            break
                    
                else:
                    #noData
                    break
        else:
            print 'no request'
            conn.close()
            break

    s.close()
    print 'finish thread'
    #allThreads[conn] = None
    allThreads.pop(conn)
    print 'over thread', len(allThreads)

    '''
    first_line = request.split('\n')[0]
    url = first_line.split(' ')[1]
 
    print 'first line'
    print first_line
    print 'url'
    print url
 
    http_pos = url.find('://')
    if http_pos == -1:
        temp = url
    else:
        temp = url[(http_pos+3):]
    
    port_pos = temp.find(":")
 
    webserver_pos = temp.find("/")
    if webserver_pos == -1:
        webserver_pos = len(temp)
    
    webserver = ""
    port = -1
 
    if port_pos == -1 or webserver_pos < port_pos:
        port = 80
        webserver = temp[:webserver_pos]
    else:
        port = int(temp[(port_pos+1):])[:webserver_pos-port_pos-1]
        webserver = temp[:port_pos]
    
    print 'Connect to', webserver, port
 
    try:
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.connect((webserver, port))
        s.send(request)
 
        while 1:
            data = s.recv(MAX_DATA_RECV)
            if len(data) > 0:
                conn.send(data)
            else:
                break
        s.close()
        conn.close()
    except socket.error, (value, message):
        if s:
            s.close()
        if conn:
            conn.close()
        print 'runtime error', message
        sys.exit(1)
    '''
 
 
if __name__ == '__main__':
    main()

testSer.py

from flask import Flask
DEBUG = True
app = Flask(__name__)
app.config.from_object(__name__)

@app.route('/')
def hello():
    return 'test success'
if __name__ == '__main__':
    app.run(port=5000, host='0.0.0.0')

使用localproxy.py 加上超时机制 来 超时 浏览器客户端 和 连接web 服务器 socket 连接超时, 当长时间没有数据的时候.