python - SocketServer rfile.read() 非常非常慢

Question

我正在构建一个可以转发所有 SSL 流量的 HTTPS 代理。它被称为透明隧道。无论如何，我对 Python 的 socketserver 有问题。当我调用 rfile.read() 时，需要很长时间才能返回。即使我使用 select 来确保 I/O 准备就绪，它仍然需要很长时间。通常为 30 秒，客户端的套接字因超时而关闭。我不能让套接字解除阻塞，因为我需要先读取数据，然后将我刚刚读取的所有数据转发到远程服务器，它必须先返回数据。

我的代码如下:

import SocketServer
import BaseHTTPServer
import socket
import threading
import httplib
import time
import os
import urllib
import ssl
import copy

from history import *
from http import *
from https import *
from logger import Logger

DEFAULT_CERT_FILE = "./cert/ncerts/proxpy.pem"

proxystate = None

class ProxyHandler(SocketServer.StreamRequestHandler):
    def __init__(self, request, client_address, server):
        self.peer = False
        self.keepalive = False
        self.target = None
        self.tunnel_mode = False
        self.forwardSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        # Just for debugging
        self.counter = 0
        self._host = None
        self._port = 0

        SocketServer.StreamRequestHandler.__init__(self, request, client_address, server)

    def createConnection(self, host, port):
        global proxystate

        if self.target and self._host == host:
            return self.target

        try:
            # If a SSL tunnel was established, create a HTTPS connection to the server
            if self.peer:
                conn = httplib.HTTPSConnection(host, port)
            else:
                # HTTP Connection
                conn = httplib.HTTPConnection(host, port)
        except HTTPException as e:
            proxystate.log.debug(e.__str__())

        # If we need a persistent connection, add the socket to the dictionary
        if self.keepalive:
            self.target = conn

        self._host = host
        self._port = port

        return conn

    def sendResponse(self, res):
        self.wfile.write(res)

    def finish(self):
        if not self.keepalive:
            if self.target:
                self.target.close()
            return SocketServer.StreamRequestHandler.finish(self)

        # Otherwise keep-alive is True, then go on and listen on the socket
        return self.handle()

    def handle(self):
        global proxystate

        if self.keepalive:
            if self.peer:
                HTTPSUtil.wait_read(self.request)
            else:
                HTTPUtil.wait_read(self.request)

            # Just debugging
            if self.counter > 0:
                proxystate.log.debug(str(self.client_address) + ' socket reused: ' + str(self.counter))
            self.counter += 1

        if self.tunnel_mode:
            HTTPUtil.wait_read(self.request)
            print "++++++++++++++++++++++++++"
            req=self.rfile.read(4096) #This is the line take very lone time!
            print "----------------------------------"
            data = self.doFORWARD(req)
            print "**************************************"
            if len(data)!=0:
                self.sendResponse(data)
                return

        try:
            req = HTTPRequest.build(self.rfile)
        except Exception as e:
            proxystate.log.debug(e.__str__() + ": Error on reading request message")
            return

        if req is None:
            return

        # Delegate request to plugin
        req = ProxyPlugin.delegate(ProxyPlugin.EVENT_MANGLE_REQUEST, req.clone())

        # if you need a persistent connection set the flag in order to save the status
        if req.isKeepAlive():
            self.keepalive = True
        else:
            self.keepalive = False

        # Target server host and port
        host, port = ProxyState.getTargetHost(req)

        if req.getMethod() == HTTPRequest.METHOD_GET:
            res = self.doGET(host, port, req)
            self.sendResponse(res)
        elif req.getMethod() == HTTPRequest.METHOD_POST:
            res = self.doPOST(host, port, req)
            self.sendResponse(res)
        elif req.getMethod() == HTTPRequest.METHOD_CONNECT:
            res = self.doCONNECT(host, port, req)

    def _request(self, conn, method, path, params, headers):
        global proxystate
        conn.putrequest(method, path, skip_host = True, skip_accept_encoding = True)
        for header,v in headers.iteritems():
            # auto-fix content-length
            if header.lower() == 'content-length':
                conn.putheader(header, str(len(params)))
            else:
                for i in v:
                    conn.putheader(header, i)
        conn.endheaders()

        if len(params) > 0:
            conn.send(params)

    def doRequest(self, conn, method, path, params, headers):
        global proxystate
        try:
            self._request(conn, method, path, params, headers)
            return True
        except IOError as e:
            proxystate.log.error("%s: %s:%d" % (e.__str__(), conn.host, conn.port))
            return False



    def doCONNECT(self, host, port, req):
        #global proxystate
        self.tunnel_mode = True
        self.tunnel_host = host
        self.tunnel_port = port
        #socket_req = self.request
        #certfilename = DEFAULT_CERT_FILE
        #socket_ssl = ssl.wrap_socket(socket_req, server_side = True, certfile = certfilename, 
                                     #ssl_version = ssl.PROTOCOL_SSLv23, do_handshake_on_connect = False)
        HTTPSRequest.sendAck(self.request)
        #print "Send ack to the peer %s on port %d for establishing SSL tunnel" % (host, port)
        print "into forward mode: %s : %s" % (host, port)
        '''
        host, port = socket_req.getpeername()
        proxystate.log.debug("Send ack to the peer %s on port %d for establishing SSL tunnel" % (host, port))

        while True:
            try:
                socket_ssl.do_handshake()
                break
            except (ssl.SSLError, IOError) as e:
                # proxystate.log.error(e.__str__())
                print e.__str__()
                return

        # Switch to new socket
        self.peer    = True
        self.request = socket_ssl
'''
        self.setup()
        #self.handle()

    def doFORWARD(self, data):
        host, port = self.request.getpeername()
        #print "client_host", host
        #print "client_port", port
        try:
            print "%s:%s===>data read, now sending..."%(host,port)
            self.forwardSocket.connect((self.tunnel_host,self.tunnel_port))
            self.forwardSocket.sendall(data)
            print data
            print "%s:%s===>sent %d bytes to server"%(host,port,len(data))
            select.select([self.forwardSocket], [], [])
            chunk = self.forwardSocket.recv(4096)
            print chunk
            print "%s:%s===>receive %d bytes from server"%(host,port,len(chunk))
            return chunk
        except socket.error as e:
            print e.__str__()
            return ''


class ThreadedHTTPProxyServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
    allow_reuse_address = True

class ProxyServer():    
    def __init__(self, init_state):
        global proxystate
        proxystate = init_state
        self.proxyServer_port = proxystate.listenport
        self.proxyServer_host = proxystate.listenaddr

    def startProxyServer(self):
        global proxystate

        self.proxyServer = ThreadedHTTPProxyServer((self.proxyServer_host, self.proxyServer_port), ProxyHandler)

        # Start a thread with the server (that thread will then spawn a worker
        # thread for each request)
        server_thread = threading.Thread(target = self.proxyServer.serve_forever)

        # Exit the server thread when the main thread terminates
        server_thread.setDaemon(True)
        proxystate.log.info("Server %s listening on port %d" % (self.proxyServer_host, self.proxyServer_port))
        server_thread.start()

        while True:
            time.sleep(0.1)

这让我发疯，因为当我转发正常的 HTTP 流量时，read() 会立即返回数据。但是每次客户端发送 SSL 流量(二进制)时，read() 都需要很长时间才能返回!而且我认为有一些机制 read() 仅在请求套接字被远程客户端关闭时才返回。有谁知道如何使 read() 快速？我尝试了 recv() 和 readline()，在处理二进制流量时，它们都和 read() 一样慢!

Answer 1

我在尝试通过 http 向其发送图像时在我的 python 服务器上遇到了类似的问题。

rfile.read 对于 800kb 的图像需要 1700ms，对于 4.5mb 的图像需要 4500ms。我也是通过 vpn 上的有限网络速度来做到这一点的。

看起来 rfile.read 下载/上传文件。通过提高客户端-服务器之间的连接/网络速度，我将 1700 毫秒的读取时间减少到不到 50 毫秒。