java使用httpclient 4.1抓取https获取bad_record_mac

Question

使用以下代码抓取https网站，在大多数网站上都可以正常工作，但在某些网站上无法正常工作，例如grubhub.com。

import java.io.IOException;
import java.security.cert.X509Certificate;

import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLException;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.conn.ssl.X509HostnameVerifier;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;

public class TestSsl {

    HttpClient httpclient = new DefaultHttpClient();

    public static void main(String[] args) throws Exception {

        TestSsl t = new TestSsl();
        t.init();
        t.queryData();
    }

    private void init() {
        httpclient = wrapClient(httpclient);
        httpclient.getParams()
                .setIntParameter("http.socket.timeout", 30 * 1000);
    }

    /**
     * 
     * @throws Exception
     */
    private void queryData() throws Exception {
        HttpGet httpget = new HttpGet("https://www.grubhub.com/browse/");
        HttpResponse response = httpclient.execute(httpget);

        HttpEntity entity = response.getEntity();
        String result = EntityUtils.toString(entity);
        System.out.println(result);
        EntityUtils.consume(entity);

    }

    /**
     * 
     * @param base
     * @return
     */
    public static HttpClient wrapClient(HttpClient base) {
        try {
            SSLContext ctx = SSLContext.getInstance("SSL");
            X509TrustManager tm = new X509TrustManager() {
                public void checkClientTrusted(X509Certificate[] xcs,
                        String string) {
                }

                public void checkServerTrusted(X509Certificate[] xcs,
                        String string) {
                }

                public X509Certificate[] getAcceptedIssuers() {
                    return null;
                }
            };
            X509HostnameVerifier verifier = new X509HostnameVerifier() {

                @Override
                public void verify(String string, SSLSocket ssls) throws IOException {
                }

                @Override
                public void verify(String string, X509Certificate xc) throws SSLException {
                }

                @Override
                public void verify(String string, String[] strings, String[] strings1) throws SSLException {
                }

                @Override
                public boolean verify(String string, SSLSession ssls) {
                    return true;
                }
            };

            ctx.init(null, new TrustManager[] { tm }, null);
            SSLSocketFactory ssf = new SSLSocketFactory(ctx);
            ssf.setHostnameVerifier(verifier);
            ClientConnectionManager ccm = base.getConnectionManager();
            SchemeRegistry sr = ccm.getSchemeRegistry();
            sr.register(new Scheme("https", ssf, 443));
            return new DefaultHttpClient(ccm, base.getParams());
        } catch (Exception ex) {
            ex.printStackTrace();
            return null;
        }
    }
}

我什至在 jvm 上设置了以下内容:-Djavax.net.debug=all -Dhttps.protocols=SSLv3 -Dforce.http.jre.executor=true

主要，阅读:SSLv3 警报，长度 = 2 主要，RECV SSLv3 警报:致命，bad_record_mac 主要，称为 closeSocket() 主要，处理异常:javax.net.ssl.SSLException:收到致命警报:bad_record_mac 主要的，称为 close() 主要的，称为 closeInternal(true) 主要的，称为 close() 主要的，称为 closeInternal(true) 主要的，称为 close() 主要的，称为 closeInternal(true) 线程“main”中的异常 javax.net.ssl.SSLException:收到致命警报:bad_record_mac

我已经研究并在代码中放入了一堆东西，但仍然没有运气。调试输出太大，无法在此处发布，但可以通过在 IDE 中创建一个新类并运行它来轻松重现。

非常感谢任何帮助。

Answer 1

您遇到一个已知问题，即 jre 试图与不支持 TLS 的服务器通信。查看更多信息 here .

建议的解决方法是强制使用 sslv3。 不幸的是，尝试使用系统属性来实现它是行不通的，因此您需要通过在每个套接字上将启用的协议(protocol)设置为 SSLv3 来实现。

这是一个适用于您的主机的小应用程序:

String urlString = "https://www.grubhub.com/browse/";
URL url = new URL(urlString);
SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault();
SSLSocket socket = (SSLSocket) factory.createSocket(url.getHost(), 443);
socket.setEnabledProtocols(new String[]{"SSLv3"}); // <--- THIS IS THE WORK-AROUND
PrintWriter out = new PrintWriter(
        new OutputStreamWriter(
                socket.getOutputStream()));
out.println("GET " + urlString + " HTTP/1.1");
out.println();
out.flush();

BufferedReader in = new BufferedReader(new InputStreamReader(socket.getInputStream()));

String line;
while ((line = in.readLine()) != null) {
    System.out.println(line);
}

out.close();
in.close();

您需要用一个像这样配置每个套接字的 SSLSocketFactory 覆盖 HttpClient 的 SSLSocketFactory:

socket.setEnabledProtocols(new String[]{"SSLv3"});