java - 未使用 HttpUrlConnection 设置 Cookie

Question

我正在尝试连接到一个站点，根据此页面:http://www.mkyong.com/java/how-to-automate-login-a-website-java-example/

但似乎 cookie 没有设置。 找不到问题...

我的目标是通过此站点的身份验证，以便获取 Html。 如果有其他更简单的解决方案，请不要犹豫。

package htmlmKyongTest;

import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;

import javax.net.ssl.HttpsURLConnection;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class HttpUrlConnectionExample {

  private List<String> cookies;
  private HttpsURLConnection conn;

  private final String USER_AGENT = "Mozilla/5.0";

  public static void main(String[] args) throws Exception {



    String url = "https://www.entea.fr/";
    String gmail = "https://www.entea.fr/etabs/0671688W/Pages/Accueil.aspx";

    HttpUrlConnectionExample http = new HttpUrlConnectionExample();

    // make sure cookies is turn on
    CookieHandler.setDefault(new CookieManager());

    // 1. Send a "GET" request, so that you can extract the form's data.
    String page = http.GetPageContent(url);
    String postParams = http.getFormParams(page, "myUsername", "myPassword");

    // 2. Construct above post's content and then send a POST request for
    // authentication
    http.sendPost(url, postParams);

    // 3. success
    String result = http.GetPageContent(gmail);
    System.out.println(result);
  }

  private void sendPost(String url, String postParams) throws Exception {

    URL obj = new URL(url);
    conn = (HttpsURLConnection) obj.openConnection();

    // Acts like a browser
    conn.setUseCaches(false);
    conn.setRequestMethod("POST");
    conn.setRequestProperty("Host", "www.entea.fr");
    conn.setRequestProperty("User-Agent", USER_AGENT);
    conn.setRequestProperty("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
    conn.setRequestProperty("Accept-Language", "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4");
    conn.setRequestProperty("Content-Type","application/x-www-form-urlencoded");
    for (String cookie : this.cookies) {
        conn.addRequestProperty("Set-Cookie", cookie.split(";", 1)[0]);
    }

    conn.setRequestProperty("Connection", "keep-alive");
    conn.setRequestProperty("Referer", "https://www.entea.fr/CookieAuth.dll?GetLogon?curl=Z2F&reason=0&formdir=7");
    conn.setRequestProperty("Accept-Encoding","gzip,deflate");
    conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
    conn.setRequestProperty("Content-Length", Integer.toString(postParams.length()));
    conn.setDoOutput(true);
    conn.setDoInput(true);

    // Send post request
    DataOutputStream wr = new DataOutputStream(conn.getOutputStream());
    wr.writeBytes(postParams);
    wr.flush();
    wr.close();

    int responseCode = conn.getResponseCode();
    System.out.println("\nSending 'POST' request to URL : " + url);
    System.out.println("Post parameters : " + postParams);
    System.out.println("Response Code : " + responseCode);

    BufferedReader in = 
             new BufferedReader(new InputStreamReader(conn.getInputStream()));
    String inputLine;
    StringBuffer response = new StringBuffer();

    while ((inputLine = in.readLine()) != null) {
        response.append(inputLine);
    }
    in.close();

  }

  private String GetPageContent(String url) throws Exception {

    URL obj = new URL(url);
    conn = (HttpsURLConnection) obj.openConnection();

    // default is GET
    conn.setRequestMethod("GET");
    conn.setUseCaches(false);

    // act like a browser
    conn.setRequestProperty("User-Agent", USER_AGENT);
    conn.setRequestProperty("Accept",
        "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
    conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");

    if (cookies != null) {
        for (String cookie : this.cookies) {
            conn.addRequestProperty("Set-Cookie", cookie.split(";", 1)[0]);
        }
    }
    int responseCode = conn.getResponseCode();
    System.out.println("\nSending 'GET' request to URL : " + url);
    System.out.println("Response Code : " + responseCode);

    BufferedReader in = 
            new BufferedReader(new InputStreamReader(conn.getInputStream()));
    String inputLine;
    StringBuffer response = new StringBuffer();

    while ((inputLine = in.readLine()) != null) {
        response.append(inputLine);
    }
    in.close();

    // Get the response cookies
    setCookies(conn.getHeaderFields().get("Set-Cookie"));
    return response.toString();

  }

  public String getFormParams(String html, String username, String password)
        throws UnsupportedEncodingException {

    System.out.println("Extracting form's data...");
    Document doc = Jsoup.parse(html);

    Element loginform = doc.getElementById("logonForm");
    Elements inputElements = loginform.getElementsByTag("input");
    List<String> paramList = new ArrayList<String>();
    for (Element inputElement : inputElements) {
        System.out.println("inputElement : " + inputElement.toString());
        String key = inputElement.attr("name");
        String value = inputElement.attr("value");

        if (key.equals("username"))
            value = username;
        else if (key.equals("password")) {
            value = password;
        }
        if (!(inputElement.attr("style").contains("none"))) paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
    }

    // build parameters list
    StringBuilder result = new StringBuilder();
    for (String param : paramList) {
        if (result.length() == 0) {
            result.append(param);
        } else {
            result.append("&" + param);
        }
    }

    System.out.println("result : " + result.toString().replace("trusted=4", "SubmitCreds.x=0&SubmitCreds.y=0"));
    return result.toString().replace("trusted=4", "SubmitCreds.x=0&SubmitCreds.y=0");
  }

  public List<String> getCookies() {
    return cookies;
  }

  public void setCookies(List<String> cookies) {
    this.cookies = cookies;
  }

}

输出如下:

Sending 'GET' request to URL : https://www.entea.fr/
Response Code : 200
Extracting form's data...
inputElement : <input type="password" id="autocompleteoff" style="display:none" />
inputElement : <input type="hidden" id="curl" name="curl" value="Z2F" />
inputElement : <input type="hidden" id="flags" name="flags" value="0" />
inputElement : <input type="hidden" id="forcedownlevel" name="forcedownlevel" value="0" />
inputElement : <input type="hidden" id="formdir" name="formdir" value="7" />
inputElement : <input type="text" id="username" name="username" />
inputElement : <input id="password" onfocus="g_fFcs=0" type="password" name="password" autocomplete="off" />
inputElement : <input type="radio" name="trusted" id="rdoPblc" value="0" onclick="clkSec()" checked="checked" />
inputElement : <input type="radio" name="trusted" id="rdoPrvt" value="4" onclick="clkSec()" />
inputElement : <input id="SubmitCreds" onclick="clkLgn()" type="image" value="Ouvrir une session" name="SubmitCreds" src="/CookieAuth.dll?GetPic?formdir=7&amp;image=encart_footer_01.png" alt="Connexion" />
result : curl=Z2F&flags=0&forcedownlevel=0&formdir=7&username=xxxxxxxxx&password=xxxxxxxxxxx*&trusted=0&SubmitCreds.x=0&SubmitCreds.y=0&SubmitCreds=Ouvrir+une+session
Exception in thread "main" java.lang.NullPointerException
    at htmlmKyongTest.HttpUrlConnectionExample.sendPost(HttpUrlConnectionExample.java:66)
    at htmlmKyongTest.HttpUrlConnectionExample.main(HttpUrlConnectionExample.java:46)

Answer 1

第一个 sendPost 的 cookies 字段未初始化。您应该使用 if (cookies != null) 保护您的 for 循环。

编辑:实际的解决方案是将初始请求 + 重定向拆分为 2 个单独的 GET 请求。

试试这个:

import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;

import javax.net.ssl.HttpsURLConnection;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class HttpUrlConnectionExample {

    private List<String> cookies;
    private HttpsURLConnection conn;

    private final String USER_AGENT = "Mozilla/5.0";

    public static void main(String[] args) throws Exception {
        String url = "https://www.entea.fr/";
        String gmail = "https://www.entea.fr/etabs/0671688W/Pages/Accueil.aspx";

        HttpUrlConnectionExample http = new HttpUrlConnectionExample();

        // make sure cookies is turn on
        CookieHandler.setDefault(new CookieManager());

        // 1. Send a "GET" request, so that you can extract the form's data.
        http.getPageContent(url);
        String page = http.getPageContent("https://www.entea.fr/CookieAuth.dll?GetLogon?curl=Z2F&reason=0&formdir=7");

        String postParams = http.getFormParams(page, "myUsername", "myPassword");

        // 2. Construct above post's content and then send a POST request for
        // authentication
        http.sendPost(url, postParams);

        // 3. success
        String result = http.getPageContent(gmail);
        System.out.println(result);
    }

    private void sendPost(String url, String postParams) throws Exception {

        URL obj = new URL(url);
        conn = (HttpsURLConnection) obj.openConnection();

        // Acts like a browser
        conn.setUseCaches(false);
        conn.setRequestMethod("POST");
        conn.setRequestProperty("Host", "www.entea.fr");
        conn.setRequestProperty("User-Agent", USER_AGENT);
        conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
        conn.setRequestProperty("Accept-Language", "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4");
        conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
        for (String cookie : this.cookies) {
            conn.addRequestProperty("Set-Cookie", cookie.split(";", 1)[0]);
        }

        conn.setRequestProperty("Connection", "keep-alive");
        conn.setRequestProperty("Referer", "https://www.entea.fr/CookieAuth.dll?GetLogon?curl=Z2F&reason=0&formdir=7");
        conn.setRequestProperty("Accept-Encoding", "gzip,deflate");
        conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
        conn.setRequestProperty("Content-Length", Integer.toString(postParams.length()));
        conn.setDoOutput(true);
        conn.setDoInput(true);

        // Send post request
        DataOutputStream wr = new DataOutputStream(conn.getOutputStream());
        wr.writeBytes(postParams);
        wr.flush();
        wr.close();

        int responseCode = conn.getResponseCode();
        System.out.println("\nSending 'POST' request to URL : " + url);
        System.out.println("Post parameters : " + postParams);
        System.out.println("Response Code : " + responseCode);

        BufferedReader in =
                new BufferedReader(new InputStreamReader(conn.getInputStream()));
        String inputLine;
        StringBuffer response = new StringBuffer();

        while ((inputLine = in.readLine()) != null) {
            response.append(inputLine);
        }
        in.close();

    }

    private String getPageContent(String url) throws Exception {

        URL obj = new URL(url);
        conn = (HttpsURLConnection) obj.openConnection();

        // default is GET
        conn.setRequestMethod("GET");
        conn.setUseCaches(false);

        // act like a browser
        conn.setRequestProperty("User-Agent", USER_AGENT);
        conn.setRequestProperty("Accept",
                "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
        conn.setInstanceFollowRedirects(false);

        if (cookies != null) {
            for (String cookie : this.cookies) {
                conn.addRequestProperty("Set-Cookie", cookie.split(";", 1)[0]);
            }
        }
        int responseCode = conn.getResponseCode();
        System.out.println("\nSending 'GET' request to URL : " + url);
        System.out.println("Response Code : " + responseCode);

        BufferedReader in =
                new BufferedReader(new InputStreamReader(conn.getInputStream()));
        String inputLine;
        StringBuffer response = new StringBuffer();

        while ((inputLine = in.readLine()) != null) {
            response.append(inputLine);
        }
        in.close();

        // Get the response cookies
        setCookies(conn.getHeaderFields().get("Set-Cookie"));
        return response.toString();

    }

    public String getFormParams(String html, String username, String password)
            throws UnsupportedEncodingException {

        System.out.println("Extracting form's data...");
        Document doc = Jsoup.parse(html);

        Element loginform = doc.getElementById("logonForm");
        Elements inputElements = loginform.getElementsByTag("input");
        List<String> paramList = new ArrayList<String>();
        for (Element inputElement : inputElements) {
            System.out.println("inputElement : " + inputElement.toString());
            String key = inputElement.attr("name");
            String value = inputElement.attr("value");

            if (key.equals("username"))
                value = username;
            else if (key.equals("password")) {
                value = password;
            }
            if (!(inputElement.attr("style").contains("none")))
                paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
        }

        // build parameters list
        StringBuilder result = new StringBuilder();
        for (String param : paramList) {
            if (result.length() == 0) {
                result.append(param);
            } else {
                result.append("&" + param);
            }
        }

        System.out.println("result : " + result.toString().replace("trusted=4", "SubmitCreds.x=0&SubmitCreds.y=0"));
        return result.toString().replace("trusted=4", "SubmitCreds.x=0&SubmitCreds.y=0");
    }

    public void setCookies(List<String> cookies) {
        if (cookies != null) {
            this.cookies = cookies;
        }
    }

}