c# - 套接字接收挂起

标签 c# .net windows sockets screen-scraping

我正在尝试下载、搜索 bing 页面,并询问是否使用套接字,我决定使用套接字,而不是 webclient。

在 bing、yahoo、google 的情况下,socket.Receive(); 在几个循环后挂起,但适用于询问。 for google loop 将接收 4 - 5 次,然后停止调用。

我不知道为什么?

public string Get(string url)
{
    Uri requestedUri = new Uri(url);
    string fulladdress = requestedUri.Host;
    IPHostEntry entry = Dns.GetHostEntry(fulladdress);
    StringBuilder sb = new StringBuilder();

    try
    {
        using (Socket socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.IP))
        {
            socket.Connect(entry.AddressList[0], 80);

            NetworkStream ns = new NetworkStream(socket);

            string part_request = string.Empty;
            string build_request = string.Empty;
            if (jar.Count != 0)
            {
                part_request = "GET {0} HTTP/1.1\r\nHost: {1} \r\nUser-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nAccept-Language: en-us,en;q=0.5\r\nAccept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\nCookie: {2}\r\nConnection: keep-alive\r\n\r\n";
                build_request = string.Format(part_request, requestedUri.PathAndQuery, requestedUri.Host, GetCookies(requestedUri));
            }
            else
            {
                part_request = "GET {0} HTTP/1.1\r\nHost: {1} \r\nUser-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nAccept-Language: en-us,en;q=0.5\r\nAccept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\nConnection: keep-alive\r\n\r\n";
                build_request = string.Format(part_request, requestedUri.PathAndQuery, requestedUri.Host);
            }

            byte[] data = Encoding.UTF8.GetBytes(build_request);
            socket.Send(data, data.Length, 0);

            byte[] bytesReceived = new byte[102400];
            int bytes = 0;

            do
            {
                bytes = socket.Receive(bytesReceived, bytesReceived.Length, 0);
                sb.Append(Encoding.ASCII.GetString(bytesReceived, 0, bytes));
            }
            while (bytes > 0);

            List<String> CookieHeaders = new List<string>();
            foreach (string header in sb.ToString().Split("\n\r".ToCharArray(), StringSplitOptions.RemoveEmptyEntries))
            {
                if (header.StartsWith("Set-Cookie"))
                {
                    CookieHeaders.Add(header.Replace("Set-Cookie: ", ""));
                }
            }

            this.AddCookies(CookieHeaders, requestedUri);

            socket.Close();
        }
    }
    catch (Exception ex)
    {
        string errorMessage = ex.Message;
    }

    return sb.ToString();
}

CookieContainer jar = new CookieContainer();

public string GetCookies(Uri _uri)
{
    StringBuilder sb = new StringBuilder();
    CookieCollection collection = jar.GetCookies(_uri);

    if (collection.Count != 0)
    {
        foreach (Cookie item in collection)
        {
            sb.Append(item.Name + "=" + item.Value + ";");
        }
    }
    return sb.ToString();
}

最佳答案

这是因为您已经读完了内容,但您仍在请求更多...

do
{
   bytes = socket.Receive(bytesReceived, bytesReceived.Length, 0);
   sb.Append(Encoding.ASCII.GetString(bytesReceived, 0, bytes));
}
while (bytes > 0);

这假设只要最后一个请求返回的字节数大于 0,就会有更多可用字节,而实际上当网络流到达末尾时,您很可能会在最后一个循环中填充一些缓冲区。 (例如 bytes > 0 但仅此而已)...所以服务器关闭连接。

尝试这样的事情......

do
{
   bytes = socket.Receive(bytesReceived, bytesReceived.Length, 0);
   sb.Append(Encoding.ASCII.GetString(bytesReceived, 0, bytes));
}
while (bytes == bytesReceived.Length);

一些服务器(ask 可能是其中之一)显然不会像您期望的那样自动关闭连接,因此它不会总是失败的原因。

:::编辑:::

我的测试样本:

加载 visual studio,创建一个新的控制台应用程序,然后将以下内容粘贴到生成的程序类中(代替所有现有代码):

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.Net.Sockets;

namespace ConsoleApplication1
{
    class Program
    {
        static void Main(string[] args)
        {
            string test = Get("http://www.google.co.uk/search?q=test&ie=utf-8&oe=utf-8&aq=t&rls=org.mozilla:en-GB:official&client=firefox-a");
            Console.Read();
        }

        public static string Get(string url)
        {
            Uri requestedUri = new Uri(url);
            string fulladdress = requestedUri.Host;
            IPHostEntry entry = Dns.GetHostEntry(fulladdress);
            StringBuilder sb = new StringBuilder();

            try
            {
                using (Socket socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.IP))
                {
                    socket.Connect(entry.AddressList[0], 80);

                    NetworkStream ns = new NetworkStream(socket);

                    string part_request = string.Empty;
                    string build_request = string.Empty;
                    if (jar.Count != 0)
                    {
                        part_request = "GET {0} HTTP/1.1\r\nHost: {1} \r\nUser-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nAccept-Language: en-us,en;q=0.5\r\nAccept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\nCookie: {2}\r\nConnection: keep-alive\r\n\r\n";
                        build_request = string.Format(part_request, requestedUri.PathAndQuery, requestedUri.Host, GetCookies(requestedUri));
                    }
                    else
                    {
                        part_request = "GET {0} HTTP/1.1\r\nHost: {1} \r\nUser-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nAccept-Language: en-us,en;q=0.5\r\nAccept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\nConnection: keep-alive\r\n\r\n";
                        build_request = string.Format(part_request, requestedUri.PathAndQuery, requestedUri.Host);
                    }

                    byte[] data = Encoding.UTF8.GetBytes(build_request);
                    socket.Send(data, data.Length, 0);

                    byte[] bytesReceived = new byte[4096];
                    int bytes = 0;
                    string currentBatch = "";

                    do
                    {
                        bytes = socket.Receive(bytesReceived);
                        currentBatch = Encoding.ASCII.GetString(bytesReceived, 0, bytes);
                        Console.Write(currentBatch);
                        sb.Append(currentBatch);
                    }
                    while (bytes == bytesReceived.Length);

                    List<String> CookieHeaders = new List<string>();
                    foreach (string header in sb.ToString().Split("\n\r".ToCharArray(), StringSplitOptions.RemoveEmptyEntries))
                    {
                        if (header.StartsWith("Set-Cookie"))
                        {
                            CookieHeaders.Add(header.Replace("Set-Cookie: ", ""));
                        }
                    }

                    //this.AddCookies(CookieHeaders, requestedUri);

                    socket.Close();
                }
            }
            catch (Exception ex)
            {
                string errorMessage = ex.Message;
            }

            return sb.ToString();
        }

        static CookieContainer jar = new CookieContainer();

        public static string GetCookies(Uri _uri)
        {
            StringBuilder sb = new StringBuilder();
            CookieCollection collection = jar.GetCookies(_uri);

            if (collection.Count != 0)
            {
                foreach (Cookie item in collection)
                {
                    sb.Append(item.Name + "=" + item.Value + ";");
                }
            }
            return sb.ToString();
        }
        }
    }

我减少了缓冲区以确保它不止一次被填充......从我的角度来看似乎没问题 这篇文章附带了我电脑上的典型作品保证:)

关于c# - 套接字接收挂起,我们在Stack Overflow上找到一个类似的问题: https://stackoverflow.com/questions/6775065/

相关文章:

c# - 在 C :\Users\[username]\AppData\Local folder? 中安装应用程序的 VS2010 设置的 DefaultLocation 属性应该是什么

c# - Xamarin Forms,如何将输入的日期从日期选择器转换为您选择的格式?

java - ldap中的别名是什么

c# - C#程序使用秒表卡住以对 Action 计时

c# - 如何在 C# 中编写事件和事件处理程序?

linux - Python远程编程/调试

python - 使用pyw文件的Windows启动不会关闭cmd

c# - 自动映射器将深对象自动映射到平面对象并返回

c# - 自定义 map 最短路径

python - Tensorflow无法导入