linux - Windows 和 Linux 之间的 Socket 性能

Question

我写了一个关于 windows 和 linux 之间通信的 demo。

在千兆网络环境下性能仅为40MB/s。

有什么方法可以提升性能吗？而且我不知道为什么这么慢。

---

windows作为客户端的代码

#include "stdafx.h"


#define WIN32_LEAN_AND_MEAN

#include <windows.h>
#include <winsock2.h>
#include <ws2tcpip.h>
#include <stdlib.h>
#include <stdio.h>


// Need to link with Ws2_32.lib, Mswsock.lib, and Advapi32.lib
#pragma comment (lib, "Ws2_32.lib")
#pragma comment (lib, "Mswsock.lib")
#pragma comment (lib, "AdvApi32.lib")


#define DEFAULT_BUFLEN 65536
#define DEFAULT_PORT "27015"

int tcpnodelay(int sock) {
    int yes=1;
    return setsockopt(sock,IPPROTO_TCP,TCP_NODELAY,(char*)&yes,sizeof(int));
}

int __cdecl main(int argc, char **argv) 
{
    WSADATA wsaData;
    SOCKET ConnectSocket = INVALID_SOCKET;
    struct addrinfo *result = NULL,
                    *ptr = NULL,
                    hints;
    char sendbuf[DEFAULT_BUFLEN];
    char recvbuf[32];
    unsigned long no;
    unsigned long iResult;
    unsigned long sent;
    int recvbuflen = 32;
    int recv_len, n;
    unsigned long recv_no;

    // Validate the parameters
    if (argc != 2) {
        printf("usage: %s server-name\n", argv[0]);
        return 1;
    }

    // Initialize Winsock
    iResult = WSAStartup(MAKEWORD(2,2), &wsaData);
    if (iResult != 0) {
        printf("WSAStartup failed with error: %d\n", iResult);
        return 1;
    }

    ZeroMemory( &hints, sizeof(hints) );
    hints.ai_family = AF_UNSPEC;
    hints.ai_socktype = SOCK_STREAM;
    hints.ai_protocol = IPPROTO_TCP;

    // Resolve the server address and port
    iResult = getaddrinfo(argv[1], DEFAULT_PORT, &hints, &result);
    if ( iResult != 0 ) {
        printf("getaddrinfo failed with error: %d\n", iResult);
        WSACleanup();
        return 1;
    }

    // Attempt to connect to an address until one succeeds
    for(ptr=result; ptr != NULL ;ptr=ptr->ai_next) {

        // Create a SOCKET for connecting to server
        ConnectSocket = socket(ptr->ai_family, ptr->ai_socktype, 
            ptr->ai_protocol);
        if (ConnectSocket == INVALID_SOCKET) {
            printf("socket failed with error: %ld\n", WSAGetLastError());
            WSACleanup();
            return 1;
        }

        // Connect to server.
        iResult = connect( ConnectSocket, ptr->ai_addr, (int)ptr->ai_addrlen);
        if (iResult == SOCKET_ERROR) {
            closesocket(ConnectSocket);
            ConnectSocket = INVALID_SOCKET;
            continue;
        }
        break;
    }

    freeaddrinfo(result);

    if (ConnectSocket == INVALID_SOCKET) {
        printf("Unable to connect to server!\n");
        WSACleanup();
        return 1;
    }

    if (tcpnodelay(ConnectSocket)<0) {
        printf("Can't set TCP_NODELAY!\n");
    }

    recv_len = sizeof(no);
    printf("recv_len: %d\n", recv_len);

    // Send an initial buffer
    memset(sendbuf, 0, DEFAULT_BUFLEN);
    for (no=0; no<1024*1024; no++) {
        sent = 0;
        while (sent < DEFAULT_BUFLEN) {
            iResult = send(ConnectSocket, sendbuf+sent, DEFAULT_BUFLEN-sent, 0);

            if (iResult == SOCKET_ERROR) {
                printf("send failed with error: %d\n", WSAGetLastError());
                closesocket(ConnectSocket);
                WSACleanup();
                return 1;
            }

            sent += iResult;
        }


        n = recv(ConnectSocket, (char *)(&recv_no), recv_len, 0);
        if (recv_len!=n || recv_no!=no) {
            printf("recv len: %d, recv no %ld", n, recv_no);
            exit(1);
        }

    }


    printf("Bytes Sent: %ld\n", iResult);

    // shutdown the connection since no more data will be sent
    iResult = shutdown(ConnectSocket, SD_SEND);
    if (iResult == SOCKET_ERROR) {
        printf("shutdown failed with error: %d\n", WSAGetLastError());
        closesocket(ConnectSocket);
        WSACleanup();
        return 1;
    }

    // cleanup
    closesocket(ConnectSocket);
    WSACleanup();

    return 0;
}

---

linux中作为服务器的代码

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h> 
#include <sys/socket.h>
#include <netinet/in.h>

int main( int argc, char *argv[] )
{
  int sockfd, newsockfd, portno, clilen;
  char buffer[65536];
  unsigned int no;
  unsigned long read_size;
  struct sockaddr_in serv_addr, cli_addr;
  int  n;
  int send_len;

  /* First call to socket() function */
  sockfd = socket(AF_INET, SOCK_STREAM, 0);
  if (sockfd < 0) 
  {
    perror("ERROR opening socket");
    exit(1);
  }
  /* Initialize socket structure */
  bzero((char *) &serv_addr, sizeof(serv_addr));
  portno = 27015;
  serv_addr.sin_family = AF_INET;
  serv_addr.sin_addr.s_addr = INADDR_ANY;
  serv_addr.sin_port = htons(portno);

  /* Now bind the host address using bind() call.*/
  if (bind(sockfd, (struct sockaddr *) &serv_addr,
           sizeof(serv_addr)) < 0)
  {
    perror("ERROR on binding");
    exit(1);
  }

  /* Now start listening for the clients, here process will
   * go in sleep mode and will wait for the incoming connection
   */
  listen(sockfd,5);
  clilen = sizeof(cli_addr);

  /* Accept actual connection from the client */
  newsockfd = accept(sockfd, (struct sockaddr *)&cli_addr, 
                     &clilen);
  if (newsockfd < 0) 
  {
    perror("ERROR on accept");
    exit(1);
  }

  send_len = sizeof(no);
  printf("send_len: %d\n", send_len);

  /* If connection is established then start communicating */
  bzero(buffer,65536);
  for (no=0; no<1024*1024; no++) {
    read_size = 0;
    while (read_size < 65536) {
      n = read( newsockfd,buffer+read_size,65536-read_size );
      if (n < 0)
      {
        perror("ERROR reading from socket");
        exit(1);
      }
      read_size += n;
    }


    n = write(newsockfd, &no, send_len);
    if (n < send_len) {
      printf("send len: %d\n", n);
      exit(1);
    }
  }
  printf("the message size: %d\n", n);

  /* Write a response to the client */
  n = write(newsockfd,"I got your message",18);
  if (n < 0)
  {
    perror("ERROR writing to socket");
    exit(1);
  }

  close(newsockfd);

  return 0; 
}

Answer 1

你自己回答了你的问题:

Yes, if I send the entire buffer, the performance will be to 90MB/s. But my work need to send every 64KB data and receive feed back, then send the next 64KB.

您可以在千兆位以太网上每秒发送大约 81,200 个全尺寸帧，分别约为 120MiB/s(其中包括 TCP 和 IP 的 header ，因此实际上您可以期望更少)。
TCP 将从几个“可调”参数(MTU、窗口大小)的次优默认值开始(对于您的千兆 LAN 是次优的，但对于“一般”未知网络是安全的)并将自适应地调整这些。这种情况发生得很快，但不会立即发生。因此，您在批量发送所有数据时看到的有效 90MiB/s 是绝对现实的。

现在的问题是，这是您可以实现的实际(或多或少是理论上的)最大值，如果您始终保持线路繁忙。

等待服务器回复是完全相反的，它让线路在两者之间“空闲”(就您发送的数据而言)至少直到收到回复并确认(在尤其是 Windows 可能需要相当非零的时间，默认情况下，ACK 仅每 200 毫秒发送一次，或者当至少有 2 个 ACK​​ 排队时——谷歌 TcpDelAckTicks 了解更多信息)。 虽然这个“空闲时间”很短，但这对您可以达到的最大吞吐量是一个非常严重的影响。相当于其他领域所谓的“流水线摊位”。

如果您的协议(protocol)允许，请尝试重叠您的发送和回复。也就是说，发送至少两个(最好是三个)你的 65kiB block ，然后才从服务器接收答案。这样，TCP 堆栈始终有它可以发送的数据，以保持线路繁忙并利用可用带宽。