In TCP socket program,client send some data, but server need read multiple times. why?

Question

I have a question about socket.I send N-size data from client to server, N-size less than 100 byte.So I think my data should not be split to multiple tcp packet.In my opinion, Client send data should be done at one times and Server can receive data at one time.But The result is not satisfactory.Real situation is the server need call read data.I don't understand it.Follow code:

epoll_server.cpp(only receive data.)

#include <sys/socket.h>
#include <stdio.h>
#include <string.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/epoll.h>
#include <fcntl.h>
#include <netdb.h>

#define BUFSIZE 1024
#define INITSIZE 1024
#define MAXEVENTCOUNT 10240

// add non-blocking to sockfd
int make_socket_non_blocking(int fd)
{
    // get initial flag
    int src_flags;
    src_flags= fcntl(fd, F_GETFL,0);           
    if(src_flags == -1)
    {                    
        perror("fcntl get error.");
        return-1;
    }

    // add non-blocking
    int new_flags = src_flags | O_NONBLOCK;
    int ret_value;
    ret_value = fcntl(fd, F_SETFL, new_flags);
    if(ret_value == -1)
    {
        perror("fcntl set error.");
        return-1;
    }

    return 0;
}


// main function
int main(int argc, char* argv[])
{
    int server_sockfd, client_sockfd;
    int server_len;
    struct sockaddr_in server_address;

    // create server socket fd
    server_sockfd = socket(AF_INET, SOCK_STREAM, 0);
    // init server address struct
    bzero(&server_address, sizeof(server_address));
    server_address.sin_family = AF_INET;
    server_address.sin_port = htons(9567);
    server_address.sin_addr.s_addr = INADDR_ANY;
    server_len = sizeof(server_address);

    // bind server address info for server fd
    if((bind(server_sockfd, (struct sockaddr*)&server_address, server_len)) == -1)
    {
        perror("bind error");
        exit(EXIT_FAILURE);
    }

    // let server is listened state
    listen(server_sockfd, 5);
    printf("server start waiting for connect...\r\n");

    // only suggestion
    int efd = epoll_create(INITSIZE);
    if(-1 == efd)
    {
        printf("epoll_create error happen.\n");
        return -1;
    }

    // set server_sockfd
    struct epoll_event server_event, event;
    server_event.data.fd = server_sockfd;
    server_event.events = EPOLLIN | EPOLLET;
    int ret_epollctl = epoll_ctl(efd, EPOLL_CTL_ADD, server_sockfd, &server_event);
    if(-1 == ret_epollctl)
    {
        printf("epoll_ctl error happen when efd is adding server_sockfd.\n");
        return -1;
    }

    /* event loop */
    struct epoll_event* return_events;
    // set timeout is 3000 ms
    int timeout_msecond = 3000;    
    return_events = (struct epoll_event*)malloc(MAXEVENTCOUNT*sizeof(struct epoll_event));
    int count = 0;
    while(1)
    {
        int ret_epollwait = epoll_wait(efd, return_events, MAXEVENTCOUNT, timeout_msecond);
        // part_1:epoll_wait error happen
        if(-1 == ret_epollwait)
        {
            printf("logged epoll_wait error happen.\n");
            continue;
        }
        // part_2:epoll_wait timeout
        if(0 == ret_epollwait)
        {
            printf("logged epoll_wait timeout.\n");
            continue;
        }
        // part_3:do some other event
        int index = 0;
        for(index = 0; index < MAXEVENTCOUNT; index++)
        {
            // part_3-1:hup ...
            if((return_events[index].events & EPOLLERR)
             || (return_events[index].events & EPOLLHUP)
             || !(return_events[index].events & EPOLLIN) )
            {                
                continue;
            }

            // part_3-2:is connection
            if(return_events[index].data.fd == server_sockfd)
            {
                struct sockaddr_in client_address;
                int client_len = sizeof(client_address);
                // server accept connection from client
                int client_sockfd = accept(server_sockfd, (struct sockaddr*)&client_address, (socklen_t*)&client_len);
                // part_3-2-1:connection error happen
                if(-1 == client_sockfd)
                {
                    if((EAGAIN == errno)
                     || (EWOULDBLOCK == errno) )
                    {                        
                        continue;
                    }
                    else
                    {
                        printf("accept error occured.\n");
                        continue;
                    }
                }
                else // part_3-2-2:normal connection
                {
                    // get clinet some information
                    char hostinfo_buf[BUFSIZE] = {0};
                    char servname_buf[BUFSIZE] = {0};
                    int tmp_ret = getnameinfo((struct sockaddr*)&client_address, client_len, hostinfo_buf, sizeof(hostinfo_buf), servname_buf, sizeof(servname_buf), NI_NUMERICHOST| NI_NUMERICSERV);
                    if(0 == tmp_ret)
                    {
                        printf("Accepted connection on descriptor %d:ip=%s, port=%s.\n", client_sockfd, hostinfo_buf, servname_buf);
                    }
                    // set client_sockfd to non-blocking
                    tmp_ret = make_socket_non_blocking(client_sockfd);
                    if(-1 == tmp_ret)
                    {
                        printf("set client_sockfd=%d to non-blocking error occured.\n", client_sockfd);
                        abort();
                    }

                    // set client_sockfd is EPOLLIN, EPOLLET
                    event.data.fd = client_sockfd;
                    event.events = EPOLLIN | EPOLLET;
                    tmp_ret = epoll_ctl(efd, EPOLL_CTL_ADD, client_sockfd, &event);
                    if(tmp_ret == -1)
                    {
                        printf("efd add %d has a error.\n", client_sockfd);
                        continue;
                    }
                    printf("add descriptor %d:ip=%s, port=%s successfully.\n", client_sockfd, hostinfo_buf, servname_buf);
                }
                continue;
            }

            // part_3-3:read data from client            
            printf("read data start++++\n");
            int temp = 0;

            // get recv_cache size start
            int recvsize = 0;
            socklen_t optlen = sizeof(recvsize);
            int err = getsockopt(return_events[index].data.fd, SOL_SOCKET, SO_RCVBUF, &recvsize, &optlen);
            printf("recv cache size :%d\n", recvsize);
            // get recv_cache size end

            while(1) // start while(1)
            {
                printf("%d times read data\n", ++temp);
                char* recv_buffer = (char*)malloc(1024+1);
                memset(recv_buffer, 0, 1025);
                // int ret_read = read(return_events[index].data.fd, recv_buffer, sizeof(recv_buffer));
                int ret_read = recv(return_events[index].data.fd, recv_buffer, sizeof(recv_buffer), 0);
                // part_3-3-1:read return error
                if(-1 == ret_read)
                {
                    if(EAGAIN != errno)
                    {
                        printf("read data from %d error occured, errno=%d, %s.\n", return_events[index].data.fd, errno, strerror(errno));
                    }
                    break;
                }
                // part_3-3-2:no data
                if(0 == ret_read)
                {
                    continue;
                }
                // part_3-3-3:output data. If data is 'bye', connection will close.
                if(ret_read > 0)
                {
                    printf("%d client's data:size=%dbyte, content=%s\n", return_events[index].data.fd, ret_read, recv_buffer);
                    // part_3-3-3-1:close connection and remove client_sockfd
                    if((recv_buffer[0] == 'b') 
                    && (recv_buffer[1] == 'y')
                    && (recv_buffer[2] == 'e') )
                    {

                        close(return_events[index].data.fd);
                        printf("close %d, ", return_events[index].data.fd);
                        int tmp_ret = epoll_ctl(efd, EPOLL_CTL_DEL, return_events[index].data.fd, NULL);
                        if(tmp_ret == -1)            
                        {     
                            printf("efd del %d has a error.\n", client_sockfd);        
                        }     
                        printf("remove descriptor %d successfully.\n", return_events[index].data.fd);
                    }
                }
            } // end of while(1)

            printf("read data finish------\n");
        }
    }

    free(return_events);
    // close server_sockfd
    shutdown(server_sockfd, 2);

    return 0;
}

epoll_client.cpp(only send data.)

#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <sys/socket.h>
#include <resolv.h>
#include <stdlib.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <fcntl.h>

#define BUFSIZE 1024

int main(int argc, char* argv[])
{
    int sock_clientfd, ret_recvsize, i;
    struct sockaddr_in dest, mine;
    char send_buffer[BUFSIZE + 1];

    // create socket fd
    if ((sock_clientfd = socket(AF_INET, SOCK_STREAM, 0)) < 0)
    {
        perror("Socket");
        exit(EXIT_FAILURE);
    }

    // init server address that client will connetct to.
    bzero(&dest, sizeof(dest));
    dest.sin_family = AF_INET;
    dest.sin_port = htons(9567);
    if(argc != 2)
    {
        printf("Usage: %s <dest ip>\n", argv[0]);
        printf("Usage: %s 127.0.0.1\n", argv[0]);
        return -1;
    }

    printf("-----\n");

    if (inet_aton(argv[1], (struct in_addr *) &dest.sin_addr.s_addr) == 0)
    {
        perror(argv[1]);
        exit(1);
    }

    // connect to server
    printf("will connect!\n");
    if (connect(sock_clientfd, (struct sockaddr *) &dest, sizeof(dest)) != 0)
    {
        perror("Connect ");
        exit(EXIT_FAILURE);
    }

    while(1)
    {
        bzero(send_buffer, BUFSIZE + 1);
        printf("input message:");
        fgets(send_buffer, BUFSIZE, stdin);
        send_buffer[strlen(send_buffer) - 1] = '\0';
        printf("%d\n", strlen(send_buffer));
        int send_retsize = send(sock_clientfd, send_buffer, strlen(send_buffer), 0);
        if(send_retsize == -1)
        {                                
            perror("send data to client error happen!");
            exit(EXIT_FAILURE);          
        }
        printf("send succ data:%s\n", send_buffer);
        if((send_buffer[0] == 'b')
         && (send_buffer[1] == 'y')
         && (send_buffer[2] == 'e') ) 
        {
            printf("client active close connect.\n");
            break;
        }
    }

    // close sock_clientfd
    close(sock_clientfd);

    return 0;
}

Follow pircture is some run info: epoll_server.png

epoll_client.png

The server read data is only 8 byte, Is the kernel design epoll is this? I guess the reasons are as follows pirture:

you may find this function useful: http://stackoverflow.com/questions/27527395/how-to-get-the-exact-message-from-recv-in-winsock-programming/27527668#27527668 — Giorgi Moniava, Sep 05 '16 at 08:28
Your opinion is of no relevance. The fact is that TCP provides no such guarantees. — user207421, Sep 05 '16 at 09:38
OT: `recv()` and `send()` return `ssize_t` not `int`. Also there is no need to cast `void`-pointers in C. — alk, Sep 05 '16 at 09:43
If `recv()` returned `0` the other side shut down the connection. Do not expect any data anymore. Leave the `recv`-loop. — alk, Sep 05 '16 at 09:50
@alk, Oh, I understand your idea. If recv returned zero, I should close connection and not read data. — study_20160808, Sep 05 '16 at 12:49

Klas Lindbäck · Accepted Answer · 2016-09-05T08:54:04.013

5

The reason you don't receive everything that is available in one read is because you only read 8 bytes at a time.

char* recv_buffer = (char*)malloc(1024+1);
int ret_read = recv(return_events[index].data.fd, recv_buffer, sizeof(recv_buffer), 0);
            // part_3-3-1:read return error

recv_buffer is a char* not an array, so sizeof recv_buffer equals the size of a pointer which in your case is 8.

Note that you should never rely on data arriving in packages. If your message protocol states that you should be getting 10 bytes never expect all 10 bytes to be available at once. You should always code in a way that can handle data being split up into multiple reads.

If the thread handles a single socket then a simple do { read... } while (total_bytes_received < expected_bytes); will suffice.

If the thread handles multiple connections, then you need to save the bytes you have read and then continue to manage other sockets that are ready before returning to your handling loop that will use select/epoll to wait for more data.

edited Sep 05 '16 at 08:54

answered Sep 05 '16 at 08:22

Klas Lindbäck

33,105
5
57
82

@ Klas Lindbäck,Oh, I got it.I have another question. In addition to my mistake, under what circumstances will appear I said that the problem described?“Stick package” or “Non complete reading”? – study_20160808 Sep 05 '16 at 08:29
@study_20160808 I've added information on how you should read from a socket to handle the case when a message is split into multiple ip packets. – Klas Lindbäck Sep 05 '16 at 08:55
@ Klas Lindbäck,I looked your updated answer.But I don't understand follow situation : my data content:N size. Final Ip pack is N + M size and N+M size less than 1500 byte.For this situation, Does my question happen? – study_20160808 Sep 05 '16 at 09:03
@study_20160808 Think about what will happen if the server is busy and several messages have arrived from the client when the server starts to read. Or if the client has sends so much that the send buffer is full. Both read and write calls need to assume that not all data can be read/sent in a single read/write. And when you read, more (or less) than one complete message may be available. – Klas Lindbäck Sep 05 '16 at 09:14
@ Klas Lindbäck, Yes, I got it, Thank you! – study_20160808 Sep 05 '16 at 12:46

In TCP socket program,client send some data, but server need read multiple times. why?

1 Answers1