Can a live TCP connection be passed to a different process without terminating the sessions in the current process?

Question

In HA proxy and other load balancing systems, we have a process called reload which forks new process and let it take up new connections and the existing connection will stick to the old process and the old process will stay alive until all the connections are drained.

Is there any way to move the existing connections from the old process to newly forked process and close the old one?

Can someone kindly explain why it cannot be done?

Answer 1

Here is an example adapted from something I did a long time ago.
The trick is to rely on the ancillary data of sendmsg()/recvmsg() system calls on a local socket to pass an open file descriptor.
This example uses regular files to illustrate fd-passing, but of course TCP sockets can be used instead (in a non-windows environment).

There are many details that are not so obvious without digging into the documentation, but once encapsulated into some functions like send_fd()/receive_fd() in this example, it is pretty straightforward to use.

Note that, when received, the file descriptor does not necessarily have the same number as when it was sent.
One should think about it as a kind of dup() between different processes: file descriptors with different numbers actually refer to the same resource.
Note also that, as soon as the file descriptor has been sent to another process, it can be closed (exactly like with dup()/dup2()) because the new file descriptor that will be received by the other process still refers to the original resource.

/**
  gcc -std=c99 -o prog_c prog_c.c \
      -pedantic -Wall -Wextra -Wconversion \
      -Wc++-compat -Wwrite-strings -Wold-style-definition -Wvla \
      -g -O0 -UNDEBUG -fsanitize=address,undefined

  $ ./prog_c
  parent-process: using fd 3
  child-process: using fd 4
  parent-process: using fd 3
  child-process: using fd 4
  parent-process: using fd 3
  child-process: using fd 4
  parent-process: using fd 3
  child-process: using fd 4
  parent-process: using fd 3
  child-process: using fd 4
  child-process: done
  $ cat file_?.txt
  file_0.txt written by parent-process when opening
  Now child-process uses this open file.
  file_1.txt written by parent-process when opening
  Now child-process uses this open file.
  file_2.txt written by parent-process when opening
  Now child-process uses this open file.
  file_3.txt written by parent-process when opening
  Now child-process uses this open file.
  file_4.txt written by parent-process when opening
  Now child-process uses this open file.
**/


#include <sys/types.h>
#include <sys/wait.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

int // 0: success    -1: error
send_fd(int control_fd,
        int fd_to_send)
{
  int dummy_msg=1234;
  struct iovec vec;
  vec.iov_base=&dummy_msg;
  vec.iov_len=sizeof(dummy_msg);
  struct msghdr msg;
  msg.msg_name=NULL;
  msg.msg_namelen=0;
  msg.msg_iov=&vec;
  msg.msg_iovlen=1;
  struct cmsghdr * cmsg;
  char buffer[CMSG_SPACE(sizeof(int))];
  msg.msg_control=buffer;
  msg.msg_controllen=CMSG_SPACE(sizeof(int));
  cmsg=CMSG_FIRSTHDR(&msg);
  cmsg->cmsg_level=SOL_SOCKET;
  cmsg->cmsg_type=SCM_RIGHTS; // fd passing
  cmsg->cmsg_len=CMSG_LEN(sizeof(int));
  *((int *)CMSG_DATA(cmsg))=fd_to_send; // send new file descriptor
  msg.msg_flags=0;
  return sendmsg(control_fd, &msg, 0)==sizeof(dummy_msg) ? 0 : -1;
}

int // fd or -1
receive_fd(int control_fd)
{
  char buffer[CMSG_SPACE(sizeof(int))];
  struct msghdr msg;
  struct iovec vec;
  struct cmsghdr *cmsg;
  int dummy_msg;
  vec.iov_base=&dummy_msg;
  vec.iov_len=sizeof(dummy_msg);
  msg.msg_name=NULL;
  msg.msg_namelen=0;
  msg.msg_iov=&vec;
  msg.msg_iovlen=1;
  msg.msg_control=buffer;
  msg.msg_controllen=CMSG_SPACE(sizeof(int));
  msg.msg_flags=0;
  if(recvmsg(control_fd, &msg, 0)!=sizeof(dummy_msg))
  {
    return -1;
  }
  int fd=-1;
  cmsg=CMSG_FIRSTHDR(&msg); // ancillary data?
  if(cmsg&&(cmsg->cmsg_len>=(socklen_t)CMSG_LEN(sizeof(int)))&&
     (cmsg->cmsg_level==SOL_SOCKET)&&
     (cmsg->cmsg_type==SCM_RIGHTS)) // fd passing?
  {
    fd=*((int *)CMSG_DATA(cmsg)); // store new file descriptor
  }
  return fd;
}

int
main(void)
{
  int control_pair[2];
  if(socketpair(PF_LOCAL, SOCK_STREAM, 0, control_pair)==-1)
  {
    perror("socketpair");
    exit(1);
  }
  pid_t p=fork();
  if(p==-1)
  {
    perror("fork");
    exit(1);
  }
  if(p==0) // child process
  {
    close(control_pair[1]); // used by parent-process (arbitrary)
    for(;;)
    {
      int fd=receive_fd(control_pair[0]);
      if(fd==-1)
      {
        printf("child-process: done\n");
        break;
      }
      printf("child-process: using fd %d\n", fd);
      const char *text="Now child-process uses this open file.\n";
      write(fd, text, strlen(text));
      close(fd);
    }
    close(control_pair[0]);
    exit(0); // child-process stops here
  }
  // parent process
  close(control_pair[0]); // used by child-process (arbitrary)
  for(int i=0; i<5; ++i)
  {
    char text[100];
    sprintf(text, "file_%d.txt", i);
    int fd=open(text, O_WRONLY|O_TRUNC|O_CREAT, 0644);
    if(fd==-1)
    {
      perror("open");
      break;
    }
    printf("parent-process: using fd %d\n", fd);
    strcat(text, " written by parent-process when opening\n");
    write(fd, text, strlen(text));
    if(send_fd(control_pair[1], fd)==-1)
    {
      perror("send_fd");
      close(fd);
      break;
    }
    close(fd);
    sleep(1);
  }
  close(control_pair[1]);
  if(waitpid(p, NULL, 0)!=p)
  {
    perror("waitpid");
    exit(1);
  }
  return 0;
}