I have two programs, a server and a client. The server opens a file, writes data to it, and then send its file descriptor to the client over a unix domain socket. Everything works fine untill I introduce a socat proxy in between.
socat -x -v UNIX-LISTEN:/tmp/unixSockSendFe,mode=775,reuseaddr,fork UNIX-CONNECT:/tmp/unixSockSendFd
Explanation
The server listens on /tmp/unixSockSendFd, socat connects to it(UNIX-CONNECT:/tmp/unixSockSendFd), and creates another Unix domain socket(UNIX-LISTEN:/tmp/unixSockSendFe,mode=775,reuseaddr,fork), on which the client connects. Any communication between the client and server gets relayed through socat, which prints the bytes sent in their binary (-x option), and ascii (-v option) form.
If I don't use socat, and client directly connects to server(on /tmp/unixSockSendFd socket), everything works fine, but when socat is used as a proxy, the client crashes with a segmentation fault.
Server
/*Server code - sendfd.c*/
#include <sys/socket.h>
#include <sys/un.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <errno.h>
char *socket_path = "/tmp/unixSockSendFd";
char *file="/tmp/abcd.txt" ;/*file whose fd is to be sent*/
int sendfd(int sock, int fd);
int recvfd(int s);
char data[]="sahil\0";
int main(int argc, char *argv[]) {
struct sockaddr_un addr;
char buf[100];
buf[0]='\n';
int fd,rc,confd;
int fd_to_send;
int temp,len;
temp=1;
fd_to_send=open(file,O_TRUNC|O_RDWR|O_CREAT,S_IRWXU|S_IRWXG|S_IRWXO);
if(fd_to_send==-1)
{
perror("file open error");
return -1;
}
if (argc > 1) socket_path=argv[1];
if ( (fd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
perror("socket error");
exit(-1);
}
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
if (*socket_path == '\0') {
*addr.sun_path = '\0';
strncpy(addr.sun_path+1, socket_path+1, sizeof(addr.sun_path)-2);
} else {
strncpy(addr.sun_path, socket_path, sizeof(addr.sun_path)-1);
}
unlink(socket_path);
if(bind(fd,(struct sockaddr*)&addr,sizeof(addr))==-1){
perror("bind error");
return -1;
}
/*Writing data to file before sending fd*/
len=write(fd_to_send,data,(int)strlen(data));
fsync(fd_to_send);
printf("(len=%d)data written in file(content between ## marks) ##%s##\n",len,data);
listen(fd,1);
for(;;){
confd=accept(fd,NULL,NULL);
if(confd==-1)
{
perror("accept error");
continue;
}
else{
printf("new client connected ... sending fd ... \n");
sendfd(confd,fd_to_send);
close(confd);
}
}
return 0;
}
int sendfd(int sock, int fd)
{
struct msghdr hdr;
struct iovec data;
char cmsgbuf[CMSG_SPACE(sizeof(int))];
char dummy = '*';
data.iov_base = &dummy;
data.iov_len = sizeof(dummy);
memset(&hdr, 0, sizeof(hdr));
hdr.msg_name = NULL;
hdr.msg_namelen = 0;
hdr.msg_iov = &data;
hdr.msg_iovlen = 1;
hdr.msg_flags = 0;
hdr.msg_control = cmsgbuf;
hdr.msg_controllen = CMSG_LEN(sizeof(int));
struct cmsghdr* cmsg = CMSG_FIRSTHDR(&hdr);
cmsg->cmsg_len = CMSG_LEN(sizeof(int));
cmsg->cmsg_level = SOL_SOCKET;
cmsg->cmsg_type = SCM_RIGHTS;
*(int*)CMSG_DATA(cmsg) = fd;
//memcpy((CMSG_DATA(cmsg)), &fd, sizeof(fd)); -- from ivshmem server code - this too works instead of previous line
int n = sendmsg(sock, &hdr, 0);
if(n == -1)
printf("sendmsg() failed: %s (socket fd = %d)\n", strerror(errno), sock);
return n;
}
int recvfd(int s)
{
int n;
int fd;
char buf[1];
struct iovec iov;
struct msghdr msg;
struct cmsghdr *cmsg;
char cms[CMSG_SPACE(sizeof(int))];
iov.iov_base = buf;
iov.iov_len = 1;
memset(&msg, 0, sizeof msg);
msg.msg_name = 0;
msg.msg_namelen = 0;
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
msg.msg_control = (caddr_t)cms;
msg.msg_controllen = sizeof cms;
if((n=recvmsg(s, &msg, 0)) < 0)
return -1;
if(n == 0){
perror("unexpected EOF");
return -1;
}
cmsg = CMSG_FIRSTHDR(&msg);
memmove(&fd, CMSG_DATA(cmsg), sizeof(int));
return fd;
}
Client
/*Client code - recvfd.c*/
#include <sys/socket.h>
#include <sys/un.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <errno.h>
char *socket_path = "/tmp/unixSockSendFe";
int sendfd(int sock, int fd);
int recvfd(int s);
int fd_received;
int main(int argc, char *argv[]) {
struct sockaddr_un addr;
char buf[100];
buf[0]='\n';
int fd,rc,confd;
int temp,len;
temp=1;
if (argc > 1) socket_path=argv[1];
if ( (fd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
perror("socket error");
exit(-1);
}
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
if (*socket_path == '\0') {
*addr.sun_path = '\0';
strncpy(addr.sun_path+1, socket_path+1, sizeof(addr.sun_path)-2);
} else {
strncpy(addr.sun_path, socket_path, sizeof(addr.sun_path)-1);
}
if(connect(fd,(struct sockaddr*)&addr,sizeof(addr))==-1)
{
perror("connect error");
exit(-1);
}
fd_received=recvfd(fd);
lseek(fd_received,0,SEEK_SET);
len=read(fd_received,buf,5);
if(len<0)
{
perror("read error");
}
printf("(fd_received=%d,len=%d) first %d characters read from the file whoes fd was received(content within ##) ##%.*s##\n",fd_received,len,5,5,buf);
return 0;
}
int sendfd(int sock, int fd)
{
struct msghdr hdr;
struct iovec data;
char cmsgbuf[CMSG_SPACE(sizeof(int))];
char dummy = '*';
data.iov_base = &dummy;
data.iov_len = sizeof(dummy);
memset(&hdr, 0, sizeof(hdr));
hdr.msg_name = NULL;
hdr.msg_namelen = 0;
hdr.msg_iov = &data;
hdr.msg_iovlen = 1;
hdr.msg_flags = 0;
hdr.msg_control = cmsgbuf;
hdr.msg_controllen = CMSG_LEN(sizeof(int));
struct cmsghdr* cmsg = CMSG_FIRSTHDR(&hdr);
cmsg->cmsg_len = CMSG_LEN(sizeof(int));
cmsg->cmsg_level = SOL_SOCKET;
cmsg->cmsg_type = SCM_RIGHTS;
*(int*)CMSG_DATA(cmsg) = fd;
int n = sendmsg(sock, &hdr, 0);
if(n == -1)
printf("sendmsg() failed: %s (socket fd = %d)\n", strerror(errno), sock);
return n;
}
int recvfd(int s)
{
int n;
int fd;
char buf[1];
struct iovec iov;
struct msghdr msg;
struct cmsghdr *cmsg;
char cms[CMSG_SPACE(sizeof(int))];
iov.iov_base = buf;
iov.iov_len = 1;
memset(&msg, 0, sizeof msg);
msg.msg_name = 0;
msg.msg_namelen = 0;
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
msg.msg_control = (caddr_t)cms;
msg.msg_controllen = sizeof cms;
if((n=recvmsg(s, &msg, 0)) < 0)
return -1;
if(n == 0){
perror("unexpected EOF");
return -1;
}
cmsg = CMSG_FIRSTHDR(&msg);
memmove(&fd, CMSG_DATA(cmsg), sizeof(int));
return fd;
}
On running client (recvfd) I get segmentation fault.
./recvfd
[1] 6104 segmentation fault (core dumped) ./recvfd
Here are lines from running gdb with coredump
Core was generated by `./recvfd'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0x0000000000400cf9 in recvfd (s=3) at recvfd.c:146
146 memmove(&fd, CMSG_DATA(cmsg), sizeof(int));
Here is the core dump - Link.
I want to sniff the communication happening between the two processes when the file descriptor is being sent. I am not able to figure out why the client crashes when run with socat, but doesn't when run without it.
Update 1
While using socat to sniff communication happening between two processes of a well established open source project (ivshmem - used for sharing memory between running virtual machines, also a part of Intel DPDK, Link), I observed the following.
- None of the processes crash on using
socat - When
socatis used, the file descriptor is not properly sent, and does not get added to the recipient process. - When
socatis not used, and the two processes are connected directly, the file descriptor gets sent properly, and gets added to the recipient process.
