When you call mq_receive it places the data at the buffer pointed to by the second argument, which you give as &resp. It does not change the pointer itself.
&resp is a fixed address in the parent, unless you change it, which appears unlikely from the posted code [which does not show the definition of resp], so:
printf("Received worker response: %p\n", &resp);
You will always get the same value.
What you [probably] want to do is print what resp contains
UPDATE:
Okay, there were a few more bugs.
The big bug is that while you can have one queue for worker-to-farmer messages (i.e. the response queue), you can not use a single queue for requests to workers. They each need their own request queue.
Otherwise, a single worker can absorb/monopolize all requests, even ones that belong to others. If that happened, the farmer would likely see messages that were stamped from only that worker.
This is what you're seeing, because, the first worker [probably #0] has its mq_receive complete first. It is, then, so fast that it does all of the mq_receive/mq_send before any others can get to them.
It will then see a "stop" message and exit. If the others are "lucky", the first worker left the remaining stop messages in the queue. But, no request messages, so they never send a response.
Also, the response queue was opened by the farmer with O_WRONLY instead of O_RDONLY.
I've produced two versions of your program. One with annotations for bugs. Another that is cleaned up and working.
Here's the annotated version [please pardon the gratuitous style cleanup]:
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <string.h>
#include <sys/wait.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
#include <unistd.h> // for execlp
#include <mqueue.h> // for mq
typedef struct {
int a;
} REQUEST;
typedef struct {
int b;
} RESPONSE;
char *pgmname;
static char mq_farmer[80];
static char mq_worker[80];
int
main(int argc,char **argv)
{
REQUEST req;
RESPONSE resp;
ssize_t sent;
pgmname = argv[0];
--argc;
++argv;
sprintf(mq_farmer,"/mq_request_%s_%d","foo",getpid());
sprintf(mq_worker,"/mq_response_%s_%d","bar",getpid());
// define attr
// NOTE/BUG: this can have random data in it
struct mq_attr attr;
attr.mq_maxmsg = 10;
// NOTE/BUG: this is _the_ big one -- we're only doing a single request
// queue -- each worker needs its _own_ request queue -- otherwise, a
// single worker can _monopolize_ all messages for the other workers
attr.mq_msgsize = sizeof(REQUEST);
mqd_t reqQueue = mq_open(mq_farmer,O_WRONLY | O_CREAT | O_EXCL,0600,&attr);
// NOTE/BUG: this should be opened for reading
attr.mq_msgsize = sizeof(RESPONSE);
mqd_t respQueue = mq_open(mq_worker,O_WRONLY | O_CREAT | O_EXCL,0600,&attr);
// create the child processes (see process_test() and message_queue_test())
int i;
// NOTE/BUG: we must remember the child pid numbers so we can do waitpid
// later
for (i = 0; i < 3; i++) {
pid_t processID = fork();
if (processID < 0) {
// error
}
else if (processID == 0) {
// some code
// NOTE/BUG: exec* takes strings so this is wrong
execlp("./worker","worker",getpid(),i,NULL);
}
}
// NOTE/BUG: on all mq_send/mq_receive, the return type is ssize_t and
// _not_ mqd_t
pid_t pid = fork();
if (pid < 0) {
// error
}
else {
// receiving done here
if (pid == 0) {
for (i = 0; i < 3; i++) {
// read the messages from the worker queue
ssize_t received = mq_receive(respQueue,(char *) &resp,
sizeof(resp),NULL);
printf("Farmer received worker response: %p with length %ld value %d\n",
&resp,received,resp.b);
// HERE &resp is always the same
}
// end worker process
req.a = -1;
sent = mq_send(reqQueue,(char *) &req,sizeof(req),0);
printf("Farmer sent stop -- sent=%ld\n",sent);
// NOTE/BUG: we need to exit here
}
// sending done here
else {
for (i = 0; i < 3; i++) {
req.a = i;
sent = mq_send(reqQueue,(char *) &req,sizeof(req),0);
printf("Farmer sent to i=%d -- sent=%ld\n",i,sent);
}
}
}
// NOTE/BUG: we're waiting on the double fork farmer, but _not_
// on the actual worker pids
waitpid(pid,NULL,0);
mq_close(reqQueue);
mq_close(respQueue);
// clean up the message queues
mq_unlink(mq_farmer);
mq_unlink(mq_worker);
return 0;
}
int
worker_main(int argc,char *argv[])
{
REQUEST req;
RESPONSE resp;
ssize_t sent;
int arg1;
// NOTE/BUG: use getppid instead
sscanf(argv[1],"%d",&arg1);
printf("worker: my index is %d ...\n",arg1);
sprintf(mq_farmer,"/mq_request_%s_%d","foo",arg1);
sprintf(mq_worker,"/mq_response_%s_%d","bar",arg1);
mqd_t reqQueue = mq_open(mq_farmer,O_RDONLY);
mqd_t respQueue = mq_open(mq_worker,O_WRONLY);
while (1) {
// receiving
ssize_t received = mq_receive(reqQueue,(char *) &req,
sizeof(req),NULL);
printf("Worker received %p with length %ld value %d\n",
&req,received,req.a);
// received stop signal
if (req.a < 0) {
printf("stopping worker\n");
break;
}
// waiting for farmer to fork
sleep(3);
// do something with request data
resp.b = req.a;
// send response
// NOTE/BUG: last argument is unsigned int and _not_ pointer
#if 0
sent = mq_send(respQueue,(char *) &resp,sizeof(resp),NULL);
#else
sent = mq_send(respQueue,(char *) &resp,sizeof(resp),0);
#endif
printf("Worker sent response %p with length %ld value %d\n",
&req,sent,req.a);
// HERE &resp is always different (doesn't print)
}
mq_close(reqQueue);
mq_close(respQueue);
// clean up the message queues
// NOTE/BUG: farmer should do this -- not worker
mq_unlink(mq_farmer);
mq_unlink(mq_worker);
return 0;
}
Here's the cleaned up and working version. Note that, for ease/simplicity, I combined both the farmer and worker programs into a single one, using a little bit of trickery in main:
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <string.h>
#include <sys/wait.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
#include <unistd.h> // for execlp
#include <mqueue.h> // for mq
typedef struct {
int a;
} REQUEST;
typedef struct {
int b;
} RESPONSE;
char *pgmname;
int opt_x;
int opt_W;
#define WORKNR 3
char mqfile_to_farmer[80];
char mqfile_to_worker[80];
struct mq_attr attr;
pid_t ppid;
// per-worker control
struct worker {
pid_t wk_pid;
mqd_t wk_req;
char wk_mqfile[80];
};
struct worker worklist[WORKNR];
void
worker(void)
{
REQUEST req;
RESPONSE resp;
ssize_t sent;
ppid = getppid();
printf("worker: my index is %d ...\n",opt_W);
sprintf(mqfile_to_farmer,"/mq_response_%d",ppid);
sprintf(mqfile_to_worker,"/mq_request_%d_%d",ppid,opt_W);
mqd_t reqQueue = mq_open(mqfile_to_worker,O_RDONLY);
mqd_t respQueue = mq_open(mqfile_to_farmer,O_WRONLY);
while (1) {
// receiving
errno = 0;
ssize_t received = mq_receive(reqQueue,(char *) &req,
sizeof(req),NULL);
printf("Worker %d received %p with length %ld value %d -- %s\n",
opt_W,&req,received,req.a,strerror(errno));
if (received < 0)
exit(77);
// received stop signal
if (req.a < 0) {
printf("stopping worker\n");
break;
}
// do something with request data
resp.b = req.a;
// send response
errno = 0;
sent = mq_send(respQueue,(char *) &resp,sizeof(resp),0);
printf("Worker %d sent response %p with length %ld value %d -- %s\n",
opt_W,&req,sent,req.a,strerror(errno));
// HERE &resp is always different (doesn't print)
if (sent < 0)
exit(78);
}
mq_close(reqQueue);
mq_close(respQueue);
exit(0);
}
void
farmer(void)
{
REQUEST req;
RESPONSE resp;
ssize_t sent;
struct worker *wk;
ppid = getpid();
sprintf(mqfile_to_farmer,"/mq_response_%d",ppid);
attr.mq_maxmsg = 10;
attr.mq_msgsize = sizeof(REQUEST);
mqd_t respQueue = mq_open(mqfile_to_farmer,
O_RDONLY | O_CREAT | O_EXCL,0600,&attr);
if (respQueue < 0) {
printf("farmer: respQueue open fault -- %s\n",strerror(errno));
exit(1);
}
// create the child processes (see process_test() and message_queue_test())
int i;
// create the separate request queues
for (i = 0; i < WORKNR; i++) {
wk = &worklist[i];
attr.mq_msgsize = sizeof(RESPONSE);
sprintf(wk->wk_mqfile,"/mq_request_%d_%d",ppid,i);
wk->wk_req = mq_open(wk->wk_mqfile,O_WRONLY | O_CREAT | O_EXCL,0600,
&attr);
if (wk->wk_req < 0) {
printf("farmer: wk_req open fault -- %s\n",strerror(errno));
exit(1);
}
}
for (i = 0; i < WORKNR; i++) {
wk = &worklist[i];
pid_t pid = fork();
if (pid < 0) {
perror("fork");
exit(9);
}
if (pid != 0) {
wk->wk_pid = pid;
continue;
}
// NOTE/FIX: exec* takes strings so this is the correct way
if (opt_x) {
char xid[20];
sprintf(xid,"-W%d",i);
execlp(pgmname,pgmname,xid,NULL);
perror("execlp");
exit(7);
}
// simulate what exec would do -- call it direct
opt_W = i;
worker();
}
pid_t pid = fork();
if (pid < 0) {
perror("fork2");
exit(5);
}
// receiving done here
if (pid == 0) {
for (i = 0; i < WORKNR; i++) {
// read the messages from the worker queue
ssize_t received = mq_receive(respQueue,(char *) &resp,
sizeof(resp),NULL);
printf("Farmer received worker response: %p with length %ld value %d\n",
&resp,received,resp.b);
// HERE &resp is always the same
}
// end worker process
for (i = 0; i < WORKNR; i++) {
wk = &worklist[i];
req.a = -1;
sent = mq_send(wk->wk_req,(char *) &req,sizeof(req),0);
printf("Farmer sent stop -- sent=%ld\n",sent);
}
// exit the farmer's receiver
printf("farmer: receiver exiting ...\n");
exit(0);
}
// sending done here
else {
for (i = 0; i < WORKNR; i++) {
wk = &worklist[i];
req.a = i;
sent = mq_send(wk->wk_req,(char *) &req,sizeof(req),0);
printf("Farmer sent to i=%d -- sent=%ld\n",i,sent);
}
// wait for farmer's receiver to complete
printf("farmer: waiting for receiver to finish ...\n");
waitpid(pid,NULL,0);
}
mq_close(respQueue);
// wait for all workers to complete
for (i = 0; i < WORKNR; i++) {
wk = &worklist[i];
printf("farmer: waiting for worker to finish ...\n");
waitpid(wk->wk_pid,NULL,0);
mq_close(wk->wk_req);
mq_unlink(wk->wk_mqfile);
}
// clean up the message queues
mq_unlink(mqfile_to_farmer);
}
int
main(int argc,char **argv)
{
char *cp;
pgmname = argv[0];
--argc;
++argv;
opt_W = -1;
for (; argc > 0; --argc, ++argv) {
cp = *argv;
if (*cp != '-')
break;
switch (cp[1]) {
case 'W':
opt_W = atoi(cp + 2);
break;
case 'x':
opt_x = ! opt_x;
break;
}
}
if (opt_W >= 0)
worker();
else
farmer();
return 0;
}
UPDATE #2:
Here's a version that demonstrates single vs. multiple request queues. The workers now check the destination id in the message they receive matches their worker number.
If you just run it with no options, you'll get multiple queues and the "good" output.
If you run it with -b [and optionally -s] you'll get a single request queue and the program will see misrouted messages (e.g. worker 0 grabs a message intended for worker 1).
Single queue is a subset. As long as workers are "equal", it's okay. But, if they're not (e.g. one worker can do things others can't), being able to queue to the correct worker is important. An example would be a network node that has special FPGA assisted calculation hardware that other ones don't and some requests need that acceleration.
Also, single queue is self balancing by the workers. That is one form of scheduling, but there are other models. (e.g. the farmer wants to retain control of the distribution of labor). Or, the farmer has to stop one worker and keep the others going (e.g. the system being stopped will be powered off for maintenance).
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <string.h>
#include <time.h>
#include <sys/wait.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
#include <unistd.h> // for execlp
#include <mqueue.h> // for mq
typedef unsigned int u32;
typedef struct {
u32 seqno; // sequence number
int toval; // destination id
int fmval; // responder worker id
} request_t;
char *pgmname;
int opt_b; // 1=broadcast
int opt_i; // 1=ignore errors
int opt_x; // 1=do execlp
int opt_s; // number of ms to sleep
int opt_S; // sequence maximum
int opt_W; // worker xid
#define WORKNR 3
#define MAXMSG 10
char mqfile_to_farmer[80];
mqd_t respQueue;
char mqfile_to_worker[80];
mqd_t reqQueue;
struct mq_attr attr;
pid_t ppid;
pid_t curpid;
pid_t pidrcvr;
// per-worker control
typedef struct {
int wk_xid;
pid_t wk_pid;
mqd_t wk_req;
u32 wk_seqno;
char wk_mqfile[80];
} worker_t;
worker_t worklist[WORKNR];
#define FORALL_WK \
wk = &worklist[0]; wk < &worklist[WORKNR]; ++wk
#define sysfault(_fmt...) \
do { \
printf(_fmt); \
if (ppid) \
kill(ppid,SIGUSR1); \
exit(1); \
} while (0)
void
_sysfault(void)
{
__asm__ __volatile__("" :::);
}
#define logprt(_fmt...) \
do { \
int sverr = errno; \
_logprt(); \
printf(_fmt); \
errno = sverr; \
} while (0)
int logxid;
double logzero;
void
loginit(int xid)
{
logxid = xid;
}
void
_logprt(void)
{
struct timespec ts;
double sec;
clock_gettime(CLOCK_REALTIME,&ts);
sec = ts.tv_nsec;
sec /= 1e9;
sec += ts.tv_sec;
if (logzero == 0)
logzero = sec;
sec -= logzero;
switch (logxid) {
case WORKNR:
printf("%.9f LOG F: ",sec);
break;
case WORKNR + 1:
printf("%.9f LOG R: ",sec);
break;
default:
printf("%.9f LOG W%d: ",sec,logxid);
break;
}
}
void
logexit(int code)
{
exit(code);
}
void
allwait(void)
{
worker_t *wk;
// wait for farmer's receiver to complete
if (pidrcvr) {
logprt("farmer: waiting for receiver to finish ...\n");
waitpid(pidrcvr,NULL,0);
pidrcvr = 0;
}
for (FORALL_WK) {
if (wk->wk_pid) {
logprt("farmer: waiting for worker %d to finish ...\n",wk->wk_xid);
waitpid(wk->wk_pid,NULL,0);
wk->wk_pid = 0;
}
if (opt_b)
continue;
logprt("farmer: closing and removing worker queue ...\n");
mq_close(wk->wk_req);
mq_unlink(wk->wk_mqfile);
}
}
void
sighdr(int signo)
{
worker_t *wk;
switch (signo) {
case SIGUSR1: // request to master
logprt("sighdr: got master stop signal ...\n");
if (pidrcvr)
kill(pidrcvr,SIGUSR2);
for (FORALL_WK) {
if (wk->wk_pid)
kill(wk->wk_pid,SIGUSR2);
}
allwait();
logprt("farmer: abnormal termination\n");
logexit(1);
break;
case SIGUSR2: // request to slaves
logexit(1);
break;
}
}
void
reqopen(mqd_t *fdp,const char *file,int flag)
{
mqd_t fd;
int err;
attr.mq_maxmsg = MAXMSG;
attr.mq_msgsize = sizeof(request_t);
fd = *fdp;
if (fd >= 0)
mq_close(fd);
fd = mq_open(file,flag | O_CREAT,0600,&attr);
if (fd < 0)
sysfault("reqopen: %s open fault -- %s\n",file,strerror(errno));
err = mq_getattr(fd,&attr);
if (err < 0)
sysfault("reqopen: %s getattr fault -- %s\n",file,strerror(errno));
if (attr.mq_msgsize != sizeof(request_t))
sysfault("reqopen: %s size fault -- mq_msgsize=%ld siz=%ld\n",
file,attr.mq_msgsize,sizeof(request_t));
logprt("reqopen: open -- file='%s' fd=%d\n",file,fd);
*fdp = fd;
}
void worker(int execflg);
void
farmer(void)
{
request_t req;
request_t resp;
ssize_t sent;
worker_t *wk;
u32 seqno;
int xid;
ppid = getpid();
curpid = ppid;
loginit(WORKNR);
sprintf(mqfile_to_farmer,"/mq_response_%d",ppid);
sprintf(mqfile_to_worker,"/mq_request_%d",ppid);
respQueue = -1;
reqopen(&respQueue,mqfile_to_farmer,O_RDONLY | O_CREAT | O_EXCL);
reqQueue = -1;
if (opt_b)
reqopen(&reqQueue,mqfile_to_worker,O_WRONLY | O_CREAT | O_EXCL);
// create the separate request queues
xid = 0;
for (FORALL_WK) {
wk->wk_xid = xid++;
if (opt_b) {
logprt("farmer: common request queue -- reqQueue=%d\n",reqQueue);
wk->wk_req = reqQueue;
continue;
}
sprintf(wk->wk_mqfile,"/mq_request_%d_%d",ppid,wk->wk_xid);
wk->wk_req = -1;
reqopen(&wk->wk_req,wk->wk_mqfile,O_WRONLY | O_CREAT | O_EXCL);
logprt("farmer: separate request queue -- wk_req=%d\n",wk->wk_req);
}
// fork the workers
for (FORALL_WK) {
pid_t pid = fork();
if (pid < 0)
sysfault("farmer: fork fault -- %s\n",strerror(errno));
if (pid != 0) {
wk->wk_pid = pid;
continue;
}
// NOTE/FIX: exec* takes strings so this is the correct way
if (opt_x) {
char opt[2][20];
sprintf(opt[0],"-b%d",opt_b);
sprintf(opt[1],"-W%d",wk->wk_xid);
execlp(pgmname,pgmname,opt[0],opt[1],NULL);
sysfault("farmer: execlp error -- %s\n",strerror(errno));
}
// simulate what exec would do -- call it direct
opt_W = wk->wk_xid;
worker(0);
}
pidrcvr = fork();
if (pidrcvr < 0)
sysfault("farmer: fork2 error -- %s\n",strerror(errno));
// receiving done here
if (pidrcvr == 0) {
curpid = getpid();
loginit(WORKNR + 1);
for (int i = 0; i < (WORKNR * opt_S); i++) {
// read the messages from the worker queue
ssize_t received = mq_receive(respQueue,(char *) &resp,
sizeof(resp),NULL);
wk = &worklist[resp.fmval];
logprt("received worker response: length %d fmval=%d seqno=%u wk_seqno=%u\n",
(int) received,resp.fmval,resp.seqno,wk->wk_seqno);
if (received < 0) {
if (! opt_i)
sysfault("farmer: received fault -- %s\n",strerror(errno));
}
if (resp.seqno != wk->wk_seqno) {
logprt("sequence fault\n");
if (! opt_i)
sysfault("farmer: sequence fault\n");
}
++wk->wk_seqno;
}
// send stop to worker processes
for (FORALL_WK) {
req.toval = -1;
sent = mq_send(wk->wk_req,(char *) &req,sizeof(req),0);
logprt("Farmer sent stop -- wk_xid=%d sent=%d\n",
wk->wk_xid,(int) sent);
if (sent < 0) {
if (! opt_i)
sysfault("farmer: send fault on stop -- %s\n",
strerror(errno));
}
}
// exit the farmer's receiver
logprt("farmer: receiver exiting ...\n");
logexit(0);
}
// sending done here
else {
for (seqno = 0; seqno < opt_S; ++seqno) {
for (FORALL_WK) {
wk->wk_seqno = seqno;
req.seqno = seqno;
req.toval = wk->wk_xid;
sent = mq_send(wk->wk_req,(char *) &req,sizeof(req),0);
logprt("Farmer sent to wk_xid=%d wk_req=%d -- sent=%d\n",
wk->wk_xid,wk->wk_req,(int) sent);
if (sent < 0) {
if (! opt_i)
sysfault("farmer: send fault -- %s\n",strerror(errno));
}
}
}
}
mq_close(respQueue);
// wait for all workers to complete
allwait();
// clean up the message queues
mq_unlink(mqfile_to_farmer);
logprt("farmer: complete\n");
logexit(0);
}
void
worker(int execflg)
{
request_t req;
request_t resp;
ssize_t sent;
u32 seqno;
int slpcnt;
if (execflg)
ppid = getppid();
curpid = getpid();
loginit(opt_W);
logprt("worker: my index is %d ...\n",opt_W);
attr.mq_maxmsg = MAXMSG;
sprintf(mqfile_to_farmer,"/mq_response_%d",ppid);
reqopen(&respQueue,mqfile_to_farmer,O_WRONLY);
if (opt_b)
sprintf(mqfile_to_worker,"/mq_request_%d",ppid);
else
sprintf(mqfile_to_worker,"/mq_request_%d_%d",ppid,opt_W);
reqopen(&reqQueue,mqfile_to_worker,O_RDONLY);
seqno = 0;
slpcnt = opt_s;
slpcnt *= 1000;
slpcnt *= opt_W;
while (1) {
if (slpcnt > 0) {
logprt("sleep %d\n",slpcnt);
usleep(slpcnt);
slpcnt = 0;
}
// receiving
errno = 0;
ssize_t received = mq_receive(reqQueue,(char *) &req,
sizeof(req),NULL);
logprt("received length %d -- seqno=%u toval=%d\n",
(int) received,req.seqno,req.toval);
if (received < 0)
sysfault("worker: mq_receive fault -- %s\n",strerror(errno));
// received stop signal
if (req.toval < 0) {
logprt("stopping ...\n");
break;
}
if (req.toval != opt_W) {
logprt("misroute\n");
if (! opt_i)
sysfault("worker: misroute fault\n");
}
if (req.seqno != seqno) {
logprt("sequence fault\n");
if (! opt_i)
sysfault("worker: sequence fault\n");
}
// do something with request data
resp.seqno = req.seqno;
resp.toval = req.toval;
resp.fmval = opt_W;
// send response
errno = 0;
sent = mq_send(respQueue,(char *) &resp,sizeof(resp),0);
logprt("sent response with length %d -- seqno=%u toval=%d\n",
(int) sent,req.seqno,resp.toval);
// HERE &resp is always different (doesn't print)
if (sent < 0)
sysfault("worker: mq_send fault -- %s\n",strerror(errno));
++seqno;
}
mq_close(reqQueue);
mq_close(respQueue);
logexit(0);
}
int
main(int argc,char **argv)
{
char *cp;
pgmname = argv[0];
--argc;
++argv;
opt_W = -1;
opt_S = 3;
reqQueue = -1;
respQueue = -1;
signal(SIGUSR1,sighdr);
signal(SIGUSR2,sighdr);
for (; argc > 0; --argc, ++argv) {
cp = *argv;
if (*cp != '-')
break;
switch (cp[1]) {
case 'b': // broadcast mode (single request queue)
cp += 2;
opt_b = (*cp != 0) ? atoi(cp) : 1;
break;
case 'i': // ignore errors
cp += 2;
opt_i = (*cp != 0) ? atoi(cp) : 1;
break;
case 'S': // sequence maximum
cp += 2;
opt_S = (*cp != 0) ? atoi(cp) : 3;
break;
case 's': // sleep mode (milliseconds)
cp += 2;
opt_s = (*cp != 0) ? atoi(cp) : 3;
break;
case 'W': // worker number
cp += 2;
opt_W = atoi(cp + 2);
break;
case 'x': // use execlp
opt_x = ! opt_x;
break;
}
}
if (opt_W >= 0)
worker(1);
else
farmer();
return 0;
}
