thread grabs other threads IPC message from queue (Linux)

Question

When I have two different IPC message queues on Linux, sometimes the messages from the wrong queue are grabbed.

The following toy program displays the problem, can be repeated on different processors.

Any help greatly appreciated!

Bert

/*
    To compile;
    gcc MM.c -o mm -fno-stack-protector -pthread

    We want Mickey to send a message to Minnie exclusively.
    We want Donald to send a message to pluto exclusively.

    Problem: Pluto intercepts Minnie's messages.

    Listing gives:

    $ ./mm
    Mickey thread successfully started.
    Minnie thread successfully started.
    Pluto thread successfully started.
    Donald thread successfully started.
    Donald sent a message to Pluto.
    Mickey sent a message to Minnie.
    Pluto received: Sit, Pluto!

    Minnie received: Hello, Minnie!

    Mickey sent a message to Minnie.  (100 times)

    Pluto received: Hello, Minnie!
*/
#include <sys/types.h>
#include <sys/msg.h>
#include <sys/ipc.h>
#include <string.h>
#include <stdio.h>

pthread_t t1,t2,t3,t4;  

// Mickey send
key_t ipcMickey;
int mqMickeyid;
char helloMickeymsg[] = {"Hello, Minnie!"};
struct { long type; char text[100]; } myMickeymsg;

// Minnie get
int mqMinnieid;
struct { long type; char text[100]; } myMinniemsg;

// Donald send
key_t ipcDonald;
int mqDonaldid;
char helloDonaldmsg[] = {"Sit, Pluto!"};
struct { long type; char text[100]; } myDonaldmsg;

// Pluto get
int mqPlutoid;
struct { long type; char text[100]; } myPlutomsg;

static void * DONALDthreadFunc(void *arg)
{
    printf("Donald thread successfully started.\n");
    char *s = (char *) arg;

    while(1)
    {
        //send
        memset(myDonaldmsg.text, 0, 100); 
        strncpy(myDonaldmsg.text, helloDonaldmsg, strlen(helloDonaldmsg));
        myDonaldmsg.type = 1;
        msgsnd(mqDonaldid, &myDonaldmsg, sizeof(myDonaldmsg), 0);
        printf("Donald sent a message to Pluto.\r\n");
        sleep(4);
    }

    /* just a formality */
    return (void *) strlen(s);
}

static void * PLUTOthreadFunc(void *arg)
{
    printf("Pluto thread successfully started.\n");
    char *s = (char *) arg;

    while(1)
    {
        //receive
        mqPlutoid = msgget(ipcDonald, 0);
        msgrcv(mqPlutoid, &myPlutomsg, sizeof(myPlutomsg), 0, 0);
        printf("Pluto received: %s\r\n\r\n", myPlutomsg.text);
        sleep(1);
    }

    /* just a formality */
    return (void *) strlen(s);
}

static void * MICKEYthreadFunc(void *arg)
{
    printf("Mickey thread successfully started.\n");
    char *s = (char *) arg;

    while(1)
    {
        //send
        memset(myMickeymsg.text, 0, 100); 
        strncpy(myMickeymsg.text, helloMickeymsg, strlen(helloMickeymsg));
        myMickeymsg.type = 1;
        msgsnd(mqMickeyid, &myMickeymsg, sizeof(myMickeymsg), 0);
        printf("Mickey sent a message to Minnie.\r\n");
        usleep(10000);
    }

    /* just a formality */
    return (void *) strlen(s);
}

static void * MINNIEthreadFunc(void *arg)
{
    printf("Minnie thread successfully started.\n");
    char *s = (char *) arg;

    while(1)
    {
        //receive
        mqMinnieid = msgget(ipcMickey, 0);
        msgrcv(mqMinnieid, &myMinniemsg, sizeof(myMinniemsg), 0, 0);
        printf("Minnie received: %s\r\n\r\n", myMinniemsg.text);
        sleep(3);
    }

    return (void *) strlen(s);
}

int main (void) 
{
        ipcMickey = ftok("/tmp/mqmickey", 63);
        mqMickeyid = msgget(ipcMickey, IPC_CREAT | 0666);

    ipcDonald = ftok("/tmp/mqdonald", 69);
        mqDonaldid = msgget(ipcDonald, IPC_CREAT | 0666);

    pthread_create(&t1, NULL, MICKEYthreadFunc, "Mickey sends\n");
    pthread_create(&t2, NULL, MINNIEthreadFunc, "Minnie replies\n");
    pthread_create(&t3, NULL, DONALDthreadFunc, "Donald sends\n");
    pthread_create(&t4, NULL, PLUTOthreadFunc, "Pluto replies\n");

    while(1)
    {
        sleep(5);
    }
}

+1 for compiling minimal example, will test – sehe Apr 12 '11 at 20:58 — sehe, Apr 12 '11 at 20:58

sehe · Answer 1 · 2011-04-12T22:07:20.223

The problem is you missed the error handling.

Include

ipcMickey = ftok("/tmp/mqmickey", 63);
if (-1==ipcMickey)
{
    perror("ipcMickey");
    exit(255);
}
ipcDonald = ftok("/tmp/mqdonald", 69);
if (-1==ipcDonald)
{
    perror("ipcDonald");
    exit(255);
}

And you'll soon find

./mm
ipcDonald: No such file or directory

The return of ftok is -1 in that case. If both files are missing, both ipc keys will be -1, meaning all traffic shares the same port :)

So

touch /tmp/mqmickey /tmp/mqdonald

fixes it. Some statistics (replacing the sleeps with usleep(random()%10000) ):

gcc MM.c -o mm -O3 -fno-stack-protector -pthread
time ./mm | { trap "" INT; sort | uniq -c | tee stats; } 
  16047 
      1 Donald sent a 
   8054 Donald sent a message to Pluto.
      1 Donald thread successfully started.
   8040 Mickey sent a message to Minnie.
      1 Mickey thread successfully started.
   8065 Minnie received: Hello, Minnie!
      1 Minnie thread successfully started.
   7982 Pluto received: Sit, Pluto!
      1 Pluto thread successfully started.


real    0m40.814s
user    0m0.168s
sys 0m0.092s

It's also improbable but far within the realm of possibility that `ftok` returns the same key for both filenames. It simply has way too few bits to avoid occasional collusions. OP should consider looking into POSIX message queues (`mq_*`), which lack this fundamental flaw. Alternatively, you could loop creating temporary files until you get unique keys and then share the keys with all threads/processes that need them, rather than calling `ftok` again. — R.. GitHub STOP HELPING ICE, Apr 12 '11 at 21:50

score 3 · Answer 2 · answered Apr 13 '11 at 13:47

Thank you VERY MUCH sehe and VJo,

Here is a modified version of the program, taking into consideration all of your comments.

Hope this will be useful for someone else.

Bert

/*
    We want Mickey to send a message to Minnie exclusively.
    We want Donald to send a message to pluto exclusively.  

    To compile:
    gcc mm.c -o mm -fno-stack-protector -pthread

    Requires: 
    syslog enabled and started.

    To test:
    time ./mm | { trap "" INT; sort | uniq -c | tee stats; } 

    Results:
    1503373 Donald sent a message to Pluto.
          1 Donald thread successfully started.
    1423964 Mickey sent a message to Minnie.
          1 Mickey thread successfully started.
    1423958  Minnie received: Hello, Minnie!
          1 Minnie thread successfully started.
    1503333 Pluto received: Sit, Pluto!
          1 Pluto thread successfully started.

    real    0m17.133s
    user    0m16.053s
    sys 0m5.248s

*/
#include <sys/types.h>
#include <sys/msg.h>
#include <sys/ipc.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <syslog.h>


#define MQMICKEY "/tmp/MQMickey"
#define MQDONALD "/tmp/MQDonald"

pthread_t t1,t2,t3,t4;
FILE *fMickeypointer;
FILE *fDonaldpointer;

// Mickey send
key_t ipcMickey;
int mqMickeyid;
char helloMickeymsg[] = {"Hello, Minnie!"};
struct { long type; char text[100]; } myMickeymsg;

// Minnie get
int mqMinnieid;
struct { long type; char text[100]; } myMinniemsg;

// Donald send
key_t ipcDonald;
int mqDonaldid;
char helloDonaldmsg[] = {"Sit, Pluto!"};
struct { long type; char text[100]; } myDonaldmsg;

// Pluto get
int mqPlutoid;
struct { long type; char text[100]; } myPlutomsg;

static void * DONALDthreadFunc(void *arg)
{
    printf("Donald thread successfully started.\n");
    char *s = (char *) arg;

    while(1)
    {
        //send
        memset(myDonaldmsg.text, 0, 100);
        strncpy(myDonaldmsg.text, helloDonaldmsg, strlen(helloDonaldmsg));
        myDonaldmsg.type = 1;
        msgsnd(mqDonaldid, &myDonaldmsg, sizeof(myDonaldmsg), 0);
        printf("Donald sent a message to Pluto.\r\n"); 
    }

    /* just a formality */
    return (void *) strlen(s);
}

static void * PLUTOthreadFunc(void *arg)
{
    printf("Pluto thread successfully started.\n");
    char *s = (char *) arg;

    while(1)
    {
        //receive
        mqPlutoid = msgget(ipcDonald, 0);
        msgrcv(mqPlutoid, &myPlutomsg, sizeof(myPlutomsg), 0, 0);
        printf("Pluto received: %s\r\n", myPlutomsg.text);
    }

    /* just a formality */
    return (void *) strlen(s);
}

static void * MICKEYthreadFunc(void *arg)
{
    printf("Mickey thread successfully started.\n");
    char *s = (char *) arg;

    while(1)
    {
        //send
        memset(myMickeymsg.text, 0, 100);
        strncpy(myMickeymsg.text, helloMickeymsg, strlen(helloMickeymsg));
        myMickeymsg.type = 1;
        msgsnd(mqMickeyid, &myMickeymsg, sizeof(myMickeymsg), 0);
        printf("Mickey sent a message to Minnie.\r\n");
    }

    /* just a formality */
    return (void *) strlen(s);
}

static void * MINNIEthreadFunc(void *arg)
{
    printf("Minnie thread successfully started.\n");
    char *s = (char *) arg;

    while(1)
    {
        //receive
        mqMinnieid = msgget(ipcMickey, 0);
        msgrcv(mqMinnieid, &myMinniemsg, sizeof(myMinniemsg), 0, 0);
        printf(" Minnie received: %s\r\n", myMinniemsg.text);
    }

    return (void *) strlen(s);
}

int main (void)
{
    /* 
        MUST create the target files first.

        The return of ftok is -1 if file is missing. If BOTH files are missing, both ipc keys 
        will be -1, meaning all traffic shares the same port. Result: one queue will acquire
        messages not intended for it!
    */
    fMickeypointer = fopen(MQMICKEY, "a");
    if(fMickeypointer == NULL) 
    {
        printf("Failed to create Mickey queue file.\n");

        /* system logging */
        setlogmask (LOG_UPTO (LOG_NOTICE));
        openlog ("MickeyDonald", LOG_CONS | LOG_PID | LOG_NDELAY, LOG_LOCAL1);
        syslog (LOG_NOTICE, "Failed to create Mickey queue file." );
        closelog ();
        exit(91);
    }

    fDonaldpointer = fopen(MQDONALD, "a");
    if(fDonaldpointer == NULL) 
    {
        printf("Failed to create Donald queue file.\r\n");

        /* system logging */
        setlogmask (LOG_UPTO (LOG_NOTICE));
        openlog ("MickeyDonald", LOG_CONS | LOG_PID | LOG_NDELAY, LOG_LOCAL1);
        syslog (LOG_NOTICE, "Failed to create Donald queue file." );
        closelog ();
        exit(92);
    }

    ipcMickey = 0;
    ipcDonald = 0;

    /* make sure we assign DIFFERENT ipc values */  
    while(ipcMickey == ipcDonald)
    {
        ipcMickey = ftok(MQMICKEY, 63);
        if (-1==ipcMickey)
        {
            printf("ipcMickey does not exist.\r\n");

            /* system logging */
            setlogmask (LOG_UPTO (LOG_NOTICE));
            openlog ("MickeyDonald", LOG_CONS | LOG_PID | LOG_NDELAY, LOG_LOCAL1);
            syslog (LOG_NOTICE, "Failed to ftok Mickey file." );
            closelog ();
            exit(93);
        }

        ipcDonald = ftok(MQDONALD, 69);
        if (-1==ipcDonald)
        {
            printf("ipcDonald does not exist.\r\n");

            /* system logging */
            setlogmask (LOG_UPTO (LOG_NOTICE));
            openlog ("MickeyDonald", LOG_CONS | LOG_PID | LOG_NDELAY, LOG_LOCAL1);
            syslog (LOG_NOTICE, "Failed to ftok Donald file." );
            closelog ();
            exit(94);
        }
    }

    mqMickeyid = msgget(ipcMickey, IPC_CREAT | 0666);
    mqDonaldid = msgget(ipcDonald, IPC_CREAT | 0666);

    pthread_create(&t1, NULL, MICKEYthreadFunc, "Mickey sends\n");
    pthread_create(&t2, NULL, MINNIEthreadFunc, "Minnie replies\n");
    pthread_create(&t3, NULL, DONALDthreadFunc, "Donald sends\n");
    pthread_create(&t4, NULL, PLUTOthreadFunc, "Pluto replies\n");

    long exitcounter = 0;
    while(exitcounter < 100000)
    {
        exitcounter +=1;        
        usleep(100);
    }
    exit(0);
}

score 0 · Answer 3 · answered Apr 13 '11 at 10:51

Here is slightly modified version of your program :

/*
    To compile;
    gcc MM.c -o mm -fno-stack-protector -pthread

    We want Mickey to send a message to Minnie exclusively.
    We want Donald to send a message to pluto exclusively.

    Problem: Pluto intercepts Minnie's messages.

    Listing gives:

    $ ./mm
    Mickey thread successfully started.
    Minnie thread successfully started.
    Pluto thread successfully started.
    Donald thread successfully started.
    Donald sent a message to Pluto.
    Mickey sent a message to Minnie.
    Pluto received: Sit, Pluto!

    Minnie received: Hello, Minnie!

    Mickey sent a message to Minnie.  (100 times)

    Pluto received: Hello, Minnie!
*/
#include <sys/types.h>
#include <sys/msg.h>
#include <sys/ipc.h>
#include <string.h>
#include <stdio.h>

pthread_t t1,t2,t3,t4;

// Mickey send
key_t ipcMickey;
int mqMickeyid;
char helloMickeymsg[] = {"Hello, Minnie!"};
struct { long type; char text[100]; } myMickeymsg;

// Minnie get
int mqMinnieid;
struct { long type; char text[100]; } myMinniemsg;

// Donald send
key_t ipcDonald;
int mqDonaldid;
char helloDonaldmsg[] = {"Sit, Pluto!"};
struct { long type; char text[100]; } myDonaldmsg;

// Pluto get
int mqPlutoid;
struct { long type; char text[100]; } myPlutomsg;

static void * DONALDthreadFunc(void *arg)
{
    printf("Donald thread successfully started.\n");
    char *s = (char *) arg;

    while(1)
    {
        //send
        memset(myDonaldmsg.text, 0, 100);
        strncpy(myDonaldmsg.text, helloDonaldmsg, strlen(helloDonaldmsg));
        myDonaldmsg.type = 1;
        msgsnd(mqDonaldid, &myDonaldmsg, sizeof(myDonaldmsg), 0);
        printf("Donald sent a message to Pluto.\r\n");
        sleep(4);
    }

    /* just a formality */
    return (void *) strlen(s);
}

static void * PLUTOthreadFunc(void *arg)
{
    printf("Pluto thread successfully started.\n");
    char *s = (char *) arg;

    while(1)
    {
        //receive
        mqPlutoid = msgget(ipcDonald, 0);
        msgrcv(mqPlutoid, &myPlutomsg, sizeof(myPlutomsg), 0, 0);
        printf("Pluto received: %s\r\nPluto uses MQ with id = %d\r\n", myPlutomsg.text,mqPlutoid);
        sleep(1);
    }

    /* just a formality */
    return (void *) strlen(s);
}

static void * MICKEYthreadFunc(void *arg)
{
    printf("Mickey thread successfully started.\n");
    char *s = (char *) arg;

    while(1)
    {
        //send
        memset(myMickeymsg.text, 0, 100);
        strncpy(myMickeymsg.text, helloMickeymsg, strlen(helloMickeymsg));
        myMickeymsg.type = 1;
        msgsnd(mqMickeyid, &myMickeymsg, sizeof(myMickeymsg), 0);
        printf("Mickey sent a message to Minnie.\r\n");
        usleep(10000);
    }

    /* just a formality */
    return (void *) strlen(s);
}

static void * MINNIEthreadFunc(void *arg)
{
    printf("Minnie thread successfully started.\n");
    char *s = (char *) arg;

    while(1)
    {
        //receive
        mqMinnieid = msgget(ipcMickey, 0);
        msgrcv(mqMinnieid, &myMinniemsg, sizeof(myMinniemsg), 0, 0);
        printf(" Minnie received: %s\r\nMinnie uses MQ with id = %d\r\n", myMinniemsg.text,mqMinnieid);
        sleep(3);
    }

    return (void *) strlen(s);
}

int main (void)
{
    ipcMickey = ftok("./mqmickey", 63);
    mqMickeyid = msgget(ipcMickey, IPC_CREAT | 0666);
    printf( "mqMickeyid=%d\n",mqMickeyid);

    ipcDonald = ftok("./mqdonald", 69);
    mqDonaldid = msgget(ipcDonald, IPC_CREAT | 0666);
    printf( "mqDonaldid=%d\n",mqDonaldid);

    pthread_create(&t1, NULL, MICKEYthreadFunc, "Mickey sends\n");
    pthread_create(&t2, NULL, MINNIEthreadFunc, "Minnie replies\n");
    pthread_create(&t3, NULL, DONALDthreadFunc, "Donald sends\n");
    pthread_create(&t4, NULL, PLUTOthreadFunc, "Pluto replies\n");

    while(1)
    {
        sleep(5);
    }
}

I found out that for some reason, msgget fails, and both MQs are having the same id, then it is a race which thread kicks in first.

With this modified version of your program, do this :

touch mqmickey
touch mqdonald

to create files first.

And you should see messages goes to correct recipients.

thread grabs other threads IPC message from queue (Linux)

3 Answers3