2

I’m writing a FastCGI for providing zlib compression on static content for web providers which doesn’t, and I’m experiencing problems withmmap()on NetBSD.

#include <magic.h>
#ifndef MADV_DONTFORK
#define MADV_DONTFORK 0
#endif
#ifndef MADV_MERGEABLE
#define MADV_MERGEABLE 0
#endif
#ifndef MADV_HUGEPAGE
#define MADV_HUGEPAGE 0
#endif
#ifndef MADV_DONTDUMP
#define MADV_DONTDUMP 0
#endif

struct stat sb;
int fd=strcatn_open((char *)REQUEST_URI,"index.html")?:strcatn_open((char *)REQUEST_URI,"index.htm")?:open(REQUEST_URI, O_RDONLY);
if (fd==0)
    HTTP_Return_Code("Status: 404 Not Found");
if(!strcmp(HTTP_CACHE_CONTROL?:"ch","no-cache")) { //avoid to compare with a NULL pointer
    if(IfModifiedSince)
        if(tdate_parse((char *)IfModifiedSince)==sb.st_mtime) // UNIX : can compare directly
            HTTP_Return_Code("Status: 304 Not Modified\nCache-Control: public; max-age=2678400");
    if(IfUnmodifiedSince)
        if(tdate_parse((char *)IfUnmodifiedSince)==sb.st_mtime) // UNIX : can compare directly
            HTTP_Return_Code("Status: 304 Not Modified\nCache-Control: public; max-age=2678400");
}
strftime(LastModified, sizeof(char)*30, "%a, %d %b %Y %H:%M:%S GMT", gmtime(&(sb.st_mtime)) ); // RFC1123 format : "%a, %d %b %Y %H:%M:%S GMT"
magic_t mime_type=magic_open(MAGIC_PRESERVE_ATIME|MAGIC_DEVICES|MAGIC_SYMLINK|MAGIC_MIME|MAGIC_COMPRESS);
magic_load(mime_type,NULL); // load default magic database
posix_fadvise(fd,0,sb.st_size,POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE | POSIX_FADV_WILLNEED);
send_HTTP_Headers("Content-encoding: gzip\nContent-Type: %s\nLast-Modified: %s\n",magic_descriptor(mime_type,fd),LastModified);
magic_close(mime_type);

//the problem is here
void * HTTP_file=mmap(NULL,sb.st_size,PROT_READ,MAP_SHARED|MAP_FILE,fd,0);
if(HTTP_file==(void *)-1)
    err(1,"errno=%d",errno);
uLong maxCompressLen=compressBound(sb.st_size);
Bytef* destbuffer=(Bytef *)malloc((size_t)maxCompressLen);
compress2(destbuffer,&maxCompressLen,(const Bytef *)HTTP_file,sb.st_size,Z_BEST_COMPRESSION);
munmap(HTTP_file,sb.st_size);
close(fd);
madvise(destbuffer,(size_t)maxCompressLen,MADV_SEQUENTIAL | MADV_WILLNEED | MADV_DONTFORK | MADV_MERGEABLE | MADV_HUGEPAGE | MADV_DONTDUMP);
write(STDOUT_FILENO, destbuffer, (size_t)maxCompressLen);

When I turn the fastCGI functions to printf()/scanf() for debugging I get this on NetBSD :

Content-encoding: gzip
Content-Type: text/html; charset=utf-8
Last-Modified: Wed, 27 May 2015 01:15:28 GMT

compressor: errno=9: Bad file descriptor which correspond to EBADF described by the mmap man page asfd is not a valid open file descriptor. Which I don’t understand since the file descriptor seems pretty valid for me since it works with fstat and libmagic. When I mount the NetBSD filesystem over sshfs and compile/run the code for Linux I got :

Content-encoding: gzip
Content-Type: text/html; charset=utf-8
Last-Modified: Wed, 27 May 2015 01:15:28 GMT

xڭW_o�6/���d�I�8��4��+4-0��E�,������(Я����3�a�&��
;R�#�v�
       �X�����~w��Ͽ��%��▒(��wd@���,%,�ڀ�Mã�5�Y[����a�k��4<SyI�H$���B�
                                                                     �=C{iAs▒S�Riې�     n�!��`��"
a��aT°�Mô(�PEC�6IZ�L�mBR��9

�cb�S:u+"���j��0:�c ��ϿSƠr��BwÐ<^jE�4���?��`�����SyL
                                        �-۾m��̘x�9E�_�ũ�\��l����2V夞%���f�
�� F�k}7A��Q���0b                                                        y)�����▒!����ހ�h�ߗCR�Ǣ▒��ҥ2`"�D�[B����s������1p�ajz#R"-y���
a*$��ZML0▒ĵ����H߶�&?�z��1��     -8y�Uq\��S�.ΉJ� !��3^8.(y_����G�?����
                                                                     �M����L���.�Цi�0zu����n�W�Nw������n��-��X v^bJZ���qc%9��   ����^V
                                                                                                                                      ��ݾ%:e[
?5Q▒�+▒$��=�9�*�&1�����0�ƺ5�:▒"�*q5��4                                                                                                       L�;D٦U46��H��j�rw�|��PH6���O9�q�U5��k���ޏ���2U6��Y����R    ��iS��'��K9)��m�(��V���qf����9՘*���
����y�����@VqɌ)��ո�������%�s��WLo��j�ǯ.p��nW_l���X
P�nK��F0���m.*�k$*$~a���9W�&                      �s%�6�ö�A�▒�� �.G��oc���
                            �(���}@▒��L
'K�����z��@�D���l6�$
         �0▒�E=@VtB�FW�������:�r�%E�M��mw�)i���;_�^0z   �AG+RRYgg˖[)�i�*BFW�I���v��3m���)�駖��i�r.)Cz��+�-������Q~\b\b%V▒u)b۠�-��N�e4Vj��Z
S�`�VW����s�(�.֪z���ܤ���h���Wm�^a0;U9�;�K�`���bR�������a��뇗A�UpMuc6
                                                                   0<�K�>��Q��pB<��NyyB�*�$<�Ǎ|��'�7XԌ��C(
�}r�|N▒٠�       0[��N�$��y���!�y���4<>L���������|�ݧNQ[�:v���Ng/Xض▒�8
^�h�s���L���Z�����Ѐ�����?ꌲ
1������R�J8���.IL��f���Wv�N��w�&c���-d����%>�_�/��4R®�S�C6a��-�1C��
                                                                   ���^�G�k~q}���Ӎ�V�@� Y�$uo▒Ĺf�4q��
                                                                                                     �Ȓ�k���
                                                                                                            Xn�'>kn�>[�WO���(P�l�[jZ�R0#�
                                                                                                                                         �>��ͨ�׸-�>�ƶ�H)+����z�S����x,�������p�▒j���ǹ���7Rg]�k~���b��֯[o�[��_���_�F^F

Which means the same code is working on Linux without any modifications.

Additional informations :

  • uname -a on NetBSD returnsNetBSD odin 6.1.2 NetBSD 6.1.2 (GENERIC) amd64
  • uname -a on Linux returnsLinux localhost.localdomain 3.16.0-rc7-ck2 #1 SMP Fri May 1 13:38:34 CEST 2015 x86_64 GNU/Linux
  • gcc-4.5.3 is the compiler version on NetBSD
  • gcc-4.9.2 is the compiler version on Linux
  • both architectures are the same and both compilers use the same generic flags with all optimizations disabled.

Update

it seems for an unknown reasonmagic_descriptor()is closing the file descriptor on NetBSD and indeed the following simple code doesn’t work :

// test case for a libmagic bug. Libmagic is part of NetBSD packages and is used with the file command
#include <errno.h>
#include <err.h>
#include <magic.h>
#include <sys/types.h>
#include <sys/mman.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
/* gcc test.c -o test
./test a_regular_file_with read permissions*/
int main(int argc,char * argv[]) {
        int fd=open(argv[1],O_RDONLY); // set fd to 8
        struct stat sb;
        fstat(fd, &sb);

        magic_t mime_type=magic_open(MAGIC_PRESERVE_ATIME|MAGIC_DEVICES|MAGIC_SYMLINK|MAGIC_MIME|MAGIC_COMPRESS);
        magic_load(mime_type,NULL); // load default libmagic database
        printf("Content-encoding: gzip\nContent-Type: %s\n\r\n",magic_descriptor(mime_type,fd)); // after this fd still ==8 but the file is closed whereas it shouldn’t. And indeed it doesn’t on SUA and Linux distros.
        //magic_close(mime_type);

        void * HTTP_file=mmap(NULL,sb.st_size,PROT_READ,MAP_SHARED|MAP_FILE,fd,0); // always returns -1
        if(HTTP_file==(void *)-1)
                err(1,"errno=%d",errno); // errno is equal to 9 which mean the file descriptor is invalid
        write(STDOUT_FILENO, HTTP_file, sb.st_size);
}
user2284570
  • 2,891
  • 3
  • 26
  • 74
  • Where and how do you open the descriptor? – Some programmer dude Jun 03 '15 at 10:53
  • Are you sure `or`ing flags for `posix_fadvise()` is permissible? – EOF Jun 03 '15 at 11:29
  • @JoachimPileborg : Just read the code I posted`int fd=strcatn_open((char *)REQUEST_URI,"index.html")?:strcatn_open((char *)REQUEST_URI,"index.htm")?:open(REQUEST_URI, O_RDONLY);`where`REQUEST_URI`is the URL sent by the web browser. – user2284570 Jun 03 '15 at 11:29
  • @EOF : Yes I’m sure of it, and indeed it doesn’t change anything if I comment out`posix_fadvise()`. – user2284570 Jun 03 '15 at 11:30
  • 1
    Well, you being sure of it isn't going to make it right: http://pubs.opengroup.org/onlinepubs/009695399/functions/posix_fadvise.html: [...]The advice to be applied to the data is specified by the advice parameter and may be **one** of the following values:[...] (emphasis mine). – EOF Jun 03 '15 at 11:33
  • 1
    Where's the call to `stat()` or `fstat()` to fill in `struct stat sb`? Also, what happens if `open()` fails - and returns `-1`? I don't know for sure, but I doubt `open()` returns `0` on failure on NetBSD. – Andrew Henle Jun 03 '15 at 11:38

1 Answers1

1

As Google could have told you, this was a mis-feature/bug in the original source:

0000276: magic_descriptor() closes underlying file descriptor (fd) when it finishes

You should probably treat this as a known behaviour of magic_descriptor() in all releases of file and libmagic prior to 5.18. I.e. don't expect the descriptor to remain open after that call unless you know you're using a more recent release.

The fixed release of file will be included in NetBSD 7.0 -- gentle prodding on the appropriate NetBSD mailing list asking for a pullup to the current release branch may also have it included in the next release of NetBSD-6.x.

Greg A. Woods
  • 2,663
  • 29
  • 26
  • This doesn’t solve the problem since my web hosting provider won’t upgrade to NetBSD 7 for some years. Sure! Moreover, I’m not root. It is like http://bugs.gw.com/view.php?id=455 – user2284570 Jun 04 '15 at 00:41
  • I don't understand the problem. You now know that this is a "feature" of `magic_descriptor()` -- deal with it! Either re-open the file, or dup the descriptor first, or whatever makes sense to you. – Greg A. Woods Jun 04 '15 at 00:44
  • or just use the source for the latest release directly – Greg A. Woods Jun 04 '15 at 00:46
  • Due too http://bugs.gw.com/view.php?id=455 I’ll re-write that part myself. Since I can’t install anything *(not to mention /bin/cat and make are not installed)*. And only then I will post my answer. – user2284570 Jun 04 '15 at 00:47
  • Excuses. You can still use the original source (once it's on the github mirror) -- you don't have to install anything, just build it (static link it) into whatever you are building! – Greg A. Woods Jun 04 '15 at 00:52
  • How am I supposed to build it without cat and make? – user2284570 Jun 04 '15 at 00:53
  • Where the heck are you compiling what you're building and testing now? Just get the file source there and build it and link with it! – Greg A. Woods Jun 04 '15 at 00:55
  • And that source file require some definition on command line which can only be detected by autotools/configure. It needs re-writing. – user2284570 Jun 04 '15 at 01:05
  • Not the "source file" -- the `file` source. Check it all out from github on the same machine where you are building your program. Build `file` in your home directory and install libmagic in your home directory (`./configure --prefix=$HOME`). Link your program with the static version of libmagic. There, done. No need to install anything new on the target machine (other than of course your program). – Greg A. Woods Jun 04 '15 at 07:01
  • if I try to run configure, it fails because it can't find the make versions and some essential commands are missing. – user2284570 Jun 04 '15 at 10:30
  • Perhaps you should start a new question? You now know all the answers and work-arounds and alternatives to solve the problem in this question. – Greg A. Woods Jun 04 '15 at 17:02
  • No this will simply wait the weeks when I’ll rewrite the code to post an answer. Or maybe shorter if someone post a such answer before me. – user2284570 Jun 04 '15 at 22:04