Read a char string as a virtual file

Question

The question may seems strange but I didn't misspelled it: I want to decompress some data I downloaded without write them on the HDD. For that, I download it in a dynamically allocated buffer and I'd like to send it to the zlib wrapper I use (miniunzip). The problem is that this implementation is quite long (2-3K lines) and I'd like to avoid to have to rewrite it just for few lines. I'd like to know if there was any way to read the buffer via a FILE* structure (miniunzip use it's own structure but I found a "fopen()" hiden under the loader). I know its length if it can help.

Thanks in advance and excuse me for my poor grammar.

I'm working on both Windows and UNIX systems (OSX/GNU Linux).

If you're able to use C++, you could use a `std::strstream`. But that gives you a C++ stream, not a C `FILE*`, and you [can't necessarily get one from the other](http://stackoverflow.com/q/109449/827263). — Keith Thompson, Dec 19 '12 at 22:04
Hi, I wrote the whole program in C so if I could use an other way, it'd be great but I keep this possibility in mind, thanks! — Taiki, Dec 19 '12 at 22:09

score 4 · Accepted Answer · answered Dec 19 '12 at 23:43

If you're talking about the minizip library that's included with zlib, you could use the unzOpen2 function which allows you to specify a structure containing the I/O functions to use. This should get you started:

struct zmem_data {
  char *buf;
  size_t length;
};

static voidpf zmemopen(voidpf opaque, const char *filename, int mode) {
  if ((mode&ZLIB_FILEFUNC_MODE_READWRITEFILTER) != ZLIB_FILEFUNC_MODE_READ) return NULL;
  uLong *pos = malloc(sizeof(uLong));
  *pos = 0;
  return pos;
}

static uLong zmemread(voidpf opaque, voidpf stream, void* buf, uLong size) {
  struct zmem_data *data = (struct zmem_data*)opaque;
  uLong *pos = (uLong*)stream;
  uLong remaining = data->length - *pos;
  uLong readlength = size < remaining ? size : remaining;
  if (*pos > data->length) return 0;
  memcpy(buf, data->buf+*pos, readlength);
  *pos += readlength;
  return readlength;
}

static uLong zmemwrite(voidpf opaque, voidpf stream, const void *buf, uLong size) {
  /* no write support for now */
  return 0;
}

static int zmemclose(voidpf opaque, voidpf stream) {
  free(stream);
  return 0;
}

static int zmemerror(voidpf opaque, voidpf stream) {
  if (stream == NULL) return 1;
  else return 0;
}

static long zmemtell(voidpf opaque, voidpf stream) {
  return *(uLong*)stream;
}

static long zmemseek(voidpf opaque, voidpf stream, uLong offset, int origin) {
  struct zmem_data *data = (struct zmem_data*)opaque;
  uLong *pos = (uLong*)stream;
  switch (origin) {
    case ZLIB_FILEFUNC_SEEK_SET:
      *pos = offset;
      break;
    case ZLIB_FILEFUNC_SEEK_CUR:
      *pos = *pos + offset;
      break;
    case ZLIB_FILEFUNC_SEEK_END:
      *pos = data->length + offset;
      break;
    default:
      return -1;
  }
  return 0;
}

static void init_zmemfile(zlib_filefunc_def *inst, char *buf, size_t length) {
  struct zmem_data *data = malloc(sizeof(struct zmem_data));
  data->buf = buf;
  data->length = length;
  inst->opaque = data;
  inst->zopen_file = zmemopen;
  inst->zread_file = zmemread;
  inst->zwrite_file = zmemwrite;
  inst->ztell_file = zmemtell;
  inst->zseek_file = zmemseek;
  inst->zclose_file = zmemclose;
  inst->zerror_file = zmemerror;
}

static void destroy_zmemfile(zlib_filefunc_def *inst) {
  free(inst->opaque);
  inst->opaque = NULL;
}

void example() {
 zlib_filefunc_dec fileops;
 init_zmemfile(&fileops, buffer, buffer_length);
 unzFile zf = unzOpen2(NULL, &fileops);
 /* ... process zip file ... */
 unzClose(zf);
 destroy_zmemfile(&fileops);
}

Thanks for this very complete answer! I try this code then accept the answer :) — Taiki, Dec 20 '12 at 06:13

score 1 · Answer 2 · answered Dec 19 '12 at 21:59

To summarize your question: You want to provide a FILE* interface for an in-memory buffer. No, you can't do that. The fread(), etc. calls actually end up making system calls that deal with an open file descriptor which you don't have.

You're over-complicating this. Decompression code almost always works from buffers that are in memory. If they have a file interface, certainly that is just a wrapper which handles reading the file into memory and then decompressing (probably in chunks to save memory). You should be able to find a decompression library with calls to decompress a buffer that you give it (just a pointer and a length).

Why you don't want to write the downloaded data to the hard drive is certainly up to you, but I hope this is for good, and not for evil.

The other option would be to open a memory-mapped file, to which you write during download, and read during decompression. There may be a way to specify that the file not be written to disk but of that I am unsure. Also, this would be very different between windows and linux.

These may help:

It's for good :) I found those buffers but it skip a lot of checks and some functions that provide interesting things like filenames use the complex structure of the implementation and won't be available using this way. :/ — Taiki, Dec 19 '12 at 22:02

Read a char string as a virtual file

2 Answers2