Reading and Storing large hex file data using C language

Question

I need to read a large Intel Hex file and based on data type, need to store the data in a string/character array to use later on. Below is the code, I am using chunk to read line from hex file, using data_type to check the data type in read line, sub to store parsed data from line and finaldata to keep adding data as I read. However the problem is size, the max character array size is 65535 (correct me if I am wrong) but my data is around 80,000 bytes (120K characters). How can I tackle this (using C language)? or it be better if I switch to C++ or C#? Thanks in advance for any help/insight you can provide.

Edit: Hex data from file looks like below: :020000040200F1 :10C00000814202D8BFF32F8F10BD441C42E8004366 I need to read this data line by line and based from data type (shown in bold, 04 in first line, 00 in second), if it's 00, parse the data from the next byte (byte after data type) and read until end except last byte (which is checksum). Then move to next line, if the data type is 00, parse the data and add it to previously read data (string concatenation), so the variable needs to store a big amount of final data (this is I where I am struggling, how to store that large amount of data in a single variable)?

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

int main(void)
{
    FILE *fp;
    fp = fopen(*filename, "rb");
    if(fp == NULL) {
        perror("Unable to open file!");
        exit(1);
    }

    char chunk[128];
    char sub[128];

    char finaldata[65535];
    finaldata[0] = '\0';
    // Store the chunks of text into a line buffer
    size_t len = sizeof(chunk);

    while(fgets(chunk, sizeof(chunk), fp) != NULL) {
        //fputs(chunk, stdout);
        int a=0;

        if((chunk[7] == '0') && (chunk[8] == '0')) {
            size_t length = strlen(chunk);

            while (a < (length-13)) {
                sub[a]=chunk[9+a];
                a++;

            }
        }
        strcat(finaldata, sub);
        fputs(finaldata, stdout);
        memset(sub,0,sizeof(sub));
         printf("\n\n");

    }

    fclose(fp);

    printf("\n\nMax line size: %zd\n", len);

    return 0;
}

Answer 1

You say :

read until end except last byte (which is checksum)

but if I apply on :10C00000814202D8BFF32F8F10BD441C42E8004366 your code

    int a=0;

    if((chunk[7] == '0') && (chunk[8] == '0')) {
        size_t length = strlen(chunk);

        while (a < (length-13)) {
            sub[a]=chunk[9+a];
            a++;
        }
    }

sub values 814202D8BFF32F8F10BD441C42E8004 so you remove 366 at the end of the line rather than only 66

---

From your remark

when defining char array using malloc() function, what size should I put in there since I don't know the exact size?

If you want to collapse all the sub strings in one string, one way is to start by an array of size 1 for the null terminating char then to increase it line per line using malloc. For instance :

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

int main(int argc, char ** argv)
{
  if (argc != 2) {
    fprintf(stderr, "Usage: %s <file>\n", *argv);
    exit(1);
  }

  FILE *fp = fopen(argv[1], "rb");

  if (fp == NULL) {
    perror("Unable to open file!");
    exit(1);
  }

  size_t sz = 0; /* without counting the char for \n */
  char * finaldata = malloc(1);
  char chunk[128];

  while (fscanf(fp, " %127s", chunk) == 1) {
    if((chunk[7] == '0') && (chunk[8] == '0')) {
      if (strlen(chunk) != 43) {
        fprintf(stderr, "unexpected line '%s'\n", chunk);
        exit(1);
      }

      chunk[41] = 0; /* remove two last chars */

      char * s = realloc(finaldata, sz + 32 +1); /* + block + \n */

      if (s == NULL) {
        fputs("not enough memory", stderr);
        free(finaldata); /* for valgrind etc */
        exit(1);
      }

      finaldata = s;
      strcpy(finaldata + sz, chunk + 9);
      sz += 32;
    }
  }

  fclose(fp);
  finaldata[sz] = '\0';

  /* debug */
  puts(finaldata);

  free(finaldata); /* for valgrind etc */

  return 0;
}

I use fscanf to bypass possible spaces including newline before and after the part to manage. In the format " %127s" notice the space before '%', and 127 which is 128 minus 1 to let place for the null terminating char.

Compilation and execution :

pi@raspberrypi:/tmp $ gcc -Wall c.c
pi@raspberrypi:/tmp $ cat f
:020000040200F1
:10C00000814202D8BFF32F8F10BD441C42E8004366
:020000040200F1
:10C00000123456789abcdef0123456789abcdef012
pi@raspberrypi:/tmp $ ./a.out f
814202D8BFF32F8F10BD441C42E80043123456789abcdef0123456789abcdef0
pi@raspberrypi:/tmp $