0

I am trying to write code to test the speed of lz4 compress and decompress However, when I add the decompress function in the code, the error "corrupted size vs. prev_size" starts to show up.

In my mind, I think the problems should be in the calloc function of src. Because the program stops at free(src) when running debug.

Her is my code.

(the lz4.h and lz4.c is on https://github.com/lz4/lz4/tree/355f60952938e5bd4d45118af720d4b8fb0c8a09

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "lz4.h"
int splitFile(char* fileIn, size_t maxSize);
#include "lz4.c"

int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity);
int LZ4_compressBound(int inputSize);

int main()
{
    clock_t start_lz4_c,end_lz4_c,start_lz4_d,end_lz4_d;
    double compress_time,large_time,decompress_time,compress_speed,decompress_speed,small_time_c,small_average_time_c,small_time_d,small_average_time_d;
    int64_t num_f=0;
    int64_t i;
    char * temp_src=0;
    
    int srcSize= 0;
    int dstCapacity=16384; 
    int lz4_compressed_data_size=0,lz4_decompressed_data_size=0;
    int size_after_comp=0;
    int size_before_comp=0;
    int size_of_file=0;
    
    char* compressed_data = malloc((size_t)dstCapacity);
    char buff[200];


    FILE * fc;
    FILE * ft;

    //split file
    num_f = splitFile("/home/ziruo/research/1stpro/test.txt",16384);
    printf("num_f=%ld\n",num_f);

    //read the length for file
    ft = fopen("/home/ziruo/research/1stpro/test.txt","r");
    fseek(ft,0,SEEK_END);
    size_of_file = ftell(ft);
    printf("file len = %d\n",size_of_file);
    fclose(ft);

    

    //main loop
    for( i = 1; i <= num_f; i++)
    {
        char* src=calloc(16384,1);
        char* regen_buffer = calloc(srcSize,1);
        

        //read in
        sprintf(buff,"/home/ziruo/research/1stpro/test.txt.%03ld",I);

        if (compressed_data == NULL)
        {
            printf("faild to generae storage\n");
        }
        
        fc = fopen(buff,"r");
        fread(src,16384,1,fc);
        srcSize=(int)(strlen(src) + 1);
        fclose(fc);
        
        small_time_c = 0;
        small_time_d = 0;

        if(size_of_file <= 100000000)
        {
            int a;
       
            for(a = 1;a<=10;a++)
            {
                start_lz4_c = clock();
                dstCapacity= LZ4_compressBound(srcSize);
                lz4_compressed_data_size = LZ4_compress_default(src,compressed_data,srcSize,dstCapacity);
                end_lz4_c = clock();

                start_lz4_d = clock();
                
                lz4_decompressed_data_size = LZ4_decompress_safe(compressed_data,regen_buffer,lz4_compressed_data_size,srcSize);
                end_lz4_d = clock();


                small_time_c += ((double)(end_lz4_c-start_lz4_c))/CLOCKS_PER_SEC;
                small_average_time_c = small_time_c/10;
                small_time_d += ((double)(end_lz4_d-start_lz4_d))/CLOCKS_PER_SEC;
                small_average_time_d = small_time_d/10;
                
            }
            printf("time %f\n",small_average_time_d);
            compress_time += small_average_time_c;
            decompress_time += small_average_time_d;
            
        }
        else
        {
            start_lz4_c = clock();
            dstCapacity= LZ4_compressBound(srcSize);
            LZ4_compress_default(src,compressed_data,srcSize,dstCapacity);
            end_lz4_c = clock();

            large_time = ((double)(end_lz4_c-start_lz4_c))/CLOCKS_PER_SEC;
            //printf("time %f\n",large_time);
            //compress_time  += large_time;
        }
        

        //calculate time & speed
        
       
        size_before_comp += srcSize;
        size_after_comp += lz4_compressed_data_size;
        printf("decompressed_data_size is %d\n",lz4_decompressed_data_size);
        printf("decompression speed: %fMB/s\n",size_after_comp/(decompress_time *1000000));
       
        free(src);
        free(regen_buffer);
        
    }

   
    printf("before %d after %d\n",size_before_comp,size_after_comp);
    printf("compression speed: %fMB/s\n",size_before_comp/(compress_time*1000000));
    printf("compression ratio: %f\n",(float) size_before_comp/size_after_comp);
    printf("time used(s): %f\n",compress_time);
    printf("decompressed_data_size is %d\n",lz4_decompressed_data_size);
    printf("decompression speed: %fMB/s\n",size_after_comp/(decompress_time *1000000));

    return 0;

}




int splitFile(char* fileIn, size_t maxSize)
{
    int result = 0;
    FILE* fIn;
    FILE* fOut;
    char buffer[1024 * 16];
    size_t size;
    size_t read;
    size_t written;


    if ((fileIn != NULL) && (maxSize > 0))
    {
        fIn = fopen(fileIn, "rb");
        if (fIn != NULL)
        {
            fOut = NULL;
            result = 1;   // we have at least one part

            while (!feof(fIn))
            {
                // initialize (next) output file if no output file opened
                if (fOut == NULL)
                {
                    sprintf(buffer, "%s.%03d", fileIn, result);
                    fOut = fopen(buffer, "wb");
                    if (fOut == NULL)
                    {
                        result = -1;
                        break;
                    }

                    size = 0;
                }

                // calculate size of data to be read from input file in order to not exceed maxSize
                read = sizeof(buffer);
                if ((size + read) > maxSize)
                {
                    read = maxSize - size;
                }

                // read data from input file
                read = fread(buffer, 1, read, fIn);
                if (read == 0)
                {
                    result = -1;
                    break;
                }

                // write data to output file
                written = fwrite(buffer, 1, read, fOut);
                if (written != read)
                {
                    result = -1;
                    break;
                }

                // update size counter of current output file
                size += written;
                if (size >= maxSize)   // next split?
                {
                    fclose(fOut);
                    fOut = NULL;
                    result++;
                }
            }

            // clean up
            if (fOut != NULL)
            {
                fclose(fOut);
            }
            fclose(fIn);
        }
    }

    return (result);
}
Ziruo Jin
  • 21
  • 3
  • Somewhere you're writing past the end (or before the beginning) of an allocated block. Rather than use `strlen` for `srcSize`, use the return value from `fread`. – 1201ProgramAlarm Jun 17 '21 at 20:26
  • 1
    `fread(src,16384,1,fc);` followed by `srcSize=(int)(strlen(src) + 1);` is a bug. `fread` does not null-terminate the data it reads, so you could easily end up thinking it's longer than your buffer size. The only way to know how much data it stored is to look at the return value from `fread`, which you're discarding. – Tom Karzes Jun 17 '21 at 20:35
  • That's not how `feof` is used: https://stackoverflow.com/questions/5431941/why-is-while-feof-file-always-wrong – William Pursell Jun 17 '21 at 21:16

0 Answers0