0

I'm trying to optimize my IO speeds for a project that needs to iterate over 50 GB of objects and I'm not getting even 10% of my SSD performance for both reads and writes. I've boiled down my code to this snippet to explore the problem. I've got buffered streams and for purposes of this experiment I can commit additional threads on a specific suggestion. While my use case is specific to object serialization it would seem to apply to file IO generally.

This is a cleanup of my first ever question here.

Starting test: 0
    File Size: 1431.03MBytes
    Total Write time: 17468.44ms
    Average write speed: 81.92MBps
    Total Read time: 13203.10ms
    Average read speed: 108.39MBps

Starting test: 1
    File Size: 1431.03MBytes
    Total Write time: 19393.04ms
    Average write speed: 73.79MBps
    Total Read time: 12729.45ms
    Average read speed: 112.42MBps

Starting test: 2
    File Size: 1431.03MBytes
    Total Write time: 19256.42ms
    Average write speed: 74.31MBps
    Total Read time: 12573.89ms
    Average read speed: 113.81MBps

Starting test: 3
    File Size: 1431.03MBytes
    Total Write time: 19479.05ms
    Average write speed: 73.46MBps
    Total Read time: 13061.15ms
    Average read speed: 109.56MBps

Starting test: 4
    File Size: 1431.03MBytes
    Total Write time: 19585.47ms
    Average write speed: 73.07MBps
    Total Read time: 12651.04ms
    Average read speed: 113.12MBps
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.LinkedList;

public class SerializationTest {

    @SuppressWarnings("unchecked")
    public static void main(String[] args) throws IOException, ClassNotFoundException {

        for(int testCount = 0; testCount < 5; testCount++) {
            System.out.println("\nStarting test: " + testCount);
            //Test Parameters
            int LIST_SIZE = 100000;
            int LIST_WRITE_COUNT = 3000;

            //Test Array with a bunch of data
            LinkedList<Integer> intList = new LinkedList<Integer>();
            for(int count = 0; count < LIST_SIZE; count++) intList.add(count);

            //Create a file and write out a bunch of data
            File tempFile = File.createTempFile("binary_data", ".SerialObjFileCollector");
            tempFile.deleteOnExit();
            FileOutputStream fileOutStream = new FileOutputStream(tempFile);
            BufferedOutputStream outputBuffer = new BufferedOutputStream(fileOutStream);
            ObjectOutputStream outObjStream = new ObjectOutputStream(outputBuffer);

            //     --------   Optimization Starts  --------------
            //Measure performance of the writes
            long writeStartTime = System.nanoTime();
            for(int count = 0; count < LIST_WRITE_COUNT; count++) outObjStream.writeUnshared(intList);

            //Clear out the write process
            outObjStream.flush();
            outputBuffer.flush();
            fileOutStream.flush();
            fileOutStream.close();

            long writeEndTime = System.nanoTime();
            //     --------   Optimization Ends  --------------


            //Create the streams to read out the serialized data
            FileInputStream fileInStream = new FileInputStream(tempFile);
            BufferedInputStream inputBuffer = new BufferedInputStream(fileInStream);
            ObjectInputStream inObjStream = new ObjectInputStream(inputBuffer);

            //    --------   Optimization Starts  --------------
            //Measure Performance of reads
            long readStartTime = System.nanoTime();
            for(int count = 0; count < LIST_WRITE_COUNT; count++) intList = (LinkedList<Integer>) inObjStream.readUnshared();
            long readEndTime = System.nanoTime();
            //    --------   Optimization Ends  --------------


            inObjStream.close();

            //Calc test results
            double fileMBytes = ((double) tempFile.length()) / Math.pow(2, 20);
            double writeTime = writeEndTime - writeStartTime;
            writeTime *= Math.pow(10, -6);//Convert to ms
            double readTime = readEndTime - readStartTime;
            readTime *= Math.pow(10, -6);//Convert to ms
            double writeSpeed = fileMBytes/(writeTime * Math.pow(10, -3));
            double readSpeed = fileMBytes/(readTime * Math.pow(10, -3));

            //Print Test Results
            NumberFormat format = new DecimalFormat("#0.00");
            System.out.println("\tFile Size: " + format.format(fileMBytes) + "MBytes");
            System.out.println("\tTotal Write time: " + format.format(writeTime) + "ms");
            System.out.println("\tAverage write speed: " + format.format(writeSpeed) + "MBps" );
            System.out.println("\tTotal Read time: " + format.format(readTime) + "ms");
            System.out.println("\tAverage read speed: " + format.format(readSpeed) + "MBps" );
        }
    }

}

I don't do this professionally and can take constructive feedback. Thank you for looking at it.

Alex Apel
  • 21
  • 3
  • I can but speculate. Did you know 1 MBps is 8 Mbps. One is megabytes per second. The other is megabits per second. – Elliott Frisch May 12 '20 at 04:05
  • @ElliottFrisch Thank you for the reply. I think I have that angle covered as File.length() returns the size of the file in bytes per https://docs.oracle.com/javase/7/docs/api/java/io/File.html#length() – Alex Apel May 12 '20 at 04:18

0 Answers0