calculating directory file size with multi-Thread

Question

I want to write a program that can calculate the size of a directory and its subdirectories with multi-thread. I write this:

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
import java.util.concurrent.ThreadPoolExecutor;

public class Innerclass {
    Semaphore semaphore = new Semaphore(1);
    private List<String> availableConnections = new ArrayList();


    public Innerclass(){
        this.availableConnections.add("A");
    }

    public String acquireConnection() throws InterruptedException {
        semaphore.acquire();
        System.out.println("Acquiring connection " + Thread.currentThread().getName());
        return availableConnections.remove(0);
    }

    static Long count = 0L;
    static File file1 ;


    public static void main(String[] args) {
        System.out.println(countFilesInDirectory(new File("target directory address")));


    }

    public static Long countFilesInDirectory(File directory) {
        Innerclass connection = new Innerclass();
        ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(20);
        Task task1 = new Task();
        //Long count = 0L;
            for (File file : directory.listFiles()) {
                if (file.isFile()) {
                    executor.execute(new Runnable() {
                        @Override
                        public void run() {
                            try {
                                String far = connection.acquireConnection();
                                count += printFileSizeNIO(String.valueOf(file));
                                connection.acquireConnection();
                            } catch (InterruptedException e) {
                                e.printStackTrace();
                            }

                            //System.out.println(printFileSizeNIO(String.valueOf(file)));
                        }
                    });
                }
                if (file.isDirectory()) {
                    count += countFilesInDirectory(file);
                }
            }
        executor.shutdown();
        //System.out.println(task1.getCount());
        //return task1.getCount();
        return count;
    }


    public static Long printFileSizeNIO(String fileName) {

        Path path = Paths.get(fileName);
        Long bytes = 0L;

        try {
            bytes = Files.size(path);
        } catch (IOException e) {
            e.printStackTrace();
        }
        return bytes;
    }

}

This program gives results close to reality but can not calculate exactly. What do you think is the problem? and with single thread this program work exactly! Receives the current path and the number of fragments from the user and performs the calculation. Do you think there is another way besides the thread pool to write this program. Thank you, dear professors.

Answer 1

With executor service, your main thread wont wait for threads inside pool to finish.

See here: Wait main thread until all the thread pools task complete of ExecutorService?

I have updated your code a little bit. It will give the same answer always.

public class TestSF
{
    Semaphore semaphore = new Semaphore(1);

    public void acquireConnection() throws InterruptedException
    {
        semaphore.acquire();
    }

    public void releaseConnection() throws InterruptedException
    {
        semaphore.release();
    }

    static AtomicLong count = new AtomicLong(0);

    public static void main(String[] args)
    {
        long bytes = countFilesInDirectory(new File("C:\\Users\\ashish\\Desktop\\Loader"));
        System.out.println(humanReadableByteCountBin(bytes));
    }

    public static Long countFilesInDirectory(File directory)
    {
        TestSF connection = new TestSF();
        ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(20);

        for (File file : Objects.requireNonNull(directory.listFiles()))
        {
            
            executor.execute(() -> {
                if (file.isFile())
                {
                    try
                    {
                        connection.acquireConnection();
                        count.addAndGet(printFileSizeNIO(String.valueOf(file)));
                        connection.releaseConnection();
                    }
                    catch (InterruptedException e)
                    {
                        e.printStackTrace();
                    }
                }
                if (file.isDirectory())
                {
                    countFilesInDirectory(file);
                }
            });
        }
        executor.shutdown();
        try {
            if (!executor.awaitTermination(60, TimeUnit.SECONDS)) {
                executor.shutdownNow();
            }
        } catch (InterruptedException ex) {
            executor.shutdownNow();
            Thread.currentThread().interrupt();
        }
        return count.get();
    }

    public static Long printFileSizeNIO(String fileName)
    {

        Path path = Paths.get(fileName);
        long bytes = 0L;

        try
        {
            bytes = Files.size(path);
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }
        return bytes;
    }

    public static String humanReadableByteCountBin(long bytes) 
    {
        long absB = bytes == Long.MIN_VALUE ? Long.MAX_VALUE : Math.abs(bytes);
        if (absB < 1024) {
            return bytes + " B";
        }
        long value = absB;
        CharacterIterator ci = new StringCharacterIterator("KMGTPE");
        for (int i = 40; i >= 0 && absB > 0xfffccccccccccccL >> i; i -= 10) 
        {
            value >>= 10;
            ci.next();
        }
        value *= Long.signum(bytes);
        return String.format("%.1f %ciB", value / 1024.0, ci.current());
    }

}