C# how do I tell when all threads or tasks have completed when traversing a file structure

Question

This app processes image files that meet certain criteria, namely that the resolution is larger than 1600X1600. There can be upwards of 8000 files in a source directory tree but not all of them meet the resolution criteria. The tree can be 4 or 5 levels deep, at least.

I've "tasked" the actual conversion process. However, I can't tell when the last task has finished.

I really don't want to create a task array because it would contains thousands of files that don't meet the resolution criteria. And it seems a waste to open the image file, check the resolution, add or not to the task array, and then reopen it again when it's time to process the file.

Here's the code.

using System;
using System.Data;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using System.Windows.Forms;

namespace ResizeImages2
{
    public partial class Form1 : Form
    {
        private string _destDir;
        private string _sourceDir;
        private StreamWriter _logfile;
        private int _numThreads;

        private void button1_Click(object sender, EventArgs e)
        {
            _numThreads = 0;
            if (_logfile is null)
                _logfile = new StreamWriter("c:\\temp\\imagesLog.txt", append: true);

            _destDir = "c:\\inetpub\\wwwroot\\mywebsite\\";
            _soureDir = "c:\\images\\"
            Directory.CreateDirectory(_destDir);
            var root = new DirectoryInfo(sourceDir);
            WalkDirectoryTree(root); // <--async so it's going to return before all the threads are complete. Can't close the logfile here
            //_logfile.Close();
        }

        private async void WalkDirectoryTree(System.IO.DirectoryInfo root)
        {
            System.IO.FileInfo[] files = null;
            System.IO.DirectoryInfo[] subDirs = null;
            files = root.GetFiles("*-*-*.jpg"); //looks like a sku and is a jpg.
            if (files == null) return;
            foreach (System.IO.FileInfo fi in files)
            {
                _numThreads++;
                await Task.Run(() =>
                {
                    CreateImage(fi);
                    _numThreads--;
                    return true;
                });
            }

            // Now find all the subdirectories under this directory.
            subDirs = root.GetDirectories();

            foreach (System.IO.DirectoryInfo dirInfo in subDirs)
            {
                WalkDirectoryTree(dirInfo);
            }
        }

        private void CreateImage(FileSystemInfo f)
        {
            var originalBitmap = new Bitmap(f.FullName);
            if (originalBitmap.Width <= 1600 || originalBitmap.Height <= 1600) return;
            using (var bm = new Bitmap(1600, 1600))
            {
                Point[] points =
                {
                    new Point(0, 0),
                    new Point(new_wid, 0),
                    new Point(0, new_hgt),
                };
                using (var gr = Graphics.FromImage(bm))
                {
                    gr.DrawImage(originalBitmap, points);
                }

                bm.SetResolution(96, 96);
                bm.Save($"{_destDir}{f.Name}", ImageFormat.Jpeg);
                bm.Dispose();
                originalBitmap.Dispose();
            }
            _logfile.WriteLine(f.Name);
        }
    }
}

UPDATE -- final result

    private async void button1_Click(Object sender, EventArgs e)
    {
        _logfile = new StreamWriter("c:\\temp\\imagesLog.txt", append: true);

        _sourceDir=$"c:\\sourceImages\\";
        _destDir = $"c:\\inetpub\\wwwroot\\images\\";
        var jpgFiles = Directory.EnumerateFiles(_sourceDir, "*-*-*.jpg", SearchOption.AllDirectories);
        var myPOpts = new ParallelOptions {MaxDegreeOfParallelism = 10};
        await Task.Run(() => //allows the user interface to be updated -- usually, file being processed
        {
            Parallel.ForEach(jpgFiles,myPOpts, f=>
            {
                CreateImage(new FileInfo(f));
            });
            _logfile.Close();
        });
    }
    private void CreateImage(FileSystemInfo f)
    {
        var originalBitmap = new Bitmap(f.FullName);
        if (originalBitmap.Width <= 1600 || originalBitmap.Height <= 1600)
        {
            originalBitmap.Dispose();
            return;
        }
        tbCurFile.Text = f.FullName;
        var new_wid = 1600;
        var new_hgt = 1600;
        using (var bm = new Bitmap(new_wid, new_hgt))
        {
            Point[] points =
            {
                new Point(0, 0),
                new Point(new_wid, 0),
                new Point(0, new_hgt),
            };
            var scount = 1;
            var saved = false;
            //why the while/try/catch? Because we are copying files across the internet from a dropbbox in Sync Only Mode.
            //this means we are only working with a pointer until we actually open the file and sometimes 
            //the app gets ahead of itself. It tries to draw the new image before the original file is completely open
            //so let's see if can't  mitigate that here.
            while (!saved && scount < 5)
            {
                try
                {
                    using (var gr = Graphics.FromImage(bm))
                    {
                        gr.DrawImage(originalBitmap, points);
                    }

                    saved = true;
                }
                catch
                {
                    scount++;
                }
            }

            bm.SetResolution(96, 96);
            scount = 1;
            saved = false;
            while (!saved && scount<5)
            {
                try
                {
                    bm.Save($"{_destDir}{f.Name}", ImageFormat.Jpeg);
                    saved = true;
                }
                catch
                {
                    scount++;
                }
            }                
            bm.Dispose();
            originalBitmap.Dispose();
        }
        _logfile.WriteLine($"{_collectionId}\\{f.Name}");
    }

Answer 1

One quick and dirty way of handling that is to use interlocked operations with a shared variable.

    private int _numberOfOutstandingOperations;
    
    private async void button1_Click(object sender, EventArgs e)
    {
        Interlocked.Exchange(ref _numberOfOutstandingOperations, 1);
        
        await WalkDirectoryTree(root);
        CompleteOperation();
    }
    
    
    private async Task WalkDirectoryTree(System.IO.DirectoryInfo root)
    {
        // ...   
        foreach (System.IO.FileInfo fi in files)
        {
            Interlocked.Increment(ref _numberOfOutstandingOperations);
            _ = Task.Run(() =>
            {
                CreateImage(fi);
                CompleteOperation();
            });
        }
        
        foreach (System.IO.DirectoryInfo dirInfo in subDirs)
        {
            await WalkDirectoryTree(dirInfo);
        }
    }
    
    private void CompleteOperation()
    {
        if (Interlocked.Decrement(ref _numberOfOutstandingOperations) == 0)
        {
            // Everything is done, signal a mutex or complete a TaskCompletionSource or whatever
        }
    }

The trick is to initialize the number of operations to 1, so that it completes only when all tasks are finished and all directories have been walked.