1

I'm trying to watch a directory for newly added files ending in .csv do something with them, delete the file and wait for a newly added file to do process the file again. I'm using a BackgroundService with the following code:

public class Worker : BackgroundService
{
    private readonly ILogger<Worker> _logger;
    private readonly IProcessFileService _processFileService;
    private ApplicationOptions _applicationOptions;
    private FileSystemWatcher _watcher = new();


    public Worker(ILogger<Worker> logger, IProcessFileService processFileService, IOptions<ApplicationOptions> applicationOptions)
    {
        _logger = logger;
        _processFileService = processFileService;
        _applicationOptions = applicationOptions.Value;
    }

    protected override Task ExecuteAsync(CancellationToken stoppingToken)
    {
        string filePath = _applicationOptions.InputDirectory;
        _watcher.Path = filePath;
        _watcher.EnableRaisingEvents = true;
        _watcher.NotifyFilter = NotifyFilters.FileName | NotifyFilters.CreationTime | NotifyFilters.LastWrite;
        _watcher.Filter = "*.csv*";
        _watcher.Created += new FileSystemEventHandler(OnFileCreated);
        return Task.CompletedTask;
    }

    private async void OnFileCreated(object sender, FileSystemEventArgs e)
    {
        if (e.ChangeType == WatcherChangeTypes.Created)
        {
            await HandleAddedFile();
        }     
    }

    private async Task HandleAddedFile()
    {
        try
        {
           await _processFileService.ProcessFile();
        }
        catch(Exception ex)
        {
            _logger.LogError("{error}", ex.Message);
        }
       _processFileService.DeleteFileAndLog();
    }
}

Works fine for the first time, but when trying inserting a second file in the directory, I get the error: The process cannot access the file "PATH" because it is being used by another process.

Processing the label includes some async work, how can I fix this issue?

Pauldb
  • 231
  • 2
  • 8
  • 2
    The file appears in the directory as soon as it is created, not after it has been closed and released by the OS. You have to wait for that release to occur before you can open the file in `ProcessFile()`. Some ideas for doing that can be found [here](https://stackoverflow.com/questions/50744/wait-until-file-is-unlocked-in-net). – Robert Harvey Dec 27 '22 at 16:14
  • When I was dealing with this, I used "created" to enqueue the file in a watcher for "changed" and only tried to process files after the last "changed" event for that file was older than x time. – Fildor Dec 27 '22 at 18:19
  • I also noticed that none of your methods use any arguments? Why is that? Can you add them? Maybe this plays a role of its own... – Fildor Dec 27 '22 at 18:28
  • And lastly: if your path can produce peeks or generally high volume of events: keep everything you do on the event handling thread as short as you possibly can. Enqueue a file name. Update a dictionary. That's it. Have anything else be handled by the threadpool. – Fildor Dec 27 '22 at 18:33
  • My team has always found FileSystemWatcher to be less than desirable. For example, if you fail to process a file when it first shows up for some reason, it gets left in the directory, then FileSystemWatcher won't know to call the event handler again. For this reason, we make our background jobs check the directory for files, process them, await a short delay (`await Task.Delay(TimeSpan.FromSeconds(5))` or something like that) and then repeat the process. Essentially polling the directory in a while loop. It's been far more reliable for us than FileSystemWatcher. – mason Dec 27 '22 at 18:37
  • P.S.: if I remember correctly, you are returning from `ExecuteAsync` early, aren't you? – Fildor Dec 27 '22 at 18:38
  • _"The implementation should return a task that **represents the lifetime** of the long running operation(s) being performed."_ - https://learn.microsoft.com/en-us/dotnet/api/microsoft.extensions.hosting.backgroundservice.executeasync?view=dotnet-plat-ext-7.0#microsoft-extensions-hosting-backgroundservice-executeasync(system-threading-cancellationtoken) – Fildor Dec 27 '22 at 18:44

1 Answers1

0

I've followed the thread that Robert linked me; I had to rewrite some stuff, but I've got it to work now using the following code. I also had to modify some functions to take in the fullpath.

public class Worker : BackgroundService
{
    private readonly ILogger<Worker> _logger;
    private readonly IProcessFileService _processFileService;
    private ApplicationOptions _applicationOptions;
    private FileSystemWatcher _watcher = new();


    public Worker(ILogger<Worker> logger, IProcessFileService processFileService, IOptions<ApplicationOptions> applicationOptions)
    {
        _logger = logger;
        _processFileService = processFileService;
        _applicationOptions = applicationOptions.Value;
    }

    protected override Task ExecuteAsync(CancellationToken stoppingToken)
    {
        string filePath = _applicationOptions.InputDirectory;
        _watcher.Path = filePath;
        _watcher.EnableRaisingEvents = true;
        _watcher.NotifyFilter = NotifyFilters.FileName | NotifyFilters.CreationTime | NotifyFilters.LastWrite;
        _watcher.Filter = "*.csv*";
        _watcher.Created += new FileSystemEventHandler(OnFileCreated);
        return Task.CompletedTask;
    }

    private async void OnFileCreated(object sender, FileSystemEventArgs e)
    {
        if (e.ChangeType == WatcherChangeTypes.Created)
        {
            await HandleAddedFile(e);
        }     
    }

    static FileStream? WaitForFile(string fullPath, FileMode mode, FileAccess access, FileShare share)
    {
        for (int numTries = 0; numTries < 10; numTries++)
        {
            FileStream? fs = null;
            try
            {
                fs = new FileStream(fullPath, mode, access, share);
                return fs;
            }
            catch (IOException)
            {
                fs?.Dispose();
                Thread.Sleep(50);
            }
        }

        return null;
    }

    private async Task HandleAddedFile(FileSystemEventArgs e)
    {
        FileStream? fileStream = null;
        try
        {
           fileStream = WaitForFile(e.FullPath, FileMode.Open, FileAccess.Read, FileShare.Read);
           await _processFileService.ProcessFile(e.FullPath);
        }
        catch(Exception ex)
        {
            _logger.LogError("{error}", ex.Message);
        }
        fileStream?.Dispose();
       _processFileService.DeleteFileAndLog(e.FullPath);
    }
}
Pauldb
  • 231
  • 2
  • 8