How to upload files to SFTP server using Parallel.ForEach loop for better perfomance

Question

I have tried Parallel.ForEach, but some files uploaded with zero sizes. Uusing normal for each loop working fine. But it is slow. There are 4 million files. Size of each file is 4 MB.

public static bool UploadFiles(
    string ftpDirectory, string filePath, ConnectionInfo coninfo, string pattern)
{
    using (var client = new SftpClient(coninfo))
    {
        client.Connect();
        client.ConnectionInfo.Timeout = TimeSpan.FromDays(2);
        client.KeepAliveInterval = TimeSpan.FromSeconds(60);
        // 4 millions files
        var files = Directory.GetFiles(filePath, pattern , SearchOption.AllDirectories);
        foreach (var file in files)
        {
            try
            {
                using (Stream inputStream = new FileStream(file, FileMode.Open))
                {
                    client.UploadFile(inputStream, ftpDirectory + Path.GetFileName(file));
                }
            }

            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
            }
        };
        client.Disconnect();
    }
    return true;
}

There is async overload: `SftpClient.BeginUploadFile`. Though you will need to use a new connection in each thread anyway. So it's not as easy as using async API. And it's probably also the reason why `Parallel.ForEach` failed for you. Contrary to what is commented above, I believe there's nothing wrong with using `Parallel.ForEach` for this, as long as you guarantee that you never use the same connection in parallel. See [Processing SFTP files using C# Parallel.ForEach loop not processing downloads](https://stackoverflow.com/q/48833005/850848). — Martin Prikryl, Dec 09 '20 at 06:39

score 1 · Accepted Answer · edited Dec 19 '20 at 09:05

I have used the WinSCP library for Parallel.ForEach loop. It is working fine. https://winscp.net/eng/docs/library_example_parallel_transfers

public void UploadFiles(string remotePath, string localPath, string pattern)
{
    try
    {
        SessionOptions sessionOptions = new SessionOptions
        {
            Protocol = Protocol.Sftp,
            HostName = _sftpConfig["Host"],
            UserName = _sftpConfig["UserName"],
            Password = "",
            SshHostKeyFingerprint = "ssh-rsa 2048 xxx",
            SshPrivateKeyPath = _sftpConfig["SftpPrivateKeyPath"]
        };

        const int batches = 3;

        DateTime started = DateTime.Now;
        int count = 0;
        Int64 bytes = 0;

        Console.WriteLine("Starting files enumeration...");
        IEnumerable<string> files = Directory.GetFiles(localPath, pattern, SearchOption.AllDirectories);

        IEnumerator<string> filesEnumerator = files.GetEnumerator();

        List<Task> tasks = new List<Task>();

        HashSet<string> existingRemotePaths = new HashSet<string>();

        for (int i = 1; i <= batches; i++)
        {
            int no = i;

            Task task = new Task(() =>
            {
                using (Session uploadSession = new Session())
                {
                    while (true)
                    {
                        string localFilePath;
                        lock (filesEnumerator)
                        {
                            if (!filesEnumerator.MoveNext())
                            {
                                break;
                            }

                            localFilePath = filesEnumerator.Current;
                            bytes += new FileInfo(localFilePath).Length;
                            count++;
                        }

                        if (!uploadSession.Opened)
                        {
                            Console.WriteLine("Starting upload {0}...", no);
                            uploadSession.Open(sessionOptions);
                        }

                        string remoteFilePath = Path.Combine(remotePath,Path.GetFileName(localFilePath)) ;

                        Console.WriteLine("Uploading {0} to {1} in {2}...",localFilePath, remoteFilePath, no);

                        uploadSession.PutFiles(
                            localFilePath, RemotePath.EscapeFileMask(remoteFilePath)).
                            Check();
                    }

                    if (uploadSession.Opened)
                    {
                        Console.WriteLine("Upload {0} done", no);
                    }
                    else
                    {
                        Console.WriteLine("Upload {0} had nothing to do", no);
                    }
                }

            });

            tasks.Add(task);
            task.Start();
        }

        Console.WriteLine("Waiting for uploads to complete...");
        Task.WaitAll(tasks.ToArray());

        Console.WriteLine("Done");

        DateTime ended = DateTime.Now;
        Console.WriteLine("Took {0}", (ended - started));
        Console.WriteLine("Uploaded {0} files, totaling {1:N0} bytes", count, bytes);
    }
    catch (Exception e)
    {
        Console.WriteLine("Error: {0}", e);
    }
}

How to upload files to SFTP server using Parallel.ForEach loop for better perfomance

1 Answers1