0

I am currently downloading file from FTP and uploading to Azure. I am using this link for uploading large files to Azure (using BlockBlob).

private async Task UploadFileAsBlockBlob(Stream stream, BlobStorageConfiguration blobStorageConfiguration, string fileName)
{
try
            {
                var storageAccount = CloudStorageAccount.Parse(blobStorageConfiguration.ConnectionString);
                var blobClient = storageAccount.CreateCloudBlobClient();
                var cloudContainer = blobClient.GetContainerReference(blobStorageConfiguration.Container);
                await cloudContainer.CreateIfNotExistsAsync().ConfigureAwait(false);

                var storageDirectory = cloudContainer.GetDirectoryReference(blobStorageConfiguration.Path);
                var blob = storageDirectory.GetBlockBlobReference(fileName);

                const long pageSizeInBytes = 1048576 * 10; // 10mb at a time

                var blocklist = new HashSet<string>();

                if (!stream.CanRead)
                {
                    var error = $"Error while reading file from ftp {fileName}";
                    return;
                }

                long prevLastByte = 0;
                long bytesRemain = stream.Length;

                do
                {
                    long bytesToCopy = Math.Min(bytesRemain, pageSizeInBytes);
                    byte[] bytesToSend = new byte[bytesToCopy];

                    // some issue might occur for very large files (above 2 gb)
                    // overflow exception will occur.
                    stream.Read(bytesToSend, 0, (int)bytesToCopy);

                    prevLastByte += bytesToCopy;
                    bytesRemain -= bytesToCopy;

                    // create blockId
                    string blockId = Guid.NewGuid().ToString();
                    string base64BlockId = Convert.ToBase64String(Encoding.UTF8.GetBytes(blockId));

                    await blob.PutBlockAsync(base64BlockId, new MemoryStream(bytesToSend, true), null).ConfigureAwait(false);

                    blocklist.Add(base64BlockId);
                }
                while (bytesRemain > 0);

                // post blocklist
                await blob.PutBlockListAsync(blocklist).ConfigureAwait(false);
            }
            catch (Exception ex)
            {
                Logger.LogCritical(ex, "Error while importing Files");
            }
}

I am using Stream from ftp. 180Mb file takes 25 mins. Any faster approach suggested to speed up?

Gauravsa
  • 6,330
  • 2
  • 21
  • 30
  • Is it the downloading or the uploading which is limiting the speed? Many internet connections have about 10x the speed for download compared to upload. – Andrew Morton Jul 16 '20 at 17:16
  • i think its the downloading from ftp. but i send it chunk by chunk.shud I save to localfile and then read from there like will it be faster – Gauravsa Jul 16 '20 at 17:19
  • You are going to have to try it for yourself and actually measure the speeds. Do you know the speed of your internet connection? Just download the file locally - how long does that take? Is [Copy data from FTP server by using Azure Data Factory](https://learn.microsoft.com/en-us/azure/data-factory/connector-ftp) an option for you? – Andrew Morton Jul 16 '20 at 18:45
  • I am saving file to disk and then reading from disk which is quite fast, then i delete the file. I look at ADF also. Thanks – Gauravsa Jul 18 '20 at 12:19

0 Answers0