Downloading multiple files from an FTP server files using Libcurl

Question

I have used the following code to get to download all the files from the FTP Server

Steps followed are: 1. Creating a FTP list of File

    getFTPList(string sHost, string sUser, string sPass, string sUri)
{
  CURL *curl;
  CURLcode res;
  FILE *ftplister;
  string host = "ftp://";
  host += sHost;
  host += "/sample/";
  string furl = host + sUri;  
  string usrpwd = sUser;
  usrpwd += ":";
  usrpwd += sPass;
  /* local file name to store the file as */ 
  ftplister = fopen("ftp-list", "wb"); /* b is binary, needed on win32 */   
  curl = curl_easy_init();
  if(curl) {
    /* Get a file listing from sunet */ 
    curl_easy_setopt(curl, CURLOPT_URL, furl.c_str() );
    curl_easy_setopt(curl, CURLOPT_USERPWD, usrpwd.c_str());
    curl_easy_setopt(curl, CURLOPT_FTPLISTONLY, TRUE);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_list);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, ftplister);
    res = curl_easy_perform(curl);
    /* Check for errors */ 
    if(res != CURLE_OK)
      fprintf(stderr, "curl_easy_perform() failed: %s\n",
              curl_easy_strerror(res));

    /* always cleanup */ 
    curl_easy_cleanup(curl);
  } 
  fclose(ftplister); /* close the local file */   
}

Use this list to download the files calling the download functions recursively

int main(){
FILE *ftpfile; 
string line;
ftpfile = fopen("ftp-list", "r"); 
ifstream infile("ftp-list");
 while ( getline(infile, line) )
 {
    string url, ofname, surl = "ftp://myhost/uploader/", sfname = "C:\\CNAP\\";        
    url = surl + line;
    ofname = sfname +line;
    cout<<url<<"   "<<ofname<<endl;
    char* theVal ;
    char* theStr ;
    theVal = new char [url.size()+1];
    theStr = new char [ofname.size()+1];
    strcpy(theVal, url.c_str());
    strcpy(theStr, ofname.c_str());       
    downloadFile(theVal, theStr);
 }
return 0;
}

Now the download function:

size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
    size_t written;
    written = fwrite(ptr, size, nmemb, stream);
    return written;
   }

   void downloadFile(const char* url, const char* ofname)
   {
    CURL *curl;
        FILE *fp;
        CURLcode res;        
        curl = curl_easy_init();
        if (curl){
            fp = fopen(ofname,"wb");
            curl_easy_setopt(curl, CURLOPT_URL, url);
            curl_easy_setopt(curl, CURLOPT_USERPWD, "user:pass");
            curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
            curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
            res = curl_easy_perform(curl);
            curl_easy_cleanup(curl);
            fclose(fp);
        }
     }

It works well when implemented but only to download text files or some files with texts, if I download an image or a docx or a a zip or rar or for that matter any file which is not text it fails, it fails to open after downloading (says invalid file).

I am not sure what am I missing, any help would be appreciated.

I know this is an inefficient way of coding, but I just need the downloads to be right (any file). Working on the efficiency is my next agenda.

PS: Used this method used here Downloading multiple files with libcurl in C++

Thank you

possible duplicate of [Image corrupted after php curl transfer FTP](http://stackoverflow.com/questions/8912058/image-corrupted-after-php-curl-transfer-ftp) — Robᵩ, Aug 24 '12 at 13:29
It was not just image files being corrupted but all the files except the txt files. I am not sure how but I did solve the problem, using the link you provided. It was the problem with not opening the files as a binary write (wb). — Vivian Lobo, Aug 24 '12 at 15:45

score 1 · Accepted Answer · answered Aug 24 '12 at 15:46

1

The file when downloading needs to be opened as a binary file

fp = fopen(ofname,"wb");

Thank you for the help Rob

answered Aug 24 '12 at 15:46

Vivian Lobo

583
10
29

Robᵩ · Answer 2 · 2012-08-24T13:36:40.713

0

This is a guess.: Try clearing the options CURLOPT_TRANSFERTEXT and CURLOPT_CRLF.

Refernce:

http://curl.haxx.se/libcurl/c/curl_easy_setopt.html

edited Aug 24 '12 at 13:36

answered Aug 24 '12 at 13:29

Robᵩ

163,533
20
239
308

score 0 · Answer 3 · answered Nov 15 '13 at 01:44

#include <stdio.h> 
#include <curl/curl.h>
#include <curl/easy.h>
#include <string>
#include <iostream>
#include <fstream>

using namespace std;

size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream);

void getFileList(
    const string &strHost, 
    const string &strUri, 
    const string &strUser, 
    const string &strPassWord, 
    const string &strTargetFile
);

void downloadAllFiles(
    const string &strFtpListFile, 
    const string &strHost, 
    const string &strUri, 
    const string &strUser, 
    const string &strPassWord, 
    const string &strSaveTargetFolder
);



int main(void) 
{
    string strHost = "ftp://192.168.0.1:22";
    string strUri = "/cobus/test/";
    string strUser = "cobus";
    string strPassWord = "password";
    string strTargetFile = "c:\\cobus\\ftpList.txt";
    string strSaveDestFolder = "c:\\cobus\\";

    getFileList(strHost, strUri, strUser, strPassWord, strTargetFile);

    downloadAllFiles(
        strTargetFile, 
        strHost, 
        strUri, 
        strUser, 
        strPassWord, 
        strSaveDestFolder
    );
}

size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) 
{
    size_t written = fwrite(ptr, size, nmemb, stream);
    return written;

}


void getFileList(const string &strHost, const string &strUri, 
const string &strUser, const string &strPassWord, const string &strTargetFile)
{
    CURL *curl;
    CURLcode res;
    FILE *ftplister;
    string strSourceFullUri = strHost + strUri;
    string strUserPwInfo = strUser + ":" + strPassWord;

    /* local file name to store the file as */ 
    ftplister = fopen(strTargetFile.c_str(), "wb"); /* b is binary, needed on win32 */

    curl = curl_easy_init();

    if(curl) 
    {
        /* Get a file listing from sunet */ 
        curl_easy_setopt(curl, CURLOPT_URL, strSourceFullUri.c_str() );
        curl_easy_setopt(curl, CURLOPT_USERPWD, strUserPwInfo.c_str());
        curl_easy_setopt(curl, CURLOPT_FTPLISTONLY, TRUE);
        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_data);
        curl_easy_setopt(curl, CURLOPT_WRITEDATA, ftplister);
        res = curl_easy_perform(curl);

        /* Check for errors */ 
        if(res != CURLE_OK)
            fprintf(stderr, "curl_easy_perform() failed: %s\n",
            curl_easy_strerror(res));

        /* always cleanup */ 
        curl_easy_cleanup(curl);
    } 

    fclose(ftplister); /* close the local file */   

}

void downloadAllFiles(const string &strFtpListFile, const string &strHost, 
    const string &strUri, const string &strUser, const string &strPassWord, 
    const string &strSaveDestFolder)
{
    FILE *ftpFile = NULL;
    string strFileName = "";

    ifstream infile(strFtpListFile.c_str());

    if(!infile.is_open())
    {
        cerr << "can not open ftpList.txt" << endl;
        return ;
    }

    /* ftpList.txt get data, line by line, processing */
    while(getline(infile, strFileName))
    {
        CURL *curl;
        FILE *destFilePath;
        CURLcode res;        

        curl = curl_easy_init();

        if (curl)
        {
            destFilePath = fopen((strSaveDestFolder+strFileName).c_str(),"wb");
            curl_easy_setopt(curl, CURLOPT_URL, (strHost+strUri+strFileName).c_str());
            curl_easy_setopt(curl, CURLOPT_USERPWD, (strUser+":"+strPassWord).c_str());
            curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
            curl_easy_setopt(curl, CURLOPT_WRITEDATA, destFilePath);
            res = curl_easy_perform(curl);

            curl_easy_cleanup(curl);

            fclose(destFilePath);
        }
    }// end while
}

I expressed as much as possible to see at once the contents of the Vivian Lobo. I tried to test to make statements Vivian Lobo. It is good working.

stackOverFlow is hard to write code. And how can I comment to Vivian lobo article?

Downloading multiple files from an FTP server files using Libcurl

3 Answers3