0

Developing from my previous question, wanted to generalize the behavior so that would download files within a specific data range.

So, modified the code to look like this

#include "pch.h"
#define _CRT_SECURE_NO_WARNINGS
#include <cstring>
#include <ctime>
#include <iostream>
#include <iostream>
#include <stdio.h>
#include <curl/curl.h>
#include <string.h>
#include <windows.h>

size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
    size_t written = fwrite(ptr, size, nmemb, stream);
    return written;
}

void downloadFile(const char* url, const char* fname) {
    CURL *curl;
    FILE *fp;
    CURLcode res;
    curl = curl_easy_init();
    if (curl) {
        fp = fopen(fname, "wb");
        curl_easy_setopt(curl, CURLOPT_URL, url);
        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
        curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
        res = curl_easy_perform(curl);
        curl_easy_cleanup(curl);
        fclose(fp);
    }
}

struct mytm : std::tm {        // inherit "tm" to extend it a little
    mytm(int year, int month, int day) : tm{}, current{} {
        tm_year = year - 1900;
        tm_mon = month - 1;
        tm_mday = day;
        tm_hour = 12;
        current = mktime(this);
    }

    void add_days(int days) {
        current += days * 60 * 60 * 24;
        std::memcpy(this, std::localtime(&current), sizeof(std::tm));
    }

    std::string strftime(const char* format) {
        std::string retval(128, ' '); // a suitable sized buffer
        std::size_t len = std::strftime(&retval[0], retval.size(), format, this);

        retval.resize(len);
        return retval;
    }

    std::time_t current;
};

int main(void) {

    mytm date(2007, 1, 1);
    
    for (int i = 0; i < 10; ++i) {
        Sleep(1000);
        std::string url =
            date.strftime("http://www.centrodeinformacao.ren.pt/_layouts/CI.GetExcel/SafeGetExcel.aspx?T=CRG&P=%d-%m-%Y&variation=PT");

        std::string file_location =
            date.strftime("C:/Users/molecoder/Desktop/file_%d_%m_%Y.xls");

        // Pass std::string to a function that needs const char* - https://stackoverflow.com/a/347959/5675325
        const char * url_final = url.c_str();
        const char * file_location_final = file_location.c_str();

        downloadFile(url_final, file_location_final);
    }
}

While this downloads the a XLS file with the right name and data (so it's requesting correctly the URL)

XLS works fine

it does only so for the initial date - the day, month and year are not changing.

molecoder
  • 423
  • 1
  • 7
  • 24
  • 1
    Why so complicated? Just `wget -c "http://www.centrodeinformacao.ren.pt/_layouts/CI.GetExcel/SafeGetExcel.aspx?T=CRG&P=1-"{1..12}"-"{2019,2020}"&variation=PT"` would be enough – sehe Jul 12 '20 at 23:42
  • I'm exploring different ways to do it @sehe (also one with Python). Tested also with wget - https://stackoverflow.com/q/62871637/9415908 – molecoder Jul 13 '20 at 14:16

2 Answers2

1

You assign to date only once:

mytm date(2007, 1, 1);

And never modify that. So of course it doesn't change.

Update date before you format the new filename.

sehe
  • 374,641
  • 47
  • 450
  • 633
  • How can that be updated to increase the date gradually? – molecoder Jul 13 '20 at 08:12
  • 1
    In anywhich way you want? You already know how to set one, so perhaps just make an array, or some loops http://coliru.stacked-crooked.com/a/1fd76e8163c2bbd5 (maybe even use that adddays function) – sehe Jul 13 '20 at 12:29
  • FWIW I can't download a single one of these, but I notice that none of `stream` or `ptr` are being checked in `write_data`. – sehe Jul 13 '20 at 16:50
  • It wasn't that easy to understand in this specific case as even the title of the answer was "Download XLS files in a Cpp loop". The files weren't being downloaded though. – molecoder Jul 13 '20 at 17:35
1

As sehe says, the problem is the date not being modified and so I needed to update date, like this

#include "pch.h"
#define _CRT_SECURE_NO_WARNINGS
#include <cstring>
#include <ctime>
#include <iostream>
#include <stdio.h>
#include <curl/curl.h>
#include <string>
#include <windows.h>
#include <cstdlib>

size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
    size_t written = fwrite(ptr, size, nmemb, stream);
    return written;
}

void downloadFile(const char* url, const char* fname) {
    CURL *curl;
    FILE *fp;
    CURLcode res;
    curl = curl_easy_init();
    if (curl) {
        fp = fopen(fname, "wb");
        curl_easy_setopt(curl, CURLOPT_URL, url);
        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
        curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
        res = curl_easy_perform(curl);
        curl_easy_cleanup(curl);
        fclose(fp);
    }
}

struct mytm : std::tm {        // inherit "tm" to extend it a little
    mytm(int year, int month, int day) : tm{}, current{} {
        tm_year = year - 1900;
        tm_mon = month - 1;
        tm_mday = day;
        tm_hour = 12;
        current = mktime(this);
    }

    void add_days(int days) {
        current += days * 60 * 60 * 24;
        std::memcpy(this, std::localtime(&current), sizeof(std::tm));
    }

    std::string strftime(const char* format) {
        std::string retval(128, ' '); // a suitable sized buffer
        std::size_t len = std::strftime(&retval[0], retval.size(), format, this);

        retval.resize(len);
        return retval;
    }

    std::time_t current;
};

int main(void) {
    
    for (int year : {2018, 2019})
        for (int month : {1, 11})
            for (int day : {1, 31}) {
                std::cout << 
                    mytm(year, month, day)
                    .strftime("http://www.centrodeinformacao.ren.pt/_layouts/CI.GetExcel/SafeGetExcel.aspx?T=CRG&P=%d-%m-%Y&variation=PT")
                    << std::endl;

                Sleep(1000);
                
                std::string url =
                    mytm(year, month, day).strftime("http://www.centrodeinformacao.ren.pt/_layouts/CI.GetExcel/SafeGetExcel.aspx?T=CRG&P=%d-%m-%Y&variation=PT");

                std::string file_location =
                    mytm(year, month, day).strftime("C:/Users/molecoder/Desktop/file_%d_%m_%Y.xls");

                // Pass std::string to a function that needs const char* - https://stackoverflow.com/a/347959/5675325
                const char * url_final = url.c_str();
                const char * file_location_final = file_location.c_str();

                downloadFile(url_final, file_location_final);
            }


}

Notice that some of the files won't have data and would get the following result

Ocorreu um erro.REN_CRG_Total::pr_REN_CRG_SelectTotalByDate_Excel::Error occured.

Hm...

molecoder
  • 423
  • 1
  • 7
  • 24