Java - Apache Post Request Doesn't Return Full Page

Question

I'm trying to fetch data from the website: http://www.wecare4air.co.uk/air-quality-data/brookgate-cb-1/ I used Charles to get the post request and headers that Firefox sends to the site to get data and I constructed exactly the same thing (same post, same headers) using the Apache libraries.

However, my Java code only get the boilerplate of the results website - i.e. it gets the site without the results. The status code of the response is 2000, however, it's size is 8kB as opposed to 20kB the site loaded by Firefox has.

Any idea what might be causing this?

Here is the code (massive due to the number of parameters that need setting):

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;

import java.io.*;
import java.util.ArrayList;
import java.util.List;

final static String WEBSITE_URL = "http://www.wecare4air.co.uk/non-    aurn-results";

private static String dateToStationRoadPostRequestFormat(String date) {
    return String.format("%s/%s/%s", date.substring(8, 10), date.substring(5, 7), date.substring(0, 4));
}

private static String timeToStationRoadPostRequestFormat(String time) {
    return time.substring(0,5);
}

/**
 *@param date is of format YYYY-MM-DD
 *@param time is of format HH:00:00
 */
private static List<NameValuePair> preparePostRequestParams (String date, String time) {
    date = dateToStationRoadPostRequestFormat(date);
    time = timeToStationRoadPostRequestFormat(time);

    List<NameValuePair> params = new ArrayList<NameValuePair>(2);
    params.add(new BasicNameValuePair("station-id", "221"));
    params.add(new BasicNameValuePair("station_name", "Brookgate CB1"));
    params.add(new BasicNameValuePair("output_type", "grid"));
    params.add(new BasicNameValuePair("report_length", "daily"));
    params.add(new BasicNameValuePair("checkboxes", "all"));
    params.add(new BasicNameValuePair("monitors[]", "5"));
    params.add(new BasicNameValuePair("monitors[]", "4"));
    params.add(new BasicNameValuePair("monitors[]", "6"));
    params.add(new BasicNameValuePair("monitors[]", "119"));
    params.add(new BasicNameValuePair("monitors[]", "48"));
    params.add(new BasicNameValuePair("monitors[]", "11"));
    params.add(new BasicNameValuePair("monitors[]", "13"));
    params.add(new BasicNameValuePair("monitors[]", "12"));
    params.add(new BasicNameValuePair("monitors[]", "161"));
    params.add(new BasicNameValuePair("monitors[]", "162"));
    params.add(new BasicNameValuePair("monitors[]", "10"));
    params.add(new BasicNameValuePair("monitors[]", "48"));
    params.add(new BasicNameValuePair("monitors[]", "11"));
    params.add(new BasicNameValuePair("monitors[]", "13"));
    params.add(new BasicNameValuePair("monitors[]", "161"));
    params.add(new BasicNameValuePair("monitors[]", "162"));
    params.add(new BasicNameValuePair("monitors[]", "94"));
    params.add(new BasicNameValuePair("monitors[]", "93"));
    params.add(new BasicNameValuePair("monitors[]", "90"));

    //Customised values
    params.add(new BasicNameValuePair("start_date", date));
    params.add(new BasicNameValuePair("start_time", time));
    params.add(new BasicNameValuePair("end_date", date));
    params.add(new BasicNameValuePair("end_time", time));

    params.add(new BasicNameValuePair("result_type", "AVG"));
    params.add(new BasicNameValuePair("time_base", "1"));
    params.add(new BasicNameValuePair("singlebutton", ""));

    return params;
}
private static HttpPost prepareHttpPost(String date, String time) throws UnsupportedEncodingException {
    HttpPost httppost = new HttpPost(WEBSITE_URL);
    // Request parameters and other properties.
    List<NameValuePair> params = preparePostRequestParams(date, time);


    httppost.setEntity(new UrlEncodedFormEntity(params));
    httppost.setHeader("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:51.0) Gecko/20100101 Firefox/51.0");
    httppost.setHeader("Referer", "http://www.wecare4air.co.uk/air-quality-data/brookgate-cb-1/");
    return httppost;
}

public static void getGrid(String date, String time) throws IOException {
    HttpClient httpclient = HttpClients.createDefault();
    HttpPost httppost = prepareHttpPost(date, time);

    //Execute and get the response.
    HttpResponse response = httpclient.execute(httppost);
    System.out.println("Response Code : " + response.getStatusLine().getStatusCode());
    HttpEntity entity = response.getEntity();


    System.out.printf("Response length is %d\n", entity.getContentLength());

    if (entity != null) {
        BufferedReader instreamReader = new BufferedReader(new InputStreamReader(entity.getContent()));
        StringBuffer result = new StringBuffer();
        String line;
        try {
            while ((line = instreamReader.readLine()) != null) {
                result.append(line+"\n");
            }
        }
        catch (Exception e) {
            e.printStackTrace();
        }
        finally {
            instreamReader.close();
            System.out.println("Response here!!:\n\n" + result);
        }
    }
}
public static void main (String[] args) {
    try {
        getGrid("2017-01-01", "23:00:00");
    }
    catch (Exception e) {
        e.printStackTrace();
    }
}

}

Answer 1

You might've a typo in your website URL above! Have you checked if the page is not loaded via AJAX requests?