HTTP Request to Wikipedia gives no result

Question

I try to fetch HTML per Code. When fetching from "http://www.google.com" for example it works perfect. When trying to fetch from "http://en.wikipedia.org/w/api.php" I do not get any results.

Does someone have any idea ?

Code:

    String sURL="http://en.wikipedia.org/w/api.php?action=query&generator=categorymembers&gcmtitle=Category:Countries&prop=info&gcmlimit=500&format=json";

    String sText=readfromURL(sURL);



public static String readfromURL(String sURL){

    URL url = null;
    try {
        url = new URL(sURL);

    } catch (MalformedURLException e1) {
        e1.printStackTrace();
    }
    URLConnection urlconnect = null;
    try {
        urlconnect = url.openConnection();

        urlconnect.setRequestProperty("User-Agent","Mozilla/5.0 (Windows NT 5.1; rv:19.0) Gecko/20100101 Firefox/19.0");

    } catch (IOException e) {
        e.printStackTrace();
    }
    BufferedReader in = null;
    try {
        in = new BufferedReader(new InputStreamReader(urlconnect.getInputStream()));
    } catch (IOException e) {
        e.printStackTrace();
    }
    String inputLine;
    String sEntireContent="";
    try {
        while ((inputLine = in.readLine()) != null) {
            System.out.println(inputLine);
        sEntireContent=sEntireContent+inputLine;
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    try {
        in.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
    return sEntireContent;
}

Andriy Kuba · Accepted Answer · 2015-10-23T09:42:47.893

It looks like the request limit. Try to check the response code.

From the documentation (https://www.mediawiki.org/wiki/API:Etiquette):

If you make your requests in series rather than in parallel (i.e. wait for the one request to finish before sending a new request, such that you're never making more than one request at the same time), then you should definitely be fine.

Be sure that you do not do few request at a time

Update

I did verification on my local your code - you are correct it does not work. Fix - you need to use https, so it would work:

https://en.wikipedia.org/w/api.php?action=query&generator=categorymembers&gcmtitle=Category:Countries&prop=info&gcmlimit=500&format=json

result:

{"batchcomplete":"","query":{"pages":{"5165":{"pageid":5165,"ns":0,"title":"Country","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-20T20:09:05Z","lastrevid":686706429,"length":12695},"5112305":{"pageid":5112305,"ns":14,"title":"Category:Countries by continent","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-18T17:31:54Z","lastrevid":681415612,"length":133},"14353213":{"pageid":14353213,"ns":14,"title":"Category:Countries by form of government","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-08-13T23:33:29Z","lastrevid":675984011,"length":261},"5112467":{"pageid":5112467,"ns":14,"title":"Category:Countries by international organization","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-18T05:11:12Z","lastrevid":686245148,"length":123},"4696391":{"pageid":4696391,"ns":14,"title":"Category:Countries by language","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-20T01:17:18Z","lastrevid":675966601,"length":333},"5112374":{"pageid":5112374,"ns":14,"title":"Category:Countries by status","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-08-13T21:05:47Z","lastrevid":675966630,"length":30},"708617":{"pageid":708617,"ns":14,"title":"Category:Lists of countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-18T05:08:45Z","lastrevid":681553760,"length":256},"46624537":{"pageid":46624537,"ns":14,"title":"Category:Caspian littoral states","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-09-23T08:40:34Z","lastrevid":663549987,"length":50},"18066512":{"pageid":18066512,"ns":14,"title":"Category:City-states","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-09-29T20:14:14Z","lastrevid":679367764,"length":145},"2019528":{"pageid":2019528,"ns":14,"title":"Category:Country classifications","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-09-25T09:09:13Z","lastrevid":675966465,"length":182},"935240":{"pageid":935240,"ns":14,"title":"Category:Country codes","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-17T06:05:53Z","lastrevid":546489724,"length":222},"36819536":{"pageid":36819536,"ns":14,"title":"Category:Countries in fiction","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-03T06:09:16Z","lastrevid":674147667,"length":169},"699787":{"pageid":699787,"ns":14,"title":"Category:Fictional countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-17T18:43:25Z","lastrevid":610289877,"length":356},"804303":{"pageid":804303,"ns":14,"title":"Category:Former countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-09-21T09:58:52Z","lastrevid":668632882,"length":403},"7213567":{"pageid":7213567,"ns":14,"title":"Category:Island countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-22T22:10:37Z","lastrevid":648502876,"length":157},"3046541":{"pageid":3046541,"ns":14,"title":"Category:Landlocked countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-04T00:45:24Z","lastrevid":648502892,"length":54},"743058":{"pageid":743058,"ns":14,"title":"Category:Middle Eastern countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-12T14:41:59Z","lastrevid":677900732,"length":495},"41711462":{"pageid":41711462,"ns":14,"title":"Category:Mongol states","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-23T07:36:21Z","lastrevid":687093637,"length":121},"30645082":{"pageid":30645082,"ns":14,"title":"Category:Country names","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-03-07T06:33:19Z","lastrevid":561256656,"length":94},"21218559":{"pageid":21218559,"ns":14,"title":"Category:Outlines of countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-07T18:04:29Z","lastrevid":645312408,"length":248},"37943702":{"pageid":37943702,"ns":14,"title":"Category:Proposed countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-20T02:30:25Z","lastrevid":668630396,"length":130},"15086044":{"pageid":15086044,"ns":14,"title":"Category:Turkic states","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-20T06:23:35Z","lastrevid":677424552,"length":114},"32809189":{"pageid":32809189,"ns":14,"title":"Category:Works about countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-17T08:45:32Z","lastrevid":620016516,"length":153},"27539189":{"pageid":27539189,"ns":14,"title":"Category:Wikipedia books on countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-04-11T05:12:25Z","lastrevid":546775798,"length":203},"35317198":{"pageid":35317198,"ns":14,"title":"Category:Wikipedia categories named after countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-17T18:35:14Z","lastrevid":641689352,"length":202}}}}
{"batchcomplete":"","query":{"pages":{"5165":{"pageid":5165,"ns":0,"title":"Country","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-20T20:09:05Z","lastrevid":686706429,"length":12695},"5112305":{"pageid":5112305,"ns":14,"title":"Category:Countries by continent","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-18T17:31:54Z","lastrevid":681415612,"length":133},"14353213":{"pageid":14353213,"ns":14,"title":"Category:Countries by form of government","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-08-13T23:33:29Z","lastrevid":675984011,"length":261},"5112467":{"pageid":5112467,"ns":14,"title":"Category:Countries by international organization","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-18T05:11:12Z","lastrevid":686245148,"length":123},"4696391":{"pageid":4696391,"ns":14,"title":"Category:Countries by language","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-20T01:17:18Z","lastrevid":675966601,"length":333},"5112374":{"pageid":5112374,"ns":14,"title":"Category:Countries by status","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-08-13T21:05:47Z","lastrevid":675966630,"length":30},"708617":{"pageid":708617,"ns":14,"title":"Category:Lists of countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-18T05:08:45Z","lastrevid":681553760,"length":256},"46624537":{"pageid":46624537,"ns":14,"title":"Category:Caspian littoral states","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-09-23T08:40:34Z","lastrevid":663549987,"length":50},"18066512":{"pageid":18066512,"ns":14,"title":"Category:City-states","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-09-29T20:14:14Z","lastrevid":679367764,"length":145},"2019528":{"pageid":2019528,"ns":14,"title":"Category:Country classifications","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-09-25T09:09:13Z","lastrevid":675966465,"length":182},"935240":{"pageid":935240,"ns":14,"title":"Category:Country codes","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-17T06:05:53Z","lastrevid":546489724,"length":222},"36819536":{"pageid":36819536,"ns":14,"title":"Category:Countries in fiction","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-03T06:09:16Z","lastrevid":674147667,"length":169},"699787":{"pageid":699787,"ns":14,"title":"Category:Fictional countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-17T18:43:25Z","lastrevid":610289877,"length":356},"804303":{"pageid":804303,"ns":14,"title":"Category:Former countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-09-21T09:58:52Z","lastrevid":668632882,"length":403},"7213567":{"pageid":7213567,"ns":14,"title":"Category:Island countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-22T22:10:37Z","lastrevid":648502876,"length":157},"3046541":{"pageid":3046541,"ns":14,"title":"Category:Landlocked countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-04T00:45:24Z","lastrevid":648502892,"length":54},"743058":{"pageid":743058,"ns":14,"title":"Category:Middle Eastern countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-12T14:41:59Z","lastrevid":677900732,"length":495},"41711462":{"pageid":41711462,"ns":14,"title":"Category:Mongol states","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-23T07:36:21Z","lastrevid":687093637,"length":121},"30645082":{"pageid":30645082,"ns":14,"title":"Category:Country names","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-03-07T06:33:19Z","lastrevid":561256656,"length":94},"21218559":{"pageid":21218559,"ns":14,"title":"Category:Outlines of countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-07T18:04:29Z","lastrevid":645312408,"length":248},"37943702":{"pageid":37943702,"ns":14,"title":"Category:Proposed countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-20T02:30:25Z","lastrevid":668630396,"length":130},"15086044":{"pageid":15086044,"ns":14,"title":"Category:Turkic states","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-20T06:23:35Z","lastrevid":677424552,"length":114},"32809189":{"pageid":32809189,"ns":14,"title":"Category:Works about countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-17T08:45:32Z","lastrevid":620016516,"length":153},"27539189":{"pageid":27539189,"ns":14,"title":"Category:Wikipedia books on countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-04-11T05:12:25Z","lastrevid":546775798,"length":203},"35317198":{"pageid":35317198,"ns":14,"title":"Category:Wikipedia categories named after countries","contentmodel":"wikitext","pagelanguage":"en","touched":"2015-10-17T18:35:14Z","lastrevid":641689352,"length":202}}}}

Thanks, but as you can see from the code, this is a onetime request. — mcfly soft, Oct 23 '15 at 09:13
form this code - yes, but I was not sure that you did not run this code in threads, or even you have 3 programs with this code that runs in the same time on your PC. Maybe you forgot to switch one and run other. So I just mark that you need to be sure it's not because you reach limit — Andriy Kuba, Oct 23 '15 at 09:23
look at my update please - you need to use https, it redirects from http to https so you receive zero in the first response. You need to follow redirect or use https — Andriy Kuba, Oct 23 '15 at 09:43
It works ! Thanks a lot. Only one additional Question for interrest. I am wondering how this works. I do not have any ServerCertificate for an SSL-Connection. (No keystore). Is the Certificate automatically loaded on Client Side ? — mcfly soft, Oct 23 '15 at 15:09
It has some number of default. Please look on the approved answer to this question: http://stackoverflow.com/questions/11143360/ssl-certificate-verification-in-java — Andriy Kuba, Oct 23 '15 at 15:33

score 0 · Answer 2 · answered Oct 23 '15 at 09:48

The reason you didn't receive the response back is due to a HTTP Redirect 3XX. Wikipedia redirects your HTTP Request. Please try the below source code to fetch Response from Redirected URL. Please refer How to send HTTP request GET/POST in Java

public static String readfromURLwithRedirect(String url) {
    String response = "";
    try {
        URL obj = new URL(url);
        HttpURLConnection conn = (HttpURLConnection) obj.openConnection();
        conn.setReadTimeout(5000);
        conn.addRequestProperty("Accept-Language", "en-US,en;q=0.8");
        conn.addRequestProperty("User-Agent", "Mozilla");
        conn.addRequestProperty("Referer", "google.com");

        System.out.println("Request URL ... " + url);

        boolean redirect = false;

        // normally, 3xx is redirect
        int status = conn.getResponseCode();
        if (status != HttpURLConnection.HTTP_OK) {
            if (status == HttpURLConnection.HTTP_MOVED_TEMP
                    || status == HttpURLConnection.HTTP_MOVED_PERM
                    || status == HttpURLConnection.HTTP_SEE_OTHER) {
                redirect = true;
            }
        }

        System.out.println("Response Code ... " + status);

        if (redirect) {

            // get redirect url from "location" header field
            String newUrl = conn.getHeaderField("Location");

            // get the cookie if need, for login
            String cookies = conn.getHeaderField("Set-Cookie");

            // open the new connnection again
            conn = (HttpURLConnection) new URL(newUrl).openConnection();
            conn.setRequestProperty("Cookie", cookies);
            conn.addRequestProperty("Accept-Language", "en-US,en;q=0.8");
            conn.addRequestProperty("User-Agent", "Mozilla");
            conn.addRequestProperty("Referer", "google.com");

            System.out.println("Redirect to URL : " + newUrl);

        }

        BufferedReader in = new BufferedReader(
                new InputStreamReader(conn.getInputStream()));
        String inputLine;
        StringBuffer responseBuffer = new StringBuffer();

        while ((inputLine = in.readLine()) != null) {
            responseBuffer.append(inputLine);
        }
        in.close();

        System.out.println("URL Content... \n" + responseBuffer.toString());
        response = responseBuffer.toString();
        System.out.println("Done");

    } catch (Exception e) {
        e.printStackTrace();
    }

    return response;
}

HTTP Request to Wikipedia gives no result

2 Answers2