Loop through paginated REST API using requests in python

Question

I'm a very unexperienced user of REST APIs, requests and python, so bear with me. I've managed to throw a GET request using requests via python towards this REST API that was developed with Postman. I get results fine, but only for the first page (only 50 items). I know that all I need to do in order to get more results is adding '?page=2' and so on to the URL, but somehow I couldn't manage the loop so far. It appears to me that this API doesn't have the "nextUrl" or total number of pages features, so I'm a bit lost. This is what I have so far:

from getpass import getpass
import requests


usr = input('Inform user: ')
pwd = getpass('Inform password : ')


url = 'https://api.inventsys.com.br/v4/login'
payload = "{\n  \"username\": \"" + usr + "\",\n  \"password\": \"" + pwd + "\"\n}"
headers = {
      'Account': 'my-user',
      'Content-Type': 'application/json',

    }
r = requests.request('POST', url, headers = headers, data = payload, allow_redirects=False)

mytoken = r.json()['token']

projectid = input('Inform project ID (10762): ')

url = 'https://api.inventsys.com.br/v4/projects/'+projectid+'/items'
payload = {}
headers = {

      'Account': 'my-user',
      'Token': mytoken,

    }

ativos = requests.request('GET', url, headers = headers, data = payload, allow_redirects=False)


print(ativos.json()['result'])


print(ativos.text)
print(ativos.headers)

Which is providing the following result (I've shortened the result so it shows only one record of the 50):

Inform project ID (10762): 10762
ok
{
    "items": [
        {
            "id": 300457,
            "item_id_parent": null,
            "reference": "",
            "subreference1": "CAMS\/2",
            "subreference2": "CAMS\/2",
            "reference_alpha": null,
            "reference_numeric": null,
            "oid": "CAMS\/2",
            "code": null,
            "code_custom": null,
            "name": "284",
            "image": "https:\/\/static.inventsys.com.br\/278\/thumb\/f-3298939-200x200c.jpg",
            "situations": [],
            "project_id": 10762,
            "project": {
                "id": 10762,
                "name": "Fauna EGR",
                "color": null
            },
            "category_id": 20685,
            "category": {
                "id": 20685,
                "name": "EGR FAUNA - Armadilhas"
            },
            "area_id": null,
            "area": null,
            "location": {
                "lat": -30.137237548828,
                "lng": -50.90788269043,
                "address": {
                    "region": "RS",
                    "city": "Viamão",
                    "district": null,
                    "zipcode": null,
                    "street": "Rodovia Tapir Rocha",
                    "street_number": null,
                    "desc": null,
                    "full": "Rodovia Tapir Rocha Viamão \/ BR"
                }
            },
            "event_last": null,
            "description": null,
            "search_terms": "CAMS\/2 284 Fauna EGR EGR FAUNA - Armadilhas Rodovia Tapir Rocha Viamão \/ BR",
            "info": [
                {
                    "id": 42725,
                    "name": "Observacoes",
                    "type": "longtext",
                    "value": "Ver observações no odk",
                    "fvalue": "Ver observações no odk",
                    "description": null,
                    "ikey": null,
                    "group": "Fauna EGR",
                    "preload": false,
                    "filling": false,
                    "primary": false,
                    "created": "2019-10-01T17:13:04+00:00"
                },
                {
                    "id": 44542,
                    "name": "Data de instalacao",
                    "type": "date",
                    "value": "2019-10-01",
                    "fvalue": "01\/10\/2019",
                    "description": null,
                    "ikey": null,
                    "group": "Fauna EGR",
                    "preload": false,
                    "filling": false,
                    "primary": false,
                    "created": "2019-10-01T17:13:04+00:00"
                },
                {
                    "id": 44543,
                    "name": "IDcartao",
                    "type": "text",
                    "value": "Ste 04",
                    "fvalue": "Ste 04",
                    "description": null,
                    "ikey": null,
                    "group": "Fauna EGR",
                    "preload": false,
                    "filling": false,
                    "primary": true,
                    "created": "2019-10-01T17:13:04+00:00"
                },
                {
                    "id": 44544,
                    "name": "IDcamera",
                    "type": "text",
                    "value": "1",
                    "fvalue": "1",
                    "description": null,
                    "ikey": null,
                    "group": "Fauna EGR",
                    "preload": false,
                    "filling": false,
                    "primary": false,
                    "created": "2019-10-01T17:13:04+00:00"
                },
                {
                    "id": 44545,
                    "name": "IDdobueiro",
                    "type": "text",
                    "value": "284",
                    "fvalue": "284",
                    "description": null,
                    "ikey": null,
                    "group": "Fauna EGR",
                    "preload": false,
                    "filling": false,
                    "primary": false,
                    "created": "2019-10-01T17:13:04+00:00"
                },
                {
                    "id": 44546,
                    "name": "Estrada",
                    "type": "longtext",
                    "value": "Ers-040",
                    "fvalue": "Ers-040",
                    "description": null,
                    "ikey": null,
                    "group": "Fauna EGR",
                    "preload": false,
                    "filling": false,
                    "primary": false,
                    "created": "2019-10-01T17:13:04+00:00"
                },
                {
                    "id": 44547,
                    "name": "Foto Armadilha",
                    "type": "photo",
                    "value": "3298938",
                    "fvalue": "3298938",
                    "description": null,
                    "ikey": null,
                    "group": "Fauna EGR",
                    "preload": false,
                    "filling": false,
                    "primary": false,
                    "created": "2019-10-01T17:13:04+00:00"
                },
                {
                    "id": 44548,
                    "name": "gps Latitude",
                    "type": "text",
                    "value": null,
                    "fvalue": null,
                    "description": null,
                    "ikey": null,
                    "group": "Fauna EGR",
                    "preload": false,
                    "filling": false,
                    "primary": false,
                    "created": "2019-10-01T17:13:04+00:00"
                },
                {
                    "id": 44549,
                    "name": "GPS Longitude",
                    "type": "text",
                    "value": null,
                    "fvalue": null,
                    "description": null,
                    "ikey": null,
                    "group": "Fauna EGR",
                    "preload": false,
                    "filling": false,
                    "primary": false,
                    "created": "2019-10-01T17:13:04+00:00"
                },
                {
                    "id": 44550,
                    "name": "gps Altitude",
                    "type": "text",
                    "value": null,
                    "fvalue": null,
                    "description": null,
                    "ikey": null,
                    "group": "Fauna EGR",
                    "preload": false,
                    "filling": false,
                    "primary": false,
                    "created": "2019-10-01T17:13:04+00:00"
                },
                {
                    "id": 44551,
                    "name": "gps Accuracy",
                    "type": "text",
                    "value": null,
                    "fvalue": null,
                    "description": null,
                    "ikey": null,
                    "group": "Fauna EGR",
                    "preload": false,
                    "filling": false,
                    "primary": false,
                    "created": "2019-10-01T17:13:04+00:00"
                }
            ],
            "files": [
                {
                    "id": 3298939,
                    "url_orig": "https:\/\/static.inventsys.com.br\/278\/file\/cb\/cb7c7aebac.jpg",
                    "url_low": "https:\/\/static.inventsys.com.br\/278\/thumb\/f-3298939-200x200c.jpg",
                    "type": "photo",
                    "format": "jpg",
                    "size": 1040359,
                    "orig_name": "12559f77-4c11-4fe1-8b86-ce299a8e3bb7.jpg",
                    "description": null
                }
            ],
            "nsubitems": null,
            "nevents": "0",
            "updated_at": "2019-10-01T17:13:04+00:00",
            "created_at": "2019-10-01T17:13:04+00:00",
            "deleted_at": null
        }
    ],
    "nitems": 1505,
    "current_time": "2020-02-08T00:53:42+00:00",
    "result": "ok"
}
{'Server': 'nginx/1.15.8', 'Date': 'Sat, 08 Feb 2020 00:53:42 GMT', 'Content-Type': 'application/json', 'Transfer-Encoding': 'chunked', 'Connection': 'keep-alive', 'Vary': 'Accept-Encoding, Accept-Encoding', 'Set-Cookie': 'PHPSESSID=pj9d35fhigr6d61nfqcfd05qc6; path=/', 'Expires': 'Thu, 19 Nov 1981 08:52:00 GMT', 'Cache-Control': 'no-store, no-cache, must-revalidate, no-cache, private', 'Pragma': 'no-cache', 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Credentials': 'true', 'Content-Encoding': 'gzip'}

How could I get all 1505 items by doing a loop? Many thanks!

It sounds like you are on exactly the right path to accomplish what you are trying to do. You just need to find the API's way of controlling the pagination of the response data. — Z4-tier, Feb 08 '20 at 02:49

loginmind · Accepted Answer · 2020-02-08T02:19:31.313

1

Since we don't know the number of pages( the available projectid), you can use try & except as below:

projectid = 0 # initiate value for projectid
while True:
    try:
        url = 'https://api.inventsys.com.br/v4/projects/'+projectid+'/items'
        ativos = requests.request('GET', url, headers=headers, data=payload, allow_redirects=False)
        # do any thing with ativos
    except HTTPError:
        # handle HTTPError
        logging.error('HTTPError')
    # ... put any other Exception you need to handle here
    except Exception as e:
        # for handle unknown exception
        logging.error('Unknown exception')
    else:
        # loop over projectid
        projectid += 1

edited Feb 08 '20 at 02:19

answered Feb 08 '20 at 01:37

loginmind

563
5
11

This answer assumes that you loop through pages by using `projectid`, if you use other variable for iterating, just replace `projectid` with that variable. – loginmind Feb 08 '20 at 01:44
using `except Exception` like this is bad practice. – AMC Feb 08 '20 at 01:46
Exactly when we catch any exception we need to handle it, but I not put it here for simplicity. – loginmind Feb 08 '20 at 01:49
The most I can do is guess which exception you're actually trying to catch. Also, see https://stackoverflow.com/questions/4990718/about-catching-any-exception. – AMC Feb 08 '20 at 01:52
Thanks for your suggestion, I have read some most voted answers and I didn't see any conflict with my answer above. Maybe the message in print statement is not meaningful. – loginmind Feb 08 '20 at 02:15
Let’s look at it another way, then, why do you want to catch all exceptions? – AMC Feb 08 '20 at 02:25
this solution did work out for me, as in the loop is operational. problem is that when pages are over, it just enters an infinte loop and keeps responding with empty arrays for the json items. any ideas on how to stop this? { "items": [], "nitems": 1505, "current_time": "2020-02-08T21:23:47+00:00", "result": "ok" } – Guilherme Iablonovski Feb 08 '20 at 21:29
just check if `len(items) == 0` the exit loop using `break` – loginmind Feb 09 '20 at 00:06

Loop through paginated REST API using requests in python

1 Answers1