How can i scrape multiple values with the same name from a json with python?

Question

Was trying to scrape the stock quantities for a product on this link: https://sizeer.ro/v2/products/searchnearpos?address=44.4199549%2C26.1238619&distance=50&product=3046070

This is the json that gives as response:

{"pos":[{"id":"1110","name":"BUCURESTI MALL SIZEER","code":"RO31","slug":"bucuresti-mall-sizeer","warehouse":"RO31 - RO31 NEW SIZEER EU - RO","warehouse_id":"1842","street":"Calea Vitan 3rd District","house_number":"55-59","apartment_number":"","postcode":"031282","city":"Bucharest","province":"Bucuresti","phone":"0771567291","latitude":"44.4199549","longitude":"26.1238619","distance":"0","availability_variants":[{"offer_id":943027879,"product_id":3046070,"size":"46,5","stock":2,"availability_id":"43","availability_name":"Disponibil","add_to_cart":true}],"products_availability":true},{"id":"703","name":"BUCURE\u0218TI PARKLAKE SIZEER","code":"RO05","slug":"bucuresti-parklake-sizeer","warehouse":"RO05 - NEW SIZEER EU - RO","warehouse_id":"1363","street":"Strada Liviu Rebreanu","house_number":"4","apartment_number":"3 district","postcode":"031783","city":"Bucure\u0219ti","province":"Bucuresti","phone":"0770314565","latitude":"44.4206244","longitude":"26.1500617","distance":"2.0820717992071485","open_hour":{"1":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"1"},"2":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"2"},"3":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"3"},"4":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"4"},"5":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"5"},"6":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"6"},"0":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"0"}},"availability_variants":[{"offer_id":943076353,"product_id":3046070,"size":"46,5","stock":1,"availability_id":"15","availability_name":"Disponibil","add_to_cart":true}],"products_availability":true},{"id":"707","name":"BUCURE\u0218TI SUN PLAZA SIZEER","code":"RO02","slug":"bucuresti-sun-plaza-sizeer","warehouse":"RO02 - NEW SIZEER EU - RO 17B2\/02 (BUKARESZT - )","warehouse_id":"1357","street":"Calea V\u0103c\u0103re\u0219ti","house_number":"391","apartment_number":"4 district","postcode":"040055","city":"Bucure\u0219ti","province":"Bucuresti","phone":"0770314860","latitude":"44.3954567","longitude":"26.1234794","distance":"2.7242449964645985","open_hour":{"1":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"1"},"2":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"2"},"3":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"3"},"4":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"4"},"5":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"5"},"6":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"6"},"0":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"0"}},"availability_variants":[{"offer_id":943076351,"product_id":3046070,"size":"46,5","stock":1,"availability_id":"15","availability_name":"Disponibil","add_to_cart":true}],"products_availability":true},{"id":"705","name":"BUCURE\u0218TI MEGA MALL SIZEER","code":"RO01","slug":"bucuresti-mega-mall-sizeer","warehouse":"RO01 - NEW SIZEER EU - RO 17B2\/01 (BUKARESZT - )","warehouse_id":"1355","street":"Bulevardul Pierre de Coubertin","house_number":"3-5","apartment_number":"2 District","postcode":"021901","city":"Bucure\u0219ti","province":"Bucuresti","phone":"0770634923","latitude":"44.4423985","longitude":"26.1525885","distance":"3.380976816641149","open_hour":{"1":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"1"},"2":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"2"},"3":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"3"},"4":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"4"},"5":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"5"},"6":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"6"},"0":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"0"}},"availability_variants":[],"products_availability":false},{"id":"1112","name":"BUCURE\u0218TI IRIS TITAN SIZEER","code":"RO38","slug":"bucuresti-iris-titan-sizeer","warehouse":"RO38 - NEW SIZEER EU - RO","warehouse_id":"2182","street":"Bulevardul1 Decembrie 1918 Sector 3","house_number":"33A","apartment_number":"","postcode":"032455","city":"Bucure\u0219ti","province":"Bucuresti","phone":"0770142178","latitude":"44.4226633","longitude":"26.1769066","distance":"4.223391316193692","availability_variants":[{"offer_id":943076359,"product_id":3046070,"size":"46,5","stock":1,"availability_id":"15","availability_name":"Disponibil","add_to_cart":true}],"products_availability":true},{"id":"1142","name":"BUCURE\u0218TI AFI SIZEER","code":"RO39","slug":"bucuresti-afi-sizeer","warehouse":"RO39 - NEW SIZEER EU - RO","warehouse_id":"2226","street":"Blvd General Vasile Milea","house_number":"4","apartment_number":"","postcode":"061344","city":"Bucure\u0219ti","province":"Bucuresti","phone":"0770448458","latitude":"44.4292343","longitude":"26.0512946","distance":"5.854383966823716","open_hour":{"1":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"1"},"2":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"2"},"3":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"3"},"4":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"4"},"5":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"5"},"6":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"6"}},"availability_variants":[],"products_availability":false},{"id":"972","name":"BUCURE\u0218TI PROMENADA SIZEER","code":"RO21","slug":"bucuresti-promenada-sizeer","warehouse":"RO21 - NEW SIZEER EU - RO","warehouse_id":"1681","street":"Calea Floreasca","house_number":"246B","apartment_number":"","postcode":"014476","city":"Bucure\u0219ti","province":"Bucuresti","phone":"0770922114","latitude":"44.478427","longitude":"26.1013649","distance":"6.742584831194388","availability_variants":[{"offer_id":943027875,"product_id":3046070,"size":"46,5","stock":1,"availability_id":"15","availability_name":"Disponibil","add_to_cart":true}],"products_availability":true},{"id":"1160","name":"BUCURE\u0218TI PALLADY SIZEER OUTLET","code":"RO40","type_code":"outlety","slug":"bucuresti-pallady-sizeer-outlet","warehouse":"RO40 - NEW SIZEER EU - RO","warehouse_id":"2228","street":"Autostrada Soarelui km 15","house_number":"","apartment_number":"","postcode":"917110","city":"Cernica","province":"Bucuresti","phone":"0770458228","latitude":"44.4064481","longitude":"26.2741342","distance":"12.029918380409567","availability_variants":[],"products_availability":false},{"id":"952","name":"BUCURE\u0218TI FASHION HOUSE SIZEER OUTLET","code":"RO18","slug":"bucuresti-fashion-house-sizeer-outlet","warehouse":"RO18 - NEW SIZEER EU - RO","warehouse_id":"1725","street":"Strada Comertului","house_number":"13A","apartment_number":"","postcode":"077090","city":"Bucure\u0219ti","province":"Bucuresti","phone":"0770591017","latitude":"44.4339543","longitude":"25.9524912","distance":"13.697139076366327","open_hour":{"1":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"1"},"2":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"2"},"3":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"3"},"4":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"4"},"5":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"5"},"6":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"6"},"0":{"open_hour_from":"10:00","open_hour_to":"20:00","weekday":"0"}},"availability_variants":[],"products_availability":false}]}

I've tried to do something like that for storing the stock quantities for each store but it doesn't work

for i in responsejs["pos"][0]["name"]:
    available_stock.append(["pos"][0]["availability_variants"][0]["stock"])

How can i scrape those multiple values?

score 0 · Answer 1 · answered Oct 22 '22 at 13:45

0

Maybe you should try css selectors. How to Scrape JSON Data Using Scrapy this link seems to be related to your problem. You can have a look.

answered Oct 22 '22 at 13:45

Yağız

61
4

1azza · Answer 2 · 2022-10-22T13:56:34.653

0

This works! :)

for i in responsejs["pos"]:
    av = i["availability_variants"]
    # Check if the stock check is available
    if av:
        # If available add stock to list
        available_stock.append(av[0]['stock'])

edited Oct 22 '22 at 13:56

answered Oct 22 '22 at 13:45

1azza

61
3

It doesn't, it scrapes only 1 value and doesn't keep doing that – Marc9s Oct 22 '22 at 13:53
Your answer could be improved with additional supporting information. Please [edit] to add further details, such as citations or documentation, so that others can confirm that your answer is correct. You can find more information on how to write good answers [in the help center](/help/how-to-answer). – user11717481 Oct 25 '22 at 08:06

score 0 · Answer 3 · answered Oct 22 '22 at 13:47

This is one way of getting that information and displaying it in a meaningful fashion:

import requests
import pandas as pd

pd.set_option('display.max_columns', None)
pd.set_option('display.max_colwidth', None)

headers = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Safari/537.36'
}

url = 'https://sizeer.ro/v2/products/searchnearpos?address=44.4199549%2C26.1238619&distance=50&product=3046070'
r = requests.get(url, headers=headers)
df = pd.json_normalize(r.json()['pos'], record_path=['availability_variants'], meta = ['name', 'code', 'warehouse'])
print(df)

Result in terminal:

offer_id    product_id  size    stock   availability_id availability_name   add_to_cart name    code    warehouse
0   943027879   3046070 46,5    2   43  Disponibil  True    BUCURESTI MALL SIZEER   RO31    RO31 - RO31 NEW SIZEER EU - RO
1   943076353   3046070 46,5    1   15  Disponibil  True    BUCUREȘTI PARKLAKE SIZEER   RO05    RO05 - NEW SIZEER EU - RO
2   943076351   3046070 46,5    1   15  Disponibil  True    BUCUREȘTI SUN PLAZA SIZEER  RO02    RO02 - NEW SIZEER EU - RO 17B2/02 (BUKARESZT - )
3   943076359   3046070 46,5    1   15  Disponibil  True    BUCUREȘTI IRIS TITAN SIZEER RO38    RO38 - NEW SIZEER EU - RO
4   943027875   3046070 46,5    1   15  Disponibil  True    BUCUREȘTI PROMENADA SIZEER  RO21    RO21 - NEW SIZEER EU - RO

You can extract more info from that json response, if you wish so.

Pandas documentation: https://pandas.pydata.org/docs/

Also Requests documentation: https://requests.readthedocs.io/en/latest/

score 0 · Answer 4 · answered Oct 22 '22 at 13:50

0

If you have your response in a dict you could do something like:

for store_info in responsejs['pos']:
    for store_availability_variant in store_info['availability_variants']:
        available_stock.append(store_availability_variant['stock'])

answered Oct 22 '22 at 13:50

Isaac Rene

386
2
6

score 0 · Answer 5 · answered Oct 22 '22 at 13:58

You can use json lib for this.

import json
json_str = json.loads("""{"pos":[{"id":"1110","name":"BUCURESTI MALL SIZEER","code":"RO31","slug":"bucuresti-mall-sizeer","warehouse":"RO31 - RO31 NEW SIZEER EU - RO","warehouse_id":"1842","street":"Calea Vitan 3rd District","house_number":"55-59","apartment_number":"","postcode":"031282","city":"Bucharest","province":"Bucuresti","phone":"0771567291","latitude":"44.4199549","longitude":"26.1238619","distance":"0","availability_variants":[{"offer_id":943027879,"product_id":3046070,"size":"46,5","stock":2,"availability_id":"43","availability_name":"Disponibil","add_to_cart":true}],"products_availability":true},{"id":"703","name":"BUCURE\u0218TI PARKLAKE SIZEER","code":"RO05","slug":"bucuresti-parklake-sizeer","warehouse":"RO05 - NEW SIZEER EU - RO","warehouse_id":"1363","street":"Strada Liviu Rebreanu","house_number":"4","apartment_number":"3 district","postcode":"031783","city":"Bucure\u0219ti","province":"Bucuresti","phone":"0770314565","latitude":"44.4206244","longitude":"26.1500617","distance":"2.0820717992071485","open_hour":{"1":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"1"},"2":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"2"},"3":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"3"},"4":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"4"},"5":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"5"},"6":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"6"},"0":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"0"}},"availability_variants":[{"offer_id":943076353,"product_id":3046070,"size":"46,5","stock":1,"availability_id":"15","availability_name":"Disponibil","add_to_cart":true}],"products_availability":true},{"id":"707","name":"BUCURE\u0218TI SUN PLAZA SIZEER","code":"RO02","slug":"bucuresti-sun-plaza-sizeer","warehouse":"RO02 - NEW SIZEER EU - RO 17B2\/02 (BUKARESZT - )","warehouse_id":"1357","street":"Calea V\u0103c\u0103re\u0219ti","house_number":"391","apartment_number":"4 district","postcode":"040055","city":"Bucure\u0219ti","province":"Bucuresti","phone":"0770314860","latitude":"44.3954567","longitude":"26.1234794","distance":"2.7242449964645985","open_hour":{"1":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"1"},"2":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"2"},"3":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"3"},"4":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"4"},"5":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"5"},"6":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"6"},"0":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"0"}},"availability_variants":[{"offer_id":943076351,"product_id":3046070,"size":"46,5","stock":1,"availability_id":"15","availability_name":"Disponibil","add_to_cart":true}],"products_availability":true},{"id":"705","name":"BUCURE\u0218TI MEGA MALL SIZEER","code":"RO01","slug":"bucuresti-mega-mall-sizeer","warehouse":"RO01 - NEW SIZEER EU - RO 17B2\/01 (BUKARESZT - )","warehouse_id":"1355","street":"Bulevardul Pierre de Coubertin","house_number":"3-5","apartment_number":"2 District","postcode":"021901","city":"Bucure\u0219ti","province":"Bucuresti","phone":"0770634923","latitude":"44.4423985","longitude":"26.1525885","distance":"3.380976816641149","open_hour":{"1":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"1"},"2":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"2"},"3":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"3"},"4":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"4"},"5":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"5"},"6":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"6"},"0":{"open_hour_from":"10:00","open_hour_to":"22:00","weekday":"0"}},"availability_variants":[],"products_availability":false},{"id":"1112","name":"BUCURE\u0218TI IRIS TITAN SIZEER","code":"RO38","slug":"bucuresti-iris-titan-sizeer","warehouse":"RO38 - NEW SIZEER EU - RO","warehouse_id":"2182","street":"Bulevardul1 Decembrie 1918 Sector 3","house_number":"33A","apartment_number":"","postcode":"032455","city":"Bucure\u0219ti","province":"Bucuresti","phone":"0770142178","latitude":"44.4226633","longitude":"26.1769066","distance":"4.223391316193692","availability_variants":[{"offer_id":943076359,"product_id":3046070,"size":"46,5","stock":1,"availability_id":"15","availability_name":"Disponibil","add_to_cart":true}],"products_availability":true},{"id":"1142","name":"BUCURE\u0218TI AFI SIZEER","code":"RO39","slug":"bucuresti-afi-sizeer","warehouse":"RO39 - NEW SIZEER EU - RO","warehouse_id":"2226","street":"Blvd General Vasile Milea","house_number":"4","apartment_number":"","postcode":"061344","city":"Bucure\u0219ti","province":"Bucuresti","phone":"0770448458","latitude":"44.4292343","longitude":"26.0512946","distance":"5.854383966823716","open_hour":{"1":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"1"},"2":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"2"},"3":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"3"},"4":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"4"},"5":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"5"},"6":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"6"}},"availability_variants":[],"products_availability":false},{"id":"972","name":"BUCURE\u0218TI PROMENADA SIZEER","code":"RO21","slug":"bucuresti-promenada-sizeer","warehouse":"RO21 - NEW SIZEER EU - RO","warehouse_id":"1681","street":"Calea Floreasca","house_number":"246B","apartment_number":"","postcode":"014476","city":"Bucure\u0219ti","province":"Bucuresti","phone":"0770922114","latitude":"44.478427","longitude":"26.1013649","distance":"6.742584831194388","availability_variants":[{"offer_id":943027875,"product_id":3046070,"size":"46,5","stock":1,"availability_id":"15","availability_name":"Disponibil","add_to_cart":true}],"products_availability":true},{"id":"1160","name":"BUCURE\u0218TI PALLADY SIZEER OUTLET","code":"RO40","type_code":"outlety","slug":"bucuresti-pallady-sizeer-outlet","warehouse":"RO40 - NEW SIZEER EU - RO","warehouse_id":"2228","street":"Autostrada Soarelui km 15","house_number":"","apartment_number":"","postcode":"917110","city":"Cernica","province":"Bucuresti","phone":"0770458228","latitude":"44.4064481","longitude":"26.2741342","distance":"12.029918380409567","availability_variants":[],"products_availability":false},{"id":"952","name":"BUCURE\u0218TI FASHION HOUSE SIZEER OUTLET","code":"RO18","slug":"bucuresti-fashion-house-sizeer-outlet","warehouse":"RO18 - NEW SIZEER EU - RO","warehouse_id":"1725","street":"Strada Comertului","house_number":"13A","apartment_number":"","postcode":"077090","city":"Bucure\u0219ti","province":"Bucuresti","phone":"0770591017","latitude":"44.4339543","longitude":"25.9524912","distance":"13.697139076366327","open_hour":{"1":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"1"},"2":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"2"},"3":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"3"},"4":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"4"},"5":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"5"},"6":{"open_hour_from":"10:00","open_hour_to":"21:00","weekday":"6"},"0":{"open_hour_from":"10:00","open_hour_to":"20:00","weekday":"0"}},"availability_variants":[],"products_availability":false}]}""")

[av["stock"] for pos in json_str["pos"] for av in pos["availability_variants"]]

[Out]:
[2, 1, 1, 1, 1]

How can i scrape multiple values with the same name from a json with python?

5 Answers5