I am trying to use the following code to access the content of the following table on this page: https://cdn.ime.co.ir
By this code:
import requests
with requests.Session() as s:
data = {'ContractCode' : 'SAFOR99' }
r = s.post('https://cdn.ime.co.ir/home/load/' , json = data ).json()
print(r)
But what I see in the result is:
JSONDecodeError: Expecting value
Please help me know how can I read the contents of this table?
The data is not in the html content but it is retrieved via an API and more specifically the protocol is websocket. You can check the frames using Chrome devtools and filter on wss to find the following url : wss://cdn.ime.co.ir/realTimeServer/connect
There are multiple necessary query parameters including connectionToken which is fetched via a rest API on https://cdn.ime.co.ir/realTimeServer/negotiate.
After you open the websocket, you don't receive much data until you perform another rest request on https://cdn.ime.co.ir/realTimeServer/start with the same ConnectionToken value. After that, the server send you the JSON data
The following code perform all the tasks described above and you get the data in the result unfiltered list :
import requests
import json
import asyncio
import websockets
import urllib
import random
from threading import Thread
connectionData = [{"name":"marketshub"}]
r = requests.get("https://cdn.ime.co.ir/realTimeServer/negotiate", params = {
"clientProtocol": "2.1",
"connectionData": json.dumps(connectionData),
})
response = r.json()
print(f'got connection token : {response["ConnectionToken"]}')
wsParams = {
"transport": "webSockets",
"clientProtocol": "2.1",
"connectionToken": response["ConnectionToken"],
"connectionData": json.dumps(connectionData),
"tid": random.randint(0,9)
}
websocketUri = f"wss://cdn.ime.co.ir/realTimeServer/connect?{urllib.parse.urlencode(wsParams)}"
def startReceiving(arg):
r = requests.get("https://cdn.ime.co.ir/realTimeServer/start", params = wsParams)
print(f'started receiving data : {r.json()}')
result = []
async def websocketConnect():
async with websockets.connect(websocketUri) as websocket:
print(f'started websocket')
thread = Thread(target = startReceiving, args = (0, ))
thread.start()
for i in range(0,10):
print("receiving")
data = await websocket.recv()
jsonData = json.loads(data)
if ("M" in jsonData and len(jsonData["M"]) > 0 and "A" in jsonData["M"][0] and len(jsonData["M"][0]["A"]) > 0):
items = jsonData["M"][0]["A"][0]
if type(items) == list and len(items) > 0:
result = items
break
thread.join()
print(json.dumps(result, indent=4, sort_keys=True))
asyncio.get_event_loop().run_until_complete(websocketConnect())
Then you can get the SAFOR99 item using :
print([i for i in result if i["ContractCode"] == "SAFOR99"])
