TWS interactive brokers API with Python. Trouble putting live data together when received by several methods methods

Question

To give more context about my problem:

I am using python to build an API connecting to the TWS of interactive brokers. I managed to build something functional and able to fetch live data from contracts using the methods given in the doc of IB. Now that I want to use all the data to build other parallel systems with it, I have encounter problems organising the data that arrives from the IB server.

My program loops a list of 30 symbols to get live data from and then I want to put the data (ex. 'HIGH', 'LOW', 'CLOSE', 'VWAP' etc) from each symbol all together in one dataframe to calculate indicators and from there come up with an alert system that is based on them indicators.

This objective I have already accomplished it using only one symbol for the whole program. Is easy to store the data in instances or variables, pass it to a DataFrame and then calculate to set up alerts.

Now when looping a list of 30 values and receiving the data of all of them I have struggled trying to store the data for each symbol together and then calculate and set up alerts. Specially when I have to use several methods to receive the data (ex. I use tickPrice for some data and tickString for some other data) this several methods execute themselves one after the other but they wont necessarily have all the data at the same time, some values take more time than others to show.

I will show an example of my code to give even more context of my objective:

This is my EWrapper class:

class IBApi(EWrapper, EClient):
    def __init__(self):
        self.syms = ['EN', 'DG', 'AI', 'ORA', 'RI', 'ENGI', 'AC', 'VIV', 'KER', 'CA', 'BN', 'WLN', 'OR', 'VIE',
                     'LR', 'ML', 'SGO', 'CAP', 'MC', 'ACA', 'ATO', 'UG', 'SU', 'HO', 'BNP', 'GLE', 'SAN', 'SW', 'AIR', 'TTE']
        EClient.__init__(self, self)
    # Reciving Real Time Data
    def tickString(self, reqId, tickType, value):
        super().tickString(reqId, tickType, value)
        try:
            if reqId == 0:
                reqId = self.syms[0]
            if reqId == 1:
                reqId = self.syms[1]
            if reqId == 2:
                reqId = self.syms[2]
            if reqId == 3:
                reqId = self.syms[3]
            if reqId == 4:
                reqId = self.syms[4]
            if reqId == 5:
                reqId = self.syms[5]
            if reqId == 6:
                reqId = self.syms[6]
            if reqId == 7:
                reqId = self.syms[7]
            if reqId == 8:
                reqId = self.syms[8]
            if reqId == 9:
                reqId = self.syms[9]
            if reqId == 10:
                reqId = self.syms[10]
            if reqId == 11:
                reqId = self.syms[11]
            if reqId == 12:
                reqId = self.syms[12]
            if reqId == 13:
                reqId = self.syms[13]
            if reqId == 14:
                reqId = self.syms[14]
            if reqId == 15:
                reqId = self.syms[15]
            if reqId == 16:
                reqId = self.syms[16]
            if reqId == 17:
                reqId = self.syms[17]
            if reqId == 18:
                reqId = self.syms[18]
            if reqId == 19:
                reqId = self.syms[19]
            if reqId == 20:
                reqId = self.syms[20]
            if reqId == 21:
                reqId = self.syms[21]
            if reqId == 22:
                reqId = self.syms[22]
            if reqId == 23:
                reqId = self.syms[23]
            if reqId == 24:
                reqId = self.syms[24]
            if reqId == 25:
                reqId = self.syms[25]
            if reqId == 26:
                reqId = self.syms[26]
            if reqId == 27:
                reqId = self.syms[27]
            if reqId == 28:
                reqId = self.syms[28]
            if reqId == 29:
                reqId = self.syms[29]
            if reqId == 30:
                reqId = self.syms[30]
            if tickType == 48 != 0.0:
                rtVolume = value.split(";")
                vwap = float(rtVolume[4])
                self.myData(reqId, TickTypeEnum.to_str(tickType), vwap)
        except Exception as e:
            print(e)

    def tickPrice(self, reqId, tickType, price, attrib):
        super().tickPrice(reqId, tickType, price, attrib)
        try:
            if reqId == 0:
                reqId = self.syms[0]
            if reqId == 1:
                reqId = self.syms[1]
            if reqId == 2:
                reqId = self.syms[2]
            if reqId == 3:
                reqId = self.syms[3]
            if reqId == 4:
                reqId = self.syms[4]
            if reqId == 5:
                reqId = self.syms[5]
            if reqId == 6:
                reqId = self.syms[6]
            if reqId == 7:
                reqId = self.syms[7]
            if reqId == 8:
                reqId = self.syms[8]
            if reqId == 9:
                reqId = self.syms[9]
            if reqId == 10:
                reqId = self.syms[10]
            if reqId == 11:
                reqId = self.syms[11]
            if reqId == 12:
                reqId = self.syms[12]
            if reqId == 13:
                reqId = self.syms[13]
            if reqId == 14:
                reqId = self.syms[14]
            if reqId == 15:
                reqId = self.syms[15]
            if reqId == 16:
                reqId = self.syms[16]
            if reqId == 17:
                reqId = self.syms[17]
            if reqId == 18:
                reqId = self.syms[18]
            if reqId == 19:
                reqId = self.syms[19]
            if reqId == 20:
                reqId = self.syms[20]
            if reqId == 21:
                reqId = self.syms[21]
            if reqId == 22:
                reqId = self.syms[22]
            if reqId == 23:
                reqId = self.syms[23]
            if reqId == 24:
                reqId = self.syms[24]
            if reqId == 25:
                reqId = self.syms[25]
            if reqId == 26:
                reqId = self.syms[26]
            if reqId == 27:
                reqId = self.syms[27]
            if reqId == 28:
                reqId = self.syms[28]
            if reqId == 29:
                reqId = self.syms[29]
            if reqId == 30:
                reqId = self.syms[30]
            self.myData(reqId, TickTypeEnum.to_str(tickType), price)
            time.sleep(0.5)
        except Exception as e:
            print(e)
    @staticmethod
    def myData(reqId, type, price):
        if type == 'RT_VOLUME':
            values = {
                'SYMBOL': [reqId],
                'TYPE': [type],
                'VWAP': [price]
            }
            print(values)
        else:
            values = {
                'SYMBOL': [reqId],
                'TYPE': [type],
                'PRICE': [price]
            }
            print(values)
    def error(self, id, errorCode, errorMsg):
        print(errorCode)
        print(errorMsg)
 

Then I have my app class:

class App:
    ib = None
    def __init__(self):
        self.ib = IBApi()
        self.ib.connect("127.0.0.1", 7496, 88)
        ib_thread = threading.Thread(target=self.run_loop, daemon=True)
        ib_thread.start()
        time.sleep(0.5)


        for sym in self.ib.syms:
            self.marketData(self.ib.syms.index(sym), self.symbolsForData(sym))

    def symbolsForData(self, mySymbol, sec_type='STK', currency='EUR', exchange='SBF'):
        contract1 = Contract()
        contract1.symbol = mySymbol.upper()
        contract1.secType = sec_type
        contract1.currency = currency
        contract1.exchange = exchange
        return contract1

    def marketData(self, req_num, contract1):
        self.ib.reqMktData(reqId=req_num,
                      contract=contract1,
                      genericTickList='233',
                      snapshot=False,
                      regulatorySnapshot=False,
                      mktDataOptions=[])

    def run_loop(self):
        self.ib.run()

# Start App
App()
 

As you could see in the EWrapper class I have the two methods to receive the data and a assign the symbol to each reqId, and then I pass a static method that will put together the data received, also there is the list of values to loop trough. Then in my App class there is the connection, the method that build the contract, the method that hold reqMktData from IB with the parameters to fetch what I need, as well as the loop that executes the reqMktData using the list from the EWrapper class.

Everything works fine this way and I have the data that comes properly in like this:

PROBLEM

So the way that my data arrives is not really useful for me in order to set up an alert system, due to the fact that I don't have all the data together for each contract and I cant just make conditions using different values and come up with the alert. At once I either have only 'HIGH' or only 'LOW' or only 'VWAP' but I struggled figuring out how to put it all together for each symbol, since I don't have everything at once and the data keeps coming every time I just cant find my way around.

I want to clarify that I am new in programming and new using python. Sorry for my noob code and probably the "obvious" question. But if anyone can help me figuring this out I would really appreciate it. And any other remark will be gratefully taken.

Kind regards

Mario

Answer 1

It's easy to create a Pandas dataframe from a Python dictionary that contains lists. For example, the following code creates a dictionary containing ticker symbols, bid prices, and ask prices:

ticker_dict = {}
ticker_dict['SYMBOL'] = []
for sym in self.ib.syms:
    ticker_dict['SYMBOL'].append(sym)
ticker_dict['BID'] = [0.0] * len(self.ib.syms)
ticker_dict['ASK'] = [0.0] * len(self.ib.syms)
...

Now suppose tickPrice is called with a reqId of 5. You could set the fifth bid price with the following code:

ticker_dict['BID'][5] = price

Once all the data is collected, you can convert the dict to a dataframe by calling from_dict.

You might want to use empty NumPy arrays instead of lists of zeros. Then when the data is collected, you can convert the arrays to lists.

Answer 2

Here's something I wrtoe a while ago to see how well tkinter could handle a data grid. I just added a dataframe with a sample alert function.

import tkinter as tk
from tkinter import ttk
import threading

from io import StringIO
import pandas as pd 

from ibapi import wrapper
from ibapi.client import EClient
from ibapi.common import *
from ibapi.ticktype import *
from ibapi.contract import Contract

specs: StringIO = StringIO("""
Sym,Mo,Yr,Exch,Type,Curr,BID,LAST,ASK,OPEN,HIGH,LOW,CLOSE,VWAP    
ES,12,2021,GLOBEX,FUT,USD,
EUR,12,2021,GLOBEX,FUT,USD
JPY,12,2021,GLOBEX,FUT,USD
cl,12,2021,nymex,FUT,USD
USD,,,IDEALPRO,CASH,CAD
""")

class Window(tk.Tk):
    def __init__(self):
        super().__init__()
        self.protocol("WM_DELETE_WINDOW", self.close)
        self.df = pd.read_csv(specs, index_col=0, dtype=str,  na_filter= False)
        self.df.columns = self.df.columns.str.strip()
        self.title('Quotes')
        self.geometry(f'{75*self.df.shape[1]+100}x{25*self.df.shape[0]+100}')
        self.tree = ttk.Treeview(self)
        cols = list(self.df.columns)
        self.tree["columns"] = cols
        self.tree.column('#0', width=75)
        self.tree.heading('#0', text='Sym', anchor='w')
        self.tree.tag_configure('alert', background='#773333')
        
        for col in cols:
            self.tree.column(col, anchor="w", width=75)
            self.tree.heading(col, text=col, anchor='w')

        for index, row in self.df.iterrows():
            self.tree.insert("",tk.END , text=index, values=list(row))
            
        self.tree.pack()

        self.client = Client(self)
        self.client.connect("127.0.0.1", 7497, clientId=123)
        thread = threading.Thread(daemon=True, target = self.client.run)
        thread.start()
        
    def start(self):
        self.client.reqData(self.df)
    
    def update(self, reqId, col, val):
        try:
            item = self.tree.get_children()[reqId]
            self.tree.set(item, column=col, value=val)
            row = self.df.iloc[reqId] # the row is the reqId
            row.at[col]=val 
            if self.alert(row):
                self.tree.selection_set(item)
        except:
            pass
            
    def alert(self,row):
        if row.at['LAST'] > row.at['VWAP']:
            return True
        return False
            
    def close(self):
        print(self.df)
        try:
            self.client.quit()
        except: 
            pass
        finally:
            self.destroy()

class Client(wrapper.EWrapper, EClient):        
    
    def __init__(self, wdow):
        self.wdow = wdow
        self.nextValidOrderId = 0
        wrapper.EWrapper.__init__(self)
        EClient.__init__(self, wrapper=self)
        
    def quit(self):
        self.disconnect()
        
    def reqData(self, df):
        reqId = 0
        for idx, row in df.iterrows():
            cont = Contract()
            cont.symbol = idx
            cont.secType = row['Type']
            cont.currency = row['Curr']
            cont.exchange = row['Exch']
            cont.lastTradeDateOrContractMonth = row['Yr']+row['Mo']
            self.reqMktData(reqId, cont, "233", False, False, None)
            reqId += 1 # the row is the reqId
        
    def cancelMktData(self, reqId:TickerId):
        self.cancelMktData(reqId)
        
    def nextValidId(self, orderId:int):
        self.nextValidOrderId = orderId
        self.wdow.start()

    def error(self, reqId:TickerId, errorCode:int, errorString:str):
        print("Error. Id: " , reqId, " Code: " , errorCode , " Msg: " , errorString)

    def tickString(self, reqId:TickerId, tickType:TickType, value:str):
        if tickType == TickTypeEnum.RT_VOLUME:
            rtVolume = value.split(";")
            vwap = float(rtVolume[4])
            self.wdow.update(reqId, 'VWAP', vwap)
    
    def tickPrice(self, reqId: TickerId, tickType: TickType, price: float, attrib: TickAttrib):
        self.wdow.update(reqId, TickTypeEnum.to_str(tickType), price)#price,size,time
    
if __name__ == '__main__':
    wdow = Window()
    wdow.mainloop()