Is there a line by line alternative for pycurl perform

Question

I have the following code to read a logfile from a "box" on our network:

import pycurl
pycurl.Curl()
pycurl_connect = pycurl.Curl()
your_url = 'http://10.0.0.1:1333/cgi/WebCGI?7043'
headers = ['Cookie: loginname=admin; password=e5T%7B%5CYnlcIX%7B; OsVer=2.17.0.32', 'Connection: keep-alive']
pycurl_connect.setopt(pycurl.URL, your_url)
pycurl_connect.setopt(pycurl.HTTPHEADER, headers)
pycurl_connect.perform()

The logfile is very long and I only need to read the lines that are not logged in my database. Is there any way to get the information on a line-by-line basis?

Thanks.

No really. What you can do is download the file, and cache the data in a sqlite database. — Games Brainiac, Jan 27 '15 at 09:03

score 0 · Answer 1 · answered Jan 28 '15 at 13:34

I ended up using a very hackish solution. I let the curl request save the data to a text-file—this is initiated as a daemon thread—whilst the rest of my program reads from that file "line-by-line".

Some gymnastics has to take place to replicate a line-by-line file read. It end up being a bit messy, but it works.

Here is the code:

def write_logfile_as_thread(filename):
    #Delete content in file
    with open(filename, 'w') as file: pass
    t = threading.Thread(target=write_logfile, args=(filename,))
    t.daemon = True
    t.start()
    return t

def write_logfile(filename):
    storage = open(os.path.join(filename), 'w')
    pycurl.Curl()
    pycurl_connect = pycurl.Curl()
    your_url = 'http://10.0.0.1:1333/cgi/WebCGI?7043'
    headers = ['Cookie: loginname=admin; password=e5T%7B%5CYnlcIX%7B; OsVer=2.17.0.32',
                'Connection: keep-alive']
    pycurl_connect.setopt(pycurl.URL, your_url)
    pycurl_connect.setopt(pycurl.HTTPHEADER, headers)
    pycurl_connect.setopt(pycurl_connect.WRITEFUNCTION, storage.write)
    pycurl_connect.perform()
    pycurl_connect.close()

def get_logfile_as_lines(filename):
    #file read logic from http://stackoverflow.com/a/3290359/1490584
    t = write_logfile_as_thread(filename)
    file = open(filename)
    text = ""
    thread_is_alive = True
    while (thread_is_alive):
        thread_is_alive = t.is_alive()
        where = file.tell()
        chunk = file.read()
        if not chunk:
            file.seek(where)
        else:
            text += chunk
            text_lines = text.split('\n')
            if len(text_lines) > 1: 
                if thread_is_alive:
                    text = text_lines[-1]
                    new_lines = text_lines[:-1]
                else: #The last line is complete
                    new_lines = text_lines 
                for line in new_lines:
                    yield line

if __name__ == "__main__":
    #print only the first 200 lines from the logfile
    i = 0
    for line in get_logfile_as_lines('logfile.txt'):
        if i >= 200: 
            break
        print line
        i+=1

Is there a line by line alternative for pycurl perform

1 Answers1