Python capture stdout from subprocess line by line

Question

I've read many of the questions related to this and learned a lot but I still haven't been able to solve my problem. I'm building a wxPython app that runs a c++ executable and displays the stdout from that executable in real time. I've run into several strange results trying to make this work. Here's my current setup/problem:

//test.cc (compiled as test.out with gcc 4.5.2)
#include <stdio.h>
int main()
{
  FILE* fh = fopen("output.txt", "w");
  for (int i = 0; i < 10000; i++)
  {
      printf("Outputting: %d\n", i);
      fprintf(fh, "Outputting: %d\n", i);
  }
  fclose(fh);
  return 0;
}

#wxPythonScript.py (running on 2.7 interpreter)
def run(self):
  self.externalBinary = subprocess.Popen(['./test.out'], shell=False, stdout=subprocess.PIPE)
  while not self.wantAbort:
      line = self.externalBinary.stdout.readline()
      wx.PostEvent(self.notifyWindow, Result_Event(line, Result_Event.EVT_STDOUT_ID))
    print('Subprocess still running')
  print('Subprocess aborted smoothly')

If I run the above code the subprocess takes a very long time to complete, even though all it has to do is write out the data and exit. However if I run the following it completes very quickly:

#wxPythonScript.py (running on 2.7 interpreter)
def run(self):
  outFile = open('output.txt', 'r+')
  self.externalBinary = subprocess.Popen(['./test.out'], shell=False, stdout=outFile)
  while not self.wantAbort:
      #line = self.externalBinary.stdout.readline()
      #wx.PostEvent(self.notifyWindow, Result_Event(line, Result_Event.EVT_STDOUT_ID))
    print('Subprocess still running')
  print('Subprocess aborted smoothly')

So basically whenever I redirect stdout from the subprocess to a PIPE it slows down/hangs, but if I write it to a file or don't redirect it at all then it's fine. Why is that?

Answer 1

I only tested this on Windows, but it works in 2.6.6, 2.7.2, and 3.2.1:

from __future__ import print_function
from subprocess import PIPE, Popen
from threading  import Thread
import sys

try:
    from Queue import Queue, Empty
except ImportError:
    from queue import Queue, Empty  # python 3.x

ON_POSIX = 'posix' in sys.builtin_module_names

def enqueue_output(out, queue):
    for line in iter(out.readline, b''):
        line = line.decode(sys.stdout.encoding)
        queue.put(line)
    out.close()

def main():
    p = Popen(['c/main.exe'], stdout=PIPE, bufsize=1, close_fds=ON_POSIX)
    q = Queue()
    t = Thread(target=enqueue_output, args=(p.stdout, q))
    t.daemon = True # thread dies with the program
    t.start()

    #initially the queue is empty and stdout is open
    #stdout is closed when enqueue_output finishes
    #then continue printing until the queue is empty 

    while not p.stdout.closed or not q.empty():
        try:
            line = q.get_nowait()
        except Empty:
            continue
        else:
            print(line, end='')
    return 0

if __name__ == '__main__':
    sys.exit(main())

Output:

Outputting: 0
Outputting: 1
Outputting: 2
...
Outputting: 9997
Outputting: 9998
Outputting: 9999

Edit:

readline() will block until the program's stdout buffer flushes, which may take a long time if the data stream is intermittent. If you can edit the source, one option is to manually call fflush(stdout), or you can instead disable buffering using setvbuf at the start of the program. For example:

#include <stdio.h>

int main() {

    setvbuf(stdout, NULL, _IONBF, 0);

    FILE* fh = fopen("output.txt", "w");
    int i;

    for (i = 0; i < 10; i++) {
        printf("Outputting: %d\n", i);
        fprintf(fh, "Outputting: %d\n", i);
        sleep(1);
    }

    fclose(fh);
    return 0;
}

Also look into using unbuffer or stdbuf to modify the output stream of an existing program.