How can I embed a python interpreter frame in python using tkinter?

Question

I want to add a control terminal widget to my pure python+tkinter application similar to the python interpreter provided in Blender. It should be running within the same context (process) so the user can add features and control the application that is currently running from the control widget. Ideally I'd like it to also "hijack" stdout and stderr of the current application so it will report any problems or debugging information within the running application.

This is what I have come up with so far. The only problems are that it isn't responding to commands, and the thread doesn't stop when the user closes the window.

import Tkinter as tk
import sys
import code
from threading import *

class Console(tk.Frame):
    def __init__(self,parent=None):
        tk.Frame.__init__(self, parent)
        self.parent = parent
        sys.stdout = self
        sys.stderr = self
        self.createWidgets()
        self.consoleThread = ConsoleThread()
        self.after(100,self.consoleThread.start)

    def write(self,string):
        self.ttyText.insert('end', string)
        self.ttyText.see('end')

    def createWidgets(self):
        self.ttyText = tk.Text(self.parent, wrap='word')
        self.ttyText.grid(row=0,column=0,sticky=tk.N+tk.S+tk.E+tk.W)


class ConsoleThread(Thread):

    def __init__(self):
        Thread.__init__(self)

    def run(self):
        vars = globals().copy()
        vars.update(locals())
        shell = code.InteractiveConsole(vars)
        shell.interact()

if __name__ == '__main__':
    root = tk.Tk()
    root.config(background="red")
    main_window = Console(root)
    main_window.mainloop()
    try:
        if root.winfo_exists():
            root.destroy()
    except:
        pass

Answer 1

I have the answer in case anyone still cares! (I have also changed to python 3, hence the import tkinter rather than import Tkinter)

I have changed the approach slightly from the original by using a separate file to run the InteractiveConsole, and then making the main file open this other file (which I have called console.py and is in the same directory) in a subprocess, linking the stdout, stderr, and stdin of this subprocess to the tkinter Text widget programatically.

Here is the code in the for the console file (if this is run normally, it acts like a normal console):

# console.py
import code

if __name__ == '__main__':
    vars = globals().copy()
    vars.update(locals())
    shell = code.InteractiveConsole(vars)
    shell.interact() 

And here is the code for the python interpreter, that runs the console inside the Text widget:

# main.py
import tkinter as tk
import subprocess
import queue
import os
from threading import Thread

class Console(tk.Frame):
    def __init__(self,parent=None):
        tk.Frame.__init__(self, parent)
        self.parent = parent
        self.createWidgets()

        # get the path to the console.py file assuming it is in the same folder
        consolePath = os.path.join(os.path.dirname(__file__),"console.py")
        # open the console.py file (replace the path to python with the correct one for your system)
        # e.g. it might be "C:\\Python35\\python"
        self.p = subprocess.Popen(["python3",consolePath],
                                  stdout=subprocess.PIPE,
                                  stdin=subprocess.PIPE,
                                  stderr=subprocess.PIPE)

        # make queues for keeping stdout and stderr whilst it is transferred between threads
        self.outQueue = queue.Queue()
        self.errQueue = queue.Queue()

        # keep track of where any line that is submitted starts
        self.line_start = 0

        # make the enter key call the self.enter function
        self.ttyText.bind("<Return>",self.enter)

        # a daemon to keep track of the threads so they can stop running
        self.alive = True
        # start the functions that get stdout and stderr in separate threads
        Thread(target=self.readFromProccessOut).start()
        Thread(target=self.readFromProccessErr).start()

        # start the write loop in the main thread
        self.writeLoop()

    def destroy(self):
        "This is the function that is automatically called when the widget is destroyed."
        self.alive=False
        # write exit() to the console in order to stop it running
        self.p.stdin.write("exit()\n".encode())
        self.p.stdin.flush()
        # call the destroy methods to properly destroy widgets
        self.ttyText.destroy()
        tk.Frame.destroy(self)
    def enter(self,e):
        "The <Return> key press handler"
        string = self.ttyText.get(1.0, tk.END)[self.line_start:]
        self.line_start+=len(string)
        self.p.stdin.write(string.encode())
        self.p.stdin.flush()

    def readFromProccessOut(self):
        "To be executed in a separate thread to make read non-blocking"
        while self.alive:
            data = self.p.stdout.raw.read(1024).decode()
            self.outQueue.put(data)

    def readFromProccessErr(self):
        "To be executed in a separate thread to make read non-blocking"
        while self.alive:
            data = self.p.stderr.raw.read(1024).decode()
            self.errQueue.put(data)

    def writeLoop(self):
        "Used to write data from stdout and stderr to the Text widget"
        # if there is anything to write from stdout or stderr, then write it
        if not self.errQueue.empty():
            self.write(self.errQueue.get())
        if not self.outQueue.empty():
            self.write(self.outQueue.get())

        # run this method again after 10ms
        if self.alive:
            self.after(10,self.writeLoop)

    def write(self,string):
        self.ttyText.insert(tk.END, string)
        self.ttyText.see(tk.END)
        self.line_start+=len(string)

    def createWidgets(self):
        self.ttyText = tk.Text(self, wrap=tk.WORD)
        self.ttyText.pack(fill=tk.BOTH,expand=True)


if __name__ == '__main__':
    root = tk.Tk()
    root.config(background="red")
    main_window = Console(root)
    main_window.pack(fill=tk.BOTH,expand=True)
    root.mainloop()

The reason that reading from stdout and stderr is in separate threads is because the read method is blocking, which causes the program to freeze until the console.py subprocess gives more output, unless these are in separate threads. The writeLoop method and the queues are needed to write to the Text widget since tkinter is not thread safe.

This certainly still has problems to be ironed out, such as the fact that any code on the Text widget is editable even once already submitted, but hopefully it answers your question.

EDIT: I've also neatened some of the tkinter such that the Console will behave more like a standard widget.

Answer 2

it isn't responding to commands

The reason it isn't responding to commands is because you haven't linked the Text widget (self.ttyText) into stdin. Currently when you type it adds text into the widget and nothing else. This linking can be done similarly to what you've already done with stdout and stderr.

When implementing this, you need to keep track of which part of the text in the widget is the text being entered by the user - this can be done using marks (as described here).

the thread doesn't stop when the user closes the window.

I don't think there is a "clean" way to solve this issue without a major code re-write, however a solution that seems to work well enough is it simply detect when the widget is destroyed and write the string "\n\nexit()" to the interpreter. This calls the exit function inside the interpreter, which causes the call to shell.interact to finish, which makes the thread finish.

So without further ado, here is the modified code:

import tkinter as tk
import sys
import code
from threading import Thread
import queue


class Console(tk.Frame):
    def __init__(self, parent, _locals, exit_callback):
        tk.Frame.__init__(self, parent)
        self.parent = parent
        self.exit_callback = exit_callback
        self.destroyed = False

        self.real_std_in_out = (sys.stdin, sys.stdout, sys.stderr)

        sys.stdout = self
        sys.stderr = self
        sys.stdin = self

        self.stdin_buffer = queue.Queue()

        self.createWidgets()

        self.consoleThread = Thread(target=lambda: self.run_interactive_console(_locals))
        self.consoleThread.start()

    def run_interactive_console(self, _locals):
        try:
            code.interact(local=_locals)
        except SystemExit:
            if not self.destroyed:
                self.after(0, self.exit_callback)

    def destroy(self):
        self.stdin_buffer.put("\n\nexit()\n")
        self.destroyed = True
        sys.stdin, sys.stdout, sys.stderr = self.real_std_in_out
        super().destroy()

    def enter(self, event):
        input_line = self.ttyText.get("input_start", "end")
        self.ttyText.mark_set("input_start", "end-1c")
        self.ttyText.mark_gravity("input_start", "left")
        self.stdin_buffer.put(input_line)

    def write(self, string):
        self.ttyText.insert('end', string)
        self.ttyText.mark_set("input_start", "end-1c")
        self.ttyText.see('end')

    def createWidgets(self):
        self.ttyText = tk.Text(self.parent, wrap='word')
        self.ttyText.grid(row=0, column=0, sticky=tk.N + tk.S + tk.E + tk.W)
        self.ttyText.bind("<Return>", self.enter)
        self.ttyText.mark_set("input_start", "end-1c")
        self.ttyText.mark_gravity("input_start", "left")

    def flush(self):
        pass

    def readline(self):
        line = self.stdin_buffer.get()
        return line


if __name__ == '__main__':
    root = tk.Tk()
    root.config(background="red")
    main_window = Console(root, locals(), root.destroy)
    main_window.mainloop()

This code has few changes other than those that solve the problems stated in the question.

The advantage of this code over my previous answer is that it works inside a single process, so can be created at any point in the application, giving the programmer more control.

I have also written a more complete version of this which also prevents the user from editing text which shouldn't be editable (e.g. the output of a print statement) and has some basic coloring: https://gist.github.com/olisolomons/e90d53191d162d48ac534bf7c02a50cd