2

I'm trying to make a simple application that communicates with a console program that operates in a similar way to the Windows classical cmd.exe.

The execution sequence is expected to be as follows:

  1. Setup a background worker and run it to read the output.
  2. Run the process and wait for a short while.
  3. Feed a command to the Process.StandardInput
  4. Wait for a few seconds
  5. Exit the background thread and kill the process

But what's going on is not as expected because:

  1. Not all the output of the cmd.exe is read. Even when excluding the step which writes to the StandardInput
  2. The string was not passed to the command from the StandardInput even though it has AutoFlush = true.

I also tried Process.StandardInput.Flush() which does nothing I also tried to pass it a string padded with spaces to a length larger than 4096 which is the buffer size with no results. Nothing happens!! Why?

I tried this on dot net 4.5.2 and 4.7.1

Similar questions exist here, here, and here but none of the answers work. Others are implemented in another language. i.e. Java, Delphi etc

This is a simplified version of my code:

BackgroundWorker _outputWorker;
Process _process;
StreamWriter _inputWriter;
TextReader _outputReader;

void Main()
{
    _outputWorker = new BackgroundWorker { WorkerSupportsCancellation = true };
    _outputWorker.DoWork += OnOutputWorkerDoWork;
    _outputWorker.RunWorkerAsync();

    _process = new Process
    {
        EnableRaisingEvents = true,
        StartInfo = new ProcessStartInfo
        {
            FileName = "cmd.exe",
            Arguments = string.Empty,
            UseShellExecute = false,
            CreateNoWindow = true,
            WindowStyle = ProcessWindowStyle.Hidden,
            WorkingDirectory = Directory.GetCurrentDirectory(),
            StandardOutputEncoding = Encoding.UTF8,
            StandardErrorEncoding = Encoding.UTF8,
            RedirectStandardInput = true,
            RedirectStandardOutput = true,
            RedirectStandardError = true
        }
    };

    Console.WriteLine("Starting...");
    if (!_process.Start()) return;
    _inputWriter = _process.StandardInput;
    _inputWriter.AutoFlush = true; // does nothing
    _outputReader = TextReader.Synchronized(_process.StandardOutput);

    // You can exclude this step too and still not get the expected output
    Thread.Sleep(500);
    _inputWriter.WriteLine("dir");
    _inputWriter.Flush(); // does nothing, private field carpos = 0
    _inputWriter.BaseStream.Flush(); // does nothing, private field carpos = 5 which is equal to length of "dir" command + 2 characters (NewLine \r\n)
    //_inputWriter.WriteLine("dir".PadLeft(4096)); // does nothing
    // also closing the stream does nothing and does something that I can't afford which is closing the exe
    // _inputWriter.Close();
    //

    _process.WaitForExit(5000);
    _outputWorker.CancelAsync();
    _process.Kill();
    Console.WriteLine("Done");
}

void OnOutput(string data)
{
    // never mind thread safety for now. It's just a single line static call
    Console.WriteLine(data);
}

void OnOutputWorkerDoWork(object sender, DoWorkEventArgs e)
{
    const int BUFFER_SIZE = 4096;

    StringBuilder builder = new StringBuilder(BUFFER_SIZE);

    while (!_outputWorker.CancellationPending)
    {
        /*
        * It'll keep on running untill it's canceled to reduce thread costs
        * because the program will run different executables sequentially which 
        * all are similar to cmd.exe.
        */
        try
        {
            // Simplified version without locking
            if (_outputReader == null) continue;

            TextReader reader = _outputReader;
            if (reader.Peek() < 1) continue;

            char[] buffer = new char[BUFFER_SIZE];

            do
            {
                int count = reader.Read(buffer, 0, buffer.Length);
                if (count > 0) builder.Append(buffer, 0, count);
            }
            while (reader.Peek() > 0);
        }
        catch (Exception ex)
        {
            // handle exception in debug mode
            Console.WriteLine(ex.Message); // no exception generated!
            continue;
        }

        if (builder.Length == 0) continue;
        OnOutput(builder.ToString());
        builder.Length = 0;
    }

    if (!IsWaitable(_process)) return;

    try
    {
        if (_outputReader == null) return;

        TextReader reader = _outputReader;
        if (reader.Peek() < 1) return;

        char[] buffer = new char[BUFFER_SIZE];

        do
        {
            int count = reader.Read(buffer, 0, buffer.Length);
            if (count > 0) builder.Append(buffer, 0, count);
        }
        while (reader.Peek() > 0);
    }
    catch (Exception ex)
    {
        // handle exception in debug mode
        Console.WriteLine(ex.Message); // no exception generated!
        return;
    }

    if (builder.Length > 0) OnOutput(builder.ToString());
}

bool IsWaitable(Process thisValue)
{
    return thisValue != null && !thisValue.HasExited;
}

I can't use Process.BeginOutputReadLine because it waits for a NewLine to be present in the stream which does not happen for the last line of output. See this and this for details

I get Microsoft Windows [Version xxxxx]

(c) copyright line

And the program does not display the command-line prompt unless more output comes from the process that contains a NewLine

The points of interest are:

1. Why does this code not read all the output as it should to the end?

After trying many things, it seems as if the buffer does not contain any more text to read and the stream seems to be missing some data from the original output of the executable. What is weird is that happens randomly. Sometimes I get the second line and sometimes not. In any case, I never got the output that should result from feeding internal commands to the cmd process.

2. The StandardInput (StreamWriter) actually does flush the buffer (or it thinks it does) and sets its charpos to zero while the BaseStream still has its charpos field = [length of buffer]. Why is this happening?

Any insights into what could be the problem or a workaround would be greatly appreciated.

1 Answers1

1

Well, I got it to work eventually after 2 days of frustration. A very simple solution:

Process _process;
StreamWriter _inputWriter;

void Main()
{
    _process = new Process
    {
        EnableRaisingEvents = true,
        StartInfo = new ProcessStartInfo
        {
            FileName = "cmd.exe",
            Arguments = string.Empty,
            UseShellExecute = false,
            CreateNoWindow = true,
            WindowStyle = ProcessWindowStyle.Hidden,
            WorkingDirectory = Directory.GetCurrentDirectory(),
            StandardOutputEncoding = Encoding.UTF8,
            StandardErrorEncoding = Encoding.UTF8,
            RedirectStandardInput = true,
            RedirectStandardOutput = true,
            RedirectStandardError = true
        }
    };

    _process.OutputDataReceived += (s, e) => // instead of using a background worker
    {
        if (e.Data == null) return;
        Console.WriteLine(e.Data);
    };

    Console.WriteLine("Starting...");
    if (!_process.Start()) return;
    _process.BeginOutputReadLine(); // <- using BeginOutputReadLine
    _inputWriter = _process.StandardInput;
    _inputWriter.AutoFlush = true;
    _inputWriter.WriteLine(); // <- my little trick here

    // using LINQPad, replace it with Console.ReadLine();
    string input = Util.ReadLine<string> ("Enter command:");
    if (!string.IsNullOrEmpty(input)) _inputWriter.WriteLine(input);
    _process.WaitForExit(5000);
    _process.Kill();
    Console.WriteLine("Done");
}

void OnOutput(string data)
{
    Console.WriteLine(data);
}

Forgot to mention that I have to wait for user input otherwise it would work normally using Process.BeginOutputReadLine() without resorting to use a BackgroundWorker.

Anyway, I hope nobody goes through the 2 days user/dev experience and waste 2 days of their lives over this.


Edit:

Unfortunately, the previous solution is flawed. The problem is caused by the internal class AsyncStreamReader. Source code can be found here

After modifying its code to skip splitting its buffer's content into lines and skip queueing strings, it's working as expected and it's even faster!

Here is the modified version:

//
//   Copyright (c) Microsoft Corporation.  All rights reserved.
//
// ==--== 
/*============================================================
** 
** Class:  AsyncStreamReader 
**
** Purpose: For reading text from streams using a particular 
** encoding in an asychronous manner used by the process class
**
**
===========================================================*/
using System;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Threading;

namespace System.Diagnostics
{
    /// <summary>
    /// http://www.dotnetframework.org/default.aspx/DotNET/DotNET/8@0/untmp/whidbey/REDBITS/ndp/fx/src/Services/Monitoring/system/Diagnosticts/AsyncStreamReader@cs/1/AsyncStreamReader@cs
    /// </summary>
    public sealed class AsyncStreamReader : DisposableBase, IDisposable
    {
        internal const int DEFAULT_BUFFER_SIZE = 4096;  // Byte buffer size
        private const int MIN_BUFFER_SIZE = 128;

        private Decoder _decoder;
        private byte[] _byteBuffer;
        private char[] _charBuffer;
        // Record the number of valid bytes in the byteBuffer, for a few checks. 

        // This is the maximum number of chars we can get from one call to
        // ReadBuffer.  Used so ReadBuffer can tell when to copy data into
        // a user's char[] directly, instead of our internal char[]. 
        private int _maxCharsPerBuffer;

        // Store a backpointer to the process class, to check for user callbacks 
        private Process _process;
        private StringBuilder _sb;

        // Delegate to call user function.
        private Action<string> _userCallBack;

        // Internal Cancel operation 
        private bool _cancelOperation;
        private ManualResetEvent _eofEvent;

        public AsyncStreamReader(Process process, Stream stream, Action<string> callback, Encoding encoding)
            : this(process, stream, callback, encoding, DEFAULT_BUFFER_SIZE)
        {
        }


        // Creates a new AsyncStreamReader for the given stream.  The 
        // character encoding is set by encoding and the buffer size,
        // in number of 16-bit characters, is set by bufferSize. 
        public AsyncStreamReader(Process process, Stream stream, Action<string> callback, Encoding encoding, int bufferSize)
        {
            Debug.Assert(process != null && stream != null && encoding != null && callback != null, "Invalid arguments!");
            Debug.Assert(stream.CanRead, "Stream must be readable!");
            Debug.Assert(bufferSize > 0, "Invalid buffer size!");
            Init(process, stream, callback, encoding, bufferSize);
        }

        private void Init(Process process, Stream stream, Action<string> callback, Encoding encoding, int bufferSize)
        {
            _process = process;
            BaseStream = stream;
            CurrentEncoding = encoding;
            _userCallBack = callback;
            _decoder = encoding.GetDecoder();
            if (bufferSize < MIN_BUFFER_SIZE) bufferSize = MIN_BUFFER_SIZE;
            _byteBuffer = new byte[bufferSize];
            _maxCharsPerBuffer = encoding.GetMaxCharCount(bufferSize);
            _charBuffer = new char[_maxCharsPerBuffer];
            _sb = new StringBuilder(_charBuffer.Length);
            _cancelOperation = false;
            _eofEvent = new ManualResetEvent(false);
        }

        public void Close()
        {
            Dispose(true);
        }

        void IDisposable.Dispose()
        {
            Dispose(true);
        }

        protected override void Dispose(bool disposing)
        {
            if (disposing)
            {
                if (BaseStream != null)
                {
                    BaseStream.Close();
                    BaseStream = null;
                }
            }

            if (BaseStream != null)
            {
                BaseStream = null;
                CurrentEncoding = null;
                _decoder = null;
                _byteBuffer = null;
                _charBuffer = null;
            }

            if (_eofEvent != null)
            {
                _eofEvent.Close();
                _eofEvent = null;
            }
        }

        public Encoding CurrentEncoding { get; private set; }

        public Stream BaseStream { get; private set; }

        // User calls BeginRead to start the asynchronous read
        public void BeginRead()
        {
            _cancelOperation = false;
            BaseStream.BeginRead(_byteBuffer, 0, _byteBuffer.Length, ReadBuffer, null);
        }

        public void CancelOperation()
        {
            _cancelOperation = true;
        }

        // This is the async callback function. Only one thread could/should call this.
        private void ReadBuffer(IAsyncResult ar)
        {
            if (_cancelOperation) return;

            int byteLen;

            try
            {
                byteLen = BaseStream.EndRead(ar);
            }
            catch (IOException)
            {
                // We should ideally consume errors from operations getting cancelled
                // so that we don't crash the unsuspecting parent with an unhandled exc. 
                // This seems to come in 2 forms of exceptions (depending on platform and scenario),
                // namely OperationCanceledException and IOException (for errorcode that we don't 
                // map explicitly). 
                byteLen = 0; // Treat this as EOF
            }
            catch (OperationCanceledException)
            {
                // We should consume any OperationCanceledException from child read here
                // so that we don't crash the parent with an unhandled exc
                byteLen = 0; // Treat this as EOF 
            }

            if (byteLen == 0)
            {
                // We're at EOF, we won't call this function again from here on.
                _eofEvent.Set();
            }
            else
            {
                int charLen = _decoder.GetChars(_byteBuffer, 0, byteLen, _charBuffer, 0);

                if (charLen > 0)
                {
                    _sb.Length = 0;
                    _sb.Append(_charBuffer, 0, charLen);
                    _userCallBack(_sb.ToString());
                }

                BaseStream.BeginRead(_byteBuffer, 0, _byteBuffer.Length, ReadBuffer, null);
            }
        }

        // Wait until we hit EOF. This is called from Process.WaitForExit 
        // We will lose some information if we don't do this.
        public void WaitUtilEof()
        {
            if (_eofEvent != null)
            {
                _eofEvent.WaitOne();
                _eofEvent.Close();
                _eofEvent = null;
            }
        }
    }
}

Usage:

Process _process;
StreamWriter _inputWriter;
AsyncStreamReader _output;

void Main()
{
    _process = new Process
    {
        EnableRaisingEvents = true,
        StartInfo = new ProcessStartInfo
        {
            FileName = "cmd.exe",
            Arguments = string.Empty,
            UseShellExecute = false,
            CreateNoWindow = true,
            WindowStyle = ProcessWindowStyle.Hidden,
            WorkingDirectory = Directory.GetCurrentDirectory(),
            StandardOutputEncoding = Encoding.UTF8,
            StandardErrorEncoding = Encoding.UTF8,
            RedirectStandardInput = true,
            RedirectStandardOutput = true,
            RedirectStandardError = true
        }
    };

    Console.WriteLine("Starting...");
    if (!_process.Start()) return;
    BeginRead();
    _inputWriter = _process.StandardInput;
    _inputWriter.AutoFlush = true;

    Thread.Sleep(500);

    string input = Util.ReadLine<string>("Type a command:");
    if (!string.IsNullOrEmpty(input)) _inputWriter.WriteLine(input);

    _process.WaitForExit(5000);
    CancelRead();
    _process.Kill();
    Console.WriteLine("Done");
}

void OnOutput(string data)
{
    Console.Write(data);
}

void BeginRead()
{
    if (_output == null) _output = new AsyncStreamReader(_process, _process.StandardOutput.BaseStream, OnOutput, _process.StandardOutput.CurrentEncoding);
    _output.BeginRead();
}

void CancelRead()
{
    _output.CancelOperation();
}