Using unittest to test argparse - exit errors

Question

Going off of Greg Haskin's answer in this question, I tried to make a unittest to check that argparse is giving the appropriate error when I pass it some args that are not present in the choices. However, unittest generates a false positive using the try/except statement below.

In addition, when I make a test using just a with assertRaises statement, argparse forces the system exit and the program does not execute any more tests.

I would like to be able to have a test for this, but maybe it's redundant given that argparse exits upon error?

#!/usr/bin/env python3

import argparse
import unittest

class sweep_test_case(unittest.TestCase):
    """Tests that the merParse class works correctly"""

    def setUp(self):
        self.parser=argparse.ArgumentParser()
        self.parser.add_argument(
            "-c", "--color",
            type=str,
            choices=["yellow", "blue"],
            required=True)

    def test_required_unknown_TE(self):
        """Try to perform sweep on something that isn't an option.
        Should return an attribute error if it fails.
        This test incorrectly shows that the test passed, even though that must
        not be true."""
        args = ["--color", "NADA"]
        try:
            self.assertRaises(argparse.ArgumentError, self.parser.parse_args(args))
        except SystemExit:
            print("should give a false positive pass")

    def test_required_unknown(self):
        """Try to perform sweep on something that isn't an option.
        Should return an attribute error if it fails.
        This test incorrectly shows that the test passed, even though that must
        not be true."""
        args = ["--color", "NADA"]
        with self.assertRaises(argparse.ArgumentError):
            self.parser.parse_args(args)

if __name__ == '__main__':
    unittest.main()

Errors:

Usage: temp.py [-h] -c {yellow,blue}
temp.py: error: argument -c/--color: invalid choice: 'NADA' (choose from 'yellow', 'blue')
E
usage: temp.py [-h] -c {yellow,blue}
temp.py: error: argument -c/--color: invalid choice: 'NADA' (choose from 'yellow', 'blue')
should give a false positive pass
.
======================================================================
ERROR: test_required_unknown (__main__.sweep_test_case)
Try to perform sweep on something that isn't an option.
----------------------------------------------------------------------
Traceback (most recent call last): #(I deleted some lines)
  File "/Users/darrin/anaconda/lib/python3.5/argparse.py", line 2310, in _check_value
    raise ArgumentError(action, msg % args)
argparse.ArgumentError: argument -c/--color: invalid choice: 'NADA' (choose from 'yellow', 'blue')

During handling of the above exception, another exception occurred:

Traceback (most recent call last): #(I deleted some lines)
  File "/anaconda/lib/python3.5/argparse.py", line 2372, in exit
    _sys.exit(status)
SystemExit: 2

Answer 1

The trick here is to catch SystemExit instead of ArgumentError. Here's your test rewritten to catch SystemExit:

#!/usr/bin/env python3

import argparse
import unittest

class SweepTestCase(unittest.TestCase):
    """Tests that the merParse class works correctly"""

    def setUp(self):
        self.parser=argparse.ArgumentParser()
        self.parser.add_argument(
            "-c", "--color",
            type=str,
            choices=["yellow", "blue"],
            required=True)

    def test_required_unknown(self):
        """ Try to perform sweep on something that isn't an option. """
        args = ["--color", "NADA"]
        with self.assertRaises(SystemExit):
            self.parser.parse_args(args)

if __name__ == '__main__':
    unittest.main()

That now runs correctly, and the test passes:

$ python scratch.py
usage: scratch.py [-h] -c {yellow,blue}
scratch.py: error: argument -c/--color: invalid choice: 'NADA' (choose from 'yellow', 'blue')
.
----------------------------------------------------------------------
Ran 1 test in 0.001s

OK

However, you can see that the usage message is getting printed, so your test output is a bit messed up. It might also be nice to check that the usage message contains "invalid choice".

You can do that by patching sys.stderr:

#!/usr/bin/env python3

import argparse
import unittest
from io import StringIO
from unittest.mock import patch


class SweepTestCase(unittest.TestCase):
    """Tests that the merParse class works correctly"""

    def setUp(self):
        self.parser=argparse.ArgumentParser()
        self.parser.add_argument(
            "-c", "--color",
            type=str,
            choices=["yellow", "blue"],
            required=True)

    @patch('sys.stderr', new_callable=StringIO)
    def test_required_unknown(self, mock_stderr):
        """ Try to perform sweep on something that isn't an option. """
        args = ["--color", "NADA"]
        with self.assertRaises(SystemExit):
            self.parser.parse_args(args)
        self.assertRegexpMatches(mock_stderr.getvalue(), r"invalid choice")


if __name__ == '__main__':
    unittest.main()

Now you only see the regular test report:

$ python scratch.py
.
----------------------------------------------------------------------
Ran 1 test in 0.002s

OK

For pytest users, here's the equivalent that doesn't check the message.

import argparse

import pytest


def test_required_unknown():
    """ Try to perform sweep on something that isn't an option. """
    parser=argparse.ArgumentParser()
    parser.add_argument(
        "-c", "--color",
        type=str,
        choices=["yellow", "blue"],
        required=True)
    args = ["--color", "NADA"]

    with pytest.raises(SystemExit):
        parser.parse_args(args)

Pytest captures stdout/stderr by default, so it doesn't pollute the test report.

$ pytest scratch.py
================================== test session starts ===================================
platform linux -- Python 3.6.7, pytest-3.5.0, py-1.7.0, pluggy-0.6.0
rootdir: /home/don/.PyCharm2018.3/config/scratches, inifile:
collected 1 item                                                                         

scratch.py .                                                                       [100%]

================================ 1 passed in 0.01 seconds ================================

You can also check the stdout/stderr contents with pytest:

import argparse

import pytest


def test_required_unknown(capsys):
    """ Try to perform sweep on something that isn't an option. """
    parser=argparse.ArgumentParser()
    parser.add_argument(
        "-c", "--color",
        type=str,
        choices=["yellow", "blue"],
        required=True)
    args = ["--color", "NADA"]

    with pytest.raises(SystemExit):
        parser.parse_args(args)

    stderr = capsys.readouterr().err
    assert 'invalid choice' in stderr

As usual, I find pytest easier to use, but you can make it work in either one.

Answer 2

While the parser may raise an ArgumentError during parsing a specific argument, that is normally trapped, and passed to parser.error and parse.exit. The result is that the usage is printed, along with an error message, and then sys.exit(2).

So asssertRaises is not a good way of testing for this kind of error in argparse. The unittest file for the module, test/test_argparse.py has an elaborate way of getting around this, the involves subclassing the ArgumentParser, redefining its error method, and redirecting output.

parser.parse_known_args (which is called by parse_args) ends with:

    try:
        namespace, args = self._parse_known_args(args, namespace)
        if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
            args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
            delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
        return namespace, args
    except ArgumentError:
        err = _sys.exc_info()[1]
        self.error(str(err))

=================

How about this test (I've borrowed several ideas from test_argparse.py:

import argparse
import unittest

class ErrorRaisingArgumentParser(argparse.ArgumentParser):
    def error(self, message):
        #print(message)
        raise ValueError(message)  # reraise an error

class sweep_test_case(unittest.TestCase):
    """Tests that the Parse class works correctly"""

    def setUp(self):
        self.parser=ErrorRaisingArgumentParser()
        self.parser.add_argument(
            "-c", "--color",
            type=str,
            choices=["yellow", "blue"],
            required=True)

    def test_required_unknown(self):
        """Try to perform sweep on something that isn't an option.
        Should pass"""
        args = ["--color", "NADA"]
        with self.assertRaises(ValueError) as cm:
            self.parser.parse_args(args)
        print('msg:',cm.exception)
        self.assertIn('invalid choice', str(cm.exception))

if __name__ == '__main__':
    unittest.main()

with a run:

1931:~/mypy$ python3 stack39028204.py 
msg: argument -c/--color: invalid choice: 'NADA' (choose from 'yellow', 'blue')
.
----------------------------------------------------------------------
Ran 1 test in 0.002s

OK

Answer 3

With many of the great answers above, I see that in the setUp method a parser instance is created inside our test and an argument is added to it, effectively causing the test to be of argparse's implementation. This, of course, could be a valid test/use case but wouldn't necessarily test a script's or application's specific use of argparse. I think Yauhen Yakimovich's answer gives good insight into how to make use of argparse in a pragmatic way. While I haven't embraced it fully, I thought a simplified test method is possible via a parser generator and an override.

I've opted for testing my code rather than argparse's implementation. To achieve this we'll want to utilize a factory to create the parser in our code that holds all the argument definitions. This facilitates testing our own parser in setUp.

// my_class.py
import argparse

class MyClass:
    def __init__(self):
        self.parser = self._create_args_parser()

    def _create_args_parser():
        parser = argparse.ArgumentParser()
        parser.add_argument('--kind', 
                             action='store',
                             dest='kind', 
                             choices=['type1', 'type2'], 
                             help='kind can be any of: type1, type2')

        return parser

In our test, we can generate our parser and test against it. We will override the error method to ensure we don't get trapped in argparse's ArgumentError evaluation.

import unittest
from my_class import MyClass

class MyClassTest(unittest.TestCase):
    def _redefine_parser_error_method(self, message):
        raise ValueError

    def setUp(self):
        parser = MyClass._create_args_parser()
        parser.error = self._redefine_parser_error_func
        self.parser = parser

    def test_override_certificate_kind_arguments(self):
        args = ['--kind', 'not-supported']
        expected_message = "argument --kind: invalid choice: 'not-supported'.*$"

        with self.assertRaisesRegex(ValueError, expected_message):
            self.parser.parse_args(args)

This might not be the absolute best answer but I find it nice to use our own parser's arguments and test that part by simply testing against an exception we know should only happen in the test itself.

Answer 4

If you look in the error log, you can see that a argparse.ArgumentError is raised and not an AttributeError. your code should look like this:

#!/usr/bin/env python3

import argparse
import unittest
from argparse import ArgumentError

class sweep_test_case(unittest.TestCase):
    """Tests that the merParse class works correctly"""

    def setUp(self):
        self.parser=argparse.ArgumentParser()
        self.parser.add_argument(
            "-c", "--color",
            type=str,
            choices=["yellow", "blue"],
            required=True)

    def test_required_unknown_TE(self):
        """Try to perform sweep on something that isn't an option.
        Should return an attribute error if it fails.
        This test incorrectly shows that the test passed, even though that must
        not be true."""
        args = ["--color", "NADA"]
        try:
            self.assertRaises(ArgumentError, self.parser.parse_args(args))
        except SystemExit:
            print("should give a false positive pass")

    def test_required_unknown(self):
        """Try to perform sweep on something that isn't an option.
        Should return an attribute error if it fails.
        This test incorrectly shows that the test passed, even though that must
        not be true."""
        args = ["--color", "NADA"]
        with self.assertRaises(ArgumentError):
            self.parser.parse_args(args)

if __name__ == '__main__':
    unittest.main()

Answer 5

If you look into the source code of argparse, in argparse.py, around line 1732 (my python version is 3.5.1), there is a method of ArgumentParser called parse_known_args. The code is:

# parse the arguments and exit if there are any errors
try:
    namespace, args = self._parse_known_args(args, namespace)
    if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
        args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
        delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
    return namespace, args
except ArgumentError:
    err = _sys.exc_info()[1]
    self.error(str(err))

So, the ArgumentError will be swallowed by argparse, and exit with an error code. If you want to test this anyway, the only way I could think out is mocking sys.exc_info.

Answer 6

I know this is an old question but just to expand on @don-kirkby's answer of looking for SystemExit – but without having to use pytest or patching – you can wrap the testcode in contextlib.redirect_stderr, if you want to assert something about the error message:

    import contextlib
    from io import StringIO
    import unittest
    class MyTest(unittest.TestCase):
        def test_foo(self):
            ioerr = StringIO()
            with contextlib.redirect_stderr(ioerr):
                with self.assertRaises(SystemExit) as err:
                    foo('bad')
            self.assertEqual(err.exception.code, 2)
            self.assertIn("That is a 'bad' thing", ioerr.getvalue())

Answer 7

I had a similar problem with the same error of argparse (exit 2) and corrected it capturing the first element of tuple that parse_known_args() return, an argparse.Namespace object.

def test_basics_options_of_parser(self):
    parser = w2ptdd.get_parser()
    # unpacking tuple
    parser_name_space,__ = parser.parse_known_args()
    args = vars(parser_name_space)
    self.assertFalse(args['unit'])
    self.assertFalse(args['functional'])