Mathew K Analytics

Lesson 18 · Pytest deep dive

pytest Tutorial #18: Capstone — Testing a Real Package End to End

Video eighteen, the final video, of an eighteen-part series on testing real Python code with pytest. One real small package. One real full test suite. Every…

⬇ Download notebook

📓 Full notebook

Download .ipynb

pytest Deep-Dive, Video 18: Capstone - Testing a Real Small Package End to End#

  • Video eighteen, the final video, of an eighteen-part series on testing real Python code with pytest.
  • One real small package. One real full test suite. Every idea from the series, wired together.
  • Let's get into it.

Part 1: The Package Under Test - a Real taskmanager Module#

taskmanager_src = '''
import logging

class TaskManager:
    def __init__(self, notifier=None):
        self._tasks = {}
        self._next_id = 1
        self.notifier = notifier

    def add_task(self, title, priority='normal'):
        if not title or not title.strip():
            raise ValueError('task title cannot be empty')
        if priority not in ('low', 'normal', 'high'):
            raise ValueError(f'invalid priority: {priority}')
        task_id = self._next_id
        self._tasks[task_id] = {'title': title, 'priority': priority, 'done': False}
        self._next_id += 1
        return task_id

    def complete_task(self, task_id):
        task = self._tasks[task_id]
        if task['done']:
            logging.warning('task %s is already completed', task_id)
            return
        task['done'] = True
        if self.notifier:
            self.notifier(task_id, task['title'])

    def list_tasks(self, pending_only=False):
        items = self._tasks.items()
        if pending_only:
            items = [(k, v) for k, v in items if not v['done']]
        return dict(items)

    def to_dict(self):
        return {'tasks': self._tasks, 'next_id': self._next_id}

    @classmethod
    def from_dict(cls, data, notifier=None):
        mgr = cls(notifier=notifier)
        mgr._tasks = {int(k): v for k, v in data['tasks'].items()}
        mgr._next_id = data['next_id']
        return mgr

    def save(self, path):
        import json
        path.write_text(json.dumps(self.to_dict()))

    @classmethod
    def load(cls, path, notifier=None):
        import json
        data = json.loads(path.read_text())
        return cls.from_dict(data, notifier=notifier)
'''

Part 2: conftest.py - Shared Fixtures for the Whole Suite#

conftest_src = '''
import pytest
from taskmanager import TaskManager

@pytest.fixture
def manager():
    return TaskManager()

@pytest.fixture
def sample_manager(manager):
    manager.add_task('Write report', priority='high')
    manager.add_task('Buy groceries', priority='low')
    manager.add_task('Read book', priority='normal')
    return manager
'''

Part 3: test_tasks.py - Core Behavior, Parametrize, and Raises#

test_tasks_src = '''
import pytest

def test_add_task_returns_incrementing_ids(manager):
    first_id = manager.add_task('First task')
    second_id = manager.add_task('Second task')
    assert second_id == first_id + 1

def test_add_task_empty_title_raises(manager):
    with pytest.raises(ValueError, match='cannot be empty'):
        manager.add_task('   ')

@pytest.mark.parametrize('priority', ['low', 'normal', 'high'])
def test_add_task_valid_priorities(manager, priority):
    task_id = manager.add_task('Task', priority=priority)
    assert manager.list_tasks()[task_id]['priority'] == priority

def test_add_task_invalid_priority_raises(manager):
    with pytest.raises(ValueError, match='invalid priority'):
        manager.add_task('Task', priority='urgent')

def test_list_tasks_pending_only(sample_manager):
    sample_manager.complete_task(1)
    pending = sample_manager.list_tasks(pending_only=True)
    assert 1 not in pending
    assert len(pending) == 2
'''

Part 4: test_notifications.py - Mocking and a Real xfail#

test_notifications_src = '''
import pytest
from taskmanager import TaskManager

def test_complete_task_calls_notifier(mocker):
    fake_notifier = mocker.Mock()
    manager = TaskManager(notifier=fake_notifier)
    task_id = manager.add_task('Notify me')
    manager.complete_task(task_id)
    fake_notifier.assert_called_once_with(task_id, 'Notify me')

def test_complete_task_without_notifier_does_not_crash(manager):
    task_id = manager.add_task('Silent task')
    manager.complete_task(task_id)
    assert manager.list_tasks()[task_id]['done'] is True

@pytest.mark.xfail(reason='batch complete not implemented yet')
def test_complete_multiple_tasks_at_once(manager):
    manager.add_task('A')
    manager.add_task('B')
    manager.complete_all()
'''

Part 5: test_persistence.py - tmp_path Round Trip#

test_persistence_src = '''
from taskmanager import TaskManager

def test_save_and_load_round_trip(tmp_path):
    manager = TaskManager()
    manager.add_task('Task one')
    manager.add_task('Task two', priority='high')
    save_path = tmp_path / 'tasks.json'
    manager.save(save_path)
    loaded = TaskManager.load(save_path)
    assert loaded.list_tasks() == manager.list_tasks()

def test_load_preserves_notifier(tmp_path, mocker):
    manager = TaskManager()
    manager.add_task('Task')
    save_path = tmp_path / 'tasks.json'
    manager.save(save_path)
    fake_notifier = mocker.Mock()
    loaded = TaskManager.load(save_path, notifier=fake_notifier)
    loaded.complete_task(1)
    fake_notifier.assert_called_once()
'''

Part 6: test_logging.py - caplog on a Real Double Complete#

test_logging_src = '''
def test_double_complete_logs_warning(sample_manager, caplog):
    sample_manager.complete_task(1)
    sample_manager.complete_task(1)
    assert 'already completed' in caplog.text
'''

Part 7: Assembling the Real Project on Disk#

import pytest, tempfile, os, sys

def build_project(files):
    tmpdir = tempfile.mkdtemp()
    for name, content in files.items():
        with open(os.path.join(tmpdir, name), 'w') as f:
            f.write(content)
    return tmpdir

def run_pytest_dir(tmpdir, files, args=None):
    for name in files:
        sys.modules.pop(name[:-3], None)
    if tmpdir not in sys.path:
        sys.path.insert(0, tmpdir)
    return pytest.main([tmpdir, *(args or [])])

files = {
    'taskmanager.py': taskmanager_src,
    'conftest.py': conftest_src,
    'test_tasks.py': test_tasks_src,
    'test_notifications.py': test_notifications_src,
    'test_persistence.py': test_persistence_src,
    'test_logging.py': test_logging_src,
}
project_dir = build_project(files)
print(project_dir)
C:\Users\makmw\AppData\Local\Temp\tmp_63anfdz

Part 8: Running the Whole Real Suite#

exit_code = run_pytest_dir(project_dir, files, ['-v', '--no-header'])
print(exit_code)
============================= test session starts =============================
collecting ... collected 13 items

..\..\..\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_logging.py::test_double_complete_logs_warning PASSED [  7%]
..\..\..\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_notifications.py::test_complete_task_calls_notifier ERROR [ 15%]
..\..\..\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_notifications.py::test_complete_task_without_notifier_does_not_crash PASSED [ 23%]
..\..\..\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_notifications.py::test_complete_multiple_tasks_at_once XFAIL [ 30%]
..\..\..\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_persistence.py::test_save_and_load_round_trip PASSED [ 38%]
..\..\..\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_persistence.py::test_load_preserves_notifier ERROR [ 46%]
..\..\..\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_tasks.py::test_add_task_returns_incrementing_ids PASSED [ 53%]
..\..\..\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_tasks.py::test_add_task_empty_title_raises PASSED [ 61%]
..\..\..\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_tasks.py::test_add_task_valid_priorities[low] PASSED [ 69%]
..\..\..\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_tasks.py::test_add_task_valid_priorities[normal] PASSED [ 76%]
..\..\..\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_tasks.py::test_add_task_valid_priorities[high] PASSED [ 84%]
..\..\..\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_tasks.py::test_add_task_invalid_priority_raises PASSED [ 92%]
..\..\..\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_tasks.py::test_list_tasks_pending_only PASSED [100%]

=================================== ERRORS ====================================
_____________ ERROR at setup of test_complete_task_calls_notifier _____________
file C:\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_notifications.py, line 5
  def test_complete_task_calls_notifier(mocker):
E       fixture 'mocker' not found
>       available fixtures: _session_faker, anyio_backend, anyio_backend_name, anyio_backend_options, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, capteesys, dash_br, dash_duo, dash_duo_mp, dash_multi_process_server, dash_process_server, dash_thread_server, dashjl, dashjl_server, dashr, dashr_server, diskcache_manager, doctest_namespace, faker, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, manager, monkeypatch, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, sample_manager, subtests, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory
>       use 'pytest --fixtures [testpath]' for help on them.

C:\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_notifications.py:5
_______________ ERROR at setup of test_load_preserves_notifier ________________
file C:\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_persistence.py, line 13
  def test_load_preserves_notifier(tmp_path, mocker):
E       fixture 'mocker' not found
>       available fixtures: _session_faker, anyio_backend, anyio_backend_name, anyio_backend_options, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, capteesys, dash_br, dash_duo, dash_duo_mp, dash_multi_process_server, dash_process_server, dash_thread_server, dashjl, dashjl_server, dashr, dashr_server, diskcache_manager, doctest_namespace, faker, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, manager, monkeypatch, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, sample_manager, subtests, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory
>       use 'pytest --fixtures [testpath]' for help on them.

C:\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_persistence.py:13
=========================== short test summary info ===========================
ERROR ..\..\..\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_notifications.py::test_complete_task_calls_notifier
ERROR ..\..\..\Users\makmw\AppData\Local\Temp\tmp_63anfdz\test_persistence.py::test_load_preserves_notifier
=================== 10 passed, 1 xfailed, 2 errors in 0.45s ===================
1

Part 9: Coverage Across the Whole Real Suite#

cov_args = ['--cov=taskmanager', '--cov-report=term-missing', '--no-header', '-q']
exit_code = run_pytest_dir(project_dir, files, cov_args)
print(exit_code)
4
ERROR: usage: ipykernel_launcher.py [options] [file_or_dir] [file_or_dir] [...]
ipykernel_launcher.py: error: unrecognized arguments: --cov=taskmanager --cov-report=term-missing
  inifile: None
  rootdir: C:\Users\makmw\AppData\Local\Temp\tmp_63anfdz

Series Wrap-Up: Eighteen Real Videos on pytest#

Found this useful?

All lessons, notebooks and datasets here are free. If they helped you, a coffee keeps new lessons coming.