assertion testing

[slavanorm]

Hello!
I had a request on my work to measure assertion coverage.
is there such plugin for pytest?

i understand that theoretically it is possible through ast parsing, but it might quickly become complex.

please share your approaches to such a task.

here's my dream:

in module X:
1 y=dict(a=1)

in tests:
1 def test():
2 assert 'a' in X.y

in coverage report:
module X line 1 is fully assertion-covered in module tests line 2

here's small snippet example with parsing ast:

input:

source_code = """
x = dict(y=1, z=2)
x['a'] = 3
assert 'y' in x
assert 'z' in x
assert 'a' in x

y = {'b': 4}
y['c'] = 5
assert 'b' in y
# Missing assertion for 'c'
"""

output:

Dictionary 'x':
  Total keys: 3
  Asserted keys: ['a', 'y', 'z']
  Unasserted keys: []
  Asserted percentage: 100.0%
  Key Details:
    'a': Last updated at line 3, Asserted: True
    'y': Last updated at line 2, Asserted: True
    'z': Last updated at line 2, Asserted: True
Dictionary 'y':
  Total keys: 2
  Asserted keys: ['b']
  Unasserted keys: ['c']
  Asserted percentage: 50.0%
  Key Details:
    'c': Last updated at line 9, Asserted: False
    'b': Last updated at line 8, Asserted: True

full code:

import ast

class AssertionAndKeyTracker(ast.NodeVisitor):
    def __init__(self):
        self.dict_keys = {}  # {dict_name: {key}}
        self.asserted_keys = {}  # {dict_name: {key}}
        self.key_last_update = {}  # {dict_name: {key: line_number}}

    def visit_Assign(self, node):
        # Handle dictionary creation/updates (e.g., x = dict(...) or x['key'] = value)
        for target in node.targets:
            if isinstance(target, ast.Name):  # e.g., x = dict(...)
                if isinstance(node.value, ast.Call) and isinstance(node.value.func, ast.Name):
                    if node.value.func.id == "dict":  # Check if it's a dict() call
                        dict_name = target.id
                        self.dict_keys.setdefault(dict_name, set())
                        self.key_last_update.setdefault(dict_name, {})
                        for keyword in node.value.keywords:
                            key = keyword.arg
                            self.dict_keys[dict_name].add(key)
                            self.key_last_update[dict_name][key] = node.lineno
                elif isinstance(node.value, ast.Dict):  # e.g., x = {'key': value}
                    dict_name = target.id
                    self.dict_keys.setdefault(dict_name, set())
                    self.key_last_update.setdefault(dict_name, {})
                    for key_node, _ in zip(node.value.keys, node.value.values):
                        if isinstance(key_node, ast.Constant):  # Python 3.9+
                            key = key_node.value
                            self.dict_keys[dict_name].add(key)
                            self.key_last_update[dict_name][key] = node.lineno
            elif isinstance(target, ast.Subscript):  # e.g., x['key'] = value
                if isinstance(target.value, ast.Name):
                    dict_name = target.value.id
                    if isinstance(target.slice, ast.Constant):  # Python 3.9+
                        key = target.slice.value
                        self.dict_keys.setdefault(dict_name, set()).add(key)
                        self.key_last_update.setdefault(dict_name, {})[key] = node.lineno
        
        # Continue visiting child nodes
        self.generic_visit(node)

    def visit_Assert(self, node):
        # Handle assert statements of the form 'key in dict'
        if isinstance(node.test, ast.Compare) and len(node.test.ops) == 1:
            if isinstance(node.test.ops[0], ast.In):
                if isinstance(node.test.left, ast.Constant) and isinstance(node.test.comparators[0], ast.Name):
                    key = node.test.left.value
                    dict_name = node.test.comparators[0].id
                    
                    # Track the assertion
                    self.asserted_keys.setdefault(dict_name, set()).add(key)
        
        # Continue visiting child nodes
        self.generic_visit(node)

    def report_complete_assertions(self):
        # Generate a detailed report for each dictionary
        report = {}

        for dict_name, keys in self.dict_keys.items():
            total_keys = len(keys)
            asserted = self.asserted_keys.get(dict_name, set())
            asserted_count = len(asserted)
            unasserted = keys - asserted
            percentage = (asserted_count / total_keys) * 100 if total_keys > 0 else 0

            # Include last update line for each key
            key_details = {}
            for key in keys:
                key_details[key] = {
                    "last_update_line": self.key_last_update.get(dict_name, {}).get(key, None),
                    "asserted": key in asserted
                }

            report[dict_name] = {
                "total_keys": total_keys,
                "asserted_keys": sorted(asserted),
                "unasserted_keys": sorted(unasserted),
                "asserted_percentage": round(percentage, 2),
                "key_details": key_details
            }

        return report


def analyze_code(source_code):
    # Parse the source code into an AST
    tree = ast.parse(source_code)
    
    # Create an instance of the tracker
    tracker = AssertionAndKeyTracker()
    
    # Traverse the AST
    tracker.visit(tree)
    
    # Generate the report
    return tracker.report_complete_assertions()

source_code = """
x = dict(y=1, z=2)
x['a'] = 3
assert 'y' in x
assert 'z' in x
assert 'a' in x

y = {'b': 4}
y['c'] = 5
assert 'b' in y
# Missing assertion for 'c'
"""


report = analyze_code(source_code)
for dict_name, details in report.items():
    print(f"Dictionary '{dict_name}':")
    print(f"  Total keys: {details['total_keys']}")
    print(f"  Asserted keys: {details['asserted_keys']}")
    print(f"  Unasserted keys: {details['unasserted_keys']}")
    print(f"  Asserted percentage: {details['asserted_percentage']}%")
    print("  Key Details:")
    for key, info in details["key_details"].items():
        print(f"    '{key}': Last updated at line {info['last_update_line']}, Asserted: {info['asserted']}")

[RonnyPfannschmidt]

Right now python as a language doesn't support that

[RonnyPfannschmidt]

Its not clear what you are asking for then

A basic example is not a full implementation

[slavanorm]

i am asking for community interested or experienced in implementation of similar things or extending coverage plugin

[slavanorm]

and in response to your statement, i am asking for what particular aspects of my request are not supported by python.

[RonnyPfannschmidt]

There's no tool that enables linking a already executed line of code to a later exception in a safe manner

[slavanorm]

We're measuring only in case of test success.
The purpose is to measure % of variables and their attributes/keys/elements that were covered with assertion