PropertyDamage.Coverage (PropertyDamage v0.1.0)

View Source

Track and report coverage metrics for property-based tests.

Coverage helps you understand how thoroughly your model is being exercised:

  • Command coverage: Which commands have been tested?
  • Transition coverage: Which command sequences have been tested?
  • State coverage: Which projection states have been reached?

Usage

# Enable coverage tracking
{:ok, result} = PropertyDamage.run(model: M, adapter: A, coverage: true)

# Get coverage report
coverage = PropertyDamage.Coverage.from_result(result)
IO.puts(PropertyDamage.Coverage.format(coverage))

# Or track across multiple runs
tracker = Coverage.new(M)
tracker = Coverage.record(tracker, result1)
tracker = Coverage.record(tracker, result2)
IO.puts(Coverage.format(tracker))

Coverage Metrics

  • Command coverage: Percentage of commands that were executed at least once
  • Command frequency: How often each command was executed
  • Transition coverage: Which command pairs (A → B) have been tested
  • State coverage: Unique projection states reached (by hash)
  • Check coverage: Which checks have been exercised

CI Integration

Use Coverage.meets_threshold?/2 to fail CI if coverage is too low:

coverage = Coverage.from_result(result)
unless Coverage.meets_threshold?(coverage, command: 80, transition: 50) do
  raise "Coverage threshold not met"
end

Summary

Functions

Get the least frequently executed commands (excluding untested).

Get command coverage percentage.

Format coverage report for display.

Format state class coverage as ASCII art.

Format the transition matrix as ASCII art.

Build coverage from a single result (convenience function).

Check if coverage meets specified thresholds.

Merge two coverage trackers.

Create a new coverage tracker for a model.

Record coverage from a test run result.

Get state class counts (requires state_classifier to be set).

Get the state class transition matrix.

Get state class transition counts (requires state_classifier to be set).

Get detailed statistics.

Export coverage data to JSON for CI integration.

Get the most frequently executed commands.

Get most frequently tested transitions.

Get transition coverage percentage.

Get the transition matrix as a map.

Get the number of unique states observed.

Get commands that haven't been tested yet.

Get transitions (command pairs) that haven't been tested yet.

Types

state_classifier()

@type state_classifier() :: (map() -> atom() | String.t()) | nil

t()

@type t() :: %PropertyDamage.Coverage{
  check_hits: %{required(module()) => non_neg_integer()},
  command_counts: %{required(module()) => non_neg_integer()},
  command_modules: MapSet.t(module()),
  failures_found: non_neg_integer(),
  last_state_class: atom() | nil,
  model: module(),
  state_class_counts: %{required(atom()) => non_neg_integer()},
  state_class_transitions: %{required({atom(), atom()}) => non_neg_integer()},
  state_classifier: state_classifier(),
  state_hashes: MapSet.t(integer()),
  total_commands: non_neg_integer(),
  total_runs: non_neg_integer(),
  transition_counts: %{required({module(), module()}) => non_neg_integer()}
}

Functions

bottom_commands(coverage, n \\ 10)

@spec bottom_commands(t(), non_neg_integer()) :: [{module(), non_neg_integer()}]

Get the least frequently executed commands (excluding untested).

command_coverage(coverage)

@spec command_coverage(t()) :: float()

Get command coverage percentage.

Returns the percentage of defined commands that were executed at least once.

format(tracker, format \\ :summary)

@spec format(t(), atom()) :: String.t()

Format coverage report for display.

Format Options

  • :summary - Brief summary (default)
  • :matrix - Transition matrix showing command pairs
  • :full - Complete report with matrix and untested transitions
  • :state_classes - State class transition matrix (requires state_classifier)

Examples

Coverage.format(tracker)                # summary
Coverage.format(tracker, :matrix)       # transition matrix only
Coverage.format(tracker, :full)         # everything
Coverage.format(tracker, :state_classes) # state class matrix only

format_state_class_matrix(tracker)

@spec format_state_class_matrix(t()) :: String.t()

Format state class coverage as ASCII art.

Shows which state class transitions have been tested.

format_transition_matrix(tracker)

@spec format_transition_matrix(t()) :: String.t()

Format the transition matrix as ASCII art.

Shows which command pairs have been tested:

  • ████ = well-tested (>10 occurrences)
  • ▓▓▓▓ = tested (>5 occurrences)
  • ░░░░ = lightly tested (1-5 occurrences)
  • = untested

Example

Transition Matrix

               Create  Credit  Debit
Create           ·          
Credit                ·     
Debit                     ·

from_result(result, model)

@spec from_result({:ok, map()} | {:error, PropertyDamage.FailureReport.t()}, module()) ::
  t()

Build coverage from a single result (convenience function).

meets_threshold?(tracker, opts \\ [])

@spec meets_threshold?(
  t(),
  keyword()
) :: boolean()

Check if coverage meets specified thresholds.

Options

  • :command - Minimum command coverage percentage (default: 0)
  • :transition - Minimum transition coverage percentage (default: 0)
  • :min_commands - Minimum total commands executed (default: 0)

Example

Coverage.meets_threshold?(coverage, command: 80, transition: 50)

merge(tracker1, tracker2)

@spec merge(t(), t()) :: t()

Merge two coverage trackers.

Useful for combining coverage from parallel test runs.

new(model, opts \\ [])

@spec new(
  module(),
  keyword()
) :: t()

Create a new coverage tracker for a model.

Options

  • :state_classifier - Function to classify states into abstract classes. The function receives the projection state map and returns an atom or string identifying the state class.

Example

# Track coverage with state classes
classifier = fn state ->
  cond do
    state.balance == 0 -> :zero_balance
    state.balance > 0 -> :positive_balance
    state.balance < 0 -> :negative_balance
  end
end

tracker = Coverage.new(MyModel, state_classifier: classifier)

record(tracker, arg)

@spec record(t(), {:ok, map()} | {:error, PropertyDamage.FailureReport.t()}) :: t()

Record coverage from a test run result.

Works with both success and failure results.

state_class_counts(coverage)

@spec state_class_counts(t()) :: %{required(atom()) => non_neg_integer()}

Get state class counts (requires state_classifier to be set).

Returns a map of %{state_class => count}.

state_class_matrix(coverage)

@spec state_class_matrix(t()) :: %{
  required(atom()) => %{required(atom()) => non_neg_integer()}
}

Get the state class transition matrix.

Returns %{from_class => %{to_class => count}}.

state_class_transitions(coverage)

@spec state_class_transitions(t()) :: %{
  required({atom(), atom()}) => non_neg_integer()
}

Get state class transition counts (requires state_classifier to be set).

Returns a map of %{{from_class, to_class} => count}.

stats(tracker)

@spec stats(t()) :: map()

Get detailed statistics.

to_json(tracker)

@spec to_json(t()) :: String.t()

Export coverage data to JSON for CI integration.

top_commands(coverage, n \\ 10)

@spec top_commands(t(), non_neg_integer()) :: [{module(), non_neg_integer()}]

Get the most frequently executed commands.

top_transitions(coverage, n \\ 10)

@spec top_transitions(t(), non_neg_integer()) :: [
  {{module(), module()}, non_neg_integer()}
]

Get most frequently tested transitions.

transition_coverage(coverage)

@spec transition_coverage(t()) :: float()

Get transition coverage percentage.

Returns the percentage of possible command pairs that were tested.

transition_matrix(coverage)

@spec transition_matrix(t()) :: %{
  required(module()) => %{required(module()) => non_neg_integer()}
}

Get the transition matrix as a map.

Returns %{from_command => %{to_command => count}}.

unique_states(coverage)

@spec unique_states(t()) :: non_neg_integer()

Get the number of unique states observed.

untested_commands(coverage)

@spec untested_commands(t()) :: [module()]

Get commands that haven't been tested yet.

untested_transitions(coverage)

@spec untested_transitions(t()) :: [{module(), module()}]

Get transitions (command pairs) that haven't been tested yet.

Returns list of {from_command, to_command} tuples.