Build a caption QA harness in Python: WER, missed entities, timing and reading rate

python3 -m venv .venv && source .venv/bin/activate
pip install jiwer whisper_normalizer webvtt-py

$ pip list | grep -Ei 'jiwer|whisper|webvtt'
jiwer               <your version>
webvtt-py           <your version>
whisper-normalizer  <your version>

# captions.py
from dataclasses import dataclass
import webvtt

@dataclass
class Cue:
    start: float
    end: float
    text: str

    @property
    def duration(self) -> float:
        return self.end - self.start

    @property
    def lines(self) -> list[str]:
        return self.text.split("\n")

    @property
    def flat(self) -> str:
        return " ".join(l.strip() for l in self.lines)

    @property
    def chars_per_second(self) -> float:
        return len(self.flat) / self.duration if self.duration > 0 else float("inf")

def _to_seconds(ts: str) -> float:
    h, m, s = ts.split(":")
    return int(h) * 3600 + int(m) * 60 + float(s)

def load_vtt(path: str) -> list[Cue]:
    # keep the line breaks: they are the thing the readability check measures
    return [
        Cue(_to_seconds(c.start), _to_seconds(c.end), c.text.strip())
        for c in webvtt.read(path)
    ]

def full_text(cues: list[Cue]) -> str:
    return " ".join(c.flat for c in cues)

# score_wer.py
import jiwer
from whisper_normalizer.english import EnglishTextNormalizer

normalizer = EnglishTextNormalizer()

def wer(reference: str, hypothesis: str) -> dict:
    ref, hyp = normalizer(reference), normalizer(hypothesis)
    out = jiwer.process_words(ref, hyp)
    return {
        "wer": out.wer,
        "substitutions": out.substitutions,
        "deletions": out.deletions,
        "insertions": out.insertions,
        "ref_words": sum(len(r) for r in out.references),
    }

$ python -c "from score_wer import wer; print(wer('the dose is fifteen milligrams','the dose is fifty milligrams'))"
{'wer': 0.2, 'substitutions': 1, 'deletions': 0, 'insertions': 0, 'ref_words': 5}

# score_entities.py
import re
from whisper_normalizer.english import EnglishTextNormalizer

normalizer = EnglishTextNormalizer()
NUMBER_RE = re.compile(r"\b\d+(?:\.\d+)?\b")

def _terms(text: str, lexicon: list[str]) -> list[str]:
    norm = normalizer(text)
    found = []
    # findall, not search: a term said three times should count three times, the same
    # way numbers do. Iterate a sorted list so `examples` is stable across runs.
    for t in sorted(lexicon):
        found += re.findall(rf"\b{re.escape(t)}\b", norm)
    found += NUMBER_RE.findall(norm)
    return found

def missed_entity_rate(reference: str, hypothesis: str, lexicon: list[str]) -> dict:
    ref_terms = _terms(reference, lexicon)
    hyp_terms = _terms(hypothesis, lexicon)

    remaining = list(hyp_terms)
    missed = []
    for t in ref_terms:
        if t in remaining:
            remaining.remove(t)
        else:
            missed.append(t)

    total = len(ref_terms)
    return {
        "entity_total": total,
        "entity_missed": len(missed),
        "missed_entity_rate": len(missed) / total if total else 0.0,
        "examples": missed[:10],
    }

# score_timing.py
import statistics
from whisper_normalizer.english import EnglishTextNormalizer
from captions import Cue

normalizer = EnglishTextNormalizer()

def _tokens(text: str) -> set[str]:
    return set(normalizer(text).split())

def cue_offsets(ref: list[Cue], hyp: list[Cue], window: float = 4.0,
                min_overlap: float = 0.6) -> dict:
    """Match cues by token overlap, not string equality.

    ASR almost never segments cues the same way a human transcriber does, so requiring
    identical normalized text would match nothing on real data. We take the best
    Jaccard overlap inside a time window instead.
    """
    offsets = []
    for r in ref:
        r_tok = _tokens(r.flat)
        if not r_tok:
            continue

        best, best_score = None, 0.0
        for h in hyp:
            if abs(h.start - r.start) > window:
                continue
            h_tok = _tokens(h.flat)
            if not h_tok:
                continue
            score = len(r_tok & h_tok) / len(r_tok | h_tok)
            if score > best_score:
                best, best_score = h, score

        if best is not None and best_score >= min_overlap:
            offsets.append(best.start - r.start)

    if not offsets:
        return {"matched_cues": 0, "median_offset_s": None, "p90_abs_offset_s": None}

    absolute = sorted(abs(o) for o in offsets)
    return {
        "matched_cues": len(offsets),
        "median_offset_s": round(statistics.median(offsets), 3),
        "p90_abs_offset_s": round(absolute[int(0.9 * (len(absolute) - 1))], 3),
    }

# score_readability.py
from captions import Cue

# Thresholds are editorial choices, not standards. These are in the range major
# streaming style guides use for adult English subtitles; set yours deliberately.
MAX_CPS = 20.0          # characters per second
MIN_DURATION = 5 / 6    # ~0.833s, below this a cue flashes
MAX_LINE_CHARS = 42

def readability(cues: list[Cue]) -> dict:
    too_fast   = [c for c in cues if c.chars_per_second > MAX_CPS]
    too_short  = [c for c in cues if c.duration < MIN_DURATION]
    too_wide   = [c for c in cues if any(len(l) > MAX_LINE_CHARS for l in c.lines)]
    rates = sorted(c.chars_per_second for c in cues if c.duration > 0)

    return {
        "cues": len(cues),
        "median_cps": round(rates[len(rates) // 2], 1) if rates else None,
        "pct_over_cps_limit": round(100 * len(too_fast) / len(cues), 1) if cues else 0.0,
        "cues_under_min_duration": len(too_short),
        "cues_over_line_length": len(too_wide),
    }

# run_eval.py
import json, pathlib, sys
from captions import load_vtt, full_text
from score_wer import wer
from score_entities import missed_entity_rate
from score_timing import cue_offsets
from score_readability import readability

LEXICON = set(json.loads(pathlib.Path("lexicon.json").read_text()))

def evaluate(ref_vtt: str, hyp_vtt: str) -> dict:
    ref, hyp = load_vtt(ref_vtt), load_vtt(hyp_vtt)
    ref_text, hyp_text = full_text(ref), full_text(hyp)
    return {
        "file": pathlib.Path(hyp_vtt).name,
        **wer(ref_text, hyp_text),
        **missed_entity_rate(ref_text, hyp_text, LEXICON),
        **cue_offsets(ref, hyp),
        **readability(hyp),
    }

if __name__ == "__main__":
    results = [evaluate(r, h) for r, h in zip(sys.argv[1::2], sys.argv[2::2])]
    print(json.dumps(results, indent=2))

$ python run_eval.py refs/clip01.vtt out/clip01.vtt
[
  {
    "file": "clip01.vtt",
    "wer": 0.043,
    "substitutions": 18,
    "deletions": 5,
    "insertions": 3,
    "ref_words": 604,
    "entity_total": 41,
    "entity_missed": 6,
    "missed_entity_rate": 0.146,
    "examples": ["nifedipine", "2026", "okonkwo"],
    "matched_cues": 71,
    "median_offset_s": 0.42,
    "p90_abs_offset_s": 0.91,
    "cues": 112,
    "median_cps": 17.4,
    "pct_over_cps_limit": 9.8,
    "cues_under_min_duration": 4,
    "cues_over_line_length": 7
  }
]