"""Repair three BPE functions; the engine uses all three in the full experiment.

python3 -B test_tokenizer.py -v
python3 -B starter.py --output my-tokenizer-report.json
"""
import tokenizer


def pair_counts(sequences):
    # Return pair -> frequency. Count overlapping adjacent pairs.
    # Keep documents separate. Empty/one-token sequences contribute no pairs.
    raise NotImplementedError("Implement pair_counts in starter.py")


def merge_pair(ids, pair, new_id):
    # Return a NEW list. Scan left-to-right, skip two IDs after a match.
    # Do not merge overlapping matches or mutate the input.
    raise NotImplementedError("Implement merge_pair in starter.py")


def apply_merges(ids, merges):
    # Start with a copy; apply learned pairs in rank order using merge_pair.
    # Rank zero gets tokenizer.MERGE_START, not byte ID 256 (reserved BOS).
    raise NotImplementedError("Implement apply_merges in starter.py")


if __name__ == "__main__":
    tokenizer.main(pair_counts, merge_pair, apply_merges)