Tokenizers and the Arabic tax

import azimuth_nb as azimuth

env = azimuth.setup(SLUG, lang=LANG, profile=PROFILE)

import random

ARABIC_RANGE = ("\u0600", "\u06ff")


def arabic_share(text):
    """Fraction of letters that are Arabic script."""
    letters = [ch for ch in text if ch.isalpha()]
    if not letters:
        return 0.0
    return sum(ARABIC_RANGE[0] <= ch <= ARABIC_RANGE[1] for ch in letters) / len(letters)


rows = []
with open(env.assets["parallel.tsv"], encoding="utf-8") as fh:
    for line in fh:
        parts = line.rstrip("\n").split("\t")
        if len(parts) >= 2 and parts[0].strip() and parts[1].strip():
            rows.append((parts[0].strip(), parts[1].strip()))

# WHICH COLUMN IS ARABIC IS DETECTED, NOT ASSUMED.
#
# An earlier run of this workshop measured a corpus whose columns were the
# other way round and reported that Arabic was CHEAPER than English — a
# confident, precise, exactly-backwards result. Nothing failed: both columns
# are text, both tokenize, and the arithmetic is identical. Only the sign of
# the conclusion changed.
#
# Reading the script itself costs one pass and removes the assumption. A
# column order is a property of whoever exported the file; the alphabet is a
# property of the language.
sample = rows[: min(200, len(rows))]
share_0 = sum(arabic_share(a) for a, _ in sample) / len(sample)
share_1 = sum(arabic_share(b) for _, b in sample) / len(sample)

if max(share_0, share_1) < 0.5:
    raise SystemExit(
        "Neither column looks like Arabic script — check the encoding of "
        "parallel.tsv before trusting anything below."
    )

if share_0 >= share_1:
    pairs_raw = rows
else:
    pairs_raw = [(b, a) for a, b in rows]
    print("note: columns were (english, arabic) — swapped to match")

# A fixed seed so the sample — and therefore every number below — is the same
# on your machine, on CI, and on the page.
random.seed(env.cfg["seed"])
limit = env.cfg["sampleSentences"]
pairs = random.sample(pairs_raw, min(limit, len(pairs_raw))) if limit else pairs_raw

# WHITESPACE, deliberately. "Word" has to mean the same operation in both
# languages or the ratio compares nothing. Whitespace undercounts Arabic
# morphology — one written word often carries article, stem and pronoun — so
# this makes the result a LOWER BOUND on the real gap rather than a flattering
# one. Overstating the case would be the easiest way to lose the argument.
ar_words = sum(len(ar.split()) for ar, _ in pairs)
en_words = sum(len(en.split()) for _, en in pairs)

if env.lang == "ar":
    print(f"أزواج: {len(pairs):,} من أصل {len(pairs_raw):,}")
    print(f"كلمات: {ar_words:,} عربية · {en_words:,} إنجليزية")
else:
    print(f"pairs: {len(pairs):,} of {len(pairs_raw):,}")
    print(f"words: {ar_words:,} Arabic · {en_words:,} English")

from tokenizers import Tokenizer

# Three different bets about which languages deserve vocabulary budget.
# Loaded from the Hub by name — small JSON files, no model weights.
SPECS = [
    ("gpt2", "openai-community/gpt2", "50k pieces, almost all English"),
    ("bloom", "bigscience/bloom", "250k pieces shared across 46 languages"),
    ("xlm-roberta", "FacebookAI/xlm-roberta-base", "250k pieces, 100 languages"),
]

tokenizers = {}
for name, repo, note in SPECS:
    try:
        tokenizers[name] = Tokenizer.from_pretrained(repo)
        print(f"  · {name:12} {note}")
    except Exception as exc:
        # A tokenizer that will not load is reported and skipped, not fatal:
        # the comparison still means something with two, and a dead Hub should
        # not cost you the whole workshop.
        print(f"  ! {name:12} unavailable ({type(exc).__name__}) — skipped")

n_tokenizers = len(tokenizers)

def count_tokens(tok, texts):
    """Total pieces across a list of strings, no special tokens."""
    return sum(len(tok.encode(t, add_special_tokens=False).ids) for t in texts)


ar_texts = [ar for ar, _ in pairs]
en_texts = [en for _, en in pairs]

table = []
for name, tok in tokenizers.items():
    ar_tokens = count_tokens(tok, ar_texts)
    en_tokens = count_tokens(tok, en_texts)
    ar_f = ar_tokens / ar_words
    en_f = en_tokens / en_words
    table.append(
        {
            "tokenizer": name,
            "ar_fertility": round(ar_f, 3),
            "en_fertility": round(en_f, 3),
            "ratio": round(ar_f / en_f, 3),
        }
    )

header = f"{'tokenizer':14}{'ar/word':>10}{'en/word':>10}{'ratio':>9}"
print("\n" + header)
print("-" * len(header))
for row in table:
    print(
        f"{row['tokenizer']:14}{row['ar_fertility']:>10.2f}"
        f"{row['en_fertility']:>10.2f}{row['ratio']:>9.2f}"
    )

# The headline numbers come from the WORST tokenizer for Arabic, because that
# is the one an Arabic reader is most likely to be paying for.
worst = max(table, key=lambda r: r["ratio"])
ar_fertility = worst["ar_fertility"]
en_fertility = worst["en_fertility"]
ratio = worst["ratio"]

env.explain("subword")


# One sentence, cut both ways. The average is an argument; this is the
# evidence for it.
#
# DECODE EACH PIECE, never print `.tokens` directly. GPT-2 is a BYTE-level
# BPE: its raw token strings are the bytes re-encoded as printable Latin-1, so
# an Arabic word shows up as `ĠاÙĦ | ت | ع` — mojibake that looks like a
# bug in the notebook rather than the finding. Decoding each id individually
# puts the actual fragments back on screen, and where a token is half a UTF-8
# character it shows as `�` — which IS the finding: the tokenizer is cutting
# below the level of a letter.
# CHOOSE a demonstrative pair; do not take pairs[0].
#
# This corpus contains rows whose Arabic column is untranslated — UN document
# titles, mostly — and the seeded sample happened to land on one. The result
# was the showcase cell printing the SAME English sentence under both labels,
# 15 tokens against 15, on the one screen the entire argument rests on. The
# averages above were right; the evidence for them was gibberish.
#
# So the pair is picked, not indexed: genuinely Arabic on one side, genuinely
# not on the other, and long enough to show fragmentation. Deterministic,
# because it scans in order and takes the first that qualifies.
def demonstrative(candidates):
    for ar, en in candidates:
        if arabic_share(ar) > 0.8 and arabic_share(en) < 0.2 and 8 <= len(ar.split()) <= 20:
            return ar, en
    return candidates[0]


sample_ar, sample_en = demonstrative(pairs)
worst_name = worst["tokenizer"]
tok = tokenizers[worst_name]


def pieces_of(tokenizer, text):
    """Decoded pieces, plus how many were not even whole characters.

    A byte-level BPE splits UTF-8, and an Arabic letter is two bytes. So a
    single token id can be HALF A LETTER, and decoding it alone yields no
    character at all — U+FFFD, the replacement character.

    That is not a corpus problem and not a rendering problem. It is the
    measurement: the same file decodes perfectly through BLOOM two blocks
    below. Counting the fragments turns the confusing symbol into the number
    it was always standing for.
    """
    ids = tokenizer.encode(text, add_special_tokens=False).ids
    parts, fragments = [], 0
    for i in ids:
        piece = tokenizer.decode([i])
        if not piece or "\ufffd" in piece:
            fragments += 1
            parts.append("\ufffd")
        else:
            parts.append(piece)
    return parts, fragments


def show(tokenizer, label, text, name):
    parts, fragments = pieces_of(tokenizer, text)
    print(f"\n{label} — {len(parts)} tokens, {len(text.split())} words  [{name}]")
    print("  " + " | ".join(parts))
    if fragments:
        share = fragments / len(parts) * 100
        if env.lang == "ar":
            print(
                f"  ← {fragments} من {len(parts)} رمزاً ({share:.0f}%) ليست حروفاً كاملة."
                " رمز \ufffd يعني قطعة أصغر من الحرف الواحد — وهذا هو القياس لا خطأ عرض."
            )
        else:
            print(
                f"  ← {fragments} of {len(parts)} tokens ({share:.0f}%) are not whole"
                " characters. A \ufffd is a piece SMALLER than one letter — that is the"
                " measurement, not a rendering fault."
            )


worst_name = worst["tokenizer"]
tok = tokenizers[worst_name]

show(tok, "العربية" if env.lang == "ar" else "Arabic", sample_ar, worst_name)
show(tok, "الإنجليزية" if env.lang == "ar" else "English", sample_en, worst_name)

# The same Arabic sentence through a tokenizer that bought vocabulary for it.
# This block is what proves the file is fine and the fragmentation is a choice.
best_name = min(table, key=lambda r: r["ratio"])["tokenizer"]
if best_name != worst_name:
    show(
        tokenizers[best_name],
        "العربية" if env.lang == "ar" else "Arabic",
        sample_ar,
        best_name,
    )

env.explain("context window")

price = env.cfg["pricePerMillionTokens"]
window = env.cfg["contextWindow"]

# What the ratio means once it leaves the spreadsheet.
ar_cost_multiple = round(ratio, 3)
effective_context_ar = int(window / ar_fertility)
effective_context_en = int(window / en_fertility)

ar_million_words_cost = (ar_fertility * 1_000_000 / 1_000_000) * price
en_million_words_cost = (en_fertility * 1_000_000 / 1_000_000) * price

if env.lang == "ar":
    print(
        f"لكل مليون كلمة: {ar_million_words_cost:.2f}$ بالعربية · {en_million_words_cost:.2f}$ بالإنجليزية"
    )
    print(f"القارئ العربي يدفع {ar_cost_multiple:.2f}× للمحتوى ذاته")
    print(
        f"نافذة {window:,} رمز تسع {effective_context_ar:,} كلمة عربية · {effective_context_en:,} كلمة إنجليزية"
    )
else:
    print(
        f"per million words: ${ar_million_words_cost:.2f} Arabic · ${en_million_words_cost:.2f} English"
    )
    print(f"an Arabic reader pays {ar_cost_multiple:.2f}× for the same content")
    print(
        f"a {window:,}-token window holds {effective_context_ar:,} Arabic words · {effective_context_en:,} English"
    )

fertility_ratio_final = ratio
ratio_ok = env.check("fertility-measured", fertility_ratio_final)
count_ok = env.check("tokenizers-compared", n_tokenizers)

receipt = env.receipt()