File size: 8,535 Bytes
68d8806 ce59834 68d8806 ce59834 68d8806 ce59834 68d8806 ce59834 68d8806 b8d6c6c 68d8806 b8d6c6c 68d8806 b8d6c6c 68d8806 b8d6c6c 68d8806 c7b1be1 68d8806 c7b1be1 68d8806 c7b1be1 68d8806 b8d6c6c c7b1be1 68d8806 b8d6c6c 68d8806 c7b1be1 b8d6c6c c7b1be1 b8d6c6c c7b1be1 68d8806 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 | """Tests for LatinBertTokenizer.
Validates that the HF wrapper produces identical output to the
original tensor2tensor SubwordTextEncoder used in Bamman & Burns (2020).
Reference IDs generated from the standalone encoder on the cluster.
"""
import os
import pytest
from pathlib import Path
VOCAB_FILE = str(Path(__file__).parent / "latin.subword.encoder")
@pytest.fixture
def tokenizer():
from latincy_latinbert import LatinBertTokenizer
return LatinBertTokenizer(vocab_file=VOCAB_FILE)
class TestSpecialTokens:
def test_special_token_ids(self, tokenizer):
"""BERT special tokens must occupy IDs 0-4."""
assert tokenizer.convert_tokens_to_ids("[PAD]") == 0
assert tokenizer.convert_tokens_to_ids("[UNK]") == 1
assert tokenizer.convert_tokens_to_ids("[CLS]") == 2
assert tokenizer.convert_tokens_to_ids("[SEP]") == 3
assert tokenizer.convert_tokens_to_ids("[MASK]") == 4
def test_special_token_strings(self, tokenizer):
assert tokenizer.pad_token == "[PAD]"
assert tokenizer.unk_token == "[UNK]"
assert tokenizer.cls_token == "[CLS]"
assert tokenizer.sep_token == "[SEP]"
assert tokenizer.mask_token == "[MASK]"
def test_vocab_size_includes_specials(self, tokenizer):
"""vocab_size = 5 special + 32895 subtokens = 32900."""
assert tokenizer.vocab_size == 32900
def test_subtoken_offset(self, tokenizer):
"""First subtoken '<pad>_' from encoder should be at ID 5, not 0."""
assert tokenizer.convert_tokens_to_ids("<pad>_") == 5
def test_add_special_tokens_encoding(self, tokenizer):
"""encode with add_special_tokens=True should wrap with [CLS]/[SEP]."""
ids = tokenizer.encode("et", add_special_tokens=True)
assert ids[0] == 2 # [CLS]
assert ids[-1] == 3 # [SEP]
class TestVocab:
def test_vocab_size(self, tokenizer):
assert tokenizer.vocab_size == 32900
def test_pad_token_id(self, tokenizer):
assert tokenizer.pad_token == "[PAD]"
assert tokenizer.convert_tokens_to_ids("[PAD]") == 0
def test_eos_token(self, tokenizer):
assert tokenizer.eos_token == "<EOS>_"
assert tokenizer.convert_tokens_to_ids("<EOS>_") == 6 # was 1, now 1+5
class TestEncoding:
"""Reference IDs from the original LatinTokenizer (with +5 offset).
The original whitespace-splits before subword encoding, so there are
NO inter-word space escapes. One clean subtoken per word here.
"""
def test_gallia(self, tokenizer):
ids = tokenizer.encode("Gallia est omnis divisa in partes tres",
add_special_tokens=False)
# do_lower_case=True → each word is a single subtoken; no space escapes
expected = [6533, 15, 343, 6773, 12, 568, 564]
assert ids == expected
def test_arma(self, tokenizer):
ids = tokenizer.encode("arma virumque cano",
add_special_tokens=False)
expected = [915, 18566, 8107, 4420]
assert ids == expected
def test_no_interword_space_escapes(self, tokenizer):
"""Full-string encoding must not inject space escapes between words.
The buggy full-string path emitted `\\ 32 ;_` (ids 32888, 7735, 13)
for every inter-word space — tokens the model never trained on.
"""
ids = tokenizer.encode("gallia est omnis divisa in partes tres",
add_special_tokens=False)
assert 32888 not in ids and 7735 not in ids and 13 not in ids
def test_fullstring_matches_per_word(self, tokenizer):
"""Native full-string tokenization == word-by-word tokenization.
This is the contract: `tokenizer(full_sentence)` reproduces the
original per-word LatinTokenizer output, so HF-native usage is
faithful without requiring is_split_into_words.
"""
text = "Gallia est omnis divisa in partes tres, quarum unam incolunt Belgae"
full = tokenizer.tokenize(text)
per_word = []
for word in text.split():
per_word.extend(tokenizer.tokenize(word))
assert full == per_word
def test_uppercase(self, tokenizer):
"""Uppercase input should be lowercased, not escaped to codepoints."""
ids = tokenizer.encode("ROMA", add_special_tokens=False)
expected = [2560] # 'roma_' — single token, not 10 escaped codepoints
assert ids == expected
def test_empty(self, tokenizer):
ids = tokenizer.encode("", add_special_tokens=False)
assert ids == []
class TestRoundtrip:
def test_decode_lowercase(self, tokenizer):
"""Lowercase text should roundtrip exactly."""
text = "gallia est omnis divisa in partes tres"
ids = tokenizer.encode(text, add_special_tokens=False)
decoded = tokenizer.decode(ids)
assert decoded == text
def test_decode_arma(self, tokenizer):
text = "arma virumque cano"
ids = tokenizer.encode(text, add_special_tokens=False)
decoded = tokenizer.decode(ids)
assert decoded == text
def test_decode_with_punctuation(self, tokenizer):
# Word boundaries decode to spaces (BERT-style), so attached
# punctuation becomes space-separated: "tres." -> "tres ."
text = "gallia est omnis divisa in partes tres."
ids = tokenizer.encode(text, add_special_tokens=False)
decoded = tokenizer.decode(ids)
assert decoded == "gallia est omnis divisa in partes tres ."
def test_decode_uppercase_lossy(self, tokenizer):
"""Uppercase input decodes to lowercase (lowercasing is lossy)."""
ids = tokenizer.encode("Gallia", add_special_tokens=False)
decoded = tokenizer.decode(ids)
assert decoded == "gallia"
class TestLowercasing:
"""Verify do_lower_case=True matches original Latin BERT behavior."""
def test_case_insensitive_ids(self, tokenizer):
"""Uppercase and lowercase input must produce identical IDs."""
assert (tokenizer.encode("gallia", add_special_tokens=False)
== tokenizer.encode("Gallia", add_special_tokens=False))
assert (tokenizer.encode("roma", add_special_tokens=False)
== tokenizer.encode("ROMA", add_special_tokens=False))
def test_no_codepoint_escapes(self, tokenizer):
"""Uppercase letters should not produce \\<ordinal>; escape sequences."""
tokens = tokenizer.tokenize("Cytherea")
# Should be clean subwords, not ['\\', '67', ';', ...]
assert tokens[0] != "\\"
assert all(not t.isdigit() or len(t) > 2 for t in tokens)
def test_reasonable_expansion_ratio(self, tokenizer):
"""With lowercasing, proper nouns should not explode into codepoint escapes.
Whitespace-splitting means no inter-word space escapes, so every
subtoken belongs to a word.
"""
text = "Cytherea Camenis Roma Gallia"
tokens = tokenizer.tokenize(text)
# No space-escape tokens should be present at all
assert all(t not in ("\\", "32", ";_") for t in tokens)
ratio = len(tokens) / len(text.split())
assert ratio < 2.5, f"Word expansion ratio {ratio:.1f}x is too high"
def test_do_lower_case_false(self):
"""With do_lower_case=False, uppercase chars are escaped (old behavior)."""
from latincy_latinbert import LatinBertTokenizer
tok = LatinBertTokenizer(vocab_file=VOCAB_FILE, do_lower_case=False)
tokens = tok.tokenize("Cytherea")
# First token should be backslash escape for uppercase C
assert tokens[0] == "\\"
def test_do_lower_case_default_true(self, tokenizer):
"""Default tokenizer has do_lower_case=True."""
assert tokenizer.do_lower_case is True
class TestSaveLoad:
def test_save_and_reload(self, tokenizer, tmp_path):
tokenizer.save_pretrained(tmp_path)
from latincy_latinbert import LatinBertTokenizer
loaded = LatinBertTokenizer.from_pretrained(tmp_path)
text = "Gallia est omnis divisa in partes tres"
assert tokenizer.encode(text) == loaded.encode(text)
def test_vocab_file_saved(self, tokenizer, tmp_path):
tokenizer.save_pretrained(tmp_path)
assert (tmp_path / "latin.subword.encoder").exists()
def test_config_saved(self, tokenizer, tmp_path):
tokenizer.save_pretrained(tmp_path)
assert (tmp_path / "tokenizer_config.json").exists()
|