fix(dictionary): strip Latin romanization debris from Russian translations and sanitize dictionary cache
This commit is contained in:
@@ -432,6 +432,22 @@ def repair_mismatched_cefr_contexts(conn: sqlite3.Connection):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Clean up any leftover Latin transliteration artifacts from offline_dictionary
|
||||
try:
|
||||
dirty_check = conn.execute(
|
||||
"SELECT 1 FROM offline_dictionary WHERE translation GLOB '*[a-zA-Záéíóúýšžčñ/\\\\]*' LIMIT 1"
|
||||
).fetchone()
|
||||
if dirty_check:
|
||||
rows = conn.execute(
|
||||
"SELECT rowid, translation FROM offline_dictionary WHERE translation GLOB '*[a-zA-Záéíóúýšžčñ/\\\\]*'"
|
||||
).fetchall()
|
||||
from app.dictionary import clean_russian_translation
|
||||
cleaned = [(clean_russian_translation(r[1]), r[0]) for r in rows]
|
||||
conn.executemany("UPDATE offline_dictionary SET translation = ? WHERE rowid = ?", cleaned)
|
||||
conn.execute("DELETE FROM dictionary_cache")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _seed_offline_dictionary(conn: sqlite3.Connection):
|
||||
"""Seeds the offline dictionary from data/offline_dict.tsv.gz if the table is empty."""
|
||||
@@ -452,12 +468,14 @@ def _seed_offline_dictionary(conn: sqlite3.Connection):
|
||||
return
|
||||
|
||||
import gzip
|
||||
from app.dictionary import clean_russian_translation
|
||||
|
||||
batch = []
|
||||
with gzip.open(archive_path, "rt", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
parts = line.rstrip("\r\n").split("\t")
|
||||
if len(parts) == 4:
|
||||
parts[1] = clean_russian_translation(parts[1])
|
||||
batch.append(parts)
|
||||
if len(batch) >= 10000:
|
||||
conn.executemany(
|
||||
|
||||
Reference in New Issue
Block a user